[
  {
    "path": ".editorconfig",
    "content": "# editorconfig.org\n\nroot = true\n\n[*]\ncharset = utf-8\nend_of_line = lf\ninsert_final_newline = true\nindent_style = space\ntrim_trailing_whitespace = true\nmax_line_length = 100\n\n[Dockerfile]\nindent_size = 4\n\n[{Makefile,*.mk}]\nindent_style = tab\nindent_size = 8\n\n[*.rb]\nindent_size = 4\n\n[*.rs]\nindent_size = 4\n\n[*.s]\nindent_style = tab\nindent_size = 8\n\n[*.sh]\nindent_size = 4\n\n[*.toml]\nindent_size = 4\n\n[*.{yml,yaml}]\nindent_size = 2\n"
  },
  {
    "path": ".githooks/pre-commit",
    "content": "#!/usr/bin/env ruby\n# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire_relative '../utils/devtool/copyright'\n\ndef copyright_check(staged_files)\n    source_files_exts = ['.S', '.rs', '.rb']\n\n    staged_files = staged_files.select do |f|\n        next if f.include?('build.rs')\n        next if f.include?('boot_test_string.rb')\n\n        f.include?('Makefile') ||\n            f.include?('Dockerfile') ||\n            source_files_exts.include?(File.extname(f))\n    end\n    return true if staged_files.empty?\n\n    copyright_check_files(staged_files)\nend\n\n## -------------------------------------------------------------------------------------------------\n## Execution starts here\n## -------------------------------------------------------------------------------------------------\nstaged_files = `git --no-pager diff --name-only --cached --diff-filter=d`.split(/\\n/)\nroot_dir = `git rev-parse --show-toplevel`.strip\n\n# Copyright must be fixed manually.\nexit(1) unless copyright_check(staged_files)\n\n# Brute-force format. Don't care if it affects non-staged files as well, since we only add back the\n# staged ones.\nDir.chdir(root_dir) { system('ruby utils/devtool.rb fmt') }\n\nstaged_files.each { |f| system(\"git add #{f}\") }\nexit(0)\n"
  },
  {
    "path": ".github/FUNDING.yml",
    "content": "custom: [\"https://github.com/rust-embedded/rust-raspberrypi-OS-tutorials/blob/master/SPONSORING.md\"]\n"
  },
  {
    "path": ".github/pull_request_template.md",
    "content": "### Description\n\n<Please describe the issues fixed by this PR>\n\nRelated Issue: <Insert link here if applicable>\n\n### Pre-commit steps\n\n - [ ] Tested on QEMU and real HW Rasperry Pi.\n     - Not needed if it is just a README change or similar.\n - [ ] Ran `./contributor_setup.sh` followed by `./devtool ready_for_publish`\n     - You'll need `Ruby` with `Bundler` and `NPM` installed locally.\n     - If no Rust-related files were changed, `./devtool ready_for_publish_no_rust` can be used instead (faster).\n     - This step is optional, but much appreciated if done.\n"
  },
  {
    "path": ".github/workflows/build_rpi3.yml",
    "content": "name: BSP-RPi3\n\non:\n    push:\n        branches:\n            - master\n        paths-ignore:\n            - \"utils/**\"\n            - \"doc/**\"\n            - \"docker/**\"\n    pull_request:\n        branches:\n            - master\n        paths-ignore:\n            - \"utils/**\"\n            - \"doc/**\"\n            - \"docker/**\"\n    schedule:\n        - cron: \"0 5 * * *\"\n\njobs:\n    build:\n        name: Build kernels\n        runs-on: ubuntu-22.04\n\n        steps:\n            - uses: actions/checkout@v3\n            - name: Set up Ruby\n              uses: ruby/setup-ruby@v1\n            - name: Set up Rust nightly\n              run: |\n                  cargo install cargo-binutils rustfilt\n            - name: Set up Ruby\n              run: |\n                  gem install bundler\n                  bundle config set without 'uart'\n                  bundle install --retry 3\n            - name: Run\n              run: |\n                  BSP=rpi3 bundle exec ruby utils/devtool.rb make\n"
  },
  {
    "path": ".github/workflows/build_rpi4.yml",
    "content": "name: BSP-RPi4\n\non:\n    push:\n        branches:\n            - master\n        paths-ignore:\n            - \"utils/**\"\n            - \"doc/**\"\n            - \"docker/**\"\n    pull_request:\n        branches:\n            - master\n        paths-ignore:\n            - \"utils/**\"\n            - \"doc/**\"\n            - \"docker/**\"\n    schedule:\n        - cron: \"0 5 * * *\"\n\njobs:\n    build:\n        name: Build kernels\n        runs-on: ubuntu-22.04\n\n        steps:\n            - uses: actions/checkout@v3\n            - name: Set up Ruby\n              uses: ruby/setup-ruby@v1\n            - name: Set up Rust nightly\n              run: |\n                  cargo install cargo-binutils rustfilt\n            - name: Set up Ruby\n              run: |\n                  gem install bundler\n                  bundle config set without 'uart'\n                  bundle install --retry 3\n            - name: Run\n              run: |\n                  BSP=rpi4 bundle exec ruby utils/devtool.rb make\n"
  },
  {
    "path": ".github/workflows/sanity.yml",
    "content": "name: Various Sanity Checks\n\non:\n    push:\n        branches:\n            - master\n    pull_request:\n        branches:\n            - master\n\njobs:\n    build:\n        name: Various Sanity Checks\n        runs-on: ubuntu-22.04\n\n        steps:\n            - uses: actions/checkout@v3\n            - name: Set up Node\n              uses: actions/setup-node@v1\n              with:\n                  node-version: \"16\"\n            - name: Set up Ruby\n              uses: ruby/setup-ruby@v1\n            - name: Set up Rust nightly\n              run: |\n                  rustup component add clippy\n            - name: Set up Bundler\n              run: |\n                  gem install bundler\n                  bundle config set without 'uart'\n                  bundle install --retry 3\n            - name: Set up Prettier\n              run: |\n                  npm install prettier\n            - name: Setup misspell\n              run: |\n                  curl -L -o ./install-misspell.sh https://raw.githubusercontent.com/client9/misspell/master/install-misspell.sh\n                  sh ./install-misspell.sh -b .vendor\n            - name: Run checks\n              run: |\n                  BSP=rpi3 bundle exec ruby utils/devtool.rb clippy\n                  BSP=rpi4 bundle exec ruby utils/devtool.rb clippy\n                  bundle exec ruby utils/devtool.rb copyright\n                  bundle exec ruby utils/devtool.rb fmt_check\n                  bundle exec ruby utils/devtool.rb misspell\n                  bundle exec ruby utils/devtool.rb rubocop\n"
  },
  {
    "path": ".github/workflows/test_integration.yml",
    "content": "name: Integration-Tests\n\non:\n    push:\n        branches:\n            - master\n        paths-ignore:\n            - \"utils/**\"\n            - \"doc/**\"\n            - \"docker/**\"\n    pull_request:\n        branches:\n            - master\n        paths-ignore:\n            - \"utils/**\"\n            - \"doc/**\"\n            - \"docker/**\"\n    schedule:\n        - cron: \"0 5 * * *\"\n\njobs:\n    build:\n        name: Run integration tests\n        runs-on: ubuntu-22.04\n\n        steps:\n            - uses: actions/checkout@v3\n            - name: Set up Ruby\n              uses: ruby/setup-ruby@v1\n            - name: Set up Rust nightly\n              run: |\n                  cargo install cargo-binutils rustfilt\n            - name: Set up Ruby\n              run: |\n                  gem install bundler\n                  bundle config set without 'uart'\n                  bundle install --retry 3\n            - name: Run\n              run: |\n                  bundle exec ruby utils/devtool.rb test_integration\n"
  },
  {
    "path": ".github/workflows/test_unit.yml",
    "content": "name: Boot-and-Unit-Tests\n\non:\n    push:\n        branches:\n            - master\n        paths-ignore:\n            - \"utils/**\"\n            - \"doc/**\"\n            - \"docker/**\"\n    pull_request:\n        branches:\n            - master\n        paths-ignore:\n            - \"utils/**\"\n            - \"doc/**\"\n            - \"docker/**\"\n    schedule:\n        - cron: \"0 5 * * *\"\n\njobs:\n    build:\n        name: Run boot and unit tests\n        runs-on: ubuntu-22.04\n\n        steps:\n            - uses: actions/checkout@v3\n            - name: Set up Ruby\n              uses: ruby/setup-ruby@v1\n            - name: Set up Rust nightly\n              run: |\n                  cargo install cargo-binutils rustfilt\n            - name: Set up Ruby\n              run: |\n                  gem install bundler\n                  bundle config set without 'uart'\n                  bundle install --retry 3\n            - name: Run\n              run: |\n                  bundle exec ruby utils/devtool.rb test_boot\n                  bundle exec ruby utils/devtool.rb test_unit\n"
  },
  {
    "path": ".github/workflows/test_xtra.yml",
    "content": "name: Xtra-Tests\n\non:\n    push:\n        branches:\n            - master\n        paths-ignore:\n            - \"utils/**\"\n            - \"doc/**\"\n            - \"docker/**\"\n    pull_request:\n        branches:\n            - master\n        paths-ignore:\n            - \"utils/**\"\n            - \"doc/**\"\n            - \"docker/**\"\n    schedule:\n        - cron: \"0 5 * * *\"\n\njobs:\n    build:\n        name: Run xtra tests\n        runs-on: ubuntu-22.04\n\n        steps:\n            - uses: actions/checkout@v3\n            - name: Set up Ruby\n              uses: ruby/setup-ruby@v1\n            - name: Set up Rust nightly\n              run: |\n                  cargo install cargo-binutils\n            - name: Set up Ruby\n              run: |\n                  gem install bundler\n                  bundle config set without 'uart'\n                  bundle install --retry 3\n            - name: Run\n              run: |\n                  bundle exec ruby utils/devtool.rb make_xtra\n"
  },
  {
    "path": ".gitignore",
    "content": "**/target/*\n**/.gdb_history\n**/kernel8.img\n\nnode_modules\n.bundle\n.vendor\n\nGemfile.lock\npackage*.json\n"
  },
  {
    "path": ".prettierignore",
    "content": "**/*\n!**/*/\n\n**/target/\nnode_modules/\n.vendor/\n\n!*.json\n!*.yml\n"
  },
  {
    "path": ".prettierrc.json",
    "content": "{\n    \"printWidth\": 100,\n    \"tabWidth\": 4,\n    \"useTabs\": false,\n    \"semi\": true,\n    \"singleQuote\": false,\n    \"trailingComma\": \"es5\",\n    \"bracketSpacing\": true,\n    \"jsxBracketSameLine\": false,\n    \"arrowParens\": \"always\",\n    \"requirePragma\": false,\n    \"insertPragma\": false,\n    \"proseWrap\": \"preserve\",\n    \"endOfLine\": \"auto\",\n    \"overrides\": [\n        {\n            \"files\": \"*.rs\",\n            \"options\": {\n                \"printWidth\": 100,\n                \"tabWidth\": 4,\n                \"useTabs\": false,\n                \"semi\": true,\n                \"singleQuote\": false,\n                \"trailingComma\": \"es5\",\n                \"bracketSpacing\": true,\n                \"jsxBracketSameLine\": false,\n                \"arrowParens\": \"always\",\n                \"requirePragma\": false,\n                \"insertPragma\": false,\n                \"proseWrap\": \"preserve\",\n                \"endOfLine\": \"auto\"\n            }\n        }\n    ]\n}\n"
  },
  {
    "path": ".rubocop.yml",
    "content": "# The behavior of RuboCop can be controlled via the .rubocop.yml\n# configuration file. It makes it possible to enable/disable\n# certain cops (checks) and to alter their behavior if they accept\n# any parameters. The file can be placed either in your home\n# directory or in some project directory.\n#\n# RuboCop will start looking for the configuration file in the directory\n# where the inspected file is and continue its way up to the root directory.\n#\n# See https://github.com/rubocop-hq/rubocop/blob/master/manual/configuration.md\n\nLayout/IndentationWidth:\n    Width: 4\n    AllowedPatterns: ['^\\s*module']\n\nLayout/LineLength:\n    Max: 100\n\nLint/UnusedMethodArgument:\n    AutoCorrect: False\n\nMetrics/AbcSize:\n    Max: 25\n\nMetrics/ClassLength:\n    Enabled: false\n\nMetrics/MethodLength:\n    Max: 20\n\nAllCops:\n    NewCops: enable\n"
  },
  {
    "path": ".ruby-version",
    "content": "3.0.2\n"
  },
  {
    "path": ".rustfmt.toml",
    "content": "newline_style = \"Unix\"\nedition = \"2021\"\nimports_granularity = \"Crate\"\nformat_code_in_doc_comments = true\nnormalize_comments = true\nwrap_comments = true\ncomment_width = 100\n"
  },
  {
    "path": ".vscode/settings.json",
    "content": "{\n    \"editor.formatOnSave\": true,\n    \"editor.rulers\": [100],\n    \"rust-analyzer.cargo.target\": \"aarch64-unknown-none-softfloat\",\n    \"rust-analyzer.cargo.features\": [\"bsp_rpi3\"],\n    \"rust-analyzer.checkOnSave.allTargets\": false,\n    \"rust-analyzer.checkOnSave.extraArgs\": [\"--lib\", \"--bins\"],\n    \"rust-analyzer.lens.debug\": false,\n    \"rust-analyzer.lens.run\": false\n}\n"
  },
  {
    "path": "00_before_we_start/README.CN.md",
    "content": "# 在我们开始之前\n\n下面的文本内容是1：1 复制的文档。在每个教程的内核主要源代码文档的头部。它描述了源代码的主要结构并且试着去传达各个方法背后的理念。请阅读文章以便你能熟悉你在教程中将会遇到的东西。这将帮助你更好的浏览代码和理解不同章节的区别和增量内容。\n\n另请注意，以下文字将引用源代码文件（例如`**/memory.rs`）或在本教程的第一批教程中尚不存在的功能。 它们将被逐渐添加到本教程。\n\n玩的开心！\n\n## 代码组织和结构\n\n代码被划分成不同的*模块*，每个模块代表一个内核典型的子系统。子系统的顶层模块文件直接位于`src`文件夹中。例如，`src/memory.rs`这个包含了所有与内存管理有关的代码。\n\n## 处理器架构代码的可见性\n\n有的内核的子系统是基于特定目标架构的处理器的低级代码。对于每种支持的处理器架构，都有一个子文件夹在`src/_arch`中。例如，`src/_arch/aarch64`\n\n子模块系统在每个处理器架构的`src`文件夹中。例如，属于内核内存子系统(`src/memory.rs`)的代码将会被放在`src/_arch/aarch64/memory.rs`.后一个文件将被直接包含和重新导出到`src/memory.rs`中，以便架构代码部分是明显的遵循代码模块化。这意味着在`src/_arch/aarch64/memory.rs`中定义的公共函数foo（）。仅可通过`crate :: memory :: foo（）`访问。\n\n`_arch`中的`_`表示此文件夹不属于标准模块层次结构。而是使用`＃[path =“ _ arch/xxx/yyy.rs”]`属性将其内容有条件地引入各自的文件中。\n\n## BSP 代码\n`BSP` 表示 Board Support Package。`BSP`代码在`src/bsp.rs`中，而且包含目标主板特殊的定义和功能。这些是诸如主板的内存映射或相应主板上的设备驱动程序实例之类的东西。\n\n就像处理器架构代码一样，`BSP`代码模块架构也是试着镜像`kernel`的子系统模块，但是这次它没有明显地包含和重新导出。这意味着必须从bsp名称空间开始调用提供的所有内容，例如，`bsp::driver::driver_manager()`。\n\n## 内核接口\n\n`arch`和`bsp`都包含根据实际目标和主板不同而编译的代码。例如，`interrupt controller`对于硬件`Raspberry Pi 3`和`Raspberry Pi 4`是不同的，但是我们想让`kernel`剩下的代码可以轻松地适配它们。\n\n为了在`arch`，`bsp`和`generic kernel code`之间提供一个清晰的抽象，`interface`特征在*在可能的情况下*和*在可能的地方*被提供。它们在各自的子系统模块中定义，并有助于将程序的调用方法体现到接口，而不是具体实现上。例如，有一个通用IRQ处理接口，由两个树莓派不同的中断控制器驱动将实现的，并且仅将接口导出到`kernel`剩下的代码中。\n\n```\n        +-------------------+\n        | Interface (Trait) |\n        |                   |\n        +--+-------------+--+\n           ^             ^\n           |             |\n           |             |\n+----------+--+       +--+----------+\n| kernel code |       |  bsp code   |\n|             |       |  arch code  |\n+-------------+       +-------------+\n```\n\n# 总结\n对于一个逻辑`kernel`子系统，相应的代码可以分布在几个物理位置。这里有个内存子系统的例子：\n- `src/memory.rs` and `src/memory/**/*`\n  - 与目标处理器体系结构和“BSP”特性无关的通用代码。\n    - 例如： 将大块内存归零的函数\n  - 通过`arch`和`BSP`代码实现的内存子系统接口。\n    - 例如： 一个`MMU`接口定义的`MMU`功能原型\n - `src/bsp/__board_name__/memory.rs` 和 `src/bsp/__board_name__/memory/**/*`\n   - `BSP` 具体代码.\n   - 例如: 主板内存映射 (DRAM物理地址 和 MMIO 设备).\n- `src/_arch/__arch_name__/memory.rs` 和 `src/_arch/__arch_name__/memory/**/*`\n  - 处理器架构的具体代码。\n  - 例如:为`__arch_name__` 处理器架构实现的`MMU`接口。\n  \n从一个命名空间角度来看，内存子系统代码在：\n  - `crate::memory::*`\n  - `crate::bsp::memory::*`\n"
  },
  {
    "path": "00_before_we_start/README.ES.md",
    "content": "# Antes de comenzar\n\nEl texto a continuación es una copia 1:1 de la documentación que \npuede ser encontrada al principio del archivo del código fuente \ndel núcleo (kernel) en cada tutorial. Esta describe la estructura \ngeneral del código fuente, e intenta transmitir la filosofía detrás\nde cada enfoque. Por favor leélo para familiarizarte\ncon lo que te vas a encontrar durante los tutoriales. Te ayudará a navegar el código de una mejor manera y a entender las diferencias y agregados entre los diferentes tutoriales.\n\nPor favor, nota también que el siguiente texto va a referenciar\nlos archivos del código fuente (p. e.j. `**/memory.rs`) o funciones que\nno van a existir aún en los primeros tutoriales. Estos archivos serán agregados \na medida que el tutorial avance.\n\n¡Diviértanse!\n\n# La estructura del código y la arquitectura\n\nEl código está dividido en diferentes módulos donde cada uno representa un\nsubsistema típico del `kernel (núcleo)`. Los módulos de más alto nivel de los subsistemas se encuentran directamente en la carpeta `src`.\nPor ejemplo, `src/memory.rs` contiene el código que está relacionado\ncon el manejo de memoria.\n\n## Visibilidad del código de arquitectura del procesador\n\nAlgunos de los subsistemas del `núcleo (kernel)` dependen del código de bajo nivel (low-level) dedicado a la arquitectura del procesador.\nPor cada arquitectura de procesador que está soportada, existe una subcarpeta en `src/_arch`, por ejemplo, `src/_arch/aarch64`.\n\nLa carpeta de arquitecturas refleja los módulos del subsistema establecidos en `src`. Por ejemplo, el código de arquitectura que pertenece al subsistema MMU del `núcleo(kernel)` (`src/memory/mmu.rs`) irá dentro de (`src/_arch/aarch64/memory/mmu.rs`).\nEste archivo puede ser cargado como un módulo en `src/memory/mmu.rs` usando el `path attribute` (atributo de ruta). Usualmente, el nombre del módulo elegido es el nombre del módulo genérico con el prefijo de `arch_`\n\nPor ejemplo, esta es la parte superior de `src/memory/mmu.rs`:\n\n```\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/memory/mmu.rs\"]\nmod arch_mmu;\n```\n\nEn muchas ocasiones, los elementos de `arch_module` serán reexportados públicamente por el módulo principal.\nDe esta manera, cada módulo específico de la arquitectura puede proporcionar su implementación de un elemento, mientras que el *invocante* no debe de preocuparse por la arquitectura que se ha compilado condicionalmente.\n\n## Código BSP\n\n`BSP` significa Board Support Package (Paquete de Soporte de la Placa).\nEl código `BSP` está dentro de `src/bsp.rs` y contiene las definiciones y funciones de la placa base específica elegida. \nEntre estas cosas se encuentran diferentes elementos como el mapa de memoria de la placa o instancias de controladores para dispositivos que se presentan en la placa elegida.\n\nJusto como el código de la arquitectura del procesador, la estructura del módulo del código `BSP` trata de reflejar los módulos del subsistema del `núcleo (kernel)`, pero no ocurre una reexportación esta vez. Eso significa que lo que sea que se esté proporcionando debe ser llamado empezando por el *namespace* (espacio de nombres) de `bsp`, p. ej. `bsp::driver::driver_manager()`.\n\n## La interfaz del núcleo (kernel)\n\nEl `arch` y el `bsp` contienen código que se compilará condicionalmente dependiendo del procesador y placa actual para la que se compila el núcleo (kernel).\nPor ejemplo, el hardware de control de interrupciones de la `Raspberry Pi 3` y la  `Raspberry Pi 4` es diferente, pero nosotros queremos que el resto del código del kernel funcione correctamente con cualquiera de los dos sin mucha complicación.\n\nPara poder dar una limpia abstracción entre `arch`, `bsp` y código genérico del núcleo, los rasgos de `interface` se proporcionan *siempre y cuando tenga sentido*. Son definidos en su módulo de subsistema correspondiente y ayuda a reforzar el patrón de programar con respecto a una interfaz, sin importar la implementación concreta.\n\nPor ejemplo, habrá una *IRQ handling interface* (interfaz de manejo de interrupciones) común, el cual los dos diferentes controladores de ambas `Raspberry` implementarán, y solo exportarán la interfaz común al resto del `núcleo (kernel)`.\n\n```\n        +-------------------+\n        | Interface (Trait) |\n        |                   |\n        +--+-------------+--+\n           ^             ^\n           |             |\n           |             |\n+----------+--+       +--+----------+\n| kernel code |       |  bsp code   |\n|             |       |  arch code  |\n+-------------+       +-------------+\n```\n\n# Resumen\n\nPara un subsistema lógico del `núcleo (kernel)`, el código correspondiente puede ser distribuido sobre diferentes localizaciones físicas. Aquí un ejemplo para el subsistema de memoria:\n\n- `src/memory.rs` y `src/memory/**/*`\n  \n  - Código común que es independiente de la arquitectura del procesador de destino y las características de la placa (`BSP`).\n    - Ejemplo: Una función para poner a cero un trozo de memoria.\n  - Las interfaces para el subsistema de la memoria que son implementados por código de `arch` o `BSP`.\n    - Ejemplo: Una interfaz `MMU` que define prototipos de función de `MMU`.\n\n- `src/bsp/__board_name__/memory.rs` y `src/bsp/__board_name__/memory/**/*`\n  \n  - Código específico de `BSP`.\n  - Ejemplo: El mapa de memoria de la placa (direcciones físicas de DRAM y dispositivos MMIO).\n\n- `src/_arch/__arch_name__/memory.rs` y `src/_arch/__arch_name__/memory/**/*`\n  \n  - El código específico de la arquitectura del procesador.\n  - Ejemplo: Implementación de la interfaz `MMU` para la arquitectura `__arch_name__`.\n\nDesde una perspectiva de *namespace*, el código del subsistema de **memoria** vive en:\n\n- `crate::memory::*`\n- `crate::bsp::memory::*`\n\n# Flujo de Boot / Boot flow\n\n1. El punto de entrada del núcleo (kernel) es la función `cpu::boot::arch_boot::_start()`.\n   - Está implementado en `src/_arch/__arch_name__/cpu/boot.s`.\n"
  },
  {
    "path": "00_before_we_start/README.md",
    "content": "# Before we start\n\nThe following text is a 1:1 copy of the documentation that can be found at the top of the kernel's\nmain source code file in each tutorial. It describes the general structure of the source code, and\ntries to convey the philosophy behind the respective approach. Please read it to make yourself\nfamiliar with what you will encounter during the tutorials. It will help you to navigate the code\nbetter and understand the differences and additions between the separate tutorials.\n\nPlease also note that the following text will reference source code files (e.g. `**/memory.rs`) or\nfunctions that won't exist yet in the first bunch of the tutorials. They will be added gradually as\nthe tutorials advance.\n\nHave fun!\n\n# Code organization and architecture\n\nThe code is divided into different *modules*, each representing a typical **subsystem** of the\n`kernel`. Top-level module files of subsystems reside directly in the `src` folder. For example,\n`src/memory.rs` contains code that is concerned with all things memory management.\n\n## Visibility of processor architecture code\n\nSome of the `kernel`'s subsystems depend on low-level code that is specific to the target processor\narchitecture. For each supported processor architecture, there exists a subfolder in `src/_arch`,\nfor example, `src/_arch/aarch64`.\n\nThe architecture folders mirror the subsystem modules laid out in `src`. For example, architectural\ncode that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go into\n`src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in `src/memory/mmu.rs`\nusing the `path attribute`. Usually, the chosen module name is the generic module's name prefixed\nwith `arch_`.\n\nFor example, this is the top of `src/memory/mmu.rs`:\n\n```\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/memory/mmu.rs\"]\nmod arch_mmu;\n```\n\nOften times, items from the `arch_ module` will be publicly reexported by the parent module. This\nway, each architecture specific module can provide its implementation of an item, while the caller\nmust not be concerned which architecture has been conditionally compiled.\n\n## BSP code\n\n`BSP` stands for Board Support Package. `BSP` code is organized under `src/bsp.rs` and contains\ntarget board specific definitions and functions. These are things such as the board's memory map or\ninstances of drivers for devices that are featured on the respective board.\n\nJust like processor architecture code, the `BSP` code's module structure tries to mirror the\n`kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is provided\nmust be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`.\n\n## Kernel interfaces\n\nBoth `arch` and `bsp` contain code that is conditionally compiled depending on the actual target and\nboard for which the kernel is compiled. For example, the `interrupt controller` hardware of the\n`Raspberry Pi 3` and the `Raspberry Pi 4` is different, but we want the rest of the `kernel` code to\nplay nicely with any of the two without much hassle.\n\nIn order to provide a clean abstraction between `arch`, `bsp` and `generic kernel code`, `interface`\ntraits are provided *whenever possible* and *where it makes sense*. They are defined in the\nrespective subsystem module and help to enforce the idiom of *program to an interface, not an\nimplementation*. For example, there will be a common IRQ handling interface which the two different\ninterrupt controller `drivers` of both Raspberrys will implement, and only export the interface to\nthe rest of the `kernel`.\n\n```\n        +-------------------+\n        | Interface (Trait) |\n        |                   |\n        +--+-------------+--+\n           ^             ^\n           |             |\n           |             |\n+----------+--+       +--+----------+\n| kernel code |       |  bsp code   |\n|             |       |  arch code  |\n+-------------+       +-------------+\n```\n\n# Summary\n\nFor a logical `kernel` subsystem, corresponding code can be distributed over several physical\nlocations. Here is an example for the **memory** subsystem:\n\n- `src/memory.rs` and `src/memory/**/*`\n  - Common code that is agnostic of target processor architecture and `BSP` characteristics.\n    - Example: A function to zero a chunk of memory.\n  - Interfaces for the memory subsystem that are implemented by `arch` or `BSP` code.\n    - Example: An `MMU` interface that defines `MMU` function prototypes.\n- `src/bsp/__board_name__/memory.rs` and `src/bsp/__board_name__/memory/**/*`\n  - `BSP` specific code.\n  - Example: The board's memory map (physical addresses of DRAM and MMIO devices).\n- `src/_arch/__arch_name__/memory.rs` and `src/_arch/__arch_name__/memory/**/*`\n  - Processor architecture specific code.\n  - Example: Implementation of the `MMU` interface for the `__arch_name__` processor\n    architecture.\n\nFrom a namespace perspective, **memory** subsystem code lives in:\n\n- `crate::memory::*`\n- `crate::bsp::memory::*`\n\n# Boot flow\n\n1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.\n    - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.\n\n\n"
  },
  {
    "path": "01_wait_forever/.vscode/settings.json",
    "content": "{\n    \"editor.formatOnSave\": true,\n    \"editor.rulers\": [100],\n    \"rust-analyzer.cargo.target\": \"aarch64-unknown-none-softfloat\",\n    \"rust-analyzer.cargo.features\": [\"bsp_rpi3\"],\n    \"rust-analyzer.checkOnSave.allTargets\": false,\n    \"rust-analyzer.checkOnSave.extraArgs\": [\"--bins\"],\n    \"rust-analyzer.lens.debug\": false,\n    \"rust-analyzer.lens.run\": false\n}\n"
  },
  {
    "path": "01_wait_forever/Cargo.toml",
    "content": "[package]\nname = \"mingo\"\nversion = \"0.1.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n\n[profile.release]\nlto = true\n\n[features]\ndefault = []\nbsp_rpi3 = []\nbsp_rpi4 = []\n\n[[bin]]\nname = \"kernel\"\npath = \"src/main.rs\"\n\n##--------------------------------------------------------------------------------------------------\n## Dependencies\n##--------------------------------------------------------------------------------------------------\n\n[dependencies]"
  },
  {
    "path": "01_wait_forever/Makefile",
    "content": "## SPDX-License-Identifier: MIT OR Apache-2.0\n##\n## Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\ninclude ../common/docker.mk\ninclude ../common/format.mk\ninclude ../common/operating_system.mk\n\n##--------------------------------------------------------------------------------------------------\n## Optional, user-provided configuration values\n##--------------------------------------------------------------------------------------------------\n\n# Default to the RPi3.\nBSP ?= rpi3\n\n\n\n##--------------------------------------------------------------------------------------------------\n## BSP-specific configuration values\n##--------------------------------------------------------------------------------------------------\nQEMU_MISSING_STRING = \"This board is not yet supported for QEMU.\"\n\nifeq ($(BSP),rpi3)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE = raspi3\n    QEMU_RELEASE_ARGS = -d in_asm -display none\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    LD_SCRIPT_PATH    = $(shell pwd)/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a53\nelse ifeq ($(BSP),rpi4)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE =\n    QEMU_RELEASE_ARGS = -d in_asm -display none\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    LD_SCRIPT_PATH    = $(shell pwd)/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a72\nendif\n\n# Export for build.rs.\nexport LD_SCRIPT_PATH\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets and Prerequisites\n##--------------------------------------------------------------------------------------------------\nKERNEL_MANIFEST      = Cargo.toml\nKERNEL_LINKER_SCRIPT = kernel.ld\nLAST_BUILD_CONFIG    = target/$(BSP).build_config\n\nKERNEL_ELF      = target/$(TARGET)/release/kernel\n# This parses cargo's dep-info file.\n# https://doc.rust-lang.org/cargo/guide/build-cache.html#dep-info-files\nKERNEL_ELF_DEPS = $(filter-out %: ,$(file < $(KERNEL_ELF).d)) $(KERNEL_MANIFEST) $(LAST_BUILD_CONFIG)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Command building blocks\n##--------------------------------------------------------------------------------------------------\nRUSTFLAGS = $(RUSTC_MISC_ARGS)                   \\\n    -C link-arg=--library-path=$(LD_SCRIPT_PATH) \\\n    -C link-arg=--script=$(KERNEL_LINKER_SCRIPT)\n\nRUSTFLAGS_PEDANTIC = $(RUSTFLAGS) \\\n    -D warnings                   \\\n    -D missing_docs\n\nFEATURES      = --features bsp_$(BSP)\nCOMPILER_ARGS = --target=$(TARGET) \\\n    $(FEATURES)                    \\\n    --release\n\nRUSTC_CMD   = cargo rustc $(COMPILER_ARGS)\nDOC_CMD     = cargo doc $(COMPILER_ARGS)\nCLIPPY_CMD  = cargo clippy $(COMPILER_ARGS)\nOBJCOPY_CMD = rust-objcopy \\\n    --strip-all            \\\n    -O binary\n\nEXEC_QEMU = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE)\n\n##------------------------------------------------------------------------------\n## Dockerization\n##------------------------------------------------------------------------------\nDOCKER_CMD          = docker run -t --rm -v $(shell pwd):/work/tutorial -w /work/tutorial\nDOCKER_CMD_INTERACT = $(DOCKER_CMD) -i\n\n# DOCKER_IMAGE defined in include file (see top of this file).\nDOCKER_QEMU  = $(DOCKER_CMD_INTERACT) $(DOCKER_IMAGE)\nDOCKER_TOOLS = $(DOCKER_CMD) $(DOCKER_IMAGE)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: all doc qemu clippy clean readelf objdump nm check\n\nall: $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Save the configuration as a file, so make understands if it changed.\n##------------------------------------------------------------------------------\n$(LAST_BUILD_CONFIG):\n\t@rm -f target/*.build_config\n\t@mkdir -p target\n\t@touch $(LAST_BUILD_CONFIG)\n\n##------------------------------------------------------------------------------\n## Compile the kernel ELF\n##------------------------------------------------------------------------------\n$(KERNEL_ELF): $(KERNEL_ELF_DEPS)\n\t$(call color_header, \"Compiling kernel ELF - $(BSP)\")\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(RUSTC_CMD)\n\n##------------------------------------------------------------------------------\n## Generate the stripped kernel binary\n##------------------------------------------------------------------------------\n$(KERNEL_BIN): $(KERNEL_ELF)\n\t$(call color_header, \"Generating stripped binary\")\n\t@$(OBJCOPY_CMD) $(KERNEL_ELF) $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Name\")\n\t@echo $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Size\")\n\t$(call disk_usage_KiB, $(KERNEL_BIN))\n\n##------------------------------------------------------------------------------\n## Generate the documentation\n##------------------------------------------------------------------------------\ndoc:\n\t$(call color_header, \"Generating docs\")\n\t@$(DOC_CMD) --document-private-items --open\n\n##------------------------------------------------------------------------------\n## Run the kernel in QEMU\n##------------------------------------------------------------------------------\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\nqemu:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\nqemu: $(KERNEL_BIN)\n\t$(call color_header, \"Launching QEMU\")\n\t@$(DOCKER_QEMU) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\nendif\n\n##------------------------------------------------------------------------------\n## Run clippy\n##------------------------------------------------------------------------------\nclippy:\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(CLIPPY_CMD)\n\n##------------------------------------------------------------------------------\n## Clean\n##------------------------------------------------------------------------------\nclean:\n\trm -rf target $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run readelf\n##------------------------------------------------------------------------------\nreadelf: $(KERNEL_ELF)\n\t$(call color_header, \"Launching readelf\")\n\t@$(DOCKER_TOOLS) $(READELF_BINARY) --headers $(KERNEL_ELF)\n\n##------------------------------------------------------------------------------\n## Run objdump\n##------------------------------------------------------------------------------\nobjdump: $(KERNEL_ELF)\n\t$(call color_header, \"Launching objdump\")\n\t@$(DOCKER_TOOLS) $(OBJDUMP_BINARY) --disassemble --demangle \\\n                --section .text   \\\n                $(KERNEL_ELF) | rustfilt\n\n##------------------------------------------------------------------------------\n## Run nm\n##------------------------------------------------------------------------------\nnm: $(KERNEL_ELF)\n\t$(call color_header, \"Launching nm\")\n\t@$(DOCKER_TOOLS) $(NM_BINARY) --demangle --print-size $(KERNEL_ELF) | sort | rustfilt\n\n"
  },
  {
    "path": "01_wait_forever/README.CN.md",
    "content": "# 教程 01 - 一直等待（死循环）\n\n## tl;dr\n\n项目框架已经构建完成；目前代码做的仅仅是挂起CPU核心执行内核代码。\n\n-  `Makefile` 目标项：\n    - `doc`: 生成文档。\n    - `qemu`: 在 QEMU 中运行 `kernel`。\n    - `clippy`\n    - `clean`\n    - `readelf`: 检查 `ELF` 输出。\n    - `objdump`: 检查汇编。\n    - `nm`: 检查符号。\n- 代码按照 `kernel`， `arch` 和 `BSP` （板级支持包）的形式组织。\n    - 条件编译会根据用户提供的参数编译各自的  `arch` 和  `BSP` 的内容。\n- 自定义 `kernel.ld` 链接脚本.\n    - 载入地址为 `0x80_000`\n    - 目前仅有 `.text` 小节（section）。\n- `main.rs`: 重要的 [inner attributes]:\n    - `#![no_std]`, `#![no_main]`\n- 汇编函数 `_start()` 会执行  `wfe` (Wait For Event)， 并挂起所有正在执行  `_start()` 的核心。\n- 我们（必须）定义一个 `#[panic_handler]` 函数。\n    - 用于等待cpu事件的发生。\n\n[inner attributes]: https://doc.rust-lang.org/reference/attributes.html\n\n### 测试一下！\n\n在项目文件夹下调用 QEMU 并观察在 `wfe` 中CPU核心的运转情况：\n```console\n» make qemu\n[...]\nIN:\n0x00080000:  d503205f  wfe\n0x00080004:  17ffffff  b        #0x80000\n```\n"
  },
  {
    "path": "01_wait_forever/README.ES.md",
    "content": "# Tutorial 01 - Esperar infinitamente\n\n## tl;dr\n\n* Se configura la estructura que tiene el proyecto.\n\n* Se ejecuta una pequeño código hecho en ensamblador que tiene como función detener todos los núcleos del procesador que están ejecutando el kernel.\n\n## Compilar\n\n* El archivo `Makefile` permite ejecutar:\n\n  * `doc`: Genera la documentación.\n\n  * `qemu`: Ejecutar el kernel en QEMU.\n\n  * `clippy`: Analiza el código y sugiere mejoras.\n\n  * `clean`: Elimina todos los archivos generados durante la compilación, etc.\n\n  * `readelf`: Inspecciona el archivo `ELF` de salida.\n\n  * `objdump`: Inspecciona el ensamblador.\n\n  * `nm`: Inspecciona los símbolos.\n\n## Código a revisar\n\n* El script para enlazado específico para la `BSP` llamado `kernel.ld`.\n\n  * Carga la dirección en `0x8_0000`.\n\n  * Solo la sección `.text`.\n\n* `main.rs`: [Atributos internos](https://doc.rust-lang.org/reference/attributes.html) importantes:\n\n  * `#![no_std]`, `#![no_main]`.\n\n* `boot.s`: La función de ensamblador `_start()` que inicia `wfe` (Wait For Event / Esperar Hasta Un Evento), detiene todos los núcleos del procesador que están ejecutando `_start()`.\n\n* Tenemos que definir una función que funcione como `#[panic_handler]` (manejador de pánico) para que el compilador no nos cause problemas.\n\n  * Hazla `unimplemented!()` porque se eliminará ya que no está siendo usada.\n\n## Pruébalo\n\nDentro de la carpeta del proyecto, ejecuta a QEMU y mira el núcleo del procesador ejecutando `wfe` en bucle:\n\n```\n$ make qemu\n[...]\nIN:\n0x00080000:  d503205f  wfe\n0x00080004:  17ffffff  b        #0x80000\n```\n"
  },
  {
    "path": "01_wait_forever/README.md",
    "content": "# Tutorial 01 - Wait Forever\n\n## tl;dr\n\n- The project skeleton is set up.\n- A small piece of assembly code runs that just halts all CPU cores executing the kernel code.\n\n## Building\n\n- `Makefile` targets:\n    - `doc`: Generate documentation.\n    - `qemu`: Run the `kernel` in QEMU\n    - `clippy`\n    - `clean`\n    - `readelf`: Inspect the `ELF` output.\n    - `objdump`: Inspect the assembly.\n    - `nm`: Inspect the symbols.\n\n## Code to look at\n\n- `BSP`-specific `kernel.ld` linker script.\n    - Load address at `0x8_0000`\n    - Only `.text` section.\n- `main.rs`: Important [inner attributes]:\n    - `#![no_std]`, `#![no_main]`\n- `boot.s`: Assembly `_start()` function that executes `wfe` (Wait For Event), halting all cores\n  that are executing `_start()`.\n- We (have to) define a `#[panic_handler]` function to make the compiler happy.\n    - Make it `unimplemented!()` because it will be stripped out since it is not used.\n\n[inner attributes]: https://doc.rust-lang.org/reference/attributes.html\n\n### Test it\n\nIn the project folder, invoke QEMU and observe the CPU core spinning on `wfe`:\n\n```console\n$ make qemu\n[...]\nIN:\n0x00080000:  d503205f  wfe\n0x00080004:  17ffffff  b        #0x80000\n```\n"
  },
  {
    "path": "01_wait_forever/build.rs",
    "content": "use std::{env, fs, process};\n\nfn main() {\n    let ld_script_path = match env::var(\"LD_SCRIPT_PATH\") {\n        Ok(var) => var,\n        _ => process::exit(0),\n    };\n\n    let files = fs::read_dir(ld_script_path).unwrap();\n    files\n        .filter_map(Result::ok)\n        .filter(|d| {\n            if let Some(e) = d.path().extension() {\n                e == \"ld\"\n            } else {\n                false\n            }\n        })\n        .for_each(|f| println!(\"cargo:rerun-if-changed={}\", f.path().display()));\n}\n"
  },
  {
    "path": "01_wait_forever/src/_arch/aarch64/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural boot code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::boot::arch_boot\n\nuse core::arch::global_asm;\n\n// Assembly counterpart to this file.\nglobal_asm!(include_str!(\"boot.s\"));\n"
  },
  {
    "path": "01_wait_forever/src/_arch/aarch64/cpu/boot.s",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n.section .text._start\n\n//------------------------------------------------------------------------------\n// fn _start()\n//------------------------------------------------------------------------------\n_start:\n\t// Infinitely wait for events (aka \"park the core\").\n.L_parking_loop:\n\twfe\n\tb\t.L_parking_loop\n\n.size\t_start, . - _start\n.type\t_start, function\n.global\t_start\n"
  },
  {
    "path": "01_wait_forever/src/bsp/raspberrypi/kernel.ld",
    "content": "/* SPDX-License-Identifier: MIT OR Apache-2.0\n *\n * Copyright (c) 2018-2022 Andre Richter <andre.o.richter@gmail.com>\n */\n\n/* The physical address at which the the kernel binary will be loaded by the Raspberry's firmware */\n__rpi_phys_binary_load_addr = 0x80000;\n\n\nENTRY(__rpi_phys_binary_load_addr)\n\n/* Flags:\n *     4 == R\n *     5 == RX\n *     6 == RW\n */\nPHDRS\n{\n    segment_code PT_LOAD FLAGS(5);\n}\n\nSECTIONS\n{\n    . =  __rpi_phys_binary_load_addr;\n\n    /***********************************************************************************************\n    * Code\n    ***********************************************************************************************/\n    .text :\n    {\n        KEEP(*(.text._start))\n    } :segment_code\n}\n"
  },
  {
    "path": "01_wait_forever/src/bsp/raspberrypi.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Top-level BSP file for the Raspberry Pi 3 and 4.\n\n// Coming soon.\n"
  },
  {
    "path": "01_wait_forever/src/bsp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Conditional reexporting of Board Support Packages.\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod raspberrypi;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use raspberrypi::*;\n"
  },
  {
    "path": "01_wait_forever/src/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Boot code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/cpu/boot.rs\"]\nmod arch_boot;\n"
  },
  {
    "path": "01_wait_forever/src/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Processor code.\n\nmod boot;\n"
  },
  {
    "path": "01_wait_forever/src/main.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n// Rust embedded logo for `make doc`.\n#![doc(\n    html_logo_url = \"https://raw.githubusercontent.com/rust-embedded/wg/master/assets/logo/ewg-logo-blue-white-on-transparent.png\"\n)]\n\n//! The `kernel` binary.\n//!\n//! # Code organization and architecture\n//!\n//! The code is divided into different *modules*, each representing a typical **subsystem** of the\n//! `kernel`. Top-level module files of subsystems reside directly in the `src` folder. For example,\n//! `src/memory.rs` contains code that is concerned with all things memory management.\n//!\n//! ## Visibility of processor architecture code\n//!\n//! Some of the `kernel`'s subsystems depend on low-level code that is specific to the target\n//! processor architecture. For each supported processor architecture, there exists a subfolder in\n//! `src/_arch`, for example, `src/_arch/aarch64`.\n//!\n//! The architecture folders mirror the subsystem modules laid out in `src`. For example,\n//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go\n//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in\n//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic\n//! module's name prefixed with `arch_`.\n//!\n//! For example, this is the top of `src/memory/mmu.rs`:\n//!\n//! ```\n//! #[cfg(target_arch = \"aarch64\")]\n//! #[path = \"../_arch/aarch64/memory/mmu.rs\"]\n//! mod arch_mmu;\n//! ```\n//!\n//! Often times, items from the `arch_ module` will be publicly reexported by the parent module.\n//! This way, each architecture specific module can provide its implementation of an item, while the\n//! caller must not be concerned which architecture has been conditionally compiled.\n//!\n//! ## BSP code\n//!\n//! `BSP` stands for Board Support Package. `BSP` code is organized under `src/bsp.rs` and contains\n//! target board specific definitions and functions. These are things such as the board's memory map\n//! or instances of drivers for devices that are featured on the respective board.\n//!\n//! Just like processor architecture code, the `BSP` code's module structure tries to mirror the\n//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is\n//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`.\n//!\n//! ## Kernel interfaces\n//!\n//! Both `arch` and `bsp` contain code that is conditionally compiled depending on the actual target\n//! and board for which the kernel is compiled. For example, the `interrupt controller` hardware of\n//! the `Raspberry Pi 3` and the `Raspberry Pi 4` is different, but we want the rest of the `kernel`\n//! code to play nicely with any of the two without much hassle.\n//!\n//! In order to provide a clean abstraction between `arch`, `bsp` and `generic kernel code`,\n//! `interface` traits are provided *whenever possible* and *where it makes sense*. They are defined\n//! in the respective subsystem module and help to enforce the idiom of *program to an interface,\n//! not an implementation*. For example, there will be a common IRQ handling interface which the two\n//! different interrupt controller `drivers` of both Raspberrys will implement, and only export the\n//! interface to the rest of the `kernel`.\n//!\n//! ```\n//!         +-------------------+\n//!         | Interface (Trait) |\n//!         |                   |\n//!         +--+-------------+--+\n//!            ^             ^\n//!            |             |\n//!            |             |\n//! +----------+--+       +--+----------+\n//! | kernel code |       |  bsp code   |\n//! |             |       |  arch code  |\n//! +-------------+       +-------------+\n//! ```\n//!\n//! # Summary\n//!\n//! For a logical `kernel` subsystem, corresponding code can be distributed over several physical\n//! locations. Here is an example for the **memory** subsystem:\n//!\n//! - `src/memory.rs` and `src/memory/**/*`\n//!   - Common code that is agnostic of target processor architecture and `BSP` characteristics.\n//!     - Example: A function to zero a chunk of memory.\n//!   - Interfaces for the memory subsystem that are implemented by `arch` or `BSP` code.\n//!     - Example: An `MMU` interface that defines `MMU` function prototypes.\n//! - `src/bsp/__board_name__/memory.rs` and `src/bsp/__board_name__/memory/**/*`\n//!   - `BSP` specific code.\n//!   - Example: The board's memory map (physical addresses of DRAM and MMIO devices).\n//! - `src/_arch/__arch_name__/memory.rs` and `src/_arch/__arch_name__/memory/**/*`\n//!   - Processor architecture specific code.\n//!   - Example: Implementation of the `MMU` interface for the `__arch_name__` processor\n//!     architecture.\n//!\n//! From a namespace perspective, **memory** subsystem code lives in:\n//!\n//! - `crate::memory::*`\n//! - `crate::bsp::memory::*`\n//!\n//! # Boot flow\n//!\n//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.\n//!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.\n\n#![no_main]\n#![no_std]\n\nmod bsp;\nmod cpu;\nmod panic_wait;\n\n// Kernel code coming next tutorial.\n"
  },
  {
    "path": "01_wait_forever/src/panic_wait.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A panic handler that infinitely waits.\n\nuse core::panic::PanicInfo;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n#[panic_handler]\nfn panic(_info: &PanicInfo) -> ! {\n    unimplemented!()\n}\n"
  },
  {
    "path": "02_runtime_init/.vscode/settings.json",
    "content": "{\n    \"editor.formatOnSave\": true,\n    \"editor.rulers\": [100],\n    \"rust-analyzer.cargo.target\": \"aarch64-unknown-none-softfloat\",\n    \"rust-analyzer.cargo.features\": [\"bsp_rpi3\"],\n    \"rust-analyzer.checkOnSave.allTargets\": false,\n    \"rust-analyzer.checkOnSave.extraArgs\": [\"--bins\"],\n    \"rust-analyzer.lens.debug\": false,\n    \"rust-analyzer.lens.run\": false\n}\n"
  },
  {
    "path": "02_runtime_init/Cargo.toml",
    "content": "[package]\nname = \"mingo\"\nversion = \"0.2.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n\n[profile.release]\nlto = true\n\n[features]\ndefault = []\nbsp_rpi3 = []\nbsp_rpi4 = []\n\n[[bin]]\nname = \"kernel\"\npath = \"src/main.rs\"\n\n##--------------------------------------------------------------------------------------------------\n## Dependencies\n##--------------------------------------------------------------------------------------------------\n\n[dependencies]\n\n# Platform specific dependencies\n[target.'cfg(target_arch = \"aarch64\")'.dependencies]\naarch64-cpu = { version = \"9.x.x\" }\n"
  },
  {
    "path": "02_runtime_init/Makefile",
    "content": "## SPDX-License-Identifier: MIT OR Apache-2.0\n##\n## Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\ninclude ../common/docker.mk\ninclude ../common/format.mk\ninclude ../common/operating_system.mk\n\n##--------------------------------------------------------------------------------------------------\n## Optional, user-provided configuration values\n##--------------------------------------------------------------------------------------------------\n\n# Default to the RPi3.\nBSP ?= rpi3\n\n\n\n##--------------------------------------------------------------------------------------------------\n## BSP-specific configuration values\n##--------------------------------------------------------------------------------------------------\nQEMU_MISSING_STRING = \"This board is not yet supported for QEMU.\"\n\nifeq ($(BSP),rpi3)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE = raspi3\n    QEMU_RELEASE_ARGS = -d in_asm -display none\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    LD_SCRIPT_PATH    = $(shell pwd)/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a53\nelse ifeq ($(BSP),rpi4)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE =\n    QEMU_RELEASE_ARGS = -d in_asm -display none\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    LD_SCRIPT_PATH    = $(shell pwd)/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a72\nendif\n\n# Export for build.rs.\nexport LD_SCRIPT_PATH\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets and Prerequisites\n##--------------------------------------------------------------------------------------------------\nKERNEL_MANIFEST      = Cargo.toml\nKERNEL_LINKER_SCRIPT = kernel.ld\nLAST_BUILD_CONFIG    = target/$(BSP).build_config\n\nKERNEL_ELF      = target/$(TARGET)/release/kernel\n# This parses cargo's dep-info file.\n# https://doc.rust-lang.org/cargo/guide/build-cache.html#dep-info-files\nKERNEL_ELF_DEPS = $(filter-out %: ,$(file < $(KERNEL_ELF).d)) $(KERNEL_MANIFEST) $(LAST_BUILD_CONFIG)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Command building blocks\n##--------------------------------------------------------------------------------------------------\nRUSTFLAGS = $(RUSTC_MISC_ARGS)                   \\\n    -C link-arg=--library-path=$(LD_SCRIPT_PATH) \\\n    -C link-arg=--script=$(KERNEL_LINKER_SCRIPT)\n\nRUSTFLAGS_PEDANTIC = $(RUSTFLAGS) \\\n    -D warnings                   \\\n    -D missing_docs\n\nFEATURES      = --features bsp_$(BSP)\nCOMPILER_ARGS = --target=$(TARGET) \\\n    $(FEATURES)                    \\\n    --release\n\nRUSTC_CMD   = cargo rustc $(COMPILER_ARGS)\nDOC_CMD     = cargo doc $(COMPILER_ARGS)\nCLIPPY_CMD  = cargo clippy $(COMPILER_ARGS)\nOBJCOPY_CMD = rust-objcopy \\\n    --strip-all            \\\n    -O binary\n\nEXEC_QEMU = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE)\n\n##------------------------------------------------------------------------------\n## Dockerization\n##------------------------------------------------------------------------------\nDOCKER_CMD          = docker run -t --rm -v $(shell pwd):/work/tutorial -w /work/tutorial\nDOCKER_CMD_INTERACT = $(DOCKER_CMD) -i\n\n# DOCKER_IMAGE defined in include file (see top of this file).\nDOCKER_QEMU  = $(DOCKER_CMD_INTERACT) $(DOCKER_IMAGE)\nDOCKER_TOOLS = $(DOCKER_CMD) $(DOCKER_IMAGE)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: all doc qemu clippy clean readelf objdump nm check\n\nall: $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Save the configuration as a file, so make understands if it changed.\n##------------------------------------------------------------------------------\n$(LAST_BUILD_CONFIG):\n\t@rm -f target/*.build_config\n\t@mkdir -p target\n\t@touch $(LAST_BUILD_CONFIG)\n\n##------------------------------------------------------------------------------\n## Compile the kernel ELF\n##------------------------------------------------------------------------------\n$(KERNEL_ELF): $(KERNEL_ELF_DEPS)\n\t$(call color_header, \"Compiling kernel ELF - $(BSP)\")\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(RUSTC_CMD)\n\n##------------------------------------------------------------------------------\n## Generate the stripped kernel binary\n##------------------------------------------------------------------------------\n$(KERNEL_BIN): $(KERNEL_ELF)\n\t$(call color_header, \"Generating stripped binary\")\n\t@$(OBJCOPY_CMD) $(KERNEL_ELF) $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Name\")\n\t@echo $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Size\")\n\t$(call disk_usage_KiB, $(KERNEL_BIN))\n\n##------------------------------------------------------------------------------\n## Generate the documentation\n##------------------------------------------------------------------------------\ndoc:\n\t$(call color_header, \"Generating docs\")\n\t@$(DOC_CMD) --document-private-items --open\n\n##------------------------------------------------------------------------------\n## Run the kernel in QEMU\n##------------------------------------------------------------------------------\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\nqemu:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\nqemu: $(KERNEL_BIN)\n\t$(call color_header, \"Launching QEMU\")\n\t@$(DOCKER_QEMU) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\nendif\n\n##------------------------------------------------------------------------------\n## Run clippy\n##------------------------------------------------------------------------------\nclippy:\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(CLIPPY_CMD)\n\n##------------------------------------------------------------------------------\n## Clean\n##------------------------------------------------------------------------------\nclean:\n\trm -rf target $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run readelf\n##------------------------------------------------------------------------------\nreadelf: $(KERNEL_ELF)\n\t$(call color_header, \"Launching readelf\")\n\t@$(DOCKER_TOOLS) $(READELF_BINARY) --headers $(KERNEL_ELF)\n\n##------------------------------------------------------------------------------\n## Run objdump\n##------------------------------------------------------------------------------\nobjdump: $(KERNEL_ELF)\n\t$(call color_header, \"Launching objdump\")\n\t@$(DOCKER_TOOLS) $(OBJDUMP_BINARY) --disassemble --demangle \\\n                --section .text   \\\n                --section .rodata \\\n                $(KERNEL_ELF) | rustfilt\n\n##------------------------------------------------------------------------------\n## Run nm\n##------------------------------------------------------------------------------\nnm: $(KERNEL_ELF)\n\t$(call color_header, \"Launching nm\")\n\t@$(DOCKER_TOOLS) $(NM_BINARY) --demangle --print-size $(KERNEL_ELF) | sort | rustfilt\n\n"
  },
  {
    "path": "02_runtime_init/README.CN.md",
    "content": "# 教程 02 - 执行初始化\n\n## tl;dr\n\n- 我们拓展了`boot.S`，在第一次启动的时候调用Rust代码。\n  在跳转到rust代码前，对运行时进行了一些初始化工作。\n- Rust通过调用`panic()`挂起CPU。\n- 再次运行`make qemu`看看新增加的代码是怎么运行的。\n\n## 值得注意的变化\n\n- 链接脚本（linker script）中的变化:\n     - 新程序段（sections）: `.rodata`, `.got`, `.data`, `.bss`.\n     - 使用一个独立的位置（`.text._start_arguments`）来保存`_start()`引导函数所使用的参数。\n- `_start()` in `_arch/__arch_name__/cpu/boot.s`:\n     1. 当核心不是`core0`第0号核心的时候，挂起该CPU核心。\n     1. 通过清零`.bss`程序段来初始化`DRAM`.\n     1. 初始化堆栈指针（`stack pointer`）.\n     1. 跳转到`arch/__arch_name__/cpu/boot.rs`文件中定义的`_start_rust()`函数\n- `_start_rust()`:\n     1. 它调用了`kernel_init()`, 这个函数又调用了`panic!()`, panic函数最终把`core0`和其他核心一样挂起了。\n- 目前依赖 [aarch64-cpu] 程序库, 这个库零成本的包装了处理 CPU 资源时的“不安全”部分。\n    - 详细请参考 `_arch/__arch_name__/cpu.rs`.\n\n[bss]: https://en.wikipedia.org/wiki/.bss\n[aarch64-cpu]: https://github.com/rust-embedded/aarch64-cpu\n\n## 相比之前的变化（diff）\n请检查[英文版本](README.md#diff-to-previous)，这是最新的。\n\n"
  },
  {
    "path": "02_runtime_init/README.ES.md",
    "content": "# Tutorial 02 - Inicialización del `runtime`\n\n## tl;dr\n\n* Extendimos la funcionalidad de `boot.s` para que sea capaz de llamar código Rust por primera vez. Antes de que el cambio a Rust ocurra, se realizan algunos trabajos de inicialización del `runtime` (soporte para ejecución de código).\n* El código Rust que es llamado solo pausa la ejecución con una llamada a `panic!()`.\n* Ejecuta `make qemu` de nuevo para que puedas ver el código adicional en acción.\n\n## Adiciones importantes\n\n* Adiciones importantes al script `kernel.ld`:\n\n  * Nuevas secciones: `.rodata`, `.got`, `.data`, `.bss`.\n\n  * Un lugar totalmente dedicado a enlazar argumentos de tiempo de arranque (boot-time) que necesitan estar listos cuando se llame a `_start()`.\n\n* `_start()` en `_arch/__arch_name__/cpu/boot.s`:\n\n  1. Para todos los núcleos expecto el núcleo 0.\n\n  2. Inicializa la [`DRAM`](https://es.wikipedia.org/wiki/DRAM) poniendo a cero la sección [`.bss`](https://en.wikipedia.org/wiki/.bss).\n\n  3. Configura el `stack pointer` (puntero a la memoria [pila](https://es.wikipedia.org/wiki/Pila_(inform%C3%A1tica))).\n\n  4. Salta hacia la función `_start_rust()`, definida en `arch/__arch_name__/cpu/boot.rs`.\n\n* `_start_rust()`:\n\n  * Llama a `kernel_init()`, que llama a `panic!()`, que al final también pone al núcleo 0 en pausa.\n\n* La librería ahora usa el crate [aarch64-cpu](https://github.com/rust-embedded/aarch64-cpu), que nos da abstracciones sin coste y envuelve las partes que hacen uso de un `unsafe` (partes con código que no es seguro y podría causar errores) cuando se trabaja directamente con los recursos del procesador.\n\n  * Lo puedes ver en acción en `_arch/__arch_name__/cpu.rs`.\n\n## Diferencia con el archivo anterior\n\nPlease check [the english version](README.md#diff-to-previous), which is kept up-to-date.\n"
  },
  {
    "path": "02_runtime_init/README.md",
    "content": "# Tutorial 02 - Runtime Init\n\n## tl;dr\n\n- We extend `boot.s` to call into Rust code for the first time. Before the jump\n  to Rust happens, a bit of runtime init work is done.\n- The Rust code being called just halts execution with a call to `panic!()`.\n- Check out `make qemu` again to see the additional code run.\n\n## Notable additions\n\n- More additions to the linker script:\n     - New sections: `.rodata`, `.got`, `.data`, `.bss`.\n     - A dedicated place for linking boot-time arguments that need to be read by `_start()`.\n- `_start()` in `_arch/__arch_name__/cpu/boot.s`:\n     1. Halts core if core != core0.\n     1. Initializes the `DRAM` by zeroing the [bss] section.\n     1. Sets up the `stack pointer`.\n     1. Jumps to the `_start_rust()` function, defined in `arch/__arch_name__/cpu/boot.rs`.\n- `_start_rust()`:\n     - Calls `kernel_init()`, which calls `panic!()`, which eventually halts core0 as well.\n- The library now uses the [aarch64-cpu] crate, which provides zero-overhead abstractions and wraps\n  `unsafe` parts when dealing with the CPU's resources.\n    - See it in action in `_arch/__arch_name__/cpu.rs`.\n\n[bss]: https://en.wikipedia.org/wiki/.bss\n[aarch64-cpu]: https://github.com/rust-embedded/aarch64-cpu\n\n## Diff to previous\n```diff\n\ndiff -uNr 01_wait_forever/Cargo.toml 02_runtime_init/Cargo.toml\n--- 01_wait_forever/Cargo.toml\n+++ 02_runtime_init/Cargo.toml\n@@ -1,6 +1,6 @@\n [package]\n name = \"mingo\"\n-version = \"0.1.0\"\n+version = \"0.2.0\"\n authors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\n edition = \"2021\"\n\n@@ -21,3 +21,7 @@\n ##--------------------------------------------------------------------------------------------------\n\n [dependencies]\n+\n+# Platform specific dependencies\n+[target.'cfg(target_arch = \"aarch64\")'.dependencies]\n+aarch64-cpu = { version = \"9.x.x\" }\n\ndiff -uNr 01_wait_forever/Makefile 02_runtime_init/Makefile\n--- 01_wait_forever/Makefile\n+++ 02_runtime_init/Makefile\n@@ -181,6 +181,7 @@\n \t$(call color_header, \"Launching objdump\")\n \t@$(DOCKER_TOOLS) $(OBJDUMP_BINARY) --disassemble --demangle \\\n                 --section .text   \\\n+                --section .rodata \\\n                 $(KERNEL_ELF) | rustfilt\n ##------------------------------------------------------------------------------\n\ndiff -uNr 01_wait_forever/src/_arch/aarch64/cpu/boot.rs 02_runtime_init/src/_arch/aarch64/cpu/boot.rs\n--- 01_wait_forever/src/_arch/aarch64/cpu/boot.rs\n+++ 02_runtime_init/src/_arch/aarch64/cpu/boot.rs\n@@ -14,4 +14,19 @@\n use core::arch::global_asm;\n\n // Assembly counterpart to this file.\n-global_asm!(include_str!(\"boot.s\"));\n+global_asm!(\n+    include_str!(\"boot.s\"),\n+    CONST_CORE_ID_MASK = const 0b11\n+);\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+/// The Rust entry of the `kernel` binary.\n+///\n+/// The function is called from the assembly `_start` function.\n+#[no_mangle]\n+pub unsafe fn _start_rust() -> ! {\n+    crate::kernel_init()\n+}\n\ndiff -uNr 01_wait_forever/src/_arch/aarch64/cpu/boot.s 02_runtime_init/src/_arch/aarch64/cpu/boot.s\n--- 01_wait_forever/src/_arch/aarch64/cpu/boot.s\n+++ 02_runtime_init/src/_arch/aarch64/cpu/boot.s\n@@ -3,6 +3,22 @@\n // Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n //--------------------------------------------------------------------------------------------------\n+// Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+// Load the address of a symbol into a register, PC-relative.\n+//\n+// The symbol must lie within +/- 4 GiB of the Program Counter.\n+//\n+// # Resources\n+//\n+// - https://sourceware.org/binutils/docs-2.36/as/AArch64_002dRelocations.html\n+.macro ADR_REL register, symbol\n+\tadrp\t\\register, \\symbol\n+\tadd\t\\register, \\register, #:lo12:\\symbol\n+.endm\n+\n+//--------------------------------------------------------------------------------------------------\n // Public Code\n //--------------------------------------------------------------------------------------------------\n .section .text._start\n@@ -11,6 +27,34 @@\n // fn _start()\n //------------------------------------------------------------------------------\n _start:\n+\t// Only proceed on the boot core. Park it otherwise.\n+\tmrs\tx0, MPIDR_EL1\n+\tand\tx0, x0, {CONST_CORE_ID_MASK}\n+\tldr\tx1, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs\n+\tcmp\tx0, x1\n+\tb.ne\t.L_parking_loop\n+\n+\t// If execution reaches here, it is the boot core.\n+\n+\t// Initialize DRAM.\n+\tADR_REL\tx0, __bss_start\n+\tADR_REL x1, __bss_end_exclusive\n+\n+.L_bss_init_loop:\n+\tcmp\tx0, x1\n+\tb.eq\t.L_prepare_rust\n+\tstp\txzr, xzr, [x0], #16\n+\tb\t.L_bss_init_loop\n+\n+\t// Prepare the jump to Rust code.\n+.L_prepare_rust:\n+\t// Set the stack pointer.\n+\tADR_REL\tx0, __boot_core_stack_end_exclusive\n+\tmov\tsp, x0\n+\n+\t// Jump to Rust code.\n+\tb\t_start_rust\n+\n \t// Infinitely wait for events (aka \"park the core\").\n .L_parking_loop:\n \twfe\n\ndiff -uNr 01_wait_forever/src/_arch/aarch64/cpu.rs 02_runtime_init/src/_arch/aarch64/cpu.rs\n--- 01_wait_forever/src/_arch/aarch64/cpu.rs\n+++ 02_runtime_init/src/_arch/aarch64/cpu.rs\n@@ -0,0 +1,26 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Architectural processor code.\n+//!\n+//! # Orientation\n+//!\n+//! Since arch modules are imported into generic modules using the path attribute, the path of this\n+//! file is:\n+//!\n+//! crate::cpu::arch_cpu\n+\n+use aarch64_cpu::asm;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Pause execution on the core.\n+#[inline(always)]\n+pub fn wait_forever() -> ! {\n+    loop {\n+        asm::wfe()\n+    }\n+}\n\ndiff -uNr 01_wait_forever/src/bsp/raspberrypi/cpu.rs 02_runtime_init/src/bsp/raspberrypi/cpu.rs\n--- 01_wait_forever/src/bsp/raspberrypi/cpu.rs\n+++ 02_runtime_init/src/bsp/raspberrypi/cpu.rs\n@@ -0,0 +1,14 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! BSP Processor code.\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Used by `arch` code to find the early boot core.\n+#[no_mangle]\n+#[link_section = \".text._start_arguments\"]\n+pub static BOOT_CORE_ID: u64 = 0;\n\ndiff -uNr 01_wait_forever/src/bsp/raspberrypi/kernel.ld 02_runtime_init/src/bsp/raspberrypi/kernel.ld\n--- 01_wait_forever/src/bsp/raspberrypi/kernel.ld\n+++ 02_runtime_init/src/bsp/raspberrypi/kernel.ld\n@@ -3,6 +3,8 @@\n  * Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n  */\n\n+__rpi_phys_dram_start_addr = 0;\n+\n /* The physical address at which the the kernel binary will be loaded by the Raspberry's firmware */\n __rpi_phys_binary_load_addr = 0x80000;\n\n@@ -13,21 +15,65 @@\n  *     4 == R\n  *     5 == RX\n  *     6 == RW\n+ *\n+ * Segments are marked PT_LOAD below so that the ELF file provides virtual and physical addresses.\n+ * It doesn't mean all of them need actually be loaded.\n  */\n PHDRS\n {\n-    segment_code PT_LOAD FLAGS(5);\n+    segment_boot_core_stack PT_LOAD FLAGS(6);\n+    segment_code            PT_LOAD FLAGS(5);\n+    segment_data            PT_LOAD FLAGS(6);\n }\n\n SECTIONS\n {\n-    . =  __rpi_phys_binary_load_addr;\n+    . =  __rpi_phys_dram_start_addr;\n+\n+    /***********************************************************************************************\n+    * Boot Core Stack\n+    ***********************************************************************************************/\n+    .boot_core_stack (NOLOAD) :\n+    {\n+                                             /*   ^             */\n+                                             /*   | stack       */\n+        . += __rpi_phys_binary_load_addr;    /*   | growth      */\n+                                             /*   | direction   */\n+        __boot_core_stack_end_exclusive = .; /*   |             */\n+    } :segment_boot_core_stack\n\n     /***********************************************************************************************\n-    * Code\n+    * Code + RO Data + Global Offset Table\n     ***********************************************************************************************/\n     .text :\n     {\n         KEEP(*(.text._start))\n+        *(.text._start_arguments) /* Constants (or statics in Rust speak) read by _start(). */\n+        *(.text._start_rust)      /* The Rust entry point */\n+        *(.text*)                 /* Everything else */\n     } :segment_code\n+\n+    .rodata : ALIGN(8) { *(.rodata*) } :segment_code\n+\n+    /***********************************************************************************************\n+    * Data + BSS\n+    ***********************************************************************************************/\n+    .data : { *(.data*) } :segment_data\n+\n+    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */\n+    .bss (NOLOAD) : ALIGN(16)\n+    {\n+        __bss_start = .;\n+        *(.bss*);\n+        . = ALIGN(16);\n+        __bss_end_exclusive = .;\n+    } :segment_data\n+\n+    /***********************************************************************************************\n+    * Misc\n+    ***********************************************************************************************/\n+    .got : { *(.got*) }\n+    ASSERT(SIZEOF(.got) == 0, \"Relocation support not expected\")\n+\n+    /DISCARD/ : { *(.comment*) }\n }\n\ndiff -uNr 01_wait_forever/src/bsp/raspberrypi.rs 02_runtime_init/src/bsp/raspberrypi.rs\n--- 01_wait_forever/src/bsp/raspberrypi.rs\n+++ 02_runtime_init/src/bsp/raspberrypi.rs\n@@ -4,4 +4,4 @@\n\n //! Top-level BSP file for the Raspberry Pi 3 and 4.\n\n-// Coming soon.\n+pub mod cpu;\n\ndiff -uNr 01_wait_forever/src/cpu.rs 02_runtime_init/src/cpu.rs\n--- 01_wait_forever/src/cpu.rs\n+++ 02_runtime_init/src/cpu.rs\n@@ -4,4 +4,13 @@\n\n //! Processor code.\n\n+#[cfg(target_arch = \"aarch64\")]\n+#[path = \"_arch/aarch64/cpu.rs\"]\n+mod arch_cpu;\n+\n mod boot;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Architectural Public Reexports\n+//--------------------------------------------------------------------------------------------------\n+pub use arch_cpu::wait_forever;\n\ndiff -uNr 01_wait_forever/src/main.rs 02_runtime_init/src/main.rs\n--- 01_wait_forever/src/main.rs\n+++ 02_runtime_init/src/main.rs\n@@ -104,7 +104,9 @@\n //!\n //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.\n //!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.\n+//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.\n\n+#![feature(asm_const)]\n #![no_main]\n #![no_std]\n\n@@ -112,4 +114,11 @@\n mod cpu;\n mod panic_wait;\n\n-// Kernel code coming next tutorial.\n+/// Early init code.\n+///\n+/// # Safety\n+///\n+/// - Only a single core must be active and running this function.\n+unsafe fn kernel_init() -> ! {\n+    panic!()\n+}\n\ndiff -uNr 01_wait_forever/src/panic_wait.rs 02_runtime_init/src/panic_wait.rs\n--- 01_wait_forever/src/panic_wait.rs\n+++ 02_runtime_init/src/panic_wait.rs\n@@ -4,6 +4,7 @@\n\n //! A panic handler that infinitely waits.\n\n+use crate::cpu;\n use core::panic::PanicInfo;\n\n //--------------------------------------------------------------------------------------------------\n@@ -12,5 +13,5 @@\n\n #[panic_handler]\n fn panic(_info: &PanicInfo) -> ! {\n-    unimplemented!()\n+    cpu::wait_forever()\n }\n\n```\n"
  },
  {
    "path": "02_runtime_init/build.rs",
    "content": "use std::{env, fs, process};\n\nfn main() {\n    let ld_script_path = match env::var(\"LD_SCRIPT_PATH\") {\n        Ok(var) => var,\n        _ => process::exit(0),\n    };\n\n    let files = fs::read_dir(ld_script_path).unwrap();\n    files\n        .filter_map(Result::ok)\n        .filter(|d| {\n            if let Some(e) = d.path().extension() {\n                e == \"ld\"\n            } else {\n                false\n            }\n        })\n        .for_each(|f| println!(\"cargo:rerun-if-changed={}\", f.path().display()));\n}\n"
  },
  {
    "path": "02_runtime_init/src/_arch/aarch64/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural boot code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::boot::arch_boot\n\nuse core::arch::global_asm;\n\n// Assembly counterpart to this file.\nglobal_asm!(\n    include_str!(\"boot.s\"),\n    CONST_CORE_ID_MASK = const 0b11\n);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The Rust entry of the `kernel` binary.\n///\n/// The function is called from the assembly `_start` function.\n#[no_mangle]\npub unsafe fn _start_rust() -> ! {\n    crate::kernel_init()\n}\n"
  },
  {
    "path": "02_runtime_init/src/_arch/aarch64/cpu/boot.s",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2022 Andre Richter <andre.o.richter@gmail.com>\n\n//--------------------------------------------------------------------------------------------------\n// Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Load the address of a symbol into a register, PC-relative.\n//\n// The symbol must lie within +/- 4 GiB of the Program Counter.\n//\n// # Resources\n//\n// - https://sourceware.org/binutils/docs-2.36/as/AArch64_002dRelocations.html\n.macro ADR_REL register, symbol\n\tadrp\t\\register, \\symbol\n\tadd\t\\register, \\register, #:lo12:\\symbol\n.endm\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n.section .text._start\n\n//------------------------------------------------------------------------------\n// fn _start()\n//------------------------------------------------------------------------------\n_start:\n\t// Only proceed on the boot core. Park it otherwise.\n\tmrs\tx0, MPIDR_EL1\n\tand\tx0, x0, {CONST_CORE_ID_MASK}\n\tldr\tx1, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs\n\tcmp\tx0, x1\n\tb.ne\t.L_parking_loop\n\n\t// If execution reaches here, it is the boot core.\n\n\t// Initialize DRAM.\n\tADR_REL\tx0, __bss_start\n\tADR_REL x1, __bss_end_exclusive\n\n.L_bss_init_loop:\n\tcmp\tx0, x1\n\tb.eq\t.L_prepare_rust\n\tstp\txzr, xzr, [x0], #16\n\tb\t.L_bss_init_loop\n\n\t// Prepare the jump to Rust code.\n.L_prepare_rust:\n\t// Set the stack pointer.\n\tADR_REL\tx0, __boot_core_stack_end_exclusive\n\tmov\tsp, x0\n\n\t// Jump to Rust code.\n\tb\t_start_rust\n\n\t// Infinitely wait for events (aka \"park the core\").\n.L_parking_loop:\n\twfe\n\tb\t.L_parking_loop\n\n.size\t_start, . - _start\n.type\t_start, function\n.global\t_start\n"
  },
  {
    "path": "02_runtime_init/src/_arch/aarch64/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural processor code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::arch_cpu\n\nuse aarch64_cpu::asm;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Pause execution on the core.\n#[inline(always)]\npub fn wait_forever() -> ! {\n    loop {\n        asm::wfe()\n    }\n}\n"
  },
  {
    "path": "02_runtime_init/src/bsp/raspberrypi/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Processor code.\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Used by `arch` code to find the early boot core.\n#[no_mangle]\n#[link_section = \".text._start_arguments\"]\npub static BOOT_CORE_ID: u64 = 0;\n"
  },
  {
    "path": "02_runtime_init/src/bsp/raspberrypi/kernel.ld",
    "content": "/* SPDX-License-Identifier: MIT OR Apache-2.0\n *\n * Copyright (c) 2018-2022 Andre Richter <andre.o.richter@gmail.com>\n */\n\n__rpi_phys_dram_start_addr = 0;\n\n/* The physical address at which the the kernel binary will be loaded by the Raspberry's firmware */\n__rpi_phys_binary_load_addr = 0x80000;\n\n\nENTRY(__rpi_phys_binary_load_addr)\n\n/* Flags:\n *     4 == R\n *     5 == RX\n *     6 == RW\n *\n * Segments are marked PT_LOAD below so that the ELF file provides virtual and physical addresses.\n * It doesn't mean all of them need actually be loaded.\n */\nPHDRS\n{\n    segment_boot_core_stack PT_LOAD FLAGS(6);\n    segment_code            PT_LOAD FLAGS(5);\n    segment_data            PT_LOAD FLAGS(6);\n}\n\nSECTIONS\n{\n    . =  __rpi_phys_dram_start_addr;\n\n    /***********************************************************************************************\n    * Boot Core Stack\n    ***********************************************************************************************/\n    .boot_core_stack (NOLOAD) :\n    {\n                                             /*   ^             */\n                                             /*   | stack       */\n        . += __rpi_phys_binary_load_addr;    /*   | growth      */\n                                             /*   | direction   */\n        __boot_core_stack_end_exclusive = .; /*   |             */\n    } :segment_boot_core_stack\n\n    /***********************************************************************************************\n    * Code + RO Data + Global Offset Table\n    ***********************************************************************************************/\n    .text :\n    {\n        KEEP(*(.text._start))\n        *(.text._start_arguments) /* Constants (or statics in Rust speak) read by _start(). */\n        *(.text._start_rust)      /* The Rust entry point */\n        *(.text*)                 /* Everything else */\n    } :segment_code\n\n    .rodata : ALIGN(8) { *(.rodata*) } :segment_code\n\n    /***********************************************************************************************\n    * Data + BSS\n    ***********************************************************************************************/\n    .data : { *(.data*) } :segment_data\n\n    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */\n    .bss (NOLOAD) : ALIGN(16)\n    {\n        __bss_start = .;\n        *(.bss*);\n        . = ALIGN(16);\n        __bss_end_exclusive = .;\n    } :segment_data\n\n    /***********************************************************************************************\n    * Misc\n    ***********************************************************************************************/\n    .got : { *(.got*) }\n    ASSERT(SIZEOF(.got) == 0, \"Relocation support not expected\")\n\n    /DISCARD/ : { *(.comment*) }\n}\n"
  },
  {
    "path": "02_runtime_init/src/bsp/raspberrypi.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Top-level BSP file for the Raspberry Pi 3 and 4.\n\npub mod cpu;\n"
  },
  {
    "path": "02_runtime_init/src/bsp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Conditional reexporting of Board Support Packages.\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod raspberrypi;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use raspberrypi::*;\n"
  },
  {
    "path": "02_runtime_init/src/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Boot code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/cpu/boot.rs\"]\nmod arch_boot;\n"
  },
  {
    "path": "02_runtime_init/src/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Processor code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/cpu.rs\"]\nmod arch_cpu;\n\nmod boot;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_cpu::wait_forever;\n"
  },
  {
    "path": "02_runtime_init/src/main.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n// Rust embedded logo for `make doc`.\n#![doc(\n    html_logo_url = \"https://raw.githubusercontent.com/rust-embedded/wg/master/assets/logo/ewg-logo-blue-white-on-transparent.png\"\n)]\n\n//! The `kernel` binary.\n//!\n//! # Code organization and architecture\n//!\n//! The code is divided into different *modules*, each representing a typical **subsystem** of the\n//! `kernel`. Top-level module files of subsystems reside directly in the `src` folder. For example,\n//! `src/memory.rs` contains code that is concerned with all things memory management.\n//!\n//! ## Visibility of processor architecture code\n//!\n//! Some of the `kernel`'s subsystems depend on low-level code that is specific to the target\n//! processor architecture. For each supported processor architecture, there exists a subfolder in\n//! `src/_arch`, for example, `src/_arch/aarch64`.\n//!\n//! The architecture folders mirror the subsystem modules laid out in `src`. For example,\n//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go\n//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in\n//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic\n//! module's name prefixed with `arch_`.\n//!\n//! For example, this is the top of `src/memory/mmu.rs`:\n//!\n//! ```\n//! #[cfg(target_arch = \"aarch64\")]\n//! #[path = \"../_arch/aarch64/memory/mmu.rs\"]\n//! mod arch_mmu;\n//! ```\n//!\n//! Often times, items from the `arch_ module` will be publicly reexported by the parent module.\n//! This way, each architecture specific module can provide its implementation of an item, while the\n//! caller must not be concerned which architecture has been conditionally compiled.\n//!\n//! ## BSP code\n//!\n//! `BSP` stands for Board Support Package. `BSP` code is organized under `src/bsp.rs` and contains\n//! target board specific definitions and functions. These are things such as the board's memory map\n//! or instances of drivers for devices that are featured on the respective board.\n//!\n//! Just like processor architecture code, the `BSP` code's module structure tries to mirror the\n//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is\n//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`.\n//!\n//! ## Kernel interfaces\n//!\n//! Both `arch` and `bsp` contain code that is conditionally compiled depending on the actual target\n//! and board for which the kernel is compiled. For example, the `interrupt controller` hardware of\n//! the `Raspberry Pi 3` and the `Raspberry Pi 4` is different, but we want the rest of the `kernel`\n//! code to play nicely with any of the two without much hassle.\n//!\n//! In order to provide a clean abstraction between `arch`, `bsp` and `generic kernel code`,\n//! `interface` traits are provided *whenever possible* and *where it makes sense*. They are defined\n//! in the respective subsystem module and help to enforce the idiom of *program to an interface,\n//! not an implementation*. For example, there will be a common IRQ handling interface which the two\n//! different interrupt controller `drivers` of both Raspberrys will implement, and only export the\n//! interface to the rest of the `kernel`.\n//!\n//! ```\n//!         +-------------------+\n//!         | Interface (Trait) |\n//!         |                   |\n//!         +--+-------------+--+\n//!            ^             ^\n//!            |             |\n//!            |             |\n//! +----------+--+       +--+----------+\n//! | kernel code |       |  bsp code   |\n//! |             |       |  arch code  |\n//! +-------------+       +-------------+\n//! ```\n//!\n//! # Summary\n//!\n//! For a logical `kernel` subsystem, corresponding code can be distributed over several physical\n//! locations. Here is an example for the **memory** subsystem:\n//!\n//! - `src/memory.rs` and `src/memory/**/*`\n//!   - Common code that is agnostic of target processor architecture and `BSP` characteristics.\n//!     - Example: A function to zero a chunk of memory.\n//!   - Interfaces for the memory subsystem that are implemented by `arch` or `BSP` code.\n//!     - Example: An `MMU` interface that defines `MMU` function prototypes.\n//! - `src/bsp/__board_name__/memory.rs` and `src/bsp/__board_name__/memory/**/*`\n//!   - `BSP` specific code.\n//!   - Example: The board's memory map (physical addresses of DRAM and MMIO devices).\n//! - `src/_arch/__arch_name__/memory.rs` and `src/_arch/__arch_name__/memory/**/*`\n//!   - Processor architecture specific code.\n//!   - Example: Implementation of the `MMU` interface for the `__arch_name__` processor\n//!     architecture.\n//!\n//! From a namespace perspective, **memory** subsystem code lives in:\n//!\n//! - `crate::memory::*`\n//! - `crate::bsp::memory::*`\n//!\n//! # Boot flow\n//!\n//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.\n//!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.\n//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.\n\n#![feature(asm_const)]\n#![no_main]\n#![no_std]\n\nmod bsp;\nmod cpu;\nmod panic_wait;\n\n/// Early init code.\n///\n/// # Safety\n///\n/// - Only a single core must be active and running this function.\nunsafe fn kernel_init() -> ! {\n    panic!()\n}\n"
  },
  {
    "path": "02_runtime_init/src/panic_wait.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A panic handler that infinitely waits.\n\nuse crate::cpu;\nuse core::panic::PanicInfo;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n#[panic_handler]\nfn panic(_info: &PanicInfo) -> ! {\n    cpu::wait_forever()\n}\n"
  },
  {
    "path": "03_hacky_hello_world/.vscode/settings.json",
    "content": "{\n    \"editor.formatOnSave\": true,\n    \"editor.rulers\": [100],\n    \"rust-analyzer.cargo.target\": \"aarch64-unknown-none-softfloat\",\n    \"rust-analyzer.cargo.features\": [\"bsp_rpi3\"],\n    \"rust-analyzer.checkOnSave.allTargets\": false,\n    \"rust-analyzer.checkOnSave.extraArgs\": [\"--bins\"],\n    \"rust-analyzer.lens.debug\": false,\n    \"rust-analyzer.lens.run\": false\n}\n"
  },
  {
    "path": "03_hacky_hello_world/Cargo.toml",
    "content": "[package]\nname = \"mingo\"\nversion = \"0.3.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n\n[profile.release]\nlto = true\n\n[features]\ndefault = []\nbsp_rpi3 = []\nbsp_rpi4 = []\n\n[[bin]]\nname = \"kernel\"\npath = \"src/main.rs\"\n\n##--------------------------------------------------------------------------------------------------\n## Dependencies\n##--------------------------------------------------------------------------------------------------\n\n[dependencies]\n\n# Platform specific dependencies\n[target.'cfg(target_arch = \"aarch64\")'.dependencies]\naarch64-cpu = { version = \"9.x.x\" }\n"
  },
  {
    "path": "03_hacky_hello_world/Makefile",
    "content": "## SPDX-License-Identifier: MIT OR Apache-2.0\n##\n## Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\ninclude ../common/docker.mk\ninclude ../common/format.mk\ninclude ../common/operating_system.mk\n\n##--------------------------------------------------------------------------------------------------\n## Optional, user-provided configuration values\n##--------------------------------------------------------------------------------------------------\n\n# Default to the RPi3.\nBSP ?= rpi3\n\n\n\n##--------------------------------------------------------------------------------------------------\n## BSP-specific configuration values\n##--------------------------------------------------------------------------------------------------\nQEMU_MISSING_STRING = \"This board is not yet supported for QEMU.\"\n\nifeq ($(BSP),rpi3)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE = raspi3\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    LD_SCRIPT_PATH    = $(shell pwd)/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a53\nelse ifeq ($(BSP),rpi4)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE =\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    LD_SCRIPT_PATH    = $(shell pwd)/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a72\nendif\n\n# Export for build.rs.\nexport LD_SCRIPT_PATH\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets and Prerequisites\n##--------------------------------------------------------------------------------------------------\nKERNEL_MANIFEST      = Cargo.toml\nKERNEL_LINKER_SCRIPT = kernel.ld\nLAST_BUILD_CONFIG    = target/$(BSP).build_config\n\nKERNEL_ELF      = target/$(TARGET)/release/kernel\n# This parses cargo's dep-info file.\n# https://doc.rust-lang.org/cargo/guide/build-cache.html#dep-info-files\nKERNEL_ELF_DEPS = $(filter-out %: ,$(file < $(KERNEL_ELF).d)) $(KERNEL_MANIFEST) $(LAST_BUILD_CONFIG)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Command building blocks\n##--------------------------------------------------------------------------------------------------\nRUSTFLAGS = $(RUSTC_MISC_ARGS)                   \\\n    -C link-arg=--library-path=$(LD_SCRIPT_PATH) \\\n    -C link-arg=--script=$(KERNEL_LINKER_SCRIPT)\n\nRUSTFLAGS_PEDANTIC = $(RUSTFLAGS) \\\n    -D warnings                   \\\n    -D missing_docs\n\nFEATURES      = --features bsp_$(BSP)\nCOMPILER_ARGS = --target=$(TARGET) \\\n    $(FEATURES)                    \\\n    --release\n\nRUSTC_CMD   = cargo rustc $(COMPILER_ARGS)\nDOC_CMD     = cargo doc $(COMPILER_ARGS)\nCLIPPY_CMD  = cargo clippy $(COMPILER_ARGS)\nOBJCOPY_CMD = rust-objcopy \\\n    --strip-all            \\\n    -O binary\n\nEXEC_QEMU          = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE)\nEXEC_TEST_DISPATCH = ruby ../common/tests/dispatch.rb\n\n##------------------------------------------------------------------------------\n## Dockerization\n##------------------------------------------------------------------------------\nDOCKER_CMD            = docker run -t --rm -v $(shell pwd):/work/tutorial -w /work/tutorial\nDOCKER_CMD_INTERACT   = $(DOCKER_CMD) -i\nDOCKER_ARG_DIR_COMMON = -v $(shell pwd)/../common:/work/common\n\n# DOCKER_IMAGE defined in include file (see top of this file).\nDOCKER_QEMU  = $(DOCKER_CMD_INTERACT) $(DOCKER_IMAGE)\nDOCKER_TOOLS = $(DOCKER_CMD) $(DOCKER_IMAGE)\nDOCKER_TEST  = $(DOCKER_CMD) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: all doc qemu clippy clean readelf objdump nm check\n\nall: $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Save the configuration as a file, so make understands if it changed.\n##------------------------------------------------------------------------------\n$(LAST_BUILD_CONFIG):\n\t@rm -f target/*.build_config\n\t@mkdir -p target\n\t@touch $(LAST_BUILD_CONFIG)\n\n##------------------------------------------------------------------------------\n## Compile the kernel ELF\n##------------------------------------------------------------------------------\n$(KERNEL_ELF): $(KERNEL_ELF_DEPS)\n\t$(call color_header, \"Compiling kernel ELF - $(BSP)\")\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(RUSTC_CMD)\n\n##------------------------------------------------------------------------------\n## Generate the stripped kernel binary\n##------------------------------------------------------------------------------\n$(KERNEL_BIN): $(KERNEL_ELF)\n\t$(call color_header, \"Generating stripped binary\")\n\t@$(OBJCOPY_CMD) $(KERNEL_ELF) $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Name\")\n\t@echo $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Size\")\n\t$(call disk_usage_KiB, $(KERNEL_BIN))\n\n##------------------------------------------------------------------------------\n## Generate the documentation\n##------------------------------------------------------------------------------\ndoc:\n\t$(call color_header, \"Generating docs\")\n\t@$(DOC_CMD) --document-private-items --open\n\n##------------------------------------------------------------------------------\n## Run the kernel in QEMU\n##------------------------------------------------------------------------------\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\nqemu:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\nqemu: $(KERNEL_BIN)\n\t$(call color_header, \"Launching QEMU\")\n\t@$(DOCKER_QEMU) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\nendif\n\n##------------------------------------------------------------------------------\n## Run clippy\n##------------------------------------------------------------------------------\nclippy:\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(CLIPPY_CMD)\n\n##------------------------------------------------------------------------------\n## Clean\n##------------------------------------------------------------------------------\nclean:\n\trm -rf target $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run readelf\n##------------------------------------------------------------------------------\nreadelf: $(KERNEL_ELF)\n\t$(call color_header, \"Launching readelf\")\n\t@$(DOCKER_TOOLS) $(READELF_BINARY) --headers $(KERNEL_ELF)\n\n##------------------------------------------------------------------------------\n## Run objdump\n##------------------------------------------------------------------------------\nobjdump: $(KERNEL_ELF)\n\t$(call color_header, \"Launching objdump\")\n\t@$(DOCKER_TOOLS) $(OBJDUMP_BINARY) --disassemble --demangle \\\n                --section .text   \\\n                --section .rodata \\\n                $(KERNEL_ELF) | rustfilt\n\n##------------------------------------------------------------------------------\n## Run nm\n##------------------------------------------------------------------------------\nnm: $(KERNEL_ELF)\n\t$(call color_header, \"Launching nm\")\n\t@$(DOCKER_TOOLS) $(NM_BINARY) --demangle --print-size $(KERNEL_ELF) | sort | rustfilt\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Testing targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: test test_boot\n\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\ntest_boot test:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\n##------------------------------------------------------------------------------\n## Run boot test\n##------------------------------------------------------------------------------\ntest_boot: $(KERNEL_BIN)\n\t$(call color_header, \"Boot test - $(BSP)\")\n\t@$(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\ntest: test_boot\n\nendif\n"
  },
  {
    "path": "03_hacky_hello_world/README.CN.md",
    "content": "# 教程 03 - Hacky Hello World\n\n## tl;dr\n\n- 介绍全局的`println!()`宏以便尽早启用\"printf debugging\"。\n- 为了保持教程长度合理，打印函数目前 \"滥用\" 了 QEMU 属性，该属性允许我们在没有正确设置的情况下使用树莓派的`UART`。\n- 在接下来的教程中将逐步使用真实硬件的`UART`。\n\n## 值得注意的补充\n\n- `src/console.rs`为控制台命令和通过`console::console()`对内核控制台的全局访问引入了接口`Traits`。\n- `src/bsp/raspberrypi/console.rs` 实现QEMU仿真UART的接口。\n- 紧急处理程序使用新的`println!()`以显示用户错误消息。\n- 有一个新的Makefile目录`make test`，用于自动测试。它在`QEMU`中引导编译后的内核，并检查内核生成的预期输出字符串。\n  - 在本教程中，它检查字符串`Stopping here`，该字符串由`panic!()`在`main.rs`的末尾。\n\n## 测试一下\n\nQEMU不再以汇编模式运行。从现在起，它将显示`console`的输出。\n\n```console\n$ make qemu\n[...]\n\nHello from Rust!\nKernel panic!\n\nPanic location:\n      File 'src/main.rs', line 126, column 5\n\nStopping here.\n```\n\n## 相比之前的变化（diff）\n请检查[英文版本](README.md#diff-to-previous)，这是最新的。\n"
  },
  {
    "path": "03_hacky_hello_world/README.ES.md",
    "content": "# Tutorial 03 - Hacky Hello World\n\n## tl;dr\n\n* Se añade la macro global `print!()` para habilitar la \"depuración basada en printf\" (\"printf debugging\") lo más pronto posible.\n* Para mantener una duración razonable en este tutorial, las funciones de impresión por el momento \"abusan\" una propiedad de QEMU que nos permite hacer uso del `UART` de la Raspberry sin haberla configurado apropiadamente.\n* El uso del hardware real de `UART` se habilitará paso por paso en los siguientes tutoriales.\n\n## Adiciones notables\n\n* `src/console.rs` introduce una interfaz con `Trait`s para comandos de consola.\n* `src/bsp/raspberrypi/console.rs` implementa la interfaz para que QEMU pueda crear una emulación de UART.\n* El *panic handler* (manejador de pánico) hace uso de la nueva macro `print!()` para mostrar mensajes de error del usuario.\n* Hay un nuevo objetivo en el Makefile: `make test`, destinado a la automatización de pruebas. Este comando inicia el kernel (núcleo) compilado en `QEMU`, y busca una cadena de  texto específica en la salida que ha sido producida por el kernel (núcleo).\n  * En este tutorial, se buscará la cadena `Stopping here`, que es creada por la macro `panic!()` al final del archivo `main.rs`.\n\n## Pruébalo\n\nQEMU ya no está siendo ejecutado en modo ensamblador. Desde ahora en adelante mostrará la salida de la `consola`.\n\n```console\n$ make qemu\n[...]\nHello from Rust!\n\nKernel panic: Stopping here.\n```\n\n### Diccionario\n\n* *Hacky:* Solución torpe o poco elegante para un problema.\n\n* *Debugging:* Proceso para identificar y corregir errores de programación.\n\n  * *printf debugging:* Usado para describir el trabajo de depuración (*debugging*) poniendo comandos que dan una salida en consola, como el de \"printf\", en diferentes lugares del programa; observando la información y tratando de deducir qué está mal en el programa basándose en la información que nos dan nuestros comandos.\n\n* *Traits:* Un *trait* le hace saber al compilador de Rust acerca de una funcionalidad que tiene un tipo de dato particular y que puede compartir con otros tipos de datos.\n\n  > NOTA: Los *traits* son similares a una característica que se le conoce comúnmente como *interfaces* en otros lenguajes, aunque con algunas diferencias.\n\n  Si deseas aprender más acerca de esto, por favor lee este capítulo del libro de Rust: [Traits: Defining Shared Behavior - The Rust Programming Language](https://doc.rust-lang.org/book/ch10-02-traits.html)\n\n## Diferencias con el archivo anterior\n\nPlease check [the english version](README.md#diff-to-previous), which is kept up-to-date.\n"
  },
  {
    "path": "03_hacky_hello_world/README.md",
    "content": "# Tutorial 03 - Hacky Hello World\n\n## tl;dr\n\n- Introducing global `println!()` macros to enable \"printf debugging\" at the earliest.\n- To keep tutorial length reasonable, printing functions for now \"abuse\" a QEMU property that lets\n  us use the Raspberry's `UART` without setting it up properly.\n- Using the real hardware `UART` is enabled step-by-step in following tutorials.\n\n## Notable additions\n\n- `src/console.rs` introduces interface `Traits` for console commands and global access to the\n  kernel's console through `console::console()`.\n- `src/bsp/raspberrypi/console.rs` implements the interface for QEMU's emulated UART.\n- The panic handler makes use of the new `println!()` to display user error messages.\n- There is a new Makefile target, `make test`, intended for automated testing. It boots the compiled\n  kernel in `QEMU`, and checks for an expected output string produced by the kernel.\n  - In this tutorial, it checks for the string `Stopping here`, which is emitted by the `panic!()`\n    at the end of `main.rs`.\n\n## Test it\n\nQEMU is no longer running in assembly mode. It will from now on show the output of the `console`.\n\n```console\n$ make qemu\n[...]\n\nHello from Rust!\nKernel panic!\n\nPanic location:\n      File 'src/main.rs', line 126, column 5\n\nStopping here.\n```\n\n## Diff to previous\n```diff\n\ndiff -uNr 02_runtime_init/Cargo.toml 03_hacky_hello_world/Cargo.toml\n--- 02_runtime_init/Cargo.toml\n+++ 03_hacky_hello_world/Cargo.toml\n@@ -1,6 +1,6 @@\n [package]\n name = \"mingo\"\n-version = \"0.2.0\"\n+version = \"0.3.0\"\n authors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\n edition = \"2021\"\n\n\ndiff -uNr 02_runtime_init/Makefile 03_hacky_hello_world/Makefile\n--- 02_runtime_init/Makefile\n+++ 03_hacky_hello_world/Makefile\n@@ -25,7 +25,7 @@\n     KERNEL_BIN        = kernel8.img\n     QEMU_BINARY       = qemu-system-aarch64\n     QEMU_MACHINE_TYPE = raspi3\n-    QEMU_RELEASE_ARGS = -d in_asm -display none\n+    QEMU_RELEASE_ARGS = -serial stdio -display none\n     OBJDUMP_BINARY    = aarch64-none-elf-objdump\n     NM_BINARY         = aarch64-none-elf-nm\n     READELF_BINARY    = aarch64-none-elf-readelf\n@@ -36,7 +36,7 @@\n     KERNEL_BIN        = kernel8.img\n     QEMU_BINARY       = qemu-system-aarch64\n     QEMU_MACHINE_TYPE =\n-    QEMU_RELEASE_ARGS = -d in_asm -display none\n+    QEMU_RELEASE_ARGS = -serial stdio -display none\n     OBJDUMP_BINARY    = aarch64-none-elf-objdump\n     NM_BINARY         = aarch64-none-elf-nm\n     READELF_BINARY    = aarch64-none-elf-readelf\n@@ -86,17 +86,20 @@\n     --strip-all            \\\n     -O binary\n\n-EXEC_QEMU = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE)\n+EXEC_QEMU          = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE)\n+EXEC_TEST_DISPATCH = ruby ../common/tests/dispatch.rb\n\n ##------------------------------------------------------------------------------\n ## Dockerization\n ##------------------------------------------------------------------------------\n-DOCKER_CMD          = docker run -t --rm -v $(shell pwd):/work/tutorial -w /work/tutorial\n-DOCKER_CMD_INTERACT = $(DOCKER_CMD) -i\n+DOCKER_CMD            = docker run -t --rm -v $(shell pwd):/work/tutorial -w /work/tutorial\n+DOCKER_CMD_INTERACT   = $(DOCKER_CMD) -i\n+DOCKER_ARG_DIR_COMMON = -v $(shell pwd)/../common:/work/common\n\n # DOCKER_IMAGE defined in include file (see top of this file).\n DOCKER_QEMU  = $(DOCKER_CMD_INTERACT) $(DOCKER_IMAGE)\n DOCKER_TOOLS = $(DOCKER_CMD) $(DOCKER_IMAGE)\n+DOCKER_TEST  = $(DOCKER_CMD) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\n\n\n\n@@ -191,3 +194,27 @@\n \t$(call color_header, \"Launching nm\")\n \t@$(DOCKER_TOOLS) $(NM_BINARY) --demangle --print-size $(KERNEL_ELF) | sort | rustfilt\n\n+\n+\n+##--------------------------------------------------------------------------------------------------\n+## Testing targets\n+##--------------------------------------------------------------------------------------------------\n+.PHONY: test test_boot\n+\n+ifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n+\n+test_boot test:\n+\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n+\n+else # QEMU is supported.\n+\n+##------------------------------------------------------------------------------\n+## Run boot test\n+##------------------------------------------------------------------------------\n+test_boot: $(KERNEL_BIN)\n+\t$(call color_header, \"Boot test - $(BSP)\")\n+\t@$(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n+\n+test: test_boot\n+\n+endif\n\ndiff -uNr 02_runtime_init/src/bsp/raspberrypi/console.rs 03_hacky_hello_world/src/bsp/raspberrypi/console.rs\n--- 02_runtime_init/src/bsp/raspberrypi/console.rs\n+++ 03_hacky_hello_world/src/bsp/raspberrypi/console.rs\n@@ -0,0 +1,47 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! BSP console facilities.\n+\n+use crate::console;\n+use core::fmt;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// A mystical, magical device for generating QEMU output out of the void.\n+struct QEMUOutput;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Code\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Implementing `core::fmt::Write` enables usage of the `format_args!` macros, which in turn are\n+/// used to implement the `kernel`'s `print!` and `println!` macros. By implementing `write_str()`,\n+/// we get `write_fmt()` automatically.\n+///\n+/// See [`src/print.rs`].\n+///\n+/// [`src/print.rs`]: ../../print/index.html\n+impl fmt::Write for QEMUOutput {\n+    fn write_str(&mut self, s: &str) -> fmt::Result {\n+        for c in s.chars() {\n+            unsafe {\n+                core::ptr::write_volatile(0x3F20_1000 as *mut u8, c as u8);\n+            }\n+        }\n+\n+        Ok(())\n+    }\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Return a reference to the console.\n+pub fn console() -> impl console::interface::Write {\n+    QEMUOutput {}\n+}\n\ndiff -uNr 02_runtime_init/src/bsp/raspberrypi.rs 03_hacky_hello_world/src/bsp/raspberrypi.rs\n--- 02_runtime_init/src/bsp/raspberrypi.rs\n+++ 03_hacky_hello_world/src/bsp/raspberrypi.rs\n@@ -4,4 +4,5 @@\n\n //! Top-level BSP file for the Raspberry Pi 3 and 4.\n\n+pub mod console;\n pub mod cpu;\n\ndiff -uNr 02_runtime_init/src/console.rs 03_hacky_hello_world/src/console.rs\n--- 02_runtime_init/src/console.rs\n+++ 03_hacky_hello_world/src/console.rs\n@@ -0,0 +1,32 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! System console.\n+\n+use crate::bsp;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Console interfaces.\n+pub mod interface {\n+    /// Console write functions.\n+    ///\n+    /// `core::fmt::Write` is exactly what we need for now. Re-export it here because\n+    /// implementing `console::Write` gives a better hint to the reader about the\n+    /// intention.\n+    pub use core::fmt::Write;\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Return a reference to the console.\n+///\n+/// This is the global console used by all printing macros.\n+pub fn console() -> impl interface::Write {\n+    bsp::console::console()\n+}\n\ndiff -uNr 02_runtime_init/src/main.rs 03_hacky_hello_world/src/main.rs\n--- 02_runtime_init/src/main.rs\n+++ 03_hacky_hello_world/src/main.rs\n@@ -107,12 +107,16 @@\n //! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.\n\n #![feature(asm_const)]\n+#![feature(format_args_nl)]\n+#![feature(panic_info_message)]\n #![no_main]\n #![no_std]\n\n mod bsp;\n+mod console;\n mod cpu;\n mod panic_wait;\n+mod print;\n\n /// Early init code.\n ///\n@@ -120,5 +124,7 @@\n ///\n /// - Only a single core must be active and running this function.\n unsafe fn kernel_init() -> ! {\n-    panic!()\n+    println!(\"Hello from Rust!\");\n+\n+    panic!(\"Stopping here.\")\n }\n\ndiff -uNr 02_runtime_init/src/panic_wait.rs 03_hacky_hello_world/src/panic_wait.rs\n--- 02_runtime_init/src/panic_wait.rs\n+++ 03_hacky_hello_world/src/panic_wait.rs\n@@ -4,14 +4,61 @@\n\n //! A panic handler that infinitely waits.\n\n-use crate::cpu;\n+use crate::{cpu, println};\n use core::panic::PanicInfo;\n\n //--------------------------------------------------------------------------------------------------\n // Private Code\n //--------------------------------------------------------------------------------------------------\n\n+/// Stop immediately if called a second time.\n+///\n+/// # Note\n+///\n+/// Using atomics here relieves us from needing to use `unsafe` for the static variable.\n+///\n+/// On `AArch64`, which is the only implemented architecture at the time of writing this,\n+/// [`AtomicBool::load`] and [`AtomicBool::store`] are lowered to ordinary load and store\n+/// instructions. They are therefore safe to use even with MMU + caching deactivated.\n+///\n+/// [`AtomicBool::load`]: core::sync::atomic::AtomicBool::load\n+/// [`AtomicBool::store`]: core::sync::atomic::AtomicBool::store\n+fn panic_prevent_reenter() {\n+    use core::sync::atomic::{AtomicBool, Ordering};\n+\n+    #[cfg(not(target_arch = \"aarch64\"))]\n+    compile_error!(\"Add the target_arch to above's check if the following code is safe to use\");\n+\n+    static PANIC_IN_PROGRESS: AtomicBool = AtomicBool::new(false);\n+\n+    if !PANIC_IN_PROGRESS.load(Ordering::Relaxed) {\n+        PANIC_IN_PROGRESS.store(true, Ordering::Relaxed);\n+\n+        return;\n+    }\n+\n+    cpu::wait_forever()\n+}\n+\n #[panic_handler]\n-fn panic(_info: &PanicInfo) -> ! {\n+fn panic(info: &PanicInfo) -> ! {\n+    // Protect against panic infinite loops if any of the following code panics itself.\n+    panic_prevent_reenter();\n+\n+    let (location, line, column) = match info.location() {\n+        Some(loc) => (loc.file(), loc.line(), loc.column()),\n+        _ => (\"???\", 0, 0),\n+    };\n+\n+    println!(\n+        \"Kernel panic!\\n\\n\\\n+        Panic location:\\n      File '{}', line {}, column {}\\n\\n\\\n+        {}\",\n+        location,\n+        line,\n+        column,\n+        info.message().unwrap_or(&format_args!(\"\")),\n+    );\n+\n     cpu::wait_forever()\n }\n\ndiff -uNr 02_runtime_init/src/print.rs 03_hacky_hello_world/src/print.rs\n--- 02_runtime_init/src/print.rs\n+++ 03_hacky_hello_world/src/print.rs\n@@ -0,0 +1,38 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Printing.\n+\n+use crate::console;\n+use core::fmt;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+#[doc(hidden)]\n+pub fn _print(args: fmt::Arguments) {\n+    use console::interface::Write;\n+\n+    console::console().write_fmt(args).unwrap();\n+}\n+\n+/// Prints without a newline.\n+///\n+/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n+#[macro_export]\n+macro_rules! print {\n+    ($($arg:tt)*) => ($crate::print::_print(format_args!($($arg)*)));\n+}\n+\n+/// Prints with a newline.\n+///\n+/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n+#[macro_export]\n+macro_rules! println {\n+    () => ($crate::print!(\"\\n\"));\n+    ($($arg:tt)*) => ({\n+        $crate::print::_print(format_args_nl!($($arg)*));\n+    })\n+}\n\ndiff -uNr 02_runtime_init/tests/boot_test_string.rb 03_hacky_hello_world/tests/boot_test_string.rb\n--- 02_runtime_init/tests/boot_test_string.rb\n+++ 03_hacky_hello_world/tests/boot_test_string.rb\n@@ -0,0 +1,3 @@\n+# frozen_string_literal: true\n+\n+EXPECTED_PRINT = 'Stopping here'\n\n```\n"
  },
  {
    "path": "03_hacky_hello_world/build.rs",
    "content": "use std::{env, fs, process};\n\nfn main() {\n    let ld_script_path = match env::var(\"LD_SCRIPT_PATH\") {\n        Ok(var) => var,\n        _ => process::exit(0),\n    };\n\n    let files = fs::read_dir(ld_script_path).unwrap();\n    files\n        .filter_map(Result::ok)\n        .filter(|d| {\n            if let Some(e) = d.path().extension() {\n                e == \"ld\"\n            } else {\n                false\n            }\n        })\n        .for_each(|f| println!(\"cargo:rerun-if-changed={}\", f.path().display()));\n}\n"
  },
  {
    "path": "03_hacky_hello_world/src/_arch/aarch64/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural boot code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::boot::arch_boot\n\nuse core::arch::global_asm;\n\n// Assembly counterpart to this file.\nglobal_asm!(\n    include_str!(\"boot.s\"),\n    CONST_CORE_ID_MASK = const 0b11\n);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The Rust entry of the `kernel` binary.\n///\n/// The function is called from the assembly `_start` function.\n#[no_mangle]\npub unsafe fn _start_rust() -> ! {\n    crate::kernel_init()\n}\n"
  },
  {
    "path": "03_hacky_hello_world/src/_arch/aarch64/cpu/boot.s",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2022 Andre Richter <andre.o.richter@gmail.com>\n\n//--------------------------------------------------------------------------------------------------\n// Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Load the address of a symbol into a register, PC-relative.\n//\n// The symbol must lie within +/- 4 GiB of the Program Counter.\n//\n// # Resources\n//\n// - https://sourceware.org/binutils/docs-2.36/as/AArch64_002dRelocations.html\n.macro ADR_REL register, symbol\n\tadrp\t\\register, \\symbol\n\tadd\t\\register, \\register, #:lo12:\\symbol\n.endm\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n.section .text._start\n\n//------------------------------------------------------------------------------\n// fn _start()\n//------------------------------------------------------------------------------\n_start:\n\t// Only proceed on the boot core. Park it otherwise.\n\tmrs\tx0, MPIDR_EL1\n\tand\tx0, x0, {CONST_CORE_ID_MASK}\n\tldr\tx1, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs\n\tcmp\tx0, x1\n\tb.ne\t.L_parking_loop\n\n\t// If execution reaches here, it is the boot core.\n\n\t// Initialize DRAM.\n\tADR_REL\tx0, __bss_start\n\tADR_REL x1, __bss_end_exclusive\n\n.L_bss_init_loop:\n\tcmp\tx0, x1\n\tb.eq\t.L_prepare_rust\n\tstp\txzr, xzr, [x0], #16\n\tb\t.L_bss_init_loop\n\n\t// Prepare the jump to Rust code.\n.L_prepare_rust:\n\t// Set the stack pointer.\n\tADR_REL\tx0, __boot_core_stack_end_exclusive\n\tmov\tsp, x0\n\n\t// Jump to Rust code.\n\tb\t_start_rust\n\n\t// Infinitely wait for events (aka \"park the core\").\n.L_parking_loop:\n\twfe\n\tb\t.L_parking_loop\n\n.size\t_start, . - _start\n.type\t_start, function\n.global\t_start\n"
  },
  {
    "path": "03_hacky_hello_world/src/_arch/aarch64/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural processor code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::arch_cpu\n\nuse aarch64_cpu::asm;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Pause execution on the core.\n#[inline(always)]\npub fn wait_forever() -> ! {\n    loop {\n        asm::wfe()\n    }\n}\n"
  },
  {
    "path": "03_hacky_hello_world/src/bsp/raspberrypi/console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP console facilities.\n\nuse crate::console;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// A mystical, magical device for generating QEMU output out of the void.\nstruct QEMUOutput;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Implementing `core::fmt::Write` enables usage of the `format_args!` macros, which in turn are\n/// used to implement the `kernel`'s `print!` and `println!` macros. By implementing `write_str()`,\n/// we get `write_fmt()` automatically.\n///\n/// See [`src/print.rs`].\n///\n/// [`src/print.rs`]: ../../print/index.html\nimpl fmt::Write for QEMUOutput {\n    fn write_str(&mut self, s: &str) -> fmt::Result {\n        for c in s.chars() {\n            unsafe {\n                core::ptr::write_volatile(0x3F20_1000 as *mut u8, c as u8);\n            }\n        }\n\n        Ok(())\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the console.\npub fn console() -> impl console::interface::Write {\n    QEMUOutput {}\n}\n"
  },
  {
    "path": "03_hacky_hello_world/src/bsp/raspberrypi/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Processor code.\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Used by `arch` code to find the early boot core.\n#[no_mangle]\n#[link_section = \".text._start_arguments\"]\npub static BOOT_CORE_ID: u64 = 0;\n"
  },
  {
    "path": "03_hacky_hello_world/src/bsp/raspberrypi/kernel.ld",
    "content": "/* SPDX-License-Identifier: MIT OR Apache-2.0\n *\n * Copyright (c) 2018-2022 Andre Richter <andre.o.richter@gmail.com>\n */\n\n__rpi_phys_dram_start_addr = 0;\n\n/* The physical address at which the the kernel binary will be loaded by the Raspberry's firmware */\n__rpi_phys_binary_load_addr = 0x80000;\n\n\nENTRY(__rpi_phys_binary_load_addr)\n\n/* Flags:\n *     4 == R\n *     5 == RX\n *     6 == RW\n *\n * Segments are marked PT_LOAD below so that the ELF file provides virtual and physical addresses.\n * It doesn't mean all of them need actually be loaded.\n */\nPHDRS\n{\n    segment_boot_core_stack PT_LOAD FLAGS(6);\n    segment_code            PT_LOAD FLAGS(5);\n    segment_data            PT_LOAD FLAGS(6);\n}\n\nSECTIONS\n{\n    . =  __rpi_phys_dram_start_addr;\n\n    /***********************************************************************************************\n    * Boot Core Stack\n    ***********************************************************************************************/\n    .boot_core_stack (NOLOAD) :\n    {\n                                             /*   ^             */\n                                             /*   | stack       */\n        . += __rpi_phys_binary_load_addr;    /*   | growth      */\n                                             /*   | direction   */\n        __boot_core_stack_end_exclusive = .; /*   |             */\n    } :segment_boot_core_stack\n\n    /***********************************************************************************************\n    * Code + RO Data + Global Offset Table\n    ***********************************************************************************************/\n    .text :\n    {\n        KEEP(*(.text._start))\n        *(.text._start_arguments) /* Constants (or statics in Rust speak) read by _start(). */\n        *(.text._start_rust)      /* The Rust entry point */\n        *(.text*)                 /* Everything else */\n    } :segment_code\n\n    .rodata : ALIGN(8) { *(.rodata*) } :segment_code\n\n    /***********************************************************************************************\n    * Data + BSS\n    ***********************************************************************************************/\n    .data : { *(.data*) } :segment_data\n\n    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */\n    .bss (NOLOAD) : ALIGN(16)\n    {\n        __bss_start = .;\n        *(.bss*);\n        . = ALIGN(16);\n        __bss_end_exclusive = .;\n    } :segment_data\n\n    /***********************************************************************************************\n    * Misc\n    ***********************************************************************************************/\n    .got : { *(.got*) }\n    ASSERT(SIZEOF(.got) == 0, \"Relocation support not expected\")\n\n    /DISCARD/ : { *(.comment*) }\n}\n"
  },
  {
    "path": "03_hacky_hello_world/src/bsp/raspberrypi.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Top-level BSP file for the Raspberry Pi 3 and 4.\n\npub mod console;\npub mod cpu;\n"
  },
  {
    "path": "03_hacky_hello_world/src/bsp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Conditional reexporting of Board Support Packages.\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod raspberrypi;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use raspberrypi::*;\n"
  },
  {
    "path": "03_hacky_hello_world/src/console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! System console.\n\nuse crate::bsp;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Console interfaces.\npub mod interface {\n    /// Console write functions.\n    ///\n    /// `core::fmt::Write` is exactly what we need for now. Re-export it here because\n    /// implementing `console::Write` gives a better hint to the reader about the\n    /// intention.\n    pub use core::fmt::Write;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the console.\n///\n/// This is the global console used by all printing macros.\npub fn console() -> impl interface::Write {\n    bsp::console::console()\n}\n"
  },
  {
    "path": "03_hacky_hello_world/src/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Boot code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/cpu/boot.rs\"]\nmod arch_boot;\n"
  },
  {
    "path": "03_hacky_hello_world/src/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Processor code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/cpu.rs\"]\nmod arch_cpu;\n\nmod boot;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_cpu::wait_forever;\n"
  },
  {
    "path": "03_hacky_hello_world/src/main.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n// Rust embedded logo for `make doc`.\n#![doc(\n    html_logo_url = \"https://raw.githubusercontent.com/rust-embedded/wg/master/assets/logo/ewg-logo-blue-white-on-transparent.png\"\n)]\n\n//! The `kernel` binary.\n//!\n//! # Code organization and architecture\n//!\n//! The code is divided into different *modules*, each representing a typical **subsystem** of the\n//! `kernel`. Top-level module files of subsystems reside directly in the `src` folder. For example,\n//! `src/memory.rs` contains code that is concerned with all things memory management.\n//!\n//! ## Visibility of processor architecture code\n//!\n//! Some of the `kernel`'s subsystems depend on low-level code that is specific to the target\n//! processor architecture. For each supported processor architecture, there exists a subfolder in\n//! `src/_arch`, for example, `src/_arch/aarch64`.\n//!\n//! The architecture folders mirror the subsystem modules laid out in `src`. For example,\n//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go\n//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in\n//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic\n//! module's name prefixed with `arch_`.\n//!\n//! For example, this is the top of `src/memory/mmu.rs`:\n//!\n//! ```\n//! #[cfg(target_arch = \"aarch64\")]\n//! #[path = \"../_arch/aarch64/memory/mmu.rs\"]\n//! mod arch_mmu;\n//! ```\n//!\n//! Often times, items from the `arch_ module` will be publicly reexported by the parent module.\n//! This way, each architecture specific module can provide its implementation of an item, while the\n//! caller must not be concerned which architecture has been conditionally compiled.\n//!\n//! ## BSP code\n//!\n//! `BSP` stands for Board Support Package. `BSP` code is organized under `src/bsp.rs` and contains\n//! target board specific definitions and functions. These are things such as the board's memory map\n//! or instances of drivers for devices that are featured on the respective board.\n//!\n//! Just like processor architecture code, the `BSP` code's module structure tries to mirror the\n//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is\n//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`.\n//!\n//! ## Kernel interfaces\n//!\n//! Both `arch` and `bsp` contain code that is conditionally compiled depending on the actual target\n//! and board for which the kernel is compiled. For example, the `interrupt controller` hardware of\n//! the `Raspberry Pi 3` and the `Raspberry Pi 4` is different, but we want the rest of the `kernel`\n//! code to play nicely with any of the two without much hassle.\n//!\n//! In order to provide a clean abstraction between `arch`, `bsp` and `generic kernel code`,\n//! `interface` traits are provided *whenever possible* and *where it makes sense*. They are defined\n//! in the respective subsystem module and help to enforce the idiom of *program to an interface,\n//! not an implementation*. For example, there will be a common IRQ handling interface which the two\n//! different interrupt controller `drivers` of both Raspberrys will implement, and only export the\n//! interface to the rest of the `kernel`.\n//!\n//! ```\n//!         +-------------------+\n//!         | Interface (Trait) |\n//!         |                   |\n//!         +--+-------------+--+\n//!            ^             ^\n//!            |             |\n//!            |             |\n//! +----------+--+       +--+----------+\n//! | kernel code |       |  bsp code   |\n//! |             |       |  arch code  |\n//! +-------------+       +-------------+\n//! ```\n//!\n//! # Summary\n//!\n//! For a logical `kernel` subsystem, corresponding code can be distributed over several physical\n//! locations. Here is an example for the **memory** subsystem:\n//!\n//! - `src/memory.rs` and `src/memory/**/*`\n//!   - Common code that is agnostic of target processor architecture and `BSP` characteristics.\n//!     - Example: A function to zero a chunk of memory.\n//!   - Interfaces for the memory subsystem that are implemented by `arch` or `BSP` code.\n//!     - Example: An `MMU` interface that defines `MMU` function prototypes.\n//! - `src/bsp/__board_name__/memory.rs` and `src/bsp/__board_name__/memory/**/*`\n//!   - `BSP` specific code.\n//!   - Example: The board's memory map (physical addresses of DRAM and MMIO devices).\n//! - `src/_arch/__arch_name__/memory.rs` and `src/_arch/__arch_name__/memory/**/*`\n//!   - Processor architecture specific code.\n//!   - Example: Implementation of the `MMU` interface for the `__arch_name__` processor\n//!     architecture.\n//!\n//! From a namespace perspective, **memory** subsystem code lives in:\n//!\n//! - `crate::memory::*`\n//! - `crate::bsp::memory::*`\n//!\n//! # Boot flow\n//!\n//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.\n//!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.\n//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.\n\n#![feature(asm_const)]\n#![feature(format_args_nl)]\n#![feature(panic_info_message)]\n#![no_main]\n#![no_std]\n\nmod bsp;\nmod console;\nmod cpu;\nmod panic_wait;\nmod print;\n\n/// Early init code.\n///\n/// # Safety\n///\n/// - Only a single core must be active and running this function.\nunsafe fn kernel_init() -> ! {\n    println!(\"Hello from Rust!\");\n\n    panic!(\"Stopping here.\")\n}\n"
  },
  {
    "path": "03_hacky_hello_world/src/panic_wait.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A panic handler that infinitely waits.\n\nuse crate::{cpu, println};\nuse core::panic::PanicInfo;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Stop immediately if called a second time.\n///\n/// # Note\n///\n/// Using atomics here relieves us from needing to use `unsafe` for the static variable.\n///\n/// On `AArch64`, which is the only implemented architecture at the time of writing this,\n/// [`AtomicBool::load`] and [`AtomicBool::store`] are lowered to ordinary load and store\n/// instructions. They are therefore safe to use even with MMU + caching deactivated.\n///\n/// [`AtomicBool::load`]: core::sync::atomic::AtomicBool::load\n/// [`AtomicBool::store`]: core::sync::atomic::AtomicBool::store\nfn panic_prevent_reenter() {\n    use core::sync::atomic::{AtomicBool, Ordering};\n\n    #[cfg(not(target_arch = \"aarch64\"))]\n    compile_error!(\"Add the target_arch to above's check if the following code is safe to use\");\n\n    static PANIC_IN_PROGRESS: AtomicBool = AtomicBool::new(false);\n\n    if !PANIC_IN_PROGRESS.load(Ordering::Relaxed) {\n        PANIC_IN_PROGRESS.store(true, Ordering::Relaxed);\n\n        return;\n    }\n\n    cpu::wait_forever()\n}\n\n#[panic_handler]\nfn panic(info: &PanicInfo) -> ! {\n    // Protect against panic infinite loops if any of the following code panics itself.\n    panic_prevent_reenter();\n\n    let (location, line, column) = match info.location() {\n        Some(loc) => (loc.file(), loc.line(), loc.column()),\n        _ => (\"???\", 0, 0),\n    };\n\n    println!(\n        \"Kernel panic!\\n\\n\\\n        Panic location:\\n      File '{}', line {}, column {}\\n\\n\\\n        {}\",\n        location,\n        line,\n        column,\n        info.message().unwrap_or(&format_args!(\"\")),\n    );\n\n    cpu::wait_forever()\n}\n"
  },
  {
    "path": "03_hacky_hello_world/src/print.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Printing.\n\nuse crate::console;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n#[doc(hidden)]\npub fn _print(args: fmt::Arguments) {\n    use console::interface::Write;\n\n    console::console().write_fmt(args).unwrap();\n}\n\n/// Prints without a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! print {\n    ($($arg:tt)*) => ($crate::print::_print(format_args!($($arg)*)));\n}\n\n/// Prints with a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! println {\n    () => ($crate::print!(\"\\n\"));\n    ($($arg:tt)*) => ({\n        $crate::print::_print(format_args_nl!($($arg)*));\n    })\n}\n"
  },
  {
    "path": "03_hacky_hello_world/tests/boot_test_string.rb",
    "content": "# frozen_string_literal: true\n\nEXPECTED_PRINT = 'Stopping here'\n"
  },
  {
    "path": "04_safe_globals/.vscode/settings.json",
    "content": "{\n    \"editor.formatOnSave\": true,\n    \"editor.rulers\": [100],\n    \"rust-analyzer.cargo.target\": \"aarch64-unknown-none-softfloat\",\n    \"rust-analyzer.cargo.features\": [\"bsp_rpi3\"],\n    \"rust-analyzer.checkOnSave.allTargets\": false,\n    \"rust-analyzer.checkOnSave.extraArgs\": [\"--bins\"],\n    \"rust-analyzer.lens.debug\": false,\n    \"rust-analyzer.lens.run\": false\n}\n"
  },
  {
    "path": "04_safe_globals/Cargo.toml",
    "content": "[package]\nname = \"mingo\"\nversion = \"0.4.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n\n[profile.release]\nlto = true\n\n[features]\ndefault = []\nbsp_rpi3 = []\nbsp_rpi4 = []\n\n[[bin]]\nname = \"kernel\"\npath = \"src/main.rs\"\n\n##--------------------------------------------------------------------------------------------------\n## Dependencies\n##--------------------------------------------------------------------------------------------------\n\n[dependencies]\n\n# Platform specific dependencies\n[target.'cfg(target_arch = \"aarch64\")'.dependencies]\naarch64-cpu = { version = \"9.x.x\" }\n"
  },
  {
    "path": "04_safe_globals/Makefile",
    "content": "## SPDX-License-Identifier: MIT OR Apache-2.0\n##\n## Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\ninclude ../common/docker.mk\ninclude ../common/format.mk\ninclude ../common/operating_system.mk\n\n##--------------------------------------------------------------------------------------------------\n## Optional, user-provided configuration values\n##--------------------------------------------------------------------------------------------------\n\n# Default to the RPi3.\nBSP ?= rpi3\n\n\n\n##--------------------------------------------------------------------------------------------------\n## BSP-specific configuration values\n##--------------------------------------------------------------------------------------------------\nQEMU_MISSING_STRING = \"This board is not yet supported for QEMU.\"\n\nifeq ($(BSP),rpi3)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE = raspi3\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    LD_SCRIPT_PATH    = $(shell pwd)/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a53\nelse ifeq ($(BSP),rpi4)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE =\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    LD_SCRIPT_PATH    = $(shell pwd)/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a72\nendif\n\n# Export for build.rs.\nexport LD_SCRIPT_PATH\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets and Prerequisites\n##--------------------------------------------------------------------------------------------------\nKERNEL_MANIFEST      = Cargo.toml\nKERNEL_LINKER_SCRIPT = kernel.ld\nLAST_BUILD_CONFIG    = target/$(BSP).build_config\n\nKERNEL_ELF      = target/$(TARGET)/release/kernel\n# This parses cargo's dep-info file.\n# https://doc.rust-lang.org/cargo/guide/build-cache.html#dep-info-files\nKERNEL_ELF_DEPS = $(filter-out %: ,$(file < $(KERNEL_ELF).d)) $(KERNEL_MANIFEST) $(LAST_BUILD_CONFIG)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Command building blocks\n##--------------------------------------------------------------------------------------------------\nRUSTFLAGS = $(RUSTC_MISC_ARGS)                   \\\n    -C link-arg=--library-path=$(LD_SCRIPT_PATH) \\\n    -C link-arg=--script=$(KERNEL_LINKER_SCRIPT)\n\nRUSTFLAGS_PEDANTIC = $(RUSTFLAGS) \\\n    -D warnings                   \\\n    -D missing_docs\n\nFEATURES      = --features bsp_$(BSP)\nCOMPILER_ARGS = --target=$(TARGET) \\\n    $(FEATURES)                    \\\n    --release\n\nRUSTC_CMD   = cargo rustc $(COMPILER_ARGS)\nDOC_CMD     = cargo doc $(COMPILER_ARGS)\nCLIPPY_CMD  = cargo clippy $(COMPILER_ARGS)\nOBJCOPY_CMD = rust-objcopy \\\n    --strip-all            \\\n    -O binary\n\nEXEC_QEMU          = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE)\nEXEC_TEST_DISPATCH = ruby ../common/tests/dispatch.rb\n\n##------------------------------------------------------------------------------\n## Dockerization\n##------------------------------------------------------------------------------\nDOCKER_CMD            = docker run -t --rm -v $(shell pwd):/work/tutorial -w /work/tutorial\nDOCKER_CMD_INTERACT   = $(DOCKER_CMD) -i\nDOCKER_ARG_DIR_COMMON = -v $(shell pwd)/../common:/work/common\n\n# DOCKER_IMAGE defined in include file (see top of this file).\nDOCKER_QEMU  = $(DOCKER_CMD_INTERACT) $(DOCKER_IMAGE)\nDOCKER_TOOLS = $(DOCKER_CMD) $(DOCKER_IMAGE)\nDOCKER_TEST  = $(DOCKER_CMD) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: all doc qemu clippy clean readelf objdump nm check\n\nall: $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Save the configuration as a file, so make understands if it changed.\n##------------------------------------------------------------------------------\n$(LAST_BUILD_CONFIG):\n\t@rm -f target/*.build_config\n\t@mkdir -p target\n\t@touch $(LAST_BUILD_CONFIG)\n\n##------------------------------------------------------------------------------\n## Compile the kernel ELF\n##------------------------------------------------------------------------------\n$(KERNEL_ELF): $(KERNEL_ELF_DEPS)\n\t$(call color_header, \"Compiling kernel ELF - $(BSP)\")\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(RUSTC_CMD)\n\n##------------------------------------------------------------------------------\n## Generate the stripped kernel binary\n##------------------------------------------------------------------------------\n$(KERNEL_BIN): $(KERNEL_ELF)\n\t$(call color_header, \"Generating stripped binary\")\n\t@$(OBJCOPY_CMD) $(KERNEL_ELF) $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Name\")\n\t@echo $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Size\")\n\t$(call disk_usage_KiB, $(KERNEL_BIN))\n\n##------------------------------------------------------------------------------\n## Generate the documentation\n##------------------------------------------------------------------------------\ndoc:\n\t$(call color_header, \"Generating docs\")\n\t@$(DOC_CMD) --document-private-items --open\n\n##------------------------------------------------------------------------------\n## Run the kernel in QEMU\n##------------------------------------------------------------------------------\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\nqemu:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\nqemu: $(KERNEL_BIN)\n\t$(call color_header, \"Launching QEMU\")\n\t@$(DOCKER_QEMU) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\nendif\n\n##------------------------------------------------------------------------------\n## Run clippy\n##------------------------------------------------------------------------------\nclippy:\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(CLIPPY_CMD)\n\n##------------------------------------------------------------------------------\n## Clean\n##------------------------------------------------------------------------------\nclean:\n\trm -rf target $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run readelf\n##------------------------------------------------------------------------------\nreadelf: $(KERNEL_ELF)\n\t$(call color_header, \"Launching readelf\")\n\t@$(DOCKER_TOOLS) $(READELF_BINARY) --headers $(KERNEL_ELF)\n\n##------------------------------------------------------------------------------\n## Run objdump\n##------------------------------------------------------------------------------\nobjdump: $(KERNEL_ELF)\n\t$(call color_header, \"Launching objdump\")\n\t@$(DOCKER_TOOLS) $(OBJDUMP_BINARY) --disassemble --demangle \\\n                --section .text   \\\n                --section .rodata \\\n                $(KERNEL_ELF) | rustfilt\n\n##------------------------------------------------------------------------------\n## Run nm\n##------------------------------------------------------------------------------\nnm: $(KERNEL_ELF)\n\t$(call color_header, \"Launching nm\")\n\t@$(DOCKER_TOOLS) $(NM_BINARY) --demangle --print-size $(KERNEL_ELF) | sort | rustfilt\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Testing targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: test test_boot\n\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\ntest_boot test:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\n##------------------------------------------------------------------------------\n## Run boot test\n##------------------------------------------------------------------------------\ntest_boot: $(KERNEL_BIN)\n\t$(call color_header, \"Boot test - $(BSP)\")\n\t@$(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\ntest: test_boot\n\nendif\n"
  },
  {
    "path": "04_safe_globals/README.CN.md",
    "content": "# 教程 04 - 全局安全\n\n## tl;dr\n\n- 引入了假的锁。\n- 这是第一次展示原始操作系统同步，并支持安全访问全局数据结构。\n\n## Rust中的全局可变\n\n当我们引入全局可用的`print!`宏在 [教程03]，我门有一点作弊。 调用\n`core::fmt`的`write_fmt()`函数，接受`&mut self`的方法之所以有效，\n是因为在每次调用时都会创建一个新的`QEMUOutput`实例。\n\n如果我们想保留一些状态，例如关于写入字符数的统计数据，\n我们需要创建`QEMUOutput`的一个全局实例 (在Rust中，使用`static`关键字).\n\n然而`static QEMU_OUTPUT`不允许调用具有`&mut self`的函数。\n为此，我们需要`static mut`，但是调用改变`static mut`状态的函数是不安全的。\n这个是Rust编译器对此的推理，它无法再阻止核心/线程同时改变数据（它是全局的，所以每个人都可以从任何地方引用它，检查程序借用在这里帮不上忙）。\n\n\n这个问题的解决方案是将全局封装到原始同步中。在我们的例子中，是一个*MUTual EXclusion*原语的变体。\n`Mutex`是`synchronization.rs`中引入的一个特性，并由同一文件中的`NullLock`实现。\n为了使代码更易于教学，它省略了用于防止并发访问的实际体系结构特定逻辑，因为只要内核仅在单个内核上执行并禁用中断，我们就不需要它。\n\n`NullLock`侧重于展示Rust内部可变性的核心概念。请务必阅读它。\n我们还建议您阅读这篇关于[Rust的引用类型的精确心智模型]文章\n\n如果要将`NullLock`与一些真实的互斥实现进行比较，可以查看\n[spin crate]或者[parking lot crate]。\n\n[教程03]: ../03_hacky_hello_world\n[内部可变性]: https://doc.rust-lang.org/std/cell/index.html\n[Rust的引用类型的精确心智模型]: https://docs.rs/dtolnay/0.0.6/dtolnay/macro._02__reference_types.html\n[spin crate]: https://github.com/mvdnes/spin-rs\n[parking lot crate]: https://github.com/Amanieu/parking_lot\n\n## 测试\n\n```console\n$ make qemu\n[...]\n\n[0] Hello from Rust!\n[1] Chars written: 22\n[2] Stopping here.\n```\n\n## 相比之前的变化（diff）\n请检查[英文版本](README.md#diff-to-previous)，这是最新的。\n"
  },
  {
    "path": "04_safe_globals/README.md",
    "content": "# Tutorial 04 - Safe Globals\n\n## tl;dr\n\n- A pseudo-lock is introduced.\n- It is a first showcase of OS synchronization primitives and enables safe access to a global data\n  structure.\n\n## Mutable globals in Rust\n\nWhen we introduced the globally usable `print!` macros in [tutorial 03], we cheated a bit. Calling\n`core::fmt`'s `write_fmt()` function, which takes an `&mut self`, was only working because on each\ncall, a new instance of `QEMUOutput` was created.\n\nIf we would want to preserve some state, e.g. statistics about the number of characters written, we\nneed to make a single global instance of `QEMUOutput` (in Rust, using the `static` keyword).\n\nA `static QEMU_OUTPUT`, however, would not allow to call functions taking `&mut self`. For that, we\nwould need a `static mut`, but calling functions that mutate state on `static mut`s is unsafe. The\nRust compiler's reasoning for this is that it can then not prevent anymore that multiple\ncores/threads are mutating the data concurrently (it is a global, so everyone can reference it from\nanywhere. The borrow checker can't help here).\n\nThe solution to this problem is to wrap the global into a synchronization primitive. In our case, a\nvariant of a *MUTual EXclusion* primitive. `Mutex` is introduced as a trait in `synchronization.rs`,\nand implemented by the `NullLock` in the same file. In order to make the code lean for teaching\npurposes, it leaves out the actual architecture-specific logic for protection against concurrent\naccess, since we don't need it as long as the kernel only executes on a single core with interrupts\ndisabled.\n\nThe `NullLock` focuses on showcasing the Rust core concept of [interior mutability]. Make sure to\nread up on it. I also recommend to read this article about an [accurate mental model for Rust's\nreference types].\n\nIf you want to compare the `NullLock` to some real-world mutex implementations, you can check out\nimplemntations in the [spin crate] or the [parking lot crate].\n\n[tutorial 03]: ../03_hacky_hello_world\n[interior mutability]: https://doc.rust-lang.org/std/cell/index.html\n[accurate mental model for Rust's reference types]: https://docs.rs/dtolnay/0.0.6/dtolnay/macro._02__reference_types.html\n[spin crate]: https://github.com/mvdnes/spin-rs\n[parking lot crate]: https://github.com/Amanieu/parking_lot\n\n## Test it\n\n```console\n$ make qemu\n[...]\n\n[0] Hello from Rust!\n[1] Chars written: 22\n[2] Stopping here.\n```\n\n## Diff to previous\n```diff\n\ndiff -uNr 03_hacky_hello_world/Cargo.toml 04_safe_globals/Cargo.toml\n--- 03_hacky_hello_world/Cargo.toml\n+++ 04_safe_globals/Cargo.toml\n@@ -1,6 +1,6 @@\n [package]\n name = \"mingo\"\n-version = \"0.3.0\"\n+version = \"0.4.0\"\n authors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\n edition = \"2021\"\n\n\ndiff -uNr 03_hacky_hello_world/src/bsp/raspberrypi/console.rs 04_safe_globals/src/bsp/raspberrypi/console.rs\n--- 03_hacky_hello_world/src/bsp/raspberrypi/console.rs\n+++ 04_safe_globals/src/bsp/raspberrypi/console.rs\n@@ -4,7 +4,7 @@\n\n //! BSP console facilities.\n\n-use crate::console;\n+use crate::{console, synchronization, synchronization::NullLock};\n use core::fmt;\n\n //--------------------------------------------------------------------------------------------------\n@@ -12,25 +12,64 @@\n //--------------------------------------------------------------------------------------------------\n\n /// A mystical, magical device for generating QEMU output out of the void.\n-struct QEMUOutput;\n+///\n+/// The mutex protected part.\n+struct QEMUOutputInner {\n+    chars_written: usize,\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// The main struct.\n+pub struct QEMUOutput {\n+    inner: NullLock<QEMUOutputInner>,\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Global instances\n+//--------------------------------------------------------------------------------------------------\n+\n+static QEMU_OUTPUT: QEMUOutput = QEMUOutput::new();\n\n //--------------------------------------------------------------------------------------------------\n // Private Code\n //--------------------------------------------------------------------------------------------------\n\n+impl QEMUOutputInner {\n+    const fn new() -> QEMUOutputInner {\n+        QEMUOutputInner { chars_written: 0 }\n+    }\n+\n+    /// Send a character.\n+    fn write_char(&mut self, c: char) {\n+        unsafe {\n+            core::ptr::write_volatile(0x3F20_1000 as *mut u8, c as u8);\n+        }\n+\n+        self.chars_written += 1;\n+    }\n+}\n+\n /// Implementing `core::fmt::Write` enables usage of the `format_args!` macros, which in turn are\n /// used to implement the `kernel`'s `print!` and `println!` macros. By implementing `write_str()`,\n /// we get `write_fmt()` automatically.\n ///\n+/// The function takes an `&mut self`, so it must be implemented for the inner struct.\n+///\n /// See [`src/print.rs`].\n ///\n /// [`src/print.rs`]: ../../print/index.html\n-impl fmt::Write for QEMUOutput {\n+impl fmt::Write for QEMUOutputInner {\n     fn write_str(&mut self, s: &str) -> fmt::Result {\n         for c in s.chars() {\n-            unsafe {\n-                core::ptr::write_volatile(0x3F20_1000 as *mut u8, c as u8);\n+            // Convert newline to carrige return + newline.\n+            if c == '\\n' {\n+                self.write_char('\\r')\n             }\n+\n+            self.write_char(c);\n         }\n\n         Ok(())\n@@ -41,7 +80,39 @@\n // Public Code\n //--------------------------------------------------------------------------------------------------\n\n+impl QEMUOutput {\n+    /// Create a new instance.\n+    pub const fn new() -> QEMUOutput {\n+        QEMUOutput {\n+            inner: NullLock::new(QEMUOutputInner::new()),\n+        }\n+    }\n+}\n+\n /// Return a reference to the console.\n-pub fn console() -> impl console::interface::Write {\n-    QEMUOutput {}\n+pub fn console() -> &'static dyn console::interface::All {\n+    &QEMU_OUTPUT\n }\n+\n+//------------------------------------------------------------------------------\n+// OS Interface Code\n+//------------------------------------------------------------------------------\n+use synchronization::interface::Mutex;\n+\n+/// Passthrough of `args` to the `core::fmt::Write` implementation, but guarded by a Mutex to\n+/// serialize access.\n+impl console::interface::Write for QEMUOutput {\n+    fn write_fmt(&self, args: core::fmt::Arguments) -> fmt::Result {\n+        // Fully qualified syntax for the call to `core::fmt::Write::write_fmt()` to increase\n+        // readability.\n+        self.inner.lock(|inner| fmt::Write::write_fmt(inner, args))\n+    }\n+}\n+\n+impl console::interface::Statistics for QEMUOutput {\n+    fn chars_written(&self) -> usize {\n+        self.inner.lock(|inner| inner.chars_written)\n+    }\n+}\n+\n+impl console::interface::All for QEMUOutput {}\n\ndiff -uNr 03_hacky_hello_world/src/console.rs 04_safe_globals/src/console.rs\n--- 03_hacky_hello_world/src/console.rs\n+++ 04_safe_globals/src/console.rs\n@@ -12,12 +12,24 @@\n\n /// Console interfaces.\n pub mod interface {\n+    use core::fmt;\n+\n     /// Console write functions.\n-    ///\n-    /// `core::fmt::Write` is exactly what we need for now. Re-export it here because\n-    /// implementing `console::Write` gives a better hint to the reader about the\n-    /// intention.\n-    pub use core::fmt::Write;\n+    pub trait Write {\n+        /// Write a Rust format string.\n+        fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result;\n+    }\n+\n+    /// Console statistics.\n+    pub trait Statistics {\n+        /// Return the number of characters written.\n+        fn chars_written(&self) -> usize {\n+            0\n+        }\n+    }\n+\n+    /// Trait alias for a full-fledged console.\n+    pub trait All: Write + Statistics {}\n }\n\n //--------------------------------------------------------------------------------------------------\n@@ -27,6 +39,6 @@\n /// Return a reference to the console.\n ///\n /// This is the global console used by all printing macros.\n-pub fn console() -> impl interface::Write {\n+pub fn console() -> &'static dyn interface::All {\n     bsp::console::console()\n }\n\ndiff -uNr 03_hacky_hello_world/src/main.rs 04_safe_globals/src/main.rs\n--- 03_hacky_hello_world/src/main.rs\n+++ 04_safe_globals/src/main.rs\n@@ -109,6 +109,7 @@\n #![feature(asm_const)]\n #![feature(format_args_nl)]\n #![feature(panic_info_message)]\n+#![feature(trait_alias)]\n #![no_main]\n #![no_std]\n\n@@ -117,6 +118,7 @@\n mod cpu;\n mod panic_wait;\n mod print;\n+mod synchronization;\n\n /// Early init code.\n ///\n@@ -124,7 +126,12 @@\n ///\n /// - Only a single core must be active and running this function.\n unsafe fn kernel_init() -> ! {\n-    println!(\"Hello from Rust!\");\n+    use console::console;\n\n-    panic!(\"Stopping here.\")\n+    println!(\"[0] Hello from Rust!\");\n+\n+    println!(\"[1] Chars written: {}\", console().chars_written());\n+\n+    println!(\"[2] Stopping here.\");\n+    cpu::wait_forever()\n }\n\ndiff -uNr 03_hacky_hello_world/src/print.rs 04_safe_globals/src/print.rs\n--- 03_hacky_hello_world/src/print.rs\n+++ 04_safe_globals/src/print.rs\n@@ -13,8 +13,6 @@\n\n #[doc(hidden)]\n pub fn _print(args: fmt::Arguments) {\n-    use console::interface::Write;\n-\n     console::console().write_fmt(args).unwrap();\n }\n\n\ndiff -uNr 03_hacky_hello_world/src/synchronization.rs 04_safe_globals/src/synchronization.rs\n--- 03_hacky_hello_world/src/synchronization.rs\n+++ 04_safe_globals/src/synchronization.rs\n@@ -0,0 +1,77 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Synchronization primitives.\n+//!\n+//! # Resources\n+//!\n+//!   - <https://doc.rust-lang.org/book/ch16-04-extensible-concurrency-sync-and-send.html>\n+//!   - <https://stackoverflow.com/questions/59428096/understanding-the-send-trait>\n+//!   - <https://doc.rust-lang.org/std/cell/index.html>\n+\n+use core::cell::UnsafeCell;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Synchronization interfaces.\n+pub mod interface {\n+\n+    /// Any object implementing this trait guarantees exclusive access to the data wrapped within\n+    /// the Mutex for the duration of the provided closure.\n+    pub trait Mutex {\n+        /// The type of the data that is wrapped by this mutex.\n+        type Data;\n+\n+        /// Locks the mutex and grants the closure temporary mutable access to the wrapped data.\n+        fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R;\n+    }\n+}\n+\n+/// A pseudo-lock for teaching purposes.\n+///\n+/// In contrast to a real Mutex implementation, does not protect against concurrent access from\n+/// other cores to the contained data. This part is preserved for later lessons.\n+///\n+/// The lock will only be used as long as it is safe to do so, i.e. as long as the kernel is\n+/// executing single-threaded, aka only running on a single core with interrupts disabled.\n+pub struct NullLock<T>\n+where\n+    T: ?Sized,\n+{\n+    data: UnsafeCell<T>,\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+unsafe impl<T> Send for NullLock<T> where T: ?Sized + Send {}\n+unsafe impl<T> Sync for NullLock<T> where T: ?Sized + Send {}\n+\n+impl<T> NullLock<T> {\n+    /// Create an instance.\n+    pub const fn new(data: T) -> Self {\n+        Self {\n+            data: UnsafeCell::new(data),\n+        }\n+    }\n+}\n+\n+//------------------------------------------------------------------------------\n+// OS Interface Code\n+//------------------------------------------------------------------------------\n+\n+impl<T> interface::Mutex for NullLock<T> {\n+    type Data = T;\n+\n+    fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R {\n+        // In a real lock, there would be code encapsulating this line that ensures that this\n+        // mutable reference will ever only be given out once at a time.\n+        let data = unsafe { &mut *self.data.get() };\n+\n+        f(data)\n+    }\n+}\n\n```\n"
  },
  {
    "path": "04_safe_globals/build.rs",
    "content": "use std::{env, fs, process};\n\nfn main() {\n    let ld_script_path = match env::var(\"LD_SCRIPT_PATH\") {\n        Ok(var) => var,\n        _ => process::exit(0),\n    };\n\n    let files = fs::read_dir(ld_script_path).unwrap();\n    files\n        .filter_map(Result::ok)\n        .filter(|d| {\n            if let Some(e) = d.path().extension() {\n                e == \"ld\"\n            } else {\n                false\n            }\n        })\n        .for_each(|f| println!(\"cargo:rerun-if-changed={}\", f.path().display()));\n}\n"
  },
  {
    "path": "04_safe_globals/src/_arch/aarch64/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural boot code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::boot::arch_boot\n\nuse core::arch::global_asm;\n\n// Assembly counterpart to this file.\nglobal_asm!(\n    include_str!(\"boot.s\"),\n    CONST_CORE_ID_MASK = const 0b11\n);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The Rust entry of the `kernel` binary.\n///\n/// The function is called from the assembly `_start` function.\n#[no_mangle]\npub unsafe fn _start_rust() -> ! {\n    crate::kernel_init()\n}\n"
  },
  {
    "path": "04_safe_globals/src/_arch/aarch64/cpu/boot.s",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2022 Andre Richter <andre.o.richter@gmail.com>\n\n//--------------------------------------------------------------------------------------------------\n// Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Load the address of a symbol into a register, PC-relative.\n//\n// The symbol must lie within +/- 4 GiB of the Program Counter.\n//\n// # Resources\n//\n// - https://sourceware.org/binutils/docs-2.36/as/AArch64_002dRelocations.html\n.macro ADR_REL register, symbol\n\tadrp\t\\register, \\symbol\n\tadd\t\\register, \\register, #:lo12:\\symbol\n.endm\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n.section .text._start\n\n//------------------------------------------------------------------------------\n// fn _start()\n//------------------------------------------------------------------------------\n_start:\n\t// Only proceed on the boot core. Park it otherwise.\n\tmrs\tx0, MPIDR_EL1\n\tand\tx0, x0, {CONST_CORE_ID_MASK}\n\tldr\tx1, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs\n\tcmp\tx0, x1\n\tb.ne\t.L_parking_loop\n\n\t// If execution reaches here, it is the boot core.\n\n\t// Initialize DRAM.\n\tADR_REL\tx0, __bss_start\n\tADR_REL x1, __bss_end_exclusive\n\n.L_bss_init_loop:\n\tcmp\tx0, x1\n\tb.eq\t.L_prepare_rust\n\tstp\txzr, xzr, [x0], #16\n\tb\t.L_bss_init_loop\n\n\t// Prepare the jump to Rust code.\n.L_prepare_rust:\n\t// Set the stack pointer.\n\tADR_REL\tx0, __boot_core_stack_end_exclusive\n\tmov\tsp, x0\n\n\t// Jump to Rust code.\n\tb\t_start_rust\n\n\t// Infinitely wait for events (aka \"park the core\").\n.L_parking_loop:\n\twfe\n\tb\t.L_parking_loop\n\n.size\t_start, . - _start\n.type\t_start, function\n.global\t_start\n"
  },
  {
    "path": "04_safe_globals/src/_arch/aarch64/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural processor code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::arch_cpu\n\nuse aarch64_cpu::asm;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Pause execution on the core.\n#[inline(always)]\npub fn wait_forever() -> ! {\n    loop {\n        asm::wfe()\n    }\n}\n"
  },
  {
    "path": "04_safe_globals/src/bsp/raspberrypi/console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP console facilities.\n\nuse crate::{console, synchronization, synchronization::NullLock};\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// A mystical, magical device for generating QEMU output out of the void.\n///\n/// The mutex protected part.\nstruct QEMUOutputInner {\n    chars_written: usize,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// The main struct.\npub struct QEMUOutput {\n    inner: NullLock<QEMUOutputInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic QEMU_OUTPUT: QEMUOutput = QEMUOutput::new();\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl QEMUOutputInner {\n    const fn new() -> QEMUOutputInner {\n        QEMUOutputInner { chars_written: 0 }\n    }\n\n    /// Send a character.\n    fn write_char(&mut self, c: char) {\n        unsafe {\n            core::ptr::write_volatile(0x3F20_1000 as *mut u8, c as u8);\n        }\n\n        self.chars_written += 1;\n    }\n}\n\n/// Implementing `core::fmt::Write` enables usage of the `format_args!` macros, which in turn are\n/// used to implement the `kernel`'s `print!` and `println!` macros. By implementing `write_str()`,\n/// we get `write_fmt()` automatically.\n///\n/// The function takes an `&mut self`, so it must be implemented for the inner struct.\n///\n/// See [`src/print.rs`].\n///\n/// [`src/print.rs`]: ../../print/index.html\nimpl fmt::Write for QEMUOutputInner {\n    fn write_str(&mut self, s: &str) -> fmt::Result {\n        for c in s.chars() {\n            // Convert newline to carrige return + newline.\n            if c == '\\n' {\n                self.write_char('\\r')\n            }\n\n            self.write_char(c);\n        }\n\n        Ok(())\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl QEMUOutput {\n    /// Create a new instance.\n    pub const fn new() -> QEMUOutput {\n        QEMUOutput {\n            inner: NullLock::new(QEMUOutputInner::new()),\n        }\n    }\n}\n\n/// Return a reference to the console.\npub fn console() -> &'static dyn console::interface::All {\n    &QEMU_OUTPUT\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\n/// Passthrough of `args` to the `core::fmt::Write` implementation, but guarded by a Mutex to\n/// serialize access.\nimpl console::interface::Write for QEMUOutput {\n    fn write_fmt(&self, args: core::fmt::Arguments) -> fmt::Result {\n        // Fully qualified syntax for the call to `core::fmt::Write::write_fmt()` to increase\n        // readability.\n        self.inner.lock(|inner| fmt::Write::write_fmt(inner, args))\n    }\n}\n\nimpl console::interface::Statistics for QEMUOutput {\n    fn chars_written(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_written)\n    }\n}\n\nimpl console::interface::All for QEMUOutput {}\n"
  },
  {
    "path": "04_safe_globals/src/bsp/raspberrypi/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Processor code.\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Used by `arch` code to find the early boot core.\n#[no_mangle]\n#[link_section = \".text._start_arguments\"]\npub static BOOT_CORE_ID: u64 = 0;\n"
  },
  {
    "path": "04_safe_globals/src/bsp/raspberrypi/kernel.ld",
    "content": "/* SPDX-License-Identifier: MIT OR Apache-2.0\n *\n * Copyright (c) 2018-2022 Andre Richter <andre.o.richter@gmail.com>\n */\n\n__rpi_phys_dram_start_addr = 0;\n\n/* The physical address at which the the kernel binary will be loaded by the Raspberry's firmware */\n__rpi_phys_binary_load_addr = 0x80000;\n\n\nENTRY(__rpi_phys_binary_load_addr)\n\n/* Flags:\n *     4 == R\n *     5 == RX\n *     6 == RW\n *\n * Segments are marked PT_LOAD below so that the ELF file provides virtual and physical addresses.\n * It doesn't mean all of them need actually be loaded.\n */\nPHDRS\n{\n    segment_boot_core_stack PT_LOAD FLAGS(6);\n    segment_code            PT_LOAD FLAGS(5);\n    segment_data            PT_LOAD FLAGS(6);\n}\n\nSECTIONS\n{\n    . =  __rpi_phys_dram_start_addr;\n\n    /***********************************************************************************************\n    * Boot Core Stack\n    ***********************************************************************************************/\n    .boot_core_stack (NOLOAD) :\n    {\n                                             /*   ^             */\n                                             /*   | stack       */\n        . += __rpi_phys_binary_load_addr;    /*   | growth      */\n                                             /*   | direction   */\n        __boot_core_stack_end_exclusive = .; /*   |             */\n    } :segment_boot_core_stack\n\n    /***********************************************************************************************\n    * Code + RO Data + Global Offset Table\n    ***********************************************************************************************/\n    .text :\n    {\n        KEEP(*(.text._start))\n        *(.text._start_arguments) /* Constants (or statics in Rust speak) read by _start(). */\n        *(.text._start_rust)      /* The Rust entry point */\n        *(.text*)                 /* Everything else */\n    } :segment_code\n\n    .rodata : ALIGN(8) { *(.rodata*) } :segment_code\n\n    /***********************************************************************************************\n    * Data + BSS\n    ***********************************************************************************************/\n    .data : { *(.data*) } :segment_data\n\n    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */\n    .bss (NOLOAD) : ALIGN(16)\n    {\n        __bss_start = .;\n        *(.bss*);\n        . = ALIGN(16);\n        __bss_end_exclusive = .;\n    } :segment_data\n\n    /***********************************************************************************************\n    * Misc\n    ***********************************************************************************************/\n    .got : { *(.got*) }\n    ASSERT(SIZEOF(.got) == 0, \"Relocation support not expected\")\n\n    /DISCARD/ : { *(.comment*) }\n}\n"
  },
  {
    "path": "04_safe_globals/src/bsp/raspberrypi.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Top-level BSP file for the Raspberry Pi 3 and 4.\n\npub mod console;\npub mod cpu;\n"
  },
  {
    "path": "04_safe_globals/src/bsp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Conditional reexporting of Board Support Packages.\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod raspberrypi;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use raspberrypi::*;\n"
  },
  {
    "path": "04_safe_globals/src/console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! System console.\n\nuse crate::bsp;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Console interfaces.\npub mod interface {\n    use core::fmt;\n\n    /// Console write functions.\n    pub trait Write {\n        /// Write a Rust format string.\n        fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result;\n    }\n\n    /// Console statistics.\n    pub trait Statistics {\n        /// Return the number of characters written.\n        fn chars_written(&self) -> usize {\n            0\n        }\n    }\n\n    /// Trait alias for a full-fledged console.\n    pub trait All: Write + Statistics {}\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the console.\n///\n/// This is the global console used by all printing macros.\npub fn console() -> &'static dyn interface::All {\n    bsp::console::console()\n}\n"
  },
  {
    "path": "04_safe_globals/src/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Boot code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/cpu/boot.rs\"]\nmod arch_boot;\n"
  },
  {
    "path": "04_safe_globals/src/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Processor code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/cpu.rs\"]\nmod arch_cpu;\n\nmod boot;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_cpu::wait_forever;\n"
  },
  {
    "path": "04_safe_globals/src/main.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n// Rust embedded logo for `make doc`.\n#![doc(\n    html_logo_url = \"https://raw.githubusercontent.com/rust-embedded/wg/master/assets/logo/ewg-logo-blue-white-on-transparent.png\"\n)]\n\n//! The `kernel` binary.\n//!\n//! # Code organization and architecture\n//!\n//! The code is divided into different *modules*, each representing a typical **subsystem** of the\n//! `kernel`. Top-level module files of subsystems reside directly in the `src` folder. For example,\n//! `src/memory.rs` contains code that is concerned with all things memory management.\n//!\n//! ## Visibility of processor architecture code\n//!\n//! Some of the `kernel`'s subsystems depend on low-level code that is specific to the target\n//! processor architecture. For each supported processor architecture, there exists a subfolder in\n//! `src/_arch`, for example, `src/_arch/aarch64`.\n//!\n//! The architecture folders mirror the subsystem modules laid out in `src`. For example,\n//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go\n//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in\n//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic\n//! module's name prefixed with `arch_`.\n//!\n//! For example, this is the top of `src/memory/mmu.rs`:\n//!\n//! ```\n//! #[cfg(target_arch = \"aarch64\")]\n//! #[path = \"../_arch/aarch64/memory/mmu.rs\"]\n//! mod arch_mmu;\n//! ```\n//!\n//! Often times, items from the `arch_ module` will be publicly reexported by the parent module.\n//! This way, each architecture specific module can provide its implementation of an item, while the\n//! caller must not be concerned which architecture has been conditionally compiled.\n//!\n//! ## BSP code\n//!\n//! `BSP` stands for Board Support Package. `BSP` code is organized under `src/bsp.rs` and contains\n//! target board specific definitions and functions. These are things such as the board's memory map\n//! or instances of drivers for devices that are featured on the respective board.\n//!\n//! Just like processor architecture code, the `BSP` code's module structure tries to mirror the\n//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is\n//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`.\n//!\n//! ## Kernel interfaces\n//!\n//! Both `arch` and `bsp` contain code that is conditionally compiled depending on the actual target\n//! and board for which the kernel is compiled. For example, the `interrupt controller` hardware of\n//! the `Raspberry Pi 3` and the `Raspberry Pi 4` is different, but we want the rest of the `kernel`\n//! code to play nicely with any of the two without much hassle.\n//!\n//! In order to provide a clean abstraction between `arch`, `bsp` and `generic kernel code`,\n//! `interface` traits are provided *whenever possible* and *where it makes sense*. They are defined\n//! in the respective subsystem module and help to enforce the idiom of *program to an interface,\n//! not an implementation*. For example, there will be a common IRQ handling interface which the two\n//! different interrupt controller `drivers` of both Raspberrys will implement, and only export the\n//! interface to the rest of the `kernel`.\n//!\n//! ```\n//!         +-------------------+\n//!         | Interface (Trait) |\n//!         |                   |\n//!         +--+-------------+--+\n//!            ^             ^\n//!            |             |\n//!            |             |\n//! +----------+--+       +--+----------+\n//! | kernel code |       |  bsp code   |\n//! |             |       |  arch code  |\n//! +-------------+       +-------------+\n//! ```\n//!\n//! # Summary\n//!\n//! For a logical `kernel` subsystem, corresponding code can be distributed over several physical\n//! locations. Here is an example for the **memory** subsystem:\n//!\n//! - `src/memory.rs` and `src/memory/**/*`\n//!   - Common code that is agnostic of target processor architecture and `BSP` characteristics.\n//!     - Example: A function to zero a chunk of memory.\n//!   - Interfaces for the memory subsystem that are implemented by `arch` or `BSP` code.\n//!     - Example: An `MMU` interface that defines `MMU` function prototypes.\n//! - `src/bsp/__board_name__/memory.rs` and `src/bsp/__board_name__/memory/**/*`\n//!   - `BSP` specific code.\n//!   - Example: The board's memory map (physical addresses of DRAM and MMIO devices).\n//! - `src/_arch/__arch_name__/memory.rs` and `src/_arch/__arch_name__/memory/**/*`\n//!   - Processor architecture specific code.\n//!   - Example: Implementation of the `MMU` interface for the `__arch_name__` processor\n//!     architecture.\n//!\n//! From a namespace perspective, **memory** subsystem code lives in:\n//!\n//! - `crate::memory::*`\n//! - `crate::bsp::memory::*`\n//!\n//! # Boot flow\n//!\n//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.\n//!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.\n//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.\n\n#![feature(asm_const)]\n#![feature(format_args_nl)]\n#![feature(panic_info_message)]\n#![feature(trait_alias)]\n#![no_main]\n#![no_std]\n\nmod bsp;\nmod console;\nmod cpu;\nmod panic_wait;\nmod print;\nmod synchronization;\n\n/// Early init code.\n///\n/// # Safety\n///\n/// - Only a single core must be active and running this function.\nunsafe fn kernel_init() -> ! {\n    use console::console;\n\n    println!(\"[0] Hello from Rust!\");\n\n    println!(\"[1] Chars written: {}\", console().chars_written());\n\n    println!(\"[2] Stopping here.\");\n    cpu::wait_forever()\n}\n"
  },
  {
    "path": "04_safe_globals/src/panic_wait.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A panic handler that infinitely waits.\n\nuse crate::{cpu, println};\nuse core::panic::PanicInfo;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Stop immediately if called a second time.\n///\n/// # Note\n///\n/// Using atomics here relieves us from needing to use `unsafe` for the static variable.\n///\n/// On `AArch64`, which is the only implemented architecture at the time of writing this,\n/// [`AtomicBool::load`] and [`AtomicBool::store`] are lowered to ordinary load and store\n/// instructions. They are therefore safe to use even with MMU + caching deactivated.\n///\n/// [`AtomicBool::load`]: core::sync::atomic::AtomicBool::load\n/// [`AtomicBool::store`]: core::sync::atomic::AtomicBool::store\nfn panic_prevent_reenter() {\n    use core::sync::atomic::{AtomicBool, Ordering};\n\n    #[cfg(not(target_arch = \"aarch64\"))]\n    compile_error!(\"Add the target_arch to above's check if the following code is safe to use\");\n\n    static PANIC_IN_PROGRESS: AtomicBool = AtomicBool::new(false);\n\n    if !PANIC_IN_PROGRESS.load(Ordering::Relaxed) {\n        PANIC_IN_PROGRESS.store(true, Ordering::Relaxed);\n\n        return;\n    }\n\n    cpu::wait_forever()\n}\n\n#[panic_handler]\nfn panic(info: &PanicInfo) -> ! {\n    // Protect against panic infinite loops if any of the following code panics itself.\n    panic_prevent_reenter();\n\n    let (location, line, column) = match info.location() {\n        Some(loc) => (loc.file(), loc.line(), loc.column()),\n        _ => (\"???\", 0, 0),\n    };\n\n    println!(\n        \"Kernel panic!\\n\\n\\\n        Panic location:\\n      File '{}', line {}, column {}\\n\\n\\\n        {}\",\n        location,\n        line,\n        column,\n        info.message().unwrap_or(&format_args!(\"\")),\n    );\n\n    cpu::wait_forever()\n}\n"
  },
  {
    "path": "04_safe_globals/src/print.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Printing.\n\nuse crate::console;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n#[doc(hidden)]\npub fn _print(args: fmt::Arguments) {\n    console::console().write_fmt(args).unwrap();\n}\n\n/// Prints without a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! print {\n    ($($arg:tt)*) => ($crate::print::_print(format_args!($($arg)*)));\n}\n\n/// Prints with a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! println {\n    () => ($crate::print!(\"\\n\"));\n    ($($arg:tt)*) => ({\n        $crate::print::_print(format_args_nl!($($arg)*));\n    })\n}\n"
  },
  {
    "path": "04_safe_globals/src/synchronization.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Synchronization primitives.\n//!\n//! # Resources\n//!\n//!   - <https://doc.rust-lang.org/book/ch16-04-extensible-concurrency-sync-and-send.html>\n//!   - <https://stackoverflow.com/questions/59428096/understanding-the-send-trait>\n//!   - <https://doc.rust-lang.org/std/cell/index.html>\n\nuse core::cell::UnsafeCell;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Synchronization interfaces.\npub mod interface {\n\n    /// Any object implementing this trait guarantees exclusive access to the data wrapped within\n    /// the Mutex for the duration of the provided closure.\n    pub trait Mutex {\n        /// The type of the data that is wrapped by this mutex.\n        type Data;\n\n        /// Locks the mutex and grants the closure temporary mutable access to the wrapped data.\n        fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R;\n    }\n}\n\n/// A pseudo-lock for teaching purposes.\n///\n/// In contrast to a real Mutex implementation, does not protect against concurrent access from\n/// other cores to the contained data. This part is preserved for later lessons.\n///\n/// The lock will only be used as long as it is safe to do so, i.e. as long as the kernel is\n/// executing single-threaded, aka only running on a single core with interrupts disabled.\npub struct NullLock<T>\nwhere\n    T: ?Sized,\n{\n    data: UnsafeCell<T>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nunsafe impl<T> Send for NullLock<T> where T: ?Sized + Send {}\nunsafe impl<T> Sync for NullLock<T> where T: ?Sized + Send {}\n\nimpl<T> NullLock<T> {\n    /// Create an instance.\n    pub const fn new(data: T) -> Self {\n        Self {\n            data: UnsafeCell::new(data),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\n\nimpl<T> interface::Mutex for NullLock<T> {\n    type Data = T;\n\n    fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R {\n        // In a real lock, there would be code encapsulating this line that ensures that this\n        // mutable reference will ever only be given out once at a time.\n        let data = unsafe { &mut *self.data.get() };\n\n        f(data)\n    }\n}\n"
  },
  {
    "path": "04_safe_globals/tests/boot_test_string.rb",
    "content": "# frozen_string_literal: true\n\nEXPECTED_PRINT = 'Stopping here'\n"
  },
  {
    "path": "05_drivers_gpio_uart/.vscode/settings.json",
    "content": "{\n    \"editor.formatOnSave\": true,\n    \"editor.rulers\": [100],\n    \"rust-analyzer.cargo.target\": \"aarch64-unknown-none-softfloat\",\n    \"rust-analyzer.cargo.features\": [\"bsp_rpi3\"],\n    \"rust-analyzer.checkOnSave.allTargets\": false,\n    \"rust-analyzer.checkOnSave.extraArgs\": [\"--bins\"],\n    \"rust-analyzer.lens.debug\": false,\n    \"rust-analyzer.lens.run\": false\n}\n"
  },
  {
    "path": "05_drivers_gpio_uart/Cargo.toml",
    "content": "[package]\nname = \"mingo\"\nversion = \"0.5.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n\n[profile.release]\nlto = true\n\n[features]\ndefault = []\nbsp_rpi3 = [\"tock-registers\"]\nbsp_rpi4 = [\"tock-registers\"]\n\n[[bin]]\nname = \"kernel\"\npath = \"src/main.rs\"\n\n##--------------------------------------------------------------------------------------------------\n## Dependencies\n##--------------------------------------------------------------------------------------------------\n\n[dependencies]\n\n# Optional dependencies\ntock-registers = { version = \"0.8.x\", default-features = false, features = [\"register_types\"], optional = true }\n\n# Platform specific dependencies\n[target.'cfg(target_arch = \"aarch64\")'.dependencies]\naarch64-cpu = { version = \"9.x.x\" }\n"
  },
  {
    "path": "05_drivers_gpio_uart/Makefile",
    "content": "## SPDX-License-Identifier: MIT OR Apache-2.0\n##\n## Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\ninclude ../common/docker.mk\ninclude ../common/format.mk\ninclude ../common/operating_system.mk\n\n##--------------------------------------------------------------------------------------------------\n## Optional, user-provided configuration values\n##--------------------------------------------------------------------------------------------------\n\n# Default to the RPi3.\nBSP ?= rpi3\n\n# Default to a serial device name that is common in Linux.\nDEV_SERIAL ?= /dev/ttyUSB0\n\n\n\n##--------------------------------------------------------------------------------------------------\n## BSP-specific configuration values\n##--------------------------------------------------------------------------------------------------\nQEMU_MISSING_STRING = \"This board is not yet supported for QEMU.\"\n\nifeq ($(BSP),rpi3)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE = raspi3\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    LD_SCRIPT_PATH    = $(shell pwd)/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a53\nelse ifeq ($(BSP),rpi4)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE =\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    LD_SCRIPT_PATH    = $(shell pwd)/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a72\nendif\n\n# Export for build.rs.\nexport LD_SCRIPT_PATH\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets and Prerequisites\n##--------------------------------------------------------------------------------------------------\nKERNEL_MANIFEST      = Cargo.toml\nKERNEL_LINKER_SCRIPT = kernel.ld\nLAST_BUILD_CONFIG    = target/$(BSP).build_config\n\nKERNEL_ELF      = target/$(TARGET)/release/kernel\n# This parses cargo's dep-info file.\n# https://doc.rust-lang.org/cargo/guide/build-cache.html#dep-info-files\nKERNEL_ELF_DEPS = $(filter-out %: ,$(file < $(KERNEL_ELF).d)) $(KERNEL_MANIFEST) $(LAST_BUILD_CONFIG)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Command building blocks\n##--------------------------------------------------------------------------------------------------\nRUSTFLAGS = $(RUSTC_MISC_ARGS)                   \\\n    -C link-arg=--library-path=$(LD_SCRIPT_PATH) \\\n    -C link-arg=--script=$(KERNEL_LINKER_SCRIPT)\n\nRUSTFLAGS_PEDANTIC = $(RUSTFLAGS) \\\n    -D warnings                   \\\n    -D missing_docs\n\nFEATURES      = --features bsp_$(BSP)\nCOMPILER_ARGS = --target=$(TARGET) \\\n    $(FEATURES)                    \\\n    --release\n\nRUSTC_CMD   = cargo rustc $(COMPILER_ARGS)\nDOC_CMD     = cargo doc $(COMPILER_ARGS)\nCLIPPY_CMD  = cargo clippy $(COMPILER_ARGS)\nOBJCOPY_CMD = rust-objcopy \\\n    --strip-all            \\\n    -O binary\n\nEXEC_QEMU          = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE)\nEXEC_TEST_DISPATCH = ruby ../common/tests/dispatch.rb\nEXEC_MINITERM      = ruby ../common/serial/miniterm.rb\n\n##------------------------------------------------------------------------------\n## Dockerization\n##------------------------------------------------------------------------------\nDOCKER_CMD            = docker run -t --rm -v $(shell pwd):/work/tutorial -w /work/tutorial\nDOCKER_CMD_INTERACT   = $(DOCKER_CMD) -i\nDOCKER_ARG_DIR_COMMON = -v $(shell pwd)/../common:/work/common\nDOCKER_ARG_DEV        = --privileged -v /dev:/dev\n\n# DOCKER_IMAGE defined in include file (see top of this file).\nDOCKER_QEMU  = $(DOCKER_CMD_INTERACT) $(DOCKER_IMAGE)\nDOCKER_TOOLS = $(DOCKER_CMD) $(DOCKER_IMAGE)\nDOCKER_TEST  = $(DOCKER_CMD) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\n\n# Dockerize commands, which require USB device passthrough, only on Linux.\nifeq ($(shell uname -s),Linux)\n    DOCKER_CMD_DEV = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_DEV)\n\n    DOCKER_MINITERM = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\nendif\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: all doc qemu miniterm clippy clean readelf objdump nm check\n\nall: $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Save the configuration as a file, so make understands if it changed.\n##------------------------------------------------------------------------------\n$(LAST_BUILD_CONFIG):\n\t@rm -f target/*.build_config\n\t@mkdir -p target\n\t@touch $(LAST_BUILD_CONFIG)\n\n##------------------------------------------------------------------------------\n## Compile the kernel ELF\n##------------------------------------------------------------------------------\n$(KERNEL_ELF): $(KERNEL_ELF_DEPS)\n\t$(call color_header, \"Compiling kernel ELF - $(BSP)\")\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(RUSTC_CMD)\n\n##------------------------------------------------------------------------------\n## Generate the stripped kernel binary\n##------------------------------------------------------------------------------\n$(KERNEL_BIN): $(KERNEL_ELF)\n\t$(call color_header, \"Generating stripped binary\")\n\t@$(OBJCOPY_CMD) $(KERNEL_ELF) $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Name\")\n\t@echo $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Size\")\n\t$(call disk_usage_KiB, $(KERNEL_BIN))\n\n##------------------------------------------------------------------------------\n## Generate the documentation\n##-----------------------------------------------------------------------------\ndoc:\n\t$(call color_header, \"Generating docs\")\n\t@$(DOC_CMD) --document-private-items --open\n\n##------------------------------------------------------------------------------\n## Run the kernel in QEMU\n##------------------------------------------------------------------------------\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\nqemu:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\nqemu: $(KERNEL_BIN)\n\t$(call color_header, \"Launching QEMU\")\n\t@$(DOCKER_QEMU) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\nendif\n\n##------------------------------------------------------------------------------\n## Connect to the target's serial\n##------------------------------------------------------------------------------\nminiterm:\n\t@$(DOCKER_MINITERM) $(EXEC_MINITERM) $(DEV_SERIAL)\n\n##------------------------------------------------------------------------------\n## Run clippy\n##------------------------------------------------------------------------------\nclippy:\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(CLIPPY_CMD)\n\n##------------------------------------------------------------------------------\n## Clean\n##------------------------------------------------------------------------------\nclean:\n\trm -rf target $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run readelf\n##------------------------------------------------------------------------------\nreadelf: $(KERNEL_ELF)\n\t$(call color_header, \"Launching readelf\")\n\t@$(DOCKER_TOOLS) $(READELF_BINARY) --headers $(KERNEL_ELF)\n\n##------------------------------------------------------------------------------\n## Run objdump\n##------------------------------------------------------------------------------\nobjdump: $(KERNEL_ELF)\n\t$(call color_header, \"Launching objdump\")\n\t@$(DOCKER_TOOLS) $(OBJDUMP_BINARY) --disassemble --demangle \\\n                --section .text   \\\n                --section .rodata \\\n                $(KERNEL_ELF) | rustfilt\n\n##------------------------------------------------------------------------------\n## Run nm\n##------------------------------------------------------------------------------\nnm: $(KERNEL_ELF)\n\t$(call color_header, \"Launching nm\")\n\t@$(DOCKER_TOOLS) $(NM_BINARY) --demangle --print-size $(KERNEL_ELF) | sort | rustfilt\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Testing targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: test test_boot\n\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\ntest_boot test:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\n##------------------------------------------------------------------------------\n## Run boot test\n##------------------------------------------------------------------------------\ntest_boot: $(KERNEL_BIN)\n\t$(call color_header, \"Boot test - $(BSP)\")\n\t@$(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\ntest: test_boot\n\nendif\n"
  },
  {
    "path": "05_drivers_gpio_uart/README.CN.md",
    "content": "# 教程 05 - 驱动程序: GPIO和UART\n\n## tl;dr\n\n- 添加了用于真实`UART`和`GPIO`控制器的驱动程序。\n- **我们将首次能够在真实硬件上运行代码** (请向下滚动查看说明)。\n\n## 简介\n\n在上一篇教程中，我们启用了全局安全变量，为添加第一个真实设备驱动程序奠定了基础。\n我们放弃了神奇的QEMU控制台，并引入了一个`驱动程序管理器`，允许`BSP`将设备驱动程序注册到`内核`中。\n\n## 驱动程序管理器\n\n第一步是向内核添加一个`driver subsystem`。相应的代码将位于`src/driver.rs`中。\n该子系统引入了`interface::DeviceDriver`，这是每个设备驱动程序都需要实现的通用特征，并为内核所知。\n在同一文件中实例化的全局`DRIVER_MANAGER`实例（类型为`DriverManager`）作为一个中央实体，可以被调用来管理内核中的所有设备驱动程序。\n例如，通过使用全局可访问的`crate::driver::driver_manager().register_driver(...)`，任何代码都可以注册一个实现了`interface::DeviceDriver`特征的具有静态生命周期的对象。\n\n在内核初始化期间，调用`crate::driver::driver_manager().init_drivers(...)`将使驱动程序管理器遍历所有已注册的驱动程序，\n并启动它们的初始化，并执行可选的`post-init callback`，该回调可以与驱动程序一起注册。\n例如，此机制用于在`UART`驱动程序初始化后将其切换为主系统控制台的驱动程序。\n\n## BSP驱动程序实现\n\n在`src/bsp/raspberrypi/driver.rs`中，函数`init()`负责注册`UART`和`GPIO`驱动程序。\n因此，在内核初始化期间，按照以下来自`main.rs`的代码，正确的顺序是：\n（i）首先初始化BSP驱动程序子系统，然后（ii）调用`driver_manager()`。\n\n```rust\nunsafe fn kernel_init() -> ! {\n    // Initialize the BSP driver subsystem.\n    if let Err(x) = bsp::driver::init() {\n        panic!(\"Error initializing BSP driver subsystem: {}\", x);\n    }\n\n    // Initialize all device drivers.\n    driver::driver_manager().init_drivers();\n    // println! is usable from here on.\n```\n\n\n\n驱动程序本身存储在`src/bsp/device_driver`中，并且可以在不同的`BSP`之间重复使用\n在这些教程中添加的第一个驱动程序是`PL011Uart`驱动程序：它实现了`console::interface::*`特征，并且从现在开始用作主系统控制台。\n第二个驱动程序是`GPIO`驱动程序，它根据需要将`RPii's`的`UART`映射（即将来自`SoC`内部的信号路由到实际的硬件引脚）。\n请注意，`GPIO`驱动程序区分**RPi 3**和**RPi 4**。它们的硬件不同，因此我们必须在软件中进行适配。\n\n现在，`BSP`还包含了一个内存映射表，位于`src/bsp/raspberrypi/memory.rs`中。它提供了树莓派的`MMIO`地址，\n`BSP`使用这些地址来实例化相应的设备驱动程序，以便驱动程序代码知道在内存中找到设备的寄存器的位置。\n\n## SD卡启动\n\n由于我们现在有了真实的`UART`输出，我们可以在真实的硬件上运行代码。\n由于前面提到的`GPIO`驱动程序的差异，构建过程在**RPi 3**和**RPi 4**之间有所区别。\n默认情况下，所有的`Makefile`目标都将为**RPi 3**构建。\n为了**RPi 4**构建，需要在每个目标前加上`BSP=rpi4`。例如：\n\n```console\n$ BSP=rpi4 make\n$ BSP=rpi4 make doc\n```\n\n不幸的是，QEMU目前还不支持**RPi 4**，因此`BSP=rpi4 make qemu`无法工作。\n\n**准备SD卡的一些步骤在RPi3和RPi4之间有所不同，请在以下操作中小心。**\n\n### 通用步骤\n\n1. 创建一个名为`boot`的`FAT32`分区。\n2. 在SD卡上生成一个名为`config.txt`的文件，并将以下内容写入其中：\n\n```txt\narm_64bit=1\ninit_uart_clock=48000000\n```\n### RPi 3\n\n3. 从[Raspberry Pi firmware repo](https://github.com/raspberrypi/firmware/tree/master/boot)中将以下文件复制到SD卡上：\n    - [bootcode.bin](https://github.com/raspberrypi/firmware/raw/master/boot/bootcode.bin)\n    - [fixup.dat](https://github.com/raspberrypi/firmware/raw/master/boot/fixup.dat)\n    - [start.elf](https://github.com/raspberrypi/firmware/raw/master/boot/start.elf)\n4. 运行`make`命令。\n\n### RPi 4\n\n3. 从[Raspberry Pi firmware repo](https://github.com/raspberrypi/firmware/tree/master/boot)中将以下文件复制到SD卡上：\n    - [fixup4.dat](https://github.com/raspberrypi/firmware/raw/master/boot/fixup4.dat)\n    - [start4.elf](https://github.com/raspberrypi/firmware/raw/master/boot/start4.elf)\n    - [bcm2711-rpi-4-b.dtb](https://github.com/raspberrypi/firmware/raw/master/boot/bcm2711-rpi-4-b.dtb)\n4. 运行`BSP=rpi4 make`命令。\n\n\n_**注意**: 如果在您的RPi4上无法正常工作，请尝试将`start4.elf`重命名为`start.elf` (不带4)\n并复制到SD卡上。_\n\n### 再次通用步骤\n\n5. 将`kernel8.img`复制到SD卡上，并将SD卡插入RPi。\n6. 运行`miniterm` target，在主机上打开UART设备：\n\n```console\n$ make miniterm\n```\n\n> ❗ **注意**: `Miniterm`假设默认的串行设备名称为`/dev/ttyUSB0`。Depending on your\n> 根据您的主机操作系统，设备名称可能会有所不同。例如，在`macOS`上，它可能是\n> `/dev/tty.usbserial-0001`之类的。在这种情况下，请明确提供设备名称：\n\n\n```console\n$ DEV_SERIAL=/dev/tty.usbserial-0001 make miniterm\n```\n\n7. 将USB串口连接到主机PC。\n    - 请参考[top-level README](../README.md#-usb-serial-output)中的接线图。\n    - **注意**: TX（发送）线连接到RX（接收）引脚。\n    - 确保您**没有**连接USB串口的电源引脚，只连接RX/TX和GND引脚。\n8. 将RPi连接到（USB）电源线，并观察输出。\n\n```console\nMiniterm 1.0\n\n[MT] ⏳ Waiting for /dev/ttyUSB0\n[MT] ✅ Serial connected\n[0] mingo version 0.5.0\n[1] Booting on: Raspberry Pi 3\n[2] Drivers loaded:\n      1. BCM PL011 UART\n      2. BCM GPIO\n[3] Chars written: 117\n[4] Echoing input now\n```\n\n8. 通过按下<kbd>ctrl-c</kbd>退出。\n\n## 相比之前的变化（diff）\n请检查[英文版本](README.md#diff-to-previous)，这是最新的。\n"
  },
  {
    "path": "05_drivers_gpio_uart/README.md",
    "content": "# Tutorial 05 - Drivers: GPIO and UART\n\n## tl;dr\n\n- Drivers for the real `UART` and the `GPIO` controller are added.\n- **For the first time, we will be able to run the code on the real hardware** (scroll down for\n  instructions).\n\n## Introduction\n\nNow that we enabled safe globals in the previous tutorial, the infrastructure is laid for adding the\nfirst real device drivers. We throw out the magic QEMU console and introduce a `driver manager`,\nwhich allows the `BSP` to register device drivers with the `kernel`.\n\n## Driver Manager\n\nThe first step consists of adding a `driver subsystem` to the kernel. The corresponding code will\nlive in `src/driver.rs`. The subsystem introduces `interface::DeviceDriver`, a common trait that\nevery device driver will need to implement and that is known to the kernel. A global\n`DRIVER_MANAGER` instance (of type `DriverManager`) that is instantiated in the same file serves as\nthe central entity that can be called to manage all things device drivers in the kernel. For\nexample, by using the globally accessible `crate::driver::driver_manager().register_driver(...)`,\nany code can can register an object with static lifetime that implements the\n`interface::DeviceDriver` trait.\n\nDuring kernel init, a call to `crate::driver::driver_manager().init_drivers(...)` will let the\ndriver manager loop over all registered drivers and kick off their initialization, and also execute\nan optional `post-init callback` that can be registered alongside the driver. For example, this\nmechanism is used to switch over to the `UART` driver as the main system console after the `UART`\ndriver has been initialized.\n\n## BSP Driver Implementation\n\nIn `src/bsp/raspberrypi/driver.rs`, the function `init()` takes care of registering the `UART` and\n`GPIO` drivers. It is therefore important that during kernel init, the correct order of (i) first\ninitializing the BSP driver subsystem, and only then (ii) calling the `driver_manager()` is\nfollowed, like the following excerpt from `main.rs` shows:\n\n```rust\nunsafe fn kernel_init() -> ! {\n    // Initialize the BSP driver subsystem.\n    if let Err(x) = bsp::driver::init() {\n        panic!(\"Error initializing BSP driver subsystem: {}\", x);\n    }\n\n    // Initialize all device drivers.\n    driver::driver_manager().init_drivers();\n    // println! is usable from here on.\n```\n\n\n\nThe drivers themselves are stored in `src/bsp/device_driver`, and can be reused between `BSP`s. The\nfirst driver added in these tutorials is the `PL011Uart` driver: It implements the\n`console::interface::*` traits and is from now on used as the main system console. The second driver\nis the `GPIO` driver, which pinmuxes (that is, routing signals from inside the `SoC` to actual HW\npins) the RPi's PL011 UART accordingly. Note how the `GPIO` driver differentiates between **RPi 3**\nand **RPi 4**. Their HW is different, so we have to account for it in SW.\n\nThe `BSP`s now also contain a memory map in `src/bsp/raspberrypi/memory.rs`. It provides the\nRaspberry's `MMIO` addresses which are used by the `BSP` to instantiate the respective device\ndrivers, so that the driver code knows where to find the device's registers in memory.\n\n## Boot it from SD card\n\nSince we have real `UART` output now, we can run the code on the real hardware. Building is\ndifferentiated between the **RPi 3** and the **RPi 4** due to before mentioned differences in the\n`GPIO` driver. By default, all `Makefile` targets will build for the **RPi 3**. In order to build\nfor the the **RPi 4**, prepend `BSP=rpi4` to each target. For example:\n\n```console\n$ BSP=rpi4 make\n$ BSP=rpi4 make doc\n```\n\nUnfortunately, QEMU does not yet support the **RPi 4**, so `BSP=rpi4 make qemu` won't work.\n\n**Some steps for preparing the SD card differ between RPi 3 and RPi 4, so be careful in the\nfollowing.**\n\n### Common for both\n\n1. Make a single `FAT32` partition named `boot`.\n2. On the card, generate a file named `config.txt` with the following contents:\n\n```txt\narm_64bit=1\ninit_uart_clock=48000000\n```\n### RPi 3\n\n3. Copy the following files from the [Raspberry Pi firmware repo](https://github.com/raspberrypi/firmware/tree/master/boot) onto the SD card:\n    - [bootcode.bin](https://github.com/raspberrypi/firmware/raw/master/boot/bootcode.bin)\n    - [fixup.dat](https://github.com/raspberrypi/firmware/raw/master/boot/fixup.dat)\n    - [start.elf](https://github.com/raspberrypi/firmware/raw/master/boot/start.elf)\n4. Run `make`.\n\n### RPi 4\n\n3. Copy the following files from the [Raspberry Pi firmware repo](https://github.com/raspberrypi/firmware/tree/master/boot) onto the SD card:\n    - [fixup4.dat](https://github.com/raspberrypi/firmware/raw/master/boot/fixup4.dat)\n    - [start4.elf](https://github.com/raspberrypi/firmware/raw/master/boot/start4.elf)\n    - [bcm2711-rpi-4-b.dtb](https://github.com/raspberrypi/firmware/raw/master/boot/bcm2711-rpi-4-b.dtb)\n4. Run `BSP=rpi4 make`.\n\n\n_**Note**: Should it not work on your RPi 4, try renaming `start4.elf` to `start.elf` (without the 4)\non the SD card._\n\n### Common again\n\n5. Copy the `kernel8.img` onto the SD card and insert it back into the RPi.\n6. Run the `miniterm` target, which opens the UART device on the host:\n\n```console\n$ make miniterm\n```\n\n> ❗ **NOTE**: `Miniterm` assumes a default serial device name of `/dev/ttyUSB0`. Depending on your\n> host operating system, the device name might differ. For example, on `macOS`, it might be\n> something like `/dev/tty.usbserial-0001`. In this case, please give the name explicitly:\n\n\n```console\n$ DEV_SERIAL=/dev/tty.usbserial-0001 make miniterm\n```\n\n7. Connect the USB serial to your host PC.\n    - Wiring diagram at [top-level README](../README.md#-usb-serial-output).\n    - **NOTE**: TX (transmit) wire connects to the RX (receive) pin.\n    - Make sure that you **DID NOT** connect the power pin of the USB serial. Only RX/TX and GND.\n8. Connect the RPi to the (USB) power cable and observe the output:\n\n```console\nMiniterm 1.0\n\n[MT] ⏳ Waiting for /dev/ttyUSB0\n[MT] ✅ Serial connected\n[0] mingo version 0.5.0\n[1] Booting on: Raspberry Pi 3\n[2] Drivers loaded:\n      1. BCM PL011 UART\n      2. BCM GPIO\n[3] Chars written: 117\n[4] Echoing input now\n```\n\n8. Exit by pressing <kbd>ctrl-c</kbd>.\n\n## Diff to previous\n```diff\n\ndiff -uNr 04_safe_globals/Cargo.toml 05_drivers_gpio_uart/Cargo.toml\n--- 04_safe_globals/Cargo.toml\n+++ 05_drivers_gpio_uart/Cargo.toml\n@@ -1,6 +1,6 @@\n [package]\n name = \"mingo\"\n-version = \"0.4.0\"\n+version = \"0.5.0\"\n authors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\n edition = \"2021\"\n\n@@ -9,8 +9,8 @@\n\n [features]\n default = []\n-bsp_rpi3 = []\n-bsp_rpi4 = []\n+bsp_rpi3 = [\"tock-registers\"]\n+bsp_rpi4 = [\"tock-registers\"]\n\n [[bin]]\n name = \"kernel\"\n@@ -22,6 +22,9 @@\n\n [dependencies]\n\n+# Optional dependencies\n+tock-registers = { version = \"0.8.x\", default-features = false, features = [\"register_types\"], optional = true }\n+\n # Platform specific dependencies\n [target.'cfg(target_arch = \"aarch64\")'.dependencies]\n aarch64-cpu = { version = \"9.x.x\" }\n\ndiff -uNr 04_safe_globals/Makefile 05_drivers_gpio_uart/Makefile\n--- 04_safe_globals/Makefile\n+++ 05_drivers_gpio_uart/Makefile\n@@ -13,6 +13,9 @@\n # Default to the RPi3.\n BSP ?= rpi3\n\n+# Default to a serial device name that is common in Linux.\n+DEV_SERIAL ?= /dev/ttyUSB0\n+\n\n\n ##--------------------------------------------------------------------------------------------------\n@@ -88,6 +91,7 @@\n\n EXEC_QEMU          = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE)\n EXEC_TEST_DISPATCH = ruby ../common/tests/dispatch.rb\n+EXEC_MINITERM      = ruby ../common/serial/miniterm.rb\n\n ##------------------------------------------------------------------------------\n ## Dockerization\n@@ -95,18 +99,26 @@\n DOCKER_CMD            = docker run -t --rm -v $(shell pwd):/work/tutorial -w /work/tutorial\n DOCKER_CMD_INTERACT   = $(DOCKER_CMD) -i\n DOCKER_ARG_DIR_COMMON = -v $(shell pwd)/../common:/work/common\n+DOCKER_ARG_DEV        = --privileged -v /dev:/dev\n\n # DOCKER_IMAGE defined in include file (see top of this file).\n DOCKER_QEMU  = $(DOCKER_CMD_INTERACT) $(DOCKER_IMAGE)\n DOCKER_TOOLS = $(DOCKER_CMD) $(DOCKER_IMAGE)\n DOCKER_TEST  = $(DOCKER_CMD) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\n\n+# Dockerize commands, which require USB device passthrough, only on Linux.\n+ifeq ($(shell uname -s),Linux)\n+    DOCKER_CMD_DEV = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_DEV)\n+\n+    DOCKER_MINITERM = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\n+endif\n+\n\n\n ##--------------------------------------------------------------------------------------------------\n ## Targets\n ##--------------------------------------------------------------------------------------------------\n-.PHONY: all doc qemu clippy clean readelf objdump nm check\n+.PHONY: all doc qemu miniterm clippy clean readelf objdump nm check\n\n all: $(KERNEL_BIN)\n\n@@ -156,9 +168,16 @@\n qemu: $(KERNEL_BIN)\n \t$(call color_header, \"Launching QEMU\")\n \t@$(DOCKER_QEMU) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n+\n endif\n\n ##------------------------------------------------------------------------------\n+## Connect to the target's serial\n+##------------------------------------------------------------------------------\n+miniterm:\n+\t@$(DOCKER_MINITERM) $(EXEC_MINITERM) $(DEV_SERIAL)\n+\n+##------------------------------------------------------------------------------\n ## Run clippy\n ##------------------------------------------------------------------------------\n clippy:\n\ndiff -uNr 04_safe_globals/src/_arch/aarch64/cpu.rs 05_drivers_gpio_uart/src/_arch/aarch64/cpu.rs\n--- 04_safe_globals/src/_arch/aarch64/cpu.rs\n+++ 05_drivers_gpio_uart/src/_arch/aarch64/cpu.rs\n@@ -17,6 +17,17 @@\n // Public Code\n //--------------------------------------------------------------------------------------------------\n\n+pub use asm::nop;\n+\n+/// Spin for `n` cycles.\n+#[cfg(feature = \"bsp_rpi3\")]\n+#[inline(always)]\n+pub fn spin_for_cycles(n: usize) {\n+    for _ in 0..n {\n+        asm::nop();\n+    }\n+}\n+\n /// Pause execution on the core.\n #[inline(always)]\n pub fn wait_forever() -> ! {\n\ndiff -uNr 04_safe_globals/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs 05_drivers_gpio_uart/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs\n--- 04_safe_globals/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs\n+++ 05_drivers_gpio_uart/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs\n@@ -0,0 +1,228 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! GPIO Driver.\n+\n+use crate::{\n+    bsp::device_driver::common::MMIODerefWrapper, driver, synchronization,\n+    synchronization::NullLock,\n+};\n+use tock_registers::{\n+    interfaces::{ReadWriteable, Writeable},\n+    register_bitfields, register_structs,\n+    registers::ReadWrite,\n+};\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+// GPIO registers.\n+//\n+// Descriptions taken from\n+// - https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf\n+// - https://datasheets.raspberrypi.org/bcm2711/bcm2711-peripherals.pdf\n+register_bitfields! {\n+    u32,\n+\n+    /// GPIO Function Select 1\n+    GPFSEL1 [\n+        /// Pin 15\n+        FSEL15 OFFSET(15) NUMBITS(3) [\n+            Input = 0b000,\n+            Output = 0b001,\n+            AltFunc0 = 0b100  // PL011 UART RX\n+\n+        ],\n+\n+        /// Pin 14\n+        FSEL14 OFFSET(12) NUMBITS(3) [\n+            Input = 0b000,\n+            Output = 0b001,\n+            AltFunc0 = 0b100  // PL011 UART TX\n+        ]\n+    ],\n+\n+    /// GPIO Pull-up/down Register\n+    ///\n+    /// BCM2837 only.\n+    GPPUD [\n+        /// Controls the actuation of the internal pull-up/down control line to ALL the GPIO pins.\n+        PUD OFFSET(0) NUMBITS(2) [\n+            Off = 0b00,\n+            PullDown = 0b01,\n+            PullUp = 0b10\n+        ]\n+    ],\n+\n+    /// GPIO Pull-up/down Clock Register 0\n+    ///\n+    /// BCM2837 only.\n+    GPPUDCLK0 [\n+        /// Pin 15\n+        PUDCLK15 OFFSET(15) NUMBITS(1) [\n+            NoEffect = 0,\n+            AssertClock = 1\n+        ],\n+\n+        /// Pin 14\n+        PUDCLK14 OFFSET(14) NUMBITS(1) [\n+            NoEffect = 0,\n+            AssertClock = 1\n+        ]\n+    ],\n+\n+    /// GPIO Pull-up / Pull-down Register 0\n+    ///\n+    /// BCM2711 only.\n+    GPIO_PUP_PDN_CNTRL_REG0 [\n+        /// Pin 15\n+        GPIO_PUP_PDN_CNTRL15 OFFSET(30) NUMBITS(2) [\n+            NoResistor = 0b00,\n+            PullUp = 0b01\n+        ],\n+\n+        /// Pin 14\n+        GPIO_PUP_PDN_CNTRL14 OFFSET(28) NUMBITS(2) [\n+            NoResistor = 0b00,\n+            PullUp = 0b01\n+        ]\n+    ]\n+}\n+\n+register_structs! {\n+    #[allow(non_snake_case)]\n+    RegisterBlock {\n+        (0x00 => _reserved1),\n+        (0x04 => GPFSEL1: ReadWrite<u32, GPFSEL1::Register>),\n+        (0x08 => _reserved2),\n+        (0x94 => GPPUD: ReadWrite<u32, GPPUD::Register>),\n+        (0x98 => GPPUDCLK0: ReadWrite<u32, GPPUDCLK0::Register>),\n+        (0x9C => _reserved3),\n+        (0xE4 => GPIO_PUP_PDN_CNTRL_REG0: ReadWrite<u32, GPIO_PUP_PDN_CNTRL_REG0::Register>),\n+        (0xE8 => @END),\n+    }\n+}\n+\n+/// Abstraction for the associated MMIO registers.\n+type Registers = MMIODerefWrapper<RegisterBlock>;\n+\n+struct GPIOInner {\n+    registers: Registers,\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Representation of the GPIO HW.\n+pub struct GPIO {\n+    inner: NullLock<GPIOInner>,\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Code\n+//--------------------------------------------------------------------------------------------------\n+\n+impl GPIOInner {\n+    /// Create an instance.\n+    ///\n+    /// # Safety\n+    ///\n+    /// - The user must ensure to provide a correct MMIO start address.\n+    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n+        Self {\n+            registers: Registers::new(mmio_start_addr),\n+        }\n+    }\n+\n+    /// Disable pull-up/down on pins 14 and 15.\n+    #[cfg(feature = \"bsp_rpi3\")]\n+    fn disable_pud_14_15_bcm2837(&mut self) {\n+        use crate::cpu;\n+\n+        // Make an educated guess for a good delay value (Sequence described in the BCM2837\n+        // peripherals PDF).\n+        //\n+        // - According to Wikipedia, the fastest RPi4 clocks around 1.5 GHz.\n+        // - The Linux 2837 GPIO driver waits 1 µs between the steps.\n+        //\n+        // So lets try to be on the safe side and default to 2000 cycles, which would equal 1 µs\n+        // would the CPU be clocked at 2 GHz.\n+        const DELAY: usize = 2000;\n+\n+        self.registers.GPPUD.write(GPPUD::PUD::Off);\n+        cpu::spin_for_cycles(DELAY);\n+\n+        self.registers\n+            .GPPUDCLK0\n+            .write(GPPUDCLK0::PUDCLK15::AssertClock + GPPUDCLK0::PUDCLK14::AssertClock);\n+        cpu::spin_for_cycles(DELAY);\n+\n+        self.registers.GPPUD.write(GPPUD::PUD::Off);\n+        self.registers.GPPUDCLK0.set(0);\n+    }\n+\n+    /// Disable pull-up/down on pins 14 and 15.\n+    #[cfg(feature = \"bsp_rpi4\")]\n+    fn disable_pud_14_15_bcm2711(&mut self) {\n+        self.registers.GPIO_PUP_PDN_CNTRL_REG0.write(\n+            GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL15::PullUp\n+                + GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL14::PullUp,\n+        );\n+    }\n+\n+    /// Map PL011 UART as standard output.\n+    ///\n+    /// TX to pin 14\n+    /// RX to pin 15\n+    pub fn map_pl011_uart(&mut self) {\n+        // Select the UART on pins 14 and 15.\n+        self.registers\n+            .GPFSEL1\n+            .modify(GPFSEL1::FSEL15::AltFunc0 + GPFSEL1::FSEL14::AltFunc0);\n+\n+        // Disable pull-up/down on pins 14 and 15.\n+        #[cfg(feature = \"bsp_rpi3\")]\n+        self.disable_pud_14_15_bcm2837();\n+\n+        #[cfg(feature = \"bsp_rpi4\")]\n+        self.disable_pud_14_15_bcm2711();\n+    }\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+impl GPIO {\n+    pub const COMPATIBLE: &'static str = \"BCM GPIO\";\n+\n+    /// Create an instance.\n+    ///\n+    /// # Safety\n+    ///\n+    /// - The user must ensure to provide a correct MMIO start address.\n+    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n+        Self {\n+            inner: NullLock::new(GPIOInner::new(mmio_start_addr)),\n+        }\n+    }\n+\n+    /// Concurrency safe version of `GPIOInner.map_pl011_uart()`\n+    pub fn map_pl011_uart(&self) {\n+        self.inner.lock(|inner| inner.map_pl011_uart())\n+    }\n+}\n+\n+//------------------------------------------------------------------------------\n+// OS Interface Code\n+//------------------------------------------------------------------------------\n+use synchronization::interface::Mutex;\n+\n+impl driver::interface::DeviceDriver for GPIO {\n+    fn compatible(&self) -> &'static str {\n+        Self::COMPATIBLE\n+    }\n+}\n\ndiff -uNr 04_safe_globals/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs 05_drivers_gpio_uart/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs\n--- 04_safe_globals/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs\n+++ 05_drivers_gpio_uart/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs\n@@ -0,0 +1,407 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! PL011 UART driver.\n+//!\n+//! # Resources\n+//!\n+//! - <https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf>\n+//! - <https://developer.arm.com/documentation/ddi0183/latest>\n+\n+use crate::{\n+    bsp::device_driver::common::MMIODerefWrapper, console, cpu, driver, synchronization,\n+    synchronization::NullLock,\n+};\n+use core::fmt;\n+use tock_registers::{\n+    interfaces::{Readable, Writeable},\n+    register_bitfields, register_structs,\n+    registers::{ReadOnly, ReadWrite, WriteOnly},\n+};\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+// PL011 UART registers.\n+//\n+// Descriptions taken from \"PrimeCell UART (PL011) Technical Reference Manual\" r1p5.\n+register_bitfields! {\n+    u32,\n+\n+    /// Flag Register.\n+    FR [\n+        /// Transmit FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n+        /// Line Control Register, LCR_H.\n+        ///\n+        /// - If the FIFO is disabled, this bit is set when the transmit holding register is empty.\n+        /// - If the FIFO is enabled, the TXFE bit is set when the transmit FIFO is empty.\n+        /// - This bit does not indicate if there is data in the transmit shift register.\n+        TXFE OFFSET(7) NUMBITS(1) [],\n+\n+        /// Transmit FIFO full. The meaning of this bit depends on the state of the FEN bit in the\n+        /// LCR_H Register.\n+        ///\n+        /// - If the FIFO is disabled, this bit is set when the transmit holding register is full.\n+        /// - If the FIFO is enabled, the TXFF bit is set when the transmit FIFO is full.\n+        TXFF OFFSET(5) NUMBITS(1) [],\n+\n+        /// Receive FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n+        /// LCR_H Register.\n+        ///\n+        /// - If the FIFO is disabled, this bit is set when the receive holding register is empty.\n+        /// - If the FIFO is enabled, the RXFE bit is set when the receive FIFO is empty.\n+        RXFE OFFSET(4) NUMBITS(1) [],\n+\n+        /// UART busy. If this bit is set to 1, the UART is busy transmitting data. This bit remains\n+        /// set until the complete byte, including all the stop bits, has been sent from the shift\n+        /// register.\n+        ///\n+        /// This bit is set as soon as the transmit FIFO becomes non-empty, regardless of whether\n+        /// the UART is enabled or not.\n+        BUSY OFFSET(3) NUMBITS(1) []\n+    ],\n+\n+    /// Integer Baud Rate Divisor.\n+    IBRD [\n+        /// The integer baud rate divisor.\n+        BAUD_DIVINT OFFSET(0) NUMBITS(16) []\n+    ],\n+\n+    /// Fractional Baud Rate Divisor.\n+    FBRD [\n+        ///  The fractional baud rate divisor.\n+        BAUD_DIVFRAC OFFSET(0) NUMBITS(6) []\n+    ],\n+\n+    /// Line Control Register.\n+    LCR_H [\n+        /// Word length. These bits indicate the number of data bits transmitted or received in a\n+        /// frame.\n+        #[allow(clippy::enum_variant_names)]\n+        WLEN OFFSET(5) NUMBITS(2) [\n+            FiveBit = 0b00,\n+            SixBit = 0b01,\n+            SevenBit = 0b10,\n+            EightBit = 0b11\n+        ],\n+\n+        /// Enable FIFOs:\n+        ///\n+        /// 0 = FIFOs are disabled (character mode) that is, the FIFOs become 1-byte-deep holding\n+        /// registers.\n+        ///\n+        /// 1 = Transmit and receive FIFO buffers are enabled (FIFO mode).\n+        FEN  OFFSET(4) NUMBITS(1) [\n+            FifosDisabled = 0,\n+            FifosEnabled = 1\n+        ]\n+    ],\n+\n+    /// Control Register.\n+    CR [\n+        /// Receive enable. If this bit is set to 1, the receive section of the UART is enabled.\n+        /// Data reception occurs for either UART signals or SIR signals depending on the setting of\n+        /// the SIREN bit. When the UART is disabled in the middle of reception, it completes the\n+        /// current character before stopping.\n+        RXE OFFSET(9) NUMBITS(1) [\n+            Disabled = 0,\n+            Enabled = 1\n+        ],\n+\n+        /// Transmit enable. If this bit is set to 1, the transmit section of the UART is enabled.\n+        /// Data transmission occurs for either UART signals, or SIR signals depending on the\n+        /// setting of the SIREN bit. When the UART is disabled in the middle of transmission, it\n+        /// completes the current character before stopping.\n+        TXE OFFSET(8) NUMBITS(1) [\n+            Disabled = 0,\n+            Enabled = 1\n+        ],\n+\n+        /// UART enable:\n+        ///\n+        /// 0 = UART is disabled. If the UART is disabled in the middle of transmission or\n+        /// reception, it completes the current character before stopping.\n+        ///\n+        /// 1 = The UART is enabled. Data transmission and reception occurs for either UART signals\n+        /// or SIR signals depending on the setting of the SIREN bit\n+        UARTEN OFFSET(0) NUMBITS(1) [\n+            /// If the UART is disabled in the middle of transmission or reception, it completes the\n+            /// current character before stopping.\n+            Disabled = 0,\n+            Enabled = 1\n+        ]\n+    ],\n+\n+    /// Interrupt Clear Register.\n+    ICR [\n+        /// Meta field for all pending interrupts.\n+        ALL OFFSET(0) NUMBITS(11) []\n+    ]\n+}\n+\n+register_structs! {\n+    #[allow(non_snake_case)]\n+    pub RegisterBlock {\n+        (0x00 => DR: ReadWrite<u32>),\n+        (0x04 => _reserved1),\n+        (0x18 => FR: ReadOnly<u32, FR::Register>),\n+        (0x1c => _reserved2),\n+        (0x24 => IBRD: WriteOnly<u32, IBRD::Register>),\n+        (0x28 => FBRD: WriteOnly<u32, FBRD::Register>),\n+        (0x2c => LCR_H: WriteOnly<u32, LCR_H::Register>),\n+        (0x30 => CR: WriteOnly<u32, CR::Register>),\n+        (0x34 => _reserved3),\n+        (0x44 => ICR: WriteOnly<u32, ICR::Register>),\n+        (0x48 => @END),\n+    }\n+}\n+\n+/// Abstraction for the associated MMIO registers.\n+type Registers = MMIODerefWrapper<RegisterBlock>;\n+\n+#[derive(PartialEq)]\n+enum BlockingMode {\n+    Blocking,\n+    NonBlocking,\n+}\n+\n+struct PL011UartInner {\n+    registers: Registers,\n+    chars_written: usize,\n+    chars_read: usize,\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Representation of the UART.\n+pub struct PL011Uart {\n+    inner: NullLock<PL011UartInner>,\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Code\n+//--------------------------------------------------------------------------------------------------\n+\n+impl PL011UartInner {\n+    /// Create an instance.\n+    ///\n+    /// # Safety\n+    ///\n+    /// - The user must ensure to provide a correct MMIO start address.\n+    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n+        Self {\n+            registers: Registers::new(mmio_start_addr),\n+            chars_written: 0,\n+            chars_read: 0,\n+        }\n+    }\n+\n+    /// Set up baud rate and characteristics.\n+    ///\n+    /// This results in 8N1 and 921_600 baud.\n+    ///\n+    /// The calculation for the BRD is (we set the clock to 48 MHz in config.txt):\n+    /// `(48_000_000 / 16) / 921_600 = 3.2552083`.\n+    ///\n+    /// This means the integer part is `3` and goes into the `IBRD`.\n+    /// The fractional part is `0.2552083`.\n+    ///\n+    /// `FBRD` calculation according to the PL011 Technical Reference Manual:\n+    /// `INTEGER((0.2552083 * 64) + 0.5) = 16`.\n+    ///\n+    /// Therefore, the generated baud rate divider is: `3 + 16/64 = 3.25`. Which results in a\n+    /// genrated baud rate of `48_000_000 / (16 * 3.25) = 923_077`.\n+    ///\n+    /// Error = `((923_077 - 921_600) / 921_600) * 100 = 0.16modulo`.\n+    pub fn init(&mut self) {\n+        // Execution can arrive here while there are still characters queued in the TX FIFO and\n+        // actively being sent out by the UART hardware. If the UART is turned off in this case,\n+        // those queued characters would be lost.\n+        //\n+        // For example, this can happen during runtime on a call to panic!(), because panic!()\n+        // initializes its own UART instance and calls init().\n+        //\n+        // Hence, flush first to ensure all pending characters are transmitted.\n+        self.flush();\n+\n+        // Turn the UART off temporarily.\n+        self.registers.CR.set(0);\n+\n+        // Clear all pending interrupts.\n+        self.registers.ICR.write(ICR::ALL::CLEAR);\n+\n+        // From the PL011 Technical Reference Manual:\n+        //\n+        // The LCR_H, IBRD, and FBRD registers form the single 30-bit wide LCR Register that is\n+        // updated on a single write strobe generated by a LCR_H write. So, to internally update the\n+        // contents of IBRD or FBRD, a LCR_H write must always be performed at the end.\n+        //\n+        // Set the baud rate, 8N1 and FIFO enabled.\n+        self.registers.IBRD.write(IBRD::BAUD_DIVINT.val(3));\n+        self.registers.FBRD.write(FBRD::BAUD_DIVFRAC.val(16));\n+        self.registers\n+            .LCR_H\n+            .write(LCR_H::WLEN::EightBit + LCR_H::FEN::FifosEnabled);\n+\n+        // Turn the UART on.\n+        self.registers\n+            .CR\n+            .write(CR::UARTEN::Enabled + CR::TXE::Enabled + CR::RXE::Enabled);\n+    }\n+\n+    /// Send a character.\n+    fn write_char(&mut self, c: char) {\n+        // Spin while TX FIFO full is set, waiting for an empty slot.\n+        while self.registers.FR.matches_all(FR::TXFF::SET) {\n+            cpu::nop();\n+        }\n+\n+        // Write the character to the buffer.\n+        self.registers.DR.set(c as u32);\n+\n+        self.chars_written += 1;\n+    }\n+\n+    /// Block execution until the last buffered character has been physically put on the TX wire.\n+    fn flush(&self) {\n+        // Spin until the busy bit is cleared.\n+        while self.registers.FR.matches_all(FR::BUSY::SET) {\n+            cpu::nop();\n+        }\n+    }\n+\n+    /// Retrieve a character.\n+    fn read_char_converting(&mut self, blocking_mode: BlockingMode) -> Option<char> {\n+        // If RX FIFO is empty,\n+        if self.registers.FR.matches_all(FR::RXFE::SET) {\n+            // immediately return in non-blocking mode.\n+            if blocking_mode == BlockingMode::NonBlocking {\n+                return None;\n+            }\n+\n+            // Otherwise, wait until a char was received.\n+            while self.registers.FR.matches_all(FR::RXFE::SET) {\n+                cpu::nop();\n+            }\n+        }\n+\n+        // Read one character.\n+        let mut ret = self.registers.DR.get() as u8 as char;\n+\n+        // Convert carrige return to newline.\n+        if ret == '\\r' {\n+            ret = '\\n'\n+        }\n+\n+        // Update statistics.\n+        self.chars_read += 1;\n+\n+        Some(ret)\n+    }\n+}\n+\n+/// Implementing `core::fmt::Write` enables usage of the `format_args!` macros, which in turn are\n+/// used to implement the `kernel`'s `print!` and `println!` macros. By implementing `write_str()`,\n+/// we get `write_fmt()` automatically.\n+///\n+/// The function takes an `&mut self`, so it must be implemented for the inner struct.\n+///\n+/// See [`src/print.rs`].\n+///\n+/// [`src/print.rs`]: ../../print/index.html\n+impl fmt::Write for PL011UartInner {\n+    fn write_str(&mut self, s: &str) -> fmt::Result {\n+        for c in s.chars() {\n+            self.write_char(c);\n+        }\n+\n+        Ok(())\n+    }\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+impl PL011Uart {\n+    pub const COMPATIBLE: &'static str = \"BCM PL011 UART\";\n+\n+    /// Create an instance.\n+    ///\n+    /// # Safety\n+    ///\n+    /// - The user must ensure to provide a correct MMIO start address.\n+    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n+        Self {\n+            inner: NullLock::new(PL011UartInner::new(mmio_start_addr)),\n+        }\n+    }\n+}\n+\n+//------------------------------------------------------------------------------\n+// OS Interface Code\n+//------------------------------------------------------------------------------\n+use synchronization::interface::Mutex;\n+\n+impl driver::interface::DeviceDriver for PL011Uart {\n+    fn compatible(&self) -> &'static str {\n+        Self::COMPATIBLE\n+    }\n+\n+    unsafe fn init(&self) -> Result<(), &'static str> {\n+        self.inner.lock(|inner| inner.init());\n+\n+        Ok(())\n+    }\n+}\n+\n+impl console::interface::Write for PL011Uart {\n+    /// Passthrough of `args` to the `core::fmt::Write` implementation, but guarded by a Mutex to\n+    /// serialize access.\n+    fn write_char(&self, c: char) {\n+        self.inner.lock(|inner| inner.write_char(c));\n+    }\n+\n+    fn write_fmt(&self, args: core::fmt::Arguments) -> fmt::Result {\n+        // Fully qualified syntax for the call to `core::fmt::Write::write_fmt()` to increase\n+        // readability.\n+        self.inner.lock(|inner| fmt::Write::write_fmt(inner, args))\n+    }\n+\n+    fn flush(&self) {\n+        // Spin until TX FIFO empty is set.\n+        self.inner.lock(|inner| inner.flush());\n+    }\n+}\n+\n+impl console::interface::Read for PL011Uart {\n+    fn read_char(&self) -> char {\n+        self.inner\n+            .lock(|inner| inner.read_char_converting(BlockingMode::Blocking).unwrap())\n+    }\n+\n+    fn clear_rx(&self) {\n+        // Read from the RX FIFO until it is indicating empty.\n+        while self\n+            .inner\n+            .lock(|inner| inner.read_char_converting(BlockingMode::NonBlocking))\n+            .is_some()\n+        {}\n+    }\n+}\n+\n+impl console::interface::Statistics for PL011Uart {\n+    fn chars_written(&self) -> usize {\n+        self.inner.lock(|inner| inner.chars_written)\n+    }\n+\n+    fn chars_read(&self) -> usize {\n+        self.inner.lock(|inner| inner.chars_read)\n+    }\n+}\n+\n+impl console::interface::All for PL011Uart {}\n\ndiff -uNr 04_safe_globals/src/bsp/device_driver/bcm.rs 05_drivers_gpio_uart/src/bsp/device_driver/bcm.rs\n--- 04_safe_globals/src/bsp/device_driver/bcm.rs\n+++ 05_drivers_gpio_uart/src/bsp/device_driver/bcm.rs\n@@ -0,0 +1,11 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! BCM driver top level.\n+\n+mod bcm2xxx_gpio;\n+mod bcm2xxx_pl011_uart;\n+\n+pub use bcm2xxx_gpio::*;\n+pub use bcm2xxx_pl011_uart::*;\n\ndiff -uNr 04_safe_globals/src/bsp/device_driver/common.rs 05_drivers_gpio_uart/src/bsp/device_driver/common.rs\n--- 04_safe_globals/src/bsp/device_driver/common.rs\n+++ 05_drivers_gpio_uart/src/bsp/device_driver/common.rs\n@@ -0,0 +1,38 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Common device driver code.\n+\n+use core::{marker::PhantomData, ops};\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+pub struct MMIODerefWrapper<T> {\n+    start_addr: usize,\n+    phantom: PhantomData<fn() -> T>,\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+impl<T> MMIODerefWrapper<T> {\n+    /// Create an instance.\n+    pub const unsafe fn new(start_addr: usize) -> Self {\n+        Self {\n+            start_addr,\n+            phantom: PhantomData,\n+        }\n+    }\n+}\n+\n+impl<T> ops::Deref for MMIODerefWrapper<T> {\n+    type Target = T;\n+\n+    fn deref(&self) -> &Self::Target {\n+        unsafe { &*(self.start_addr as *const _) }\n+    }\n+}\n\ndiff -uNr 04_safe_globals/src/bsp/device_driver.rs 05_drivers_gpio_uart/src/bsp/device_driver.rs\n--- 04_safe_globals/src/bsp/device_driver.rs\n+++ 05_drivers_gpio_uart/src/bsp/device_driver.rs\n@@ -0,0 +1,12 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Device driver.\n+\n+#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\n+mod bcm;\n+mod common;\n+\n+#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\n+pub use bcm::*;\n\ndiff -uNr 04_safe_globals/src/bsp/raspberrypi/console.rs 05_drivers_gpio_uart/src/bsp/raspberrypi/console.rs\n--- 04_safe_globals/src/bsp/raspberrypi/console.rs\n+++ 05_drivers_gpio_uart/src/bsp/raspberrypi/console.rs\n@@ -4,115 +4,13 @@\n\n //! BSP console facilities.\n\n-use crate::{console, synchronization, synchronization::NullLock};\n-use core::fmt;\n-\n-//--------------------------------------------------------------------------------------------------\n-// Private Definitions\n-//--------------------------------------------------------------------------------------------------\n-\n-/// A mystical, magical device for generating QEMU output out of the void.\n-///\n-/// The mutex protected part.\n-struct QEMUOutputInner {\n-    chars_written: usize,\n-}\n-\n-//--------------------------------------------------------------------------------------------------\n-// Public Definitions\n-//--------------------------------------------------------------------------------------------------\n-\n-/// The main struct.\n-pub struct QEMUOutput {\n-    inner: NullLock<QEMUOutputInner>,\n-}\n-\n-//--------------------------------------------------------------------------------------------------\n-// Global instances\n-//--------------------------------------------------------------------------------------------------\n-\n-static QEMU_OUTPUT: QEMUOutput = QEMUOutput::new();\n-\n-//--------------------------------------------------------------------------------------------------\n-// Private Code\n-//--------------------------------------------------------------------------------------------------\n-\n-impl QEMUOutputInner {\n-    const fn new() -> QEMUOutputInner {\n-        QEMUOutputInner { chars_written: 0 }\n-    }\n-\n-    /// Send a character.\n-    fn write_char(&mut self, c: char) {\n-        unsafe {\n-            core::ptr::write_volatile(0x3F20_1000 as *mut u8, c as u8);\n-        }\n-\n-        self.chars_written += 1;\n-    }\n-}\n-\n-/// Implementing `core::fmt::Write` enables usage of the `format_args!` macros, which in turn are\n-/// used to implement the `kernel`'s `print!` and `println!` macros. By implementing `write_str()`,\n-/// we get `write_fmt()` automatically.\n-///\n-/// The function takes an `&mut self`, so it must be implemented for the inner struct.\n-///\n-/// See [`src/print.rs`].\n-///\n-/// [`src/print.rs`]: ../../print/index.html\n-impl fmt::Write for QEMUOutputInner {\n-    fn write_str(&mut self, s: &str) -> fmt::Result {\n-        for c in s.chars() {\n-            // Convert newline to carrige return + newline.\n-            if c == '\\n' {\n-                self.write_char('\\r')\n-            }\n-\n-            self.write_char(c);\n-        }\n-\n-        Ok(())\n-    }\n-}\n+use crate::console;\n\n //--------------------------------------------------------------------------------------------------\n // Public Code\n //--------------------------------------------------------------------------------------------------\n\n-impl QEMUOutput {\n-    /// Create a new instance.\n-    pub const fn new() -> QEMUOutput {\n-        QEMUOutput {\n-            inner: NullLock::new(QEMUOutputInner::new()),\n-        }\n-    }\n-}\n-\n /// Return a reference to the console.\n pub fn console() -> &'static dyn console::interface::All {\n-    &QEMU_OUTPUT\n-}\n-\n-//------------------------------------------------------------------------------\n-// OS Interface Code\n-//------------------------------------------------------------------------------\n-use synchronization::interface::Mutex;\n-\n-/// Passthrough of `args` to the `core::fmt::Write` implementation, but guarded by a Mutex to\n-/// serialize access.\n-impl console::interface::Write for QEMUOutput {\n-    fn write_fmt(&self, args: core::fmt::Arguments) -> fmt::Result {\n-        // Fully qualified syntax for the call to `core::fmt::Write::write_fmt()` to increase\n-        // readability.\n-        self.inner.lock(|inner| fmt::Write::write_fmt(inner, args))\n-    }\n-}\n-\n-impl console::interface::Statistics for QEMUOutput {\n-    fn chars_written(&self) -> usize {\n-        self.inner.lock(|inner| inner.chars_written)\n-    }\n+    &super::driver::PL011_UART\n }\n-\n-impl console::interface::All for QEMUOutput {}\n\ndiff -uNr 04_safe_globals/src/bsp/raspberrypi/driver.rs 05_drivers_gpio_uart/src/bsp/raspberrypi/driver.rs\n--- 04_safe_globals/src/bsp/raspberrypi/driver.rs\n+++ 05_drivers_gpio_uart/src/bsp/raspberrypi/driver.rs\n@@ -0,0 +1,71 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! BSP driver support.\n+\n+use super::memory::map::mmio;\n+use crate::{bsp::device_driver, console, driver as generic_driver};\n+use core::sync::atomic::{AtomicBool, Ordering};\n+\n+//--------------------------------------------------------------------------------------------------\n+// Global instances\n+//--------------------------------------------------------------------------------------------------\n+\n+static PL011_UART: device_driver::PL011Uart =\n+    unsafe { device_driver::PL011Uart::new(mmio::PL011_UART_START) };\n+static GPIO: device_driver::GPIO = unsafe { device_driver::GPIO::new(mmio::GPIO_START) };\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Code\n+//--------------------------------------------------------------------------------------------------\n+\n+/// This must be called only after successful init of the UART driver.\n+fn post_init_uart() -> Result<(), &'static str> {\n+    console::register_console(&PL011_UART);\n+\n+    Ok(())\n+}\n+\n+/// This must be called only after successful init of the GPIO driver.\n+fn post_init_gpio() -> Result<(), &'static str> {\n+    GPIO.map_pl011_uart();\n+    Ok(())\n+}\n+\n+fn driver_uart() -> Result<(), &'static str> {\n+    let uart_descriptor =\n+        generic_driver::DeviceDriverDescriptor::new(&PL011_UART, Some(post_init_uart));\n+    generic_driver::driver_manager().register_driver(uart_descriptor);\n+\n+    Ok(())\n+}\n+\n+fn driver_gpio() -> Result<(), &'static str> {\n+    let gpio_descriptor = generic_driver::DeviceDriverDescriptor::new(&GPIO, Some(post_init_gpio));\n+    generic_driver::driver_manager().register_driver(gpio_descriptor);\n+\n+    Ok(())\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Initialize the driver subsystem.\n+///\n+/// # Safety\n+///\n+/// See child function calls.\n+pub unsafe fn init() -> Result<(), &'static str> {\n+    static INIT_DONE: AtomicBool = AtomicBool::new(false);\n+    if INIT_DONE.load(Ordering::Relaxed) {\n+        return Err(\"Init already done\");\n+    }\n+\n+    driver_uart()?;\n+    driver_gpio()?;\n+\n+    INIT_DONE.store(true, Ordering::Relaxed);\n+    Ok(())\n+}\n\ndiff -uNr 04_safe_globals/src/bsp/raspberrypi/memory.rs 05_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs\n--- 04_safe_globals/src/bsp/raspberrypi/memory.rs\n+++ 05_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs\n@@ -0,0 +1,37 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! BSP Memory Management.\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// The board's physical memory map.\n+#[rustfmt::skip]\n+pub(super) mod map {\n+\n+    pub const GPIO_OFFSET:         usize = 0x0020_0000;\n+    pub const UART_OFFSET:         usize = 0x0020_1000;\n+\n+    /// Physical devices.\n+    #[cfg(feature = \"bsp_rpi3\")]\n+    pub mod mmio {\n+        use super::*;\n+\n+        pub const START:            usize =         0x3F00_0000;\n+        pub const GPIO_START:       usize = START + GPIO_OFFSET;\n+        pub const PL011_UART_START: usize = START + UART_OFFSET;\n+    }\n+\n+    /// Physical devices.\n+    #[cfg(feature = \"bsp_rpi4\")]\n+    pub mod mmio {\n+        use super::*;\n+\n+        pub const START:            usize =         0xFE00_0000;\n+        pub const GPIO_START:       usize = START + GPIO_OFFSET;\n+        pub const PL011_UART_START: usize = START + UART_OFFSET;\n+    }\n+}\n\ndiff -uNr 04_safe_globals/src/bsp/raspberrypi.rs 05_drivers_gpio_uart/src/bsp/raspberrypi.rs\n--- 04_safe_globals/src/bsp/raspberrypi.rs\n+++ 05_drivers_gpio_uart/src/bsp/raspberrypi.rs\n@@ -4,5 +4,23 @@\n\n //! Top-level BSP file for the Raspberry Pi 3 and 4.\n\n-pub mod console;\n pub mod cpu;\n+pub mod driver;\n+pub mod memory;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Board identification.\n+pub fn board_name() -> &'static str {\n+    #[cfg(feature = \"bsp_rpi3\")]\n+    {\n+        \"Raspberry Pi 3\"\n+    }\n+\n+    #[cfg(feature = \"bsp_rpi4\")]\n+    {\n+        \"Raspberry Pi 4\"\n+    }\n+}\n\ndiff -uNr 04_safe_globals/src/bsp.rs 05_drivers_gpio_uart/src/bsp.rs\n--- 04_safe_globals/src/bsp.rs\n+++ 05_drivers_gpio_uart/src/bsp.rs\n@@ -4,6 +4,8 @@\n\n //! Conditional reexporting of Board Support Packages.\n\n+mod device_driver;\n+\n #[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\n mod raspberrypi;\n\n\ndiff -uNr 04_safe_globals/src/console/null_console.rs 05_drivers_gpio_uart/src/console/null_console.rs\n--- 04_safe_globals/src/console/null_console.rs\n+++ 05_drivers_gpio_uart/src/console/null_console.rs\n@@ -0,0 +1,41 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Null console.\n+\n+use super::interface;\n+use core::fmt;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+pub struct NullConsole;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Global instances\n+//--------------------------------------------------------------------------------------------------\n+\n+pub static NULL_CONSOLE: NullConsole = NullConsole {};\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+impl interface::Write for NullConsole {\n+    fn write_char(&self, _c: char) {}\n+\n+    fn write_fmt(&self, _args: fmt::Arguments) -> fmt::Result {\n+        fmt::Result::Ok(())\n+    }\n+\n+    fn flush(&self) {}\n+}\n+\n+impl interface::Read for NullConsole {\n+    fn clear_rx(&self) {}\n+}\n+\n+impl interface::Statistics for NullConsole {}\n+impl interface::All for NullConsole {}\n\ndiff -uNr 04_safe_globals/src/console.rs 05_drivers_gpio_uart/src/console.rs\n--- 04_safe_globals/src/console.rs\n+++ 05_drivers_gpio_uart/src/console.rs\n@@ -4,7 +4,9 @@\n\n //! System console.\n\n-use crate::bsp;\n+mod null_console;\n+\n+use crate::synchronization::{self, NullLock};\n\n //--------------------------------------------------------------------------------------------------\n // Public Definitions\n@@ -16,8 +18,25 @@\n\n     /// Console write functions.\n     pub trait Write {\n+        /// Write a single character.\n+        fn write_char(&self, c: char);\n+\n         /// Write a Rust format string.\n         fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result;\n+\n+        /// Block until the last buffered character has been physically put on the TX wire.\n+        fn flush(&self);\n+    }\n+\n+    /// Console read functions.\n+    pub trait Read {\n+        /// Read a single character.\n+        fn read_char(&self) -> char {\n+            ' '\n+        }\n+\n+        /// Clear RX buffers, if any.\n+        fn clear_rx(&self);\n     }\n\n     /// Console statistics.\n@@ -26,19 +45,37 @@\n         fn chars_written(&self) -> usize {\n             0\n         }\n+\n+        /// Return the number of characters read.\n+        fn chars_read(&self) -> usize {\n+            0\n+        }\n     }\n\n     /// Trait alias for a full-fledged console.\n-    pub trait All: Write + Statistics {}\n+    pub trait All: Write + Read + Statistics {}\n }\n\n //--------------------------------------------------------------------------------------------------\n+// Global instances\n+//--------------------------------------------------------------------------------------------------\n+\n+static CUR_CONSOLE: NullLock<&'static (dyn interface::All + Sync)> =\n+    NullLock::new(&null_console::NULL_CONSOLE);\n+\n+//--------------------------------------------------------------------------------------------------\n // Public Code\n //--------------------------------------------------------------------------------------------------\n+use synchronization::interface::Mutex;\n+\n+/// Register a new console.\n+pub fn register_console(new_console: &'static (dyn interface::All + Sync)) {\n+    CUR_CONSOLE.lock(|con| *con = new_console);\n+}\n\n-/// Return a reference to the console.\n+/// Return a reference to the currently registered console.\n ///\n /// This is the global console used by all printing macros.\n pub fn console() -> &'static dyn interface::All {\n-    bsp::console::console()\n+    CUR_CONSOLE.lock(|con| *con)\n }\n\ndiff -uNr 04_safe_globals/src/cpu.rs 05_drivers_gpio_uart/src/cpu.rs\n--- 04_safe_globals/src/cpu.rs\n+++ 05_drivers_gpio_uart/src/cpu.rs\n@@ -13,4 +13,7 @@\n //--------------------------------------------------------------------------------------------------\n // Architectural Public Reexports\n //--------------------------------------------------------------------------------------------------\n-pub use arch_cpu::wait_forever;\n+pub use arch_cpu::{nop, wait_forever};\n+\n+#[cfg(feature = \"bsp_rpi3\")]\n+pub use arch_cpu::spin_for_cycles;\n\ndiff -uNr 04_safe_globals/src/driver.rs 05_drivers_gpio_uart/src/driver.rs\n--- 04_safe_globals/src/driver.rs\n+++ 05_drivers_gpio_uart/src/driver.rs\n@@ -0,0 +1,167 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Driver support.\n+\n+use crate::{\n+    println,\n+    synchronization::{interface::Mutex, NullLock},\n+};\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+const NUM_DRIVERS: usize = 5;\n+\n+struct DriverManagerInner {\n+    next_index: usize,\n+    descriptors: [Option<DeviceDriverDescriptor>; NUM_DRIVERS],\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Driver interfaces.\n+pub mod interface {\n+    /// Device Driver functions.\n+    pub trait DeviceDriver {\n+        /// Return a compatibility string for identifying the driver.\n+        fn compatible(&self) -> &'static str;\n+\n+        /// Called by the kernel to bring up the device.\n+        ///\n+        /// # Safety\n+        ///\n+        /// - During init, drivers might do stuff with system-wide impact.\n+        unsafe fn init(&self) -> Result<(), &'static str> {\n+            Ok(())\n+        }\n+    }\n+}\n+\n+/// Tpye to be used as an optional callback after a driver's init() has run.\n+pub type DeviceDriverPostInitCallback = unsafe fn() -> Result<(), &'static str>;\n+\n+/// A descriptor for device drivers.\n+#[derive(Copy, Clone)]\n+pub struct DeviceDriverDescriptor {\n+    device_driver: &'static (dyn interface::DeviceDriver + Sync),\n+    post_init_callback: Option<DeviceDriverPostInitCallback>,\n+}\n+\n+/// Provides device driver management functions.\n+pub struct DriverManager {\n+    inner: NullLock<DriverManagerInner>,\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Global instances\n+//--------------------------------------------------------------------------------------------------\n+\n+static DRIVER_MANAGER: DriverManager = DriverManager::new();\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Code\n+//--------------------------------------------------------------------------------------------------\n+\n+impl DriverManagerInner {\n+    /// Create an instance.\n+    pub const fn new() -> Self {\n+        Self {\n+            next_index: 0,\n+            descriptors: [None; NUM_DRIVERS],\n+        }\n+    }\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+impl DeviceDriverDescriptor {\n+    /// Create an instance.\n+    pub fn new(\n+        device_driver: &'static (dyn interface::DeviceDriver + Sync),\n+        post_init_callback: Option<DeviceDriverPostInitCallback>,\n+    ) -> Self {\n+        Self {\n+            device_driver,\n+            post_init_callback,\n+        }\n+    }\n+}\n+\n+/// Return a reference to the global DriverManager.\n+pub fn driver_manager() -> &'static DriverManager {\n+    &DRIVER_MANAGER\n+}\n+\n+impl DriverManager {\n+    /// Create an instance.\n+    pub const fn new() -> Self {\n+        Self {\n+            inner: NullLock::new(DriverManagerInner::new()),\n+        }\n+    }\n+\n+    /// Register a device driver with the kernel.\n+    pub fn register_driver(&self, descriptor: DeviceDriverDescriptor) {\n+        self.inner.lock(|inner| {\n+            inner.descriptors[inner.next_index] = Some(descriptor);\n+            inner.next_index += 1;\n+        })\n+    }\n+\n+    /// Helper for iterating over registered drivers.\n+    fn for_each_descriptor<'a>(&'a self, f: impl FnMut(&'a DeviceDriverDescriptor)) {\n+        self.inner.lock(|inner| {\n+            inner\n+                .descriptors\n+                .iter()\n+                .filter_map(|x| x.as_ref())\n+                .for_each(f)\n+        })\n+    }\n+\n+    /// Fully initialize all drivers.\n+    ///\n+    /// # Safety\n+    ///\n+    /// - During init, drivers might do stuff with system-wide impact.\n+    pub unsafe fn init_drivers(&self) {\n+        self.for_each_descriptor(|descriptor| {\n+            // 1. Initialize driver.\n+            if let Err(x) = descriptor.device_driver.init() {\n+                panic!(\n+                    \"Error initializing driver: {}: {}\",\n+                    descriptor.device_driver.compatible(),\n+                    x\n+                );\n+            }\n+\n+            // 2. Call corresponding post init callback.\n+            if let Some(callback) = &descriptor.post_init_callback {\n+                if let Err(x) = callback() {\n+                    panic!(\n+                        \"Error during driver post-init callback: {}: {}\",\n+                        descriptor.device_driver.compatible(),\n+                        x\n+                    );\n+                }\n+            }\n+        });\n+    }\n+\n+    /// Enumerate all registered device drivers.\n+    pub fn enumerate(&self) {\n+        let mut i: usize = 1;\n+        self.for_each_descriptor(|descriptor| {\n+            println!(\"      {}. {}\", i, descriptor.device_driver.compatible());\n+\n+            i += 1;\n+        });\n+    }\n+}\n\ndiff -uNr 04_safe_globals/src/main.rs 05_drivers_gpio_uart/src/main.rs\n--- 04_safe_globals/src/main.rs\n+++ 05_drivers_gpio_uart/src/main.rs\n@@ -106,6 +106,7 @@\n //!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.\n //! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.\n\n+#![allow(clippy::upper_case_acronyms)]\n #![feature(asm_const)]\n #![feature(format_args_nl)]\n #![feature(panic_info_message)]\n@@ -116,6 +117,7 @@\n mod bsp;\n mod console;\n mod cpu;\n+mod driver;\n mod panic_wait;\n mod print;\n mod synchronization;\n@@ -125,13 +127,42 @@\n /// # Safety\n ///\n /// - Only a single core must be active and running this function.\n+/// - The init calls in this function must appear in the correct order.\n unsafe fn kernel_init() -> ! {\n-    use console::console;\n+    // Initialize the BSP driver subsystem.\n+    if let Err(x) = bsp::driver::init() {\n+        panic!(\"Error initializing BSP driver subsystem: {}\", x);\n+    }\n+\n+    // Initialize all device drivers.\n+    driver::driver_manager().init_drivers();\n+    // println! is usable from here on.\n\n-    println!(\"[0] Hello from Rust!\");\n+    // Transition from unsafe to safe.\n+    kernel_main()\n+}\n\n-    println!(\"[1] Chars written: {}\", console().chars_written());\n+/// The main function running after the early init.\n+fn kernel_main() -> ! {\n+    use console::console;\n\n-    println!(\"[2] Stopping here.\");\n-    cpu::wait_forever()\n+    println!(\n+        \"[0] {} version {}\",\n+        env!(\"CARGO_PKG_NAME\"),\n+        env!(\"CARGO_PKG_VERSION\")\n+    );\n+    println!(\"[1] Booting on: {}\", bsp::board_name());\n+\n+    println!(\"[2] Drivers loaded:\");\n+    driver::driver_manager().enumerate();\n+\n+    println!(\"[3] Chars written: {}\", console().chars_written());\n+    println!(\"[4] Echoing input now\");\n+\n+    // Discard any spurious received characters before going into echo mode.\n+    console().clear_rx();\n+    loop {\n+        let c = console().read_char();\n+        console().write_char(c);\n+    }\n }\n\ndiff -uNr 04_safe_globals/tests/boot_test_string.rb 05_drivers_gpio_uart/tests/boot_test_string.rb\n--- 04_safe_globals/tests/boot_test_string.rb\n+++ 05_drivers_gpio_uart/tests/boot_test_string.rb\n@@ -1,3 +1,3 @@\n # frozen_string_literal: true\n\n-EXPECTED_PRINT = 'Stopping here'\n+EXPECTED_PRINT = 'Echoing input now'\n\n```\n"
  },
  {
    "path": "05_drivers_gpio_uart/build.rs",
    "content": "use std::{env, fs, process};\n\nfn main() {\n    let ld_script_path = match env::var(\"LD_SCRIPT_PATH\") {\n        Ok(var) => var,\n        _ => process::exit(0),\n    };\n\n    let files = fs::read_dir(ld_script_path).unwrap();\n    files\n        .filter_map(Result::ok)\n        .filter(|d| {\n            if let Some(e) = d.path().extension() {\n                e == \"ld\"\n            } else {\n                false\n            }\n        })\n        .for_each(|f| println!(\"cargo:rerun-if-changed={}\", f.path().display()));\n}\n"
  },
  {
    "path": "05_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural boot code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::boot::arch_boot\n\nuse core::arch::global_asm;\n\n// Assembly counterpart to this file.\nglobal_asm!(\n    include_str!(\"boot.s\"),\n    CONST_CORE_ID_MASK = const 0b11\n);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The Rust entry of the `kernel` binary.\n///\n/// The function is called from the assembly `_start` function.\n#[no_mangle]\npub unsafe fn _start_rust() -> ! {\n    crate::kernel_init()\n}\n"
  },
  {
    "path": "05_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.s",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2022 Andre Richter <andre.o.richter@gmail.com>\n\n//--------------------------------------------------------------------------------------------------\n// Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Load the address of a symbol into a register, PC-relative.\n//\n// The symbol must lie within +/- 4 GiB of the Program Counter.\n//\n// # Resources\n//\n// - https://sourceware.org/binutils/docs-2.36/as/AArch64_002dRelocations.html\n.macro ADR_REL register, symbol\n\tadrp\t\\register, \\symbol\n\tadd\t\\register, \\register, #:lo12:\\symbol\n.endm\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n.section .text._start\n\n//------------------------------------------------------------------------------\n// fn _start()\n//------------------------------------------------------------------------------\n_start:\n\t// Only proceed on the boot core. Park it otherwise.\n\tmrs\tx0, MPIDR_EL1\n\tand\tx0, x0, {CONST_CORE_ID_MASK}\n\tldr\tx1, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs\n\tcmp\tx0, x1\n\tb.ne\t.L_parking_loop\n\n\t// If execution reaches here, it is the boot core.\n\n\t// Initialize DRAM.\n\tADR_REL\tx0, __bss_start\n\tADR_REL x1, __bss_end_exclusive\n\n.L_bss_init_loop:\n\tcmp\tx0, x1\n\tb.eq\t.L_prepare_rust\n\tstp\txzr, xzr, [x0], #16\n\tb\t.L_bss_init_loop\n\n\t// Prepare the jump to Rust code.\n.L_prepare_rust:\n\t// Set the stack pointer.\n\tADR_REL\tx0, __boot_core_stack_end_exclusive\n\tmov\tsp, x0\n\n\t// Jump to Rust code.\n\tb\t_start_rust\n\n\t// Infinitely wait for events (aka \"park the core\").\n.L_parking_loop:\n\twfe\n\tb\t.L_parking_loop\n\n.size\t_start, . - _start\n.type\t_start, function\n.global\t_start\n"
  },
  {
    "path": "05_drivers_gpio_uart/src/_arch/aarch64/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural processor code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::arch_cpu\n\nuse aarch64_cpu::asm;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\npub use asm::nop;\n\n/// Spin for `n` cycles.\n#[cfg(feature = \"bsp_rpi3\")]\n#[inline(always)]\npub fn spin_for_cycles(n: usize) {\n    for _ in 0..n {\n        asm::nop();\n    }\n}\n\n/// Pause execution on the core.\n#[inline(always)]\npub fn wait_forever() -> ! {\n    loop {\n        asm::wfe()\n    }\n}\n"
  },
  {
    "path": "05_drivers_gpio_uart/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GPIO Driver.\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper, driver, synchronization,\n    synchronization::NullLock,\n};\nuse tock_registers::{\n    interfaces::{ReadWriteable, Writeable},\n    register_bitfields, register_structs,\n    registers::ReadWrite,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// GPIO registers.\n//\n// Descriptions taken from\n// - https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf\n// - https://datasheets.raspberrypi.org/bcm2711/bcm2711-peripherals.pdf\nregister_bitfields! {\n    u32,\n\n    /// GPIO Function Select 1\n    GPFSEL1 [\n        /// Pin 15\n        FSEL15 OFFSET(15) NUMBITS(3) [\n            Input = 0b000,\n            Output = 0b001,\n            AltFunc0 = 0b100  // PL011 UART RX\n\n        ],\n\n        /// Pin 14\n        FSEL14 OFFSET(12) NUMBITS(3) [\n            Input = 0b000,\n            Output = 0b001,\n            AltFunc0 = 0b100  // PL011 UART TX\n        ]\n    ],\n\n    /// GPIO Pull-up/down Register\n    ///\n    /// BCM2837 only.\n    GPPUD [\n        /// Controls the actuation of the internal pull-up/down control line to ALL the GPIO pins.\n        PUD OFFSET(0) NUMBITS(2) [\n            Off = 0b00,\n            PullDown = 0b01,\n            PullUp = 0b10\n        ]\n    ],\n\n    /// GPIO Pull-up/down Clock Register 0\n    ///\n    /// BCM2837 only.\n    GPPUDCLK0 [\n        /// Pin 15\n        PUDCLK15 OFFSET(15) NUMBITS(1) [\n            NoEffect = 0,\n            AssertClock = 1\n        ],\n\n        /// Pin 14\n        PUDCLK14 OFFSET(14) NUMBITS(1) [\n            NoEffect = 0,\n            AssertClock = 1\n        ]\n    ],\n\n    /// GPIO Pull-up / Pull-down Register 0\n    ///\n    /// BCM2711 only.\n    GPIO_PUP_PDN_CNTRL_REG0 [\n        /// Pin 15\n        GPIO_PUP_PDN_CNTRL15 OFFSET(30) NUMBITS(2) [\n            NoResistor = 0b00,\n            PullUp = 0b01\n        ],\n\n        /// Pin 14\n        GPIO_PUP_PDN_CNTRL14 OFFSET(28) NUMBITS(2) [\n            NoResistor = 0b00,\n            PullUp = 0b01\n        ]\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    RegisterBlock {\n        (0x00 => _reserved1),\n        (0x04 => GPFSEL1: ReadWrite<u32, GPFSEL1::Register>),\n        (0x08 => _reserved2),\n        (0x94 => GPPUD: ReadWrite<u32, GPPUD::Register>),\n        (0x98 => GPPUDCLK0: ReadWrite<u32, GPPUDCLK0::Register>),\n        (0x9C => _reserved3),\n        (0xE4 => GPIO_PUP_PDN_CNTRL_REG0: ReadWrite<u32, GPIO_PUP_PDN_CNTRL_REG0::Register>),\n        (0xE8 => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\nstruct GPIOInner {\n    registers: Registers,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the GPIO HW.\npub struct GPIO {\n    inner: NullLock<GPIOInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GPIOInner {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n        }\n    }\n\n    /// Disable pull-up/down on pins 14 and 15.\n    #[cfg(feature = \"bsp_rpi3\")]\n    fn disable_pud_14_15_bcm2837(&mut self) {\n        use crate::cpu;\n\n        // Make an educated guess for a good delay value (Sequence described in the BCM2837\n        // peripherals PDF).\n        //\n        // - According to Wikipedia, the fastest RPi4 clocks around 1.5 GHz.\n        // - The Linux 2837 GPIO driver waits 1 µs between the steps.\n        //\n        // So lets try to be on the safe side and default to 2000 cycles, which would equal 1 µs\n        // would the CPU be clocked at 2 GHz.\n        const DELAY: usize = 2000;\n\n        self.registers.GPPUD.write(GPPUD::PUD::Off);\n        cpu::spin_for_cycles(DELAY);\n\n        self.registers\n            .GPPUDCLK0\n            .write(GPPUDCLK0::PUDCLK15::AssertClock + GPPUDCLK0::PUDCLK14::AssertClock);\n        cpu::spin_for_cycles(DELAY);\n\n        self.registers.GPPUD.write(GPPUD::PUD::Off);\n        self.registers.GPPUDCLK0.set(0);\n    }\n\n    /// Disable pull-up/down on pins 14 and 15.\n    #[cfg(feature = \"bsp_rpi4\")]\n    fn disable_pud_14_15_bcm2711(&mut self) {\n        self.registers.GPIO_PUP_PDN_CNTRL_REG0.write(\n            GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL15::PullUp\n                + GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL14::PullUp,\n        );\n    }\n\n    /// Map PL011 UART as standard output.\n    ///\n    /// TX to pin 14\n    /// RX to pin 15\n    pub fn map_pl011_uart(&mut self) {\n        // Select the UART on pins 14 and 15.\n        self.registers\n            .GPFSEL1\n            .modify(GPFSEL1::FSEL15::AltFunc0 + GPFSEL1::FSEL14::AltFunc0);\n\n        // Disable pull-up/down on pins 14 and 15.\n        #[cfg(feature = \"bsp_rpi3\")]\n        self.disable_pud_14_15_bcm2837();\n\n        #[cfg(feature = \"bsp_rpi4\")]\n        self.disable_pud_14_15_bcm2711();\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GPIO {\n    pub const COMPATIBLE: &'static str = \"BCM GPIO\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            inner: NullLock::new(GPIOInner::new(mmio_start_addr)),\n        }\n    }\n\n    /// Concurrency safe version of `GPIOInner.map_pl011_uart()`\n    pub fn map_pl011_uart(&self) {\n        self.inner.lock(|inner| inner.map_pl011_uart())\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl driver::interface::DeviceDriver for GPIO {\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n}\n"
  },
  {
    "path": "05_drivers_gpio_uart/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! PL011 UART driver.\n//!\n//! # Resources\n//!\n//! - <https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf>\n//! - <https://developer.arm.com/documentation/ddi0183/latest>\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper, console, cpu, driver, synchronization,\n    synchronization::NullLock,\n};\nuse core::fmt;\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields, register_structs,\n    registers::{ReadOnly, ReadWrite, WriteOnly},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// PL011 UART registers.\n//\n// Descriptions taken from \"PrimeCell UART (PL011) Technical Reference Manual\" r1p5.\nregister_bitfields! {\n    u32,\n\n    /// Flag Register.\n    FR [\n        /// Transmit FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n        /// Line Control Register, LCR_H.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the transmit holding register is empty.\n        /// - If the FIFO is enabled, the TXFE bit is set when the transmit FIFO is empty.\n        /// - This bit does not indicate if there is data in the transmit shift register.\n        TXFE OFFSET(7) NUMBITS(1) [],\n\n        /// Transmit FIFO full. The meaning of this bit depends on the state of the FEN bit in the\n        /// LCR_H Register.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the transmit holding register is full.\n        /// - If the FIFO is enabled, the TXFF bit is set when the transmit FIFO is full.\n        TXFF OFFSET(5) NUMBITS(1) [],\n\n        /// Receive FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n        /// LCR_H Register.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the receive holding register is empty.\n        /// - If the FIFO is enabled, the RXFE bit is set when the receive FIFO is empty.\n        RXFE OFFSET(4) NUMBITS(1) [],\n\n        /// UART busy. If this bit is set to 1, the UART is busy transmitting data. This bit remains\n        /// set until the complete byte, including all the stop bits, has been sent from the shift\n        /// register.\n        ///\n        /// This bit is set as soon as the transmit FIFO becomes non-empty, regardless of whether\n        /// the UART is enabled or not.\n        BUSY OFFSET(3) NUMBITS(1) []\n    ],\n\n    /// Integer Baud Rate Divisor.\n    IBRD [\n        /// The integer baud rate divisor.\n        BAUD_DIVINT OFFSET(0) NUMBITS(16) []\n    ],\n\n    /// Fractional Baud Rate Divisor.\n    FBRD [\n        ///  The fractional baud rate divisor.\n        BAUD_DIVFRAC OFFSET(0) NUMBITS(6) []\n    ],\n\n    /// Line Control Register.\n    LCR_H [\n        /// Word length. These bits indicate the number of data bits transmitted or received in a\n        /// frame.\n        #[allow(clippy::enum_variant_names)]\n        WLEN OFFSET(5) NUMBITS(2) [\n            FiveBit = 0b00,\n            SixBit = 0b01,\n            SevenBit = 0b10,\n            EightBit = 0b11\n        ],\n\n        /// Enable FIFOs:\n        ///\n        /// 0 = FIFOs are disabled (character mode) that is, the FIFOs become 1-byte-deep holding\n        /// registers.\n        ///\n        /// 1 = Transmit and receive FIFO buffers are enabled (FIFO mode).\n        FEN  OFFSET(4) NUMBITS(1) [\n            FifosDisabled = 0,\n            FifosEnabled = 1\n        ]\n    ],\n\n    /// Control Register.\n    CR [\n        /// Receive enable. If this bit is set to 1, the receive section of the UART is enabled.\n        /// Data reception occurs for either UART signals or SIR signals depending on the setting of\n        /// the SIREN bit. When the UART is disabled in the middle of reception, it completes the\n        /// current character before stopping.\n        RXE OFFSET(9) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// Transmit enable. If this bit is set to 1, the transmit section of the UART is enabled.\n        /// Data transmission occurs for either UART signals, or SIR signals depending on the\n        /// setting of the SIREN bit. When the UART is disabled in the middle of transmission, it\n        /// completes the current character before stopping.\n        TXE OFFSET(8) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// UART enable:\n        ///\n        /// 0 = UART is disabled. If the UART is disabled in the middle of transmission or\n        /// reception, it completes the current character before stopping.\n        ///\n        /// 1 = The UART is enabled. Data transmission and reception occurs for either UART signals\n        /// or SIR signals depending on the setting of the SIREN bit\n        UARTEN OFFSET(0) NUMBITS(1) [\n            /// If the UART is disabled in the middle of transmission or reception, it completes the\n            /// current character before stopping.\n            Disabled = 0,\n            Enabled = 1\n        ]\n    ],\n\n    /// Interrupt Clear Register.\n    ICR [\n        /// Meta field for all pending interrupts.\n        ALL OFFSET(0) NUMBITS(11) []\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    pub RegisterBlock {\n        (0x00 => DR: ReadWrite<u32>),\n        (0x04 => _reserved1),\n        (0x18 => FR: ReadOnly<u32, FR::Register>),\n        (0x1c => _reserved2),\n        (0x24 => IBRD: WriteOnly<u32, IBRD::Register>),\n        (0x28 => FBRD: WriteOnly<u32, FBRD::Register>),\n        (0x2c => LCR_H: WriteOnly<u32, LCR_H::Register>),\n        (0x30 => CR: WriteOnly<u32, CR::Register>),\n        (0x34 => _reserved3),\n        (0x44 => ICR: WriteOnly<u32, ICR::Register>),\n        (0x48 => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\n#[derive(PartialEq)]\nenum BlockingMode {\n    Blocking,\n    NonBlocking,\n}\n\nstruct PL011UartInner {\n    registers: Registers,\n    chars_written: usize,\n    chars_read: usize,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the UART.\npub struct PL011Uart {\n    inner: NullLock<PL011UartInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PL011UartInner {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n            chars_written: 0,\n            chars_read: 0,\n        }\n    }\n\n    /// Set up baud rate and characteristics.\n    ///\n    /// This results in 8N1 and 921_600 baud.\n    ///\n    /// The calculation for the BRD is (we set the clock to 48 MHz in config.txt):\n    /// `(48_000_000 / 16) / 921_600 = 3.2552083`.\n    ///\n    /// This means the integer part is `3` and goes into the `IBRD`.\n    /// The fractional part is `0.2552083`.\n    ///\n    /// `FBRD` calculation according to the PL011 Technical Reference Manual:\n    /// `INTEGER((0.2552083 * 64) + 0.5) = 16`.\n    ///\n    /// Therefore, the generated baud rate divider is: `3 + 16/64 = 3.25`. Which results in a\n    /// genrated baud rate of `48_000_000 / (16 * 3.25) = 923_077`.\n    ///\n    /// Error = `((923_077 - 921_600) / 921_600) * 100 = 0.16%`.\n    pub fn init(&mut self) {\n        // Execution can arrive here while there are still characters queued in the TX FIFO and\n        // actively being sent out by the UART hardware. If the UART is turned off in this case,\n        // those queued characters would be lost.\n        //\n        // For example, this can happen during runtime on a call to panic!(), because panic!()\n        // initializes its own UART instance and calls init().\n        //\n        // Hence, flush first to ensure all pending characters are transmitted.\n        self.flush();\n\n        // Turn the UART off temporarily.\n        self.registers.CR.set(0);\n\n        // Clear all pending interrupts.\n        self.registers.ICR.write(ICR::ALL::CLEAR);\n\n        // From the PL011 Technical Reference Manual:\n        //\n        // The LCR_H, IBRD, and FBRD registers form the single 30-bit wide LCR Register that is\n        // updated on a single write strobe generated by a LCR_H write. So, to internally update the\n        // contents of IBRD or FBRD, a LCR_H write must always be performed at the end.\n        //\n        // Set the baud rate, 8N1 and FIFO enabled.\n        self.registers.IBRD.write(IBRD::BAUD_DIVINT.val(3));\n        self.registers.FBRD.write(FBRD::BAUD_DIVFRAC.val(16));\n        self.registers\n            .LCR_H\n            .write(LCR_H::WLEN::EightBit + LCR_H::FEN::FifosEnabled);\n\n        // Turn the UART on.\n        self.registers\n            .CR\n            .write(CR::UARTEN::Enabled + CR::TXE::Enabled + CR::RXE::Enabled);\n    }\n\n    /// Send a character.\n    fn write_char(&mut self, c: char) {\n        // Spin while TX FIFO full is set, waiting for an empty slot.\n        while self.registers.FR.matches_all(FR::TXFF::SET) {\n            cpu::nop();\n        }\n\n        // Write the character to the buffer.\n        self.registers.DR.set(c as u32);\n\n        self.chars_written += 1;\n    }\n\n    /// Block execution until the last buffered character has been physically put on the TX wire.\n    fn flush(&self) {\n        // Spin until the busy bit is cleared.\n        while self.registers.FR.matches_all(FR::BUSY::SET) {\n            cpu::nop();\n        }\n    }\n\n    /// Retrieve a character.\n    fn read_char_converting(&mut self, blocking_mode: BlockingMode) -> Option<char> {\n        // If RX FIFO is empty,\n        if self.registers.FR.matches_all(FR::RXFE::SET) {\n            // immediately return in non-blocking mode.\n            if blocking_mode == BlockingMode::NonBlocking {\n                return None;\n            }\n\n            // Otherwise, wait until a char was received.\n            while self.registers.FR.matches_all(FR::RXFE::SET) {\n                cpu::nop();\n            }\n        }\n\n        // Read one character.\n        let mut ret = self.registers.DR.get() as u8 as char;\n\n        // Convert carrige return to newline.\n        if ret == '\\r' {\n            ret = '\\n'\n        }\n\n        // Update statistics.\n        self.chars_read += 1;\n\n        Some(ret)\n    }\n}\n\n/// Implementing `core::fmt::Write` enables usage of the `format_args!` macros, which in turn are\n/// used to implement the `kernel`'s `print!` and `println!` macros. By implementing `write_str()`,\n/// we get `write_fmt()` automatically.\n///\n/// The function takes an `&mut self`, so it must be implemented for the inner struct.\n///\n/// See [`src/print.rs`].\n///\n/// [`src/print.rs`]: ../../print/index.html\nimpl fmt::Write for PL011UartInner {\n    fn write_str(&mut self, s: &str) -> fmt::Result {\n        for c in s.chars() {\n            self.write_char(c);\n        }\n\n        Ok(())\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PL011Uart {\n    pub const COMPATIBLE: &'static str = \"BCM PL011 UART\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            inner: NullLock::new(PL011UartInner::new(mmio_start_addr)),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl driver::interface::DeviceDriver for PL011Uart {\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n\n    unsafe fn init(&self) -> Result<(), &'static str> {\n        self.inner.lock(|inner| inner.init());\n\n        Ok(())\n    }\n}\n\nimpl console::interface::Write for PL011Uart {\n    /// Passthrough of `args` to the `core::fmt::Write` implementation, but guarded by a Mutex to\n    /// serialize access.\n    fn write_char(&self, c: char) {\n        self.inner.lock(|inner| inner.write_char(c));\n    }\n\n    fn write_fmt(&self, args: core::fmt::Arguments) -> fmt::Result {\n        // Fully qualified syntax for the call to `core::fmt::Write::write_fmt()` to increase\n        // readability.\n        self.inner.lock(|inner| fmt::Write::write_fmt(inner, args))\n    }\n\n    fn flush(&self) {\n        // Spin until TX FIFO empty is set.\n        self.inner.lock(|inner| inner.flush());\n    }\n}\n\nimpl console::interface::Read for PL011Uart {\n    fn read_char(&self) -> char {\n        self.inner\n            .lock(|inner| inner.read_char_converting(BlockingMode::Blocking).unwrap())\n    }\n\n    fn clear_rx(&self) {\n        // Read from the RX FIFO until it is indicating empty.\n        while self\n            .inner\n            .lock(|inner| inner.read_char_converting(BlockingMode::NonBlocking))\n            .is_some()\n        {}\n    }\n}\n\nimpl console::interface::Statistics for PL011Uart {\n    fn chars_written(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_written)\n    }\n\n    fn chars_read(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_read)\n    }\n}\n\nimpl console::interface::All for PL011Uart {}\n"
  },
  {
    "path": "05_drivers_gpio_uart/src/bsp/device_driver/bcm.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BCM driver top level.\n\nmod bcm2xxx_gpio;\nmod bcm2xxx_pl011_uart;\n\npub use bcm2xxx_gpio::*;\npub use bcm2xxx_pl011_uart::*;\n"
  },
  {
    "path": "05_drivers_gpio_uart/src/bsp/device_driver/common.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Common device driver code.\n\nuse core::{marker::PhantomData, ops};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct MMIODerefWrapper<T> {\n    start_addr: usize,\n    phantom: PhantomData<fn() -> T>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T> MMIODerefWrapper<T> {\n    /// Create an instance.\n    pub const unsafe fn new(start_addr: usize) -> Self {\n        Self {\n            start_addr,\n            phantom: PhantomData,\n        }\n    }\n}\n\nimpl<T> ops::Deref for MMIODerefWrapper<T> {\n    type Target = T;\n\n    fn deref(&self) -> &Self::Target {\n        unsafe { &*(self.start_addr as *const _) }\n    }\n}\n"
  },
  {
    "path": "05_drivers_gpio_uart/src/bsp/device_driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Device driver.\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod bcm;\nmod common;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use bcm::*;\n"
  },
  {
    "path": "05_drivers_gpio_uart/src/bsp/raspberrypi/console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP console facilities.\n\nuse crate::console;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the console.\npub fn console() -> &'static dyn console::interface::All {\n    &super::driver::PL011_UART\n}\n"
  },
  {
    "path": "05_drivers_gpio_uart/src/bsp/raspberrypi/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Processor code.\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Used by `arch` code to find the early boot core.\n#[no_mangle]\n#[link_section = \".text._start_arguments\"]\npub static BOOT_CORE_ID: u64 = 0;\n"
  },
  {
    "path": "05_drivers_gpio_uart/src/bsp/raspberrypi/driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP driver support.\n\nuse super::memory::map::mmio;\nuse crate::{bsp::device_driver, console, driver as generic_driver};\nuse core::sync::atomic::{AtomicBool, Ordering};\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic PL011_UART: device_driver::PL011Uart =\n    unsafe { device_driver::PL011Uart::new(mmio::PL011_UART_START) };\nstatic GPIO: device_driver::GPIO = unsafe { device_driver::GPIO::new(mmio::GPIO_START) };\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// This must be called only after successful init of the UART driver.\nfn post_init_uart() -> Result<(), &'static str> {\n    console::register_console(&PL011_UART);\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the GPIO driver.\nfn post_init_gpio() -> Result<(), &'static str> {\n    GPIO.map_pl011_uart();\n    Ok(())\n}\n\nfn driver_uart() -> Result<(), &'static str> {\n    let uart_descriptor =\n        generic_driver::DeviceDriverDescriptor::new(&PL011_UART, Some(post_init_uart));\n    generic_driver::driver_manager().register_driver(uart_descriptor);\n\n    Ok(())\n}\n\nfn driver_gpio() -> Result<(), &'static str> {\n    let gpio_descriptor = generic_driver::DeviceDriverDescriptor::new(&GPIO, Some(post_init_gpio));\n    generic_driver::driver_manager().register_driver(gpio_descriptor);\n\n    Ok(())\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Initialize the driver subsystem.\n///\n/// # Safety\n///\n/// See child function calls.\npub unsafe fn init() -> Result<(), &'static str> {\n    static INIT_DONE: AtomicBool = AtomicBool::new(false);\n    if INIT_DONE.load(Ordering::Relaxed) {\n        return Err(\"Init already done\");\n    }\n\n    driver_uart()?;\n    driver_gpio()?;\n\n    INIT_DONE.store(true, Ordering::Relaxed);\n    Ok(())\n}\n"
  },
  {
    "path": "05_drivers_gpio_uart/src/bsp/raspberrypi/kernel.ld",
    "content": "/* SPDX-License-Identifier: MIT OR Apache-2.0\n *\n * Copyright (c) 2018-2022 Andre Richter <andre.o.richter@gmail.com>\n */\n\n__rpi_phys_dram_start_addr = 0;\n\n/* The physical address at which the the kernel binary will be loaded by the Raspberry's firmware */\n__rpi_phys_binary_load_addr = 0x80000;\n\n\nENTRY(__rpi_phys_binary_load_addr)\n\n/* Flags:\n *     4 == R\n *     5 == RX\n *     6 == RW\n *\n * Segments are marked PT_LOAD below so that the ELF file provides virtual and physical addresses.\n * It doesn't mean all of them need actually be loaded.\n */\nPHDRS\n{\n    segment_boot_core_stack PT_LOAD FLAGS(6);\n    segment_code            PT_LOAD FLAGS(5);\n    segment_data            PT_LOAD FLAGS(6);\n}\n\nSECTIONS\n{\n    . =  __rpi_phys_dram_start_addr;\n\n    /***********************************************************************************************\n    * Boot Core Stack\n    ***********************************************************************************************/\n    .boot_core_stack (NOLOAD) :\n    {\n                                             /*   ^             */\n                                             /*   | stack       */\n        . += __rpi_phys_binary_load_addr;    /*   | growth      */\n                                             /*   | direction   */\n        __boot_core_stack_end_exclusive = .; /*   |             */\n    } :segment_boot_core_stack\n\n    /***********************************************************************************************\n    * Code + RO Data + Global Offset Table\n    ***********************************************************************************************/\n    .text :\n    {\n        KEEP(*(.text._start))\n        *(.text._start_arguments) /* Constants (or statics in Rust speak) read by _start(). */\n        *(.text._start_rust)      /* The Rust entry point */\n        *(.text*)                 /* Everything else */\n    } :segment_code\n\n    .rodata : ALIGN(8) { *(.rodata*) } :segment_code\n\n    /***********************************************************************************************\n    * Data + BSS\n    ***********************************************************************************************/\n    .data : { *(.data*) } :segment_data\n\n    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */\n    .bss (NOLOAD) : ALIGN(16)\n    {\n        __bss_start = .;\n        *(.bss*);\n        . = ALIGN(16);\n        __bss_end_exclusive = .;\n    } :segment_data\n\n    /***********************************************************************************************\n    * Misc\n    ***********************************************************************************************/\n    .got : { *(.got*) }\n    ASSERT(SIZEOF(.got) == 0, \"Relocation support not expected\")\n\n    /DISCARD/ : { *(.comment*) }\n}\n"
  },
  {
    "path": "05_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Memory Management.\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// The board's physical memory map.\n#[rustfmt::skip]\npub(super) mod map {\n\n    pub const GPIO_OFFSET:         usize = 0x0020_0000;\n    pub const UART_OFFSET:         usize = 0x0020_1000;\n\n    /// Physical devices.\n    #[cfg(feature = \"bsp_rpi3\")]\n    pub mod mmio {\n        use super::*;\n\n        pub const START:            usize =         0x3F00_0000;\n        pub const GPIO_START:       usize = START + GPIO_OFFSET;\n        pub const PL011_UART_START: usize = START + UART_OFFSET;\n    }\n\n    /// Physical devices.\n    #[cfg(feature = \"bsp_rpi4\")]\n    pub mod mmio {\n        use super::*;\n\n        pub const START:            usize =         0xFE00_0000;\n        pub const GPIO_START:       usize = START + GPIO_OFFSET;\n        pub const PL011_UART_START: usize = START + UART_OFFSET;\n    }\n}\n"
  },
  {
    "path": "05_drivers_gpio_uart/src/bsp/raspberrypi.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Top-level BSP file for the Raspberry Pi 3 and 4.\n\npub mod cpu;\npub mod driver;\npub mod memory;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Board identification.\npub fn board_name() -> &'static str {\n    #[cfg(feature = \"bsp_rpi3\")]\n    {\n        \"Raspberry Pi 3\"\n    }\n\n    #[cfg(feature = \"bsp_rpi4\")]\n    {\n        \"Raspberry Pi 4\"\n    }\n}\n"
  },
  {
    "path": "05_drivers_gpio_uart/src/bsp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Conditional reexporting of Board Support Packages.\n\nmod device_driver;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod raspberrypi;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use raspberrypi::*;\n"
  },
  {
    "path": "05_drivers_gpio_uart/src/console/null_console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Null console.\n\nuse super::interface;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct NullConsole;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\npub static NULL_CONSOLE: NullConsole = NullConsole {};\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl interface::Write for NullConsole {\n    fn write_char(&self, _c: char) {}\n\n    fn write_fmt(&self, _args: fmt::Arguments) -> fmt::Result {\n        fmt::Result::Ok(())\n    }\n\n    fn flush(&self) {}\n}\n\nimpl interface::Read for NullConsole {\n    fn clear_rx(&self) {}\n}\n\nimpl interface::Statistics for NullConsole {}\nimpl interface::All for NullConsole {}\n"
  },
  {
    "path": "05_drivers_gpio_uart/src/console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! System console.\n\nmod null_console;\n\nuse crate::synchronization::{self, NullLock};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Console interfaces.\npub mod interface {\n    use core::fmt;\n\n    /// Console write functions.\n    pub trait Write {\n        /// Write a single character.\n        fn write_char(&self, c: char);\n\n        /// Write a Rust format string.\n        fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result;\n\n        /// Block until the last buffered character has been physically put on the TX wire.\n        fn flush(&self);\n    }\n\n    /// Console read functions.\n    pub trait Read {\n        /// Read a single character.\n        fn read_char(&self) -> char {\n            ' '\n        }\n\n        /// Clear RX buffers, if any.\n        fn clear_rx(&self);\n    }\n\n    /// Console statistics.\n    pub trait Statistics {\n        /// Return the number of characters written.\n        fn chars_written(&self) -> usize {\n            0\n        }\n\n        /// Return the number of characters read.\n        fn chars_read(&self) -> usize {\n            0\n        }\n    }\n\n    /// Trait alias for a full-fledged console.\n    pub trait All: Write + Read + Statistics {}\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic CUR_CONSOLE: NullLock<&'static (dyn interface::All + Sync)> =\n    NullLock::new(&null_console::NULL_CONSOLE);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\n/// Register a new console.\npub fn register_console(new_console: &'static (dyn interface::All + Sync)) {\n    CUR_CONSOLE.lock(|con| *con = new_console);\n}\n\n/// Return a reference to the currently registered console.\n///\n/// This is the global console used by all printing macros.\npub fn console() -> &'static dyn interface::All {\n    CUR_CONSOLE.lock(|con| *con)\n}\n"
  },
  {
    "path": "05_drivers_gpio_uart/src/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Boot code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/cpu/boot.rs\"]\nmod arch_boot;\n"
  },
  {
    "path": "05_drivers_gpio_uart/src/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Processor code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/cpu.rs\"]\nmod arch_cpu;\n\nmod boot;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_cpu::{nop, wait_forever};\n\n#[cfg(feature = \"bsp_rpi3\")]\npub use arch_cpu::spin_for_cycles;\n"
  },
  {
    "path": "05_drivers_gpio_uart/src/driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Driver support.\n\nuse crate::{\n    println,\n    synchronization::{interface::Mutex, NullLock},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nconst NUM_DRIVERS: usize = 5;\n\nstruct DriverManagerInner {\n    next_index: usize,\n    descriptors: [Option<DeviceDriverDescriptor>; NUM_DRIVERS],\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Driver interfaces.\npub mod interface {\n    /// Device Driver functions.\n    pub trait DeviceDriver {\n        /// Return a compatibility string for identifying the driver.\n        fn compatible(&self) -> &'static str;\n\n        /// Called by the kernel to bring up the device.\n        ///\n        /// # Safety\n        ///\n        /// - During init, drivers might do stuff with system-wide impact.\n        unsafe fn init(&self) -> Result<(), &'static str> {\n            Ok(())\n        }\n    }\n}\n\n/// Tpye to be used as an optional callback after a driver's init() has run.\npub type DeviceDriverPostInitCallback = unsafe fn() -> Result<(), &'static str>;\n\n/// A descriptor for device drivers.\n#[derive(Copy, Clone)]\npub struct DeviceDriverDescriptor {\n    device_driver: &'static (dyn interface::DeviceDriver + Sync),\n    post_init_callback: Option<DeviceDriverPostInitCallback>,\n}\n\n/// Provides device driver management functions.\npub struct DriverManager {\n    inner: NullLock<DriverManagerInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic DRIVER_MANAGER: DriverManager = DriverManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl DriverManagerInner {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            next_index: 0,\n            descriptors: [None; NUM_DRIVERS],\n        }\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl DeviceDriverDescriptor {\n    /// Create an instance.\n    pub fn new(\n        device_driver: &'static (dyn interface::DeviceDriver + Sync),\n        post_init_callback: Option<DeviceDriverPostInitCallback>,\n    ) -> Self {\n        Self {\n            device_driver,\n            post_init_callback,\n        }\n    }\n}\n\n/// Return a reference to the global DriverManager.\npub fn driver_manager() -> &'static DriverManager {\n    &DRIVER_MANAGER\n}\n\nimpl DriverManager {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            inner: NullLock::new(DriverManagerInner::new()),\n        }\n    }\n\n    /// Register a device driver with the kernel.\n    pub fn register_driver(&self, descriptor: DeviceDriverDescriptor) {\n        self.inner.lock(|inner| {\n            inner.descriptors[inner.next_index] = Some(descriptor);\n            inner.next_index += 1;\n        })\n    }\n\n    /// Helper for iterating over registered drivers.\n    fn for_each_descriptor<'a>(&'a self, f: impl FnMut(&'a DeviceDriverDescriptor)) {\n        self.inner.lock(|inner| {\n            inner\n                .descriptors\n                .iter()\n                .filter_map(|x| x.as_ref())\n                .for_each(f)\n        })\n    }\n\n    /// Fully initialize all drivers.\n    ///\n    /// # Safety\n    ///\n    /// - During init, drivers might do stuff with system-wide impact.\n    pub unsafe fn init_drivers(&self) {\n        self.for_each_descriptor(|descriptor| {\n            // 1. Initialize driver.\n            if let Err(x) = descriptor.device_driver.init() {\n                panic!(\n                    \"Error initializing driver: {}: {}\",\n                    descriptor.device_driver.compatible(),\n                    x\n                );\n            }\n\n            // 2. Call corresponding post init callback.\n            if let Some(callback) = &descriptor.post_init_callback {\n                if let Err(x) = callback() {\n                    panic!(\n                        \"Error during driver post-init callback: {}: {}\",\n                        descriptor.device_driver.compatible(),\n                        x\n                    );\n                }\n            }\n        });\n    }\n\n    /// Enumerate all registered device drivers.\n    pub fn enumerate(&self) {\n        let mut i: usize = 1;\n        self.for_each_descriptor(|descriptor| {\n            println!(\"      {}. {}\", i, descriptor.device_driver.compatible());\n\n            i += 1;\n        });\n    }\n}\n"
  },
  {
    "path": "05_drivers_gpio_uart/src/main.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n// Rust embedded logo for `make doc`.\n#![doc(\n    html_logo_url = \"https://raw.githubusercontent.com/rust-embedded/wg/master/assets/logo/ewg-logo-blue-white-on-transparent.png\"\n)]\n\n//! The `kernel` binary.\n//!\n//! # Code organization and architecture\n//!\n//! The code is divided into different *modules*, each representing a typical **subsystem** of the\n//! `kernel`. Top-level module files of subsystems reside directly in the `src` folder. For example,\n//! `src/memory.rs` contains code that is concerned with all things memory management.\n//!\n//! ## Visibility of processor architecture code\n//!\n//! Some of the `kernel`'s subsystems depend on low-level code that is specific to the target\n//! processor architecture. For each supported processor architecture, there exists a subfolder in\n//! `src/_arch`, for example, `src/_arch/aarch64`.\n//!\n//! The architecture folders mirror the subsystem modules laid out in `src`. For example,\n//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go\n//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in\n//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic\n//! module's name prefixed with `arch_`.\n//!\n//! For example, this is the top of `src/memory/mmu.rs`:\n//!\n//! ```\n//! #[cfg(target_arch = \"aarch64\")]\n//! #[path = \"../_arch/aarch64/memory/mmu.rs\"]\n//! mod arch_mmu;\n//! ```\n//!\n//! Often times, items from the `arch_ module` will be publicly reexported by the parent module.\n//! This way, each architecture specific module can provide its implementation of an item, while the\n//! caller must not be concerned which architecture has been conditionally compiled.\n//!\n//! ## BSP code\n//!\n//! `BSP` stands for Board Support Package. `BSP` code is organized under `src/bsp.rs` and contains\n//! target board specific definitions and functions. These are things such as the board's memory map\n//! or instances of drivers for devices that are featured on the respective board.\n//!\n//! Just like processor architecture code, the `BSP` code's module structure tries to mirror the\n//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is\n//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`.\n//!\n//! ## Kernel interfaces\n//!\n//! Both `arch` and `bsp` contain code that is conditionally compiled depending on the actual target\n//! and board for which the kernel is compiled. For example, the `interrupt controller` hardware of\n//! the `Raspberry Pi 3` and the `Raspberry Pi 4` is different, but we want the rest of the `kernel`\n//! code to play nicely with any of the two without much hassle.\n//!\n//! In order to provide a clean abstraction between `arch`, `bsp` and `generic kernel code`,\n//! `interface` traits are provided *whenever possible* and *where it makes sense*. They are defined\n//! in the respective subsystem module and help to enforce the idiom of *program to an interface,\n//! not an implementation*. For example, there will be a common IRQ handling interface which the two\n//! different interrupt controller `drivers` of both Raspberrys will implement, and only export the\n//! interface to the rest of the `kernel`.\n//!\n//! ```\n//!         +-------------------+\n//!         | Interface (Trait) |\n//!         |                   |\n//!         +--+-------------+--+\n//!            ^             ^\n//!            |             |\n//!            |             |\n//! +----------+--+       +--+----------+\n//! | kernel code |       |  bsp code   |\n//! |             |       |  arch code  |\n//! +-------------+       +-------------+\n//! ```\n//!\n//! # Summary\n//!\n//! For a logical `kernel` subsystem, corresponding code can be distributed over several physical\n//! locations. Here is an example for the **memory** subsystem:\n//!\n//! - `src/memory.rs` and `src/memory/**/*`\n//!   - Common code that is agnostic of target processor architecture and `BSP` characteristics.\n//!     - Example: A function to zero a chunk of memory.\n//!   - Interfaces for the memory subsystem that are implemented by `arch` or `BSP` code.\n//!     - Example: An `MMU` interface that defines `MMU` function prototypes.\n//! - `src/bsp/__board_name__/memory.rs` and `src/bsp/__board_name__/memory/**/*`\n//!   - `BSP` specific code.\n//!   - Example: The board's memory map (physical addresses of DRAM and MMIO devices).\n//! - `src/_arch/__arch_name__/memory.rs` and `src/_arch/__arch_name__/memory/**/*`\n//!   - Processor architecture specific code.\n//!   - Example: Implementation of the `MMU` interface for the `__arch_name__` processor\n//!     architecture.\n//!\n//! From a namespace perspective, **memory** subsystem code lives in:\n//!\n//! - `crate::memory::*`\n//! - `crate::bsp::memory::*`\n//!\n//! # Boot flow\n//!\n//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.\n//!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.\n//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.\n\n#![allow(clippy::upper_case_acronyms)]\n#![feature(asm_const)]\n#![feature(format_args_nl)]\n#![feature(panic_info_message)]\n#![feature(trait_alias)]\n#![no_main]\n#![no_std]\n\nmod bsp;\nmod console;\nmod cpu;\nmod driver;\nmod panic_wait;\nmod print;\nmod synchronization;\n\n/// Early init code.\n///\n/// # Safety\n///\n/// - Only a single core must be active and running this function.\n/// - The init calls in this function must appear in the correct order.\nunsafe fn kernel_init() -> ! {\n    // Initialize the BSP driver subsystem.\n    if let Err(x) = bsp::driver::init() {\n        panic!(\"Error initializing BSP driver subsystem: {}\", x);\n    }\n\n    // Initialize all device drivers.\n    driver::driver_manager().init_drivers();\n    // println! is usable from here on.\n\n    // Transition from unsafe to safe.\n    kernel_main()\n}\n\n/// The main function running after the early init.\nfn kernel_main() -> ! {\n    use console::console;\n\n    println!(\n        \"[0] {} version {}\",\n        env!(\"CARGO_PKG_NAME\"),\n        env!(\"CARGO_PKG_VERSION\")\n    );\n    println!(\"[1] Booting on: {}\", bsp::board_name());\n\n    println!(\"[2] Drivers loaded:\");\n    driver::driver_manager().enumerate();\n\n    println!(\"[3] Chars written: {}\", console().chars_written());\n    println!(\"[4] Echoing input now\");\n\n    // Discard any spurious received characters before going into echo mode.\n    console().clear_rx();\n    loop {\n        let c = console().read_char();\n        console().write_char(c);\n    }\n}\n"
  },
  {
    "path": "05_drivers_gpio_uart/src/panic_wait.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A panic handler that infinitely waits.\n\nuse crate::{cpu, println};\nuse core::panic::PanicInfo;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Stop immediately if called a second time.\n///\n/// # Note\n///\n/// Using atomics here relieves us from needing to use `unsafe` for the static variable.\n///\n/// On `AArch64`, which is the only implemented architecture at the time of writing this,\n/// [`AtomicBool::load`] and [`AtomicBool::store`] are lowered to ordinary load and store\n/// instructions. They are therefore safe to use even with MMU + caching deactivated.\n///\n/// [`AtomicBool::load`]: core::sync::atomic::AtomicBool::load\n/// [`AtomicBool::store`]: core::sync::atomic::AtomicBool::store\nfn panic_prevent_reenter() {\n    use core::sync::atomic::{AtomicBool, Ordering};\n\n    #[cfg(not(target_arch = \"aarch64\"))]\n    compile_error!(\"Add the target_arch to above's check if the following code is safe to use\");\n\n    static PANIC_IN_PROGRESS: AtomicBool = AtomicBool::new(false);\n\n    if !PANIC_IN_PROGRESS.load(Ordering::Relaxed) {\n        PANIC_IN_PROGRESS.store(true, Ordering::Relaxed);\n\n        return;\n    }\n\n    cpu::wait_forever()\n}\n\n#[panic_handler]\nfn panic(info: &PanicInfo) -> ! {\n    // Protect against panic infinite loops if any of the following code panics itself.\n    panic_prevent_reenter();\n\n    let (location, line, column) = match info.location() {\n        Some(loc) => (loc.file(), loc.line(), loc.column()),\n        _ => (\"???\", 0, 0),\n    };\n\n    println!(\n        \"Kernel panic!\\n\\n\\\n        Panic location:\\n      File '{}', line {}, column {}\\n\\n\\\n        {}\",\n        location,\n        line,\n        column,\n        info.message().unwrap_or(&format_args!(\"\")),\n    );\n\n    cpu::wait_forever()\n}\n"
  },
  {
    "path": "05_drivers_gpio_uart/src/print.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Printing.\n\nuse crate::console;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n#[doc(hidden)]\npub fn _print(args: fmt::Arguments) {\n    console::console().write_fmt(args).unwrap();\n}\n\n/// Prints without a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! print {\n    ($($arg:tt)*) => ($crate::print::_print(format_args!($($arg)*)));\n}\n\n/// Prints with a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! println {\n    () => ($crate::print!(\"\\n\"));\n    ($($arg:tt)*) => ({\n        $crate::print::_print(format_args_nl!($($arg)*));\n    })\n}\n"
  },
  {
    "path": "05_drivers_gpio_uart/src/synchronization.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Synchronization primitives.\n//!\n//! # Resources\n//!\n//!   - <https://doc.rust-lang.org/book/ch16-04-extensible-concurrency-sync-and-send.html>\n//!   - <https://stackoverflow.com/questions/59428096/understanding-the-send-trait>\n//!   - <https://doc.rust-lang.org/std/cell/index.html>\n\nuse core::cell::UnsafeCell;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Synchronization interfaces.\npub mod interface {\n\n    /// Any object implementing this trait guarantees exclusive access to the data wrapped within\n    /// the Mutex for the duration of the provided closure.\n    pub trait Mutex {\n        /// The type of the data that is wrapped by this mutex.\n        type Data;\n\n        /// Locks the mutex and grants the closure temporary mutable access to the wrapped data.\n        fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R;\n    }\n}\n\n/// A pseudo-lock for teaching purposes.\n///\n/// In contrast to a real Mutex implementation, does not protect against concurrent access from\n/// other cores to the contained data. This part is preserved for later lessons.\n///\n/// The lock will only be used as long as it is safe to do so, i.e. as long as the kernel is\n/// executing single-threaded, aka only running on a single core with interrupts disabled.\npub struct NullLock<T>\nwhere\n    T: ?Sized,\n{\n    data: UnsafeCell<T>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nunsafe impl<T> Send for NullLock<T> where T: ?Sized + Send {}\nunsafe impl<T> Sync for NullLock<T> where T: ?Sized + Send {}\n\nimpl<T> NullLock<T> {\n    /// Create an instance.\n    pub const fn new(data: T) -> Self {\n        Self {\n            data: UnsafeCell::new(data),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\n\nimpl<T> interface::Mutex for NullLock<T> {\n    type Data = T;\n\n    fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R {\n        // In a real lock, there would be code encapsulating this line that ensures that this\n        // mutable reference will ever only be given out once at a time.\n        let data = unsafe { &mut *self.data.get() };\n\n        f(data)\n    }\n}\n"
  },
  {
    "path": "05_drivers_gpio_uart/tests/boot_test_string.rb",
    "content": "# frozen_string_literal: true\n\nEXPECTED_PRINT = 'Echoing input now'\n"
  },
  {
    "path": "06_uart_chainloader/.vscode/settings.json",
    "content": "{\n    \"editor.formatOnSave\": true,\n    \"editor.rulers\": [100],\n    \"rust-analyzer.cargo.target\": \"aarch64-unknown-none-softfloat\",\n    \"rust-analyzer.cargo.features\": [\"bsp_rpi3\"],\n    \"rust-analyzer.checkOnSave.allTargets\": false,\n    \"rust-analyzer.checkOnSave.extraArgs\": [\"--bins\"],\n    \"rust-analyzer.lens.debug\": false,\n    \"rust-analyzer.lens.run\": false\n}\n"
  },
  {
    "path": "06_uart_chainloader/Cargo.toml",
    "content": "[package]\nname = \"mingo\"\nversion = \"0.6.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n\n[profile.release]\nlto = true\n\n[features]\ndefault = []\nbsp_rpi3 = [\"tock-registers\"]\nbsp_rpi4 = [\"tock-registers\"]\n\n[[bin]]\nname = \"kernel\"\npath = \"src/main.rs\"\n\n##--------------------------------------------------------------------------------------------------\n## Dependencies\n##--------------------------------------------------------------------------------------------------\n\n[dependencies]\n\n# Optional dependencies\ntock-registers = { version = \"0.8.x\", default-features = false, features = [\"register_types\"], optional = true }\n\n# Platform specific dependencies\n[target.'cfg(target_arch = \"aarch64\")'.dependencies]\naarch64-cpu = { version = \"9.x.x\" }\n"
  },
  {
    "path": "06_uart_chainloader/Makefile",
    "content": "## SPDX-License-Identifier: MIT OR Apache-2.0\n##\n## Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\ninclude ../common/docker.mk\ninclude ../common/format.mk\ninclude ../common/operating_system.mk\n\n##--------------------------------------------------------------------------------------------------\n## Optional, user-provided configuration values\n##--------------------------------------------------------------------------------------------------\n\n# Default to the RPi3.\nBSP ?= rpi3\n\n# Default to a serial device name that is common in Linux.\nDEV_SERIAL ?= /dev/ttyUSB0\n\n\n\n##--------------------------------------------------------------------------------------------------\n## BSP-specific configuration values\n##--------------------------------------------------------------------------------------------------\nQEMU_MISSING_STRING = \"This board is not yet supported for QEMU.\"\n\nifeq ($(BSP),rpi3)\n    TARGET                 = aarch64-unknown-none-softfloat\n    KERNEL_BIN             = kernel8.img\n    QEMU_BINARY            = qemu-system-aarch64\n    QEMU_MACHINE_TYPE      = raspi3\n    QEMU_RELEASE_ARGS      = -serial stdio -display none\n    OBJDUMP_BINARY         = aarch64-none-elf-objdump\n    NM_BINARY              = aarch64-none-elf-nm\n    READELF_BINARY         = aarch64-none-elf-readelf\n    LD_SCRIPT_PATH         = $(shell pwd)/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS        = -C target-cpu=cortex-a53\n    CHAINBOOT_DEMO_PAYLOAD = demo_payload_rpi3.img\nelse ifeq ($(BSP),rpi4)\n    TARGET                 = aarch64-unknown-none-softfloat\n    KERNEL_BIN             = kernel8.img\n    QEMU_BINARY            = qemu-system-aarch64\n    QEMU_MACHINE_TYPE      =\n    QEMU_RELEASE_ARGS      = -serial stdio -display none\n    OBJDUMP_BINARY         = aarch64-none-elf-objdump\n    NM_BINARY              = aarch64-none-elf-nm\n    READELF_BINARY         = aarch64-none-elf-readelf\n    LD_SCRIPT_PATH         = $(shell pwd)/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS        = -C target-cpu=cortex-a72\n    CHAINBOOT_DEMO_PAYLOAD = demo_payload_rpi4.img\nendif\n\n# Export for build.rs.\nexport LD_SCRIPT_PATH\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets and Prerequisites\n##--------------------------------------------------------------------------------------------------\nKERNEL_MANIFEST      = Cargo.toml\nKERNEL_LINKER_SCRIPT = kernel.ld\nLAST_BUILD_CONFIG    = target/$(BSP).build_config\n\nKERNEL_ELF      = target/$(TARGET)/release/kernel\n# This parses cargo's dep-info file.\n# https://doc.rust-lang.org/cargo/guide/build-cache.html#dep-info-files\nKERNEL_ELF_DEPS = $(filter-out %: ,$(file < $(KERNEL_ELF).d)) $(KERNEL_MANIFEST) $(LAST_BUILD_CONFIG)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Command building blocks\n##--------------------------------------------------------------------------------------------------\nRUSTFLAGS = $(RUSTC_MISC_ARGS)                   \\\n    -C link-arg=--library-path=$(LD_SCRIPT_PATH) \\\n    -C link-arg=--script=$(KERNEL_LINKER_SCRIPT)\n\nRUSTFLAGS_PEDANTIC = $(RUSTFLAGS) \\\n    -D warnings                   \\\n    -D missing_docs\n\nFEATURES      = --features bsp_$(BSP)\nCOMPILER_ARGS = --target=$(TARGET) \\\n    $(FEATURES)                    \\\n    --release\n\nRUSTC_CMD   = cargo rustc $(COMPILER_ARGS)\nDOC_CMD     = cargo doc $(COMPILER_ARGS)\nCLIPPY_CMD  = cargo clippy $(COMPILER_ARGS)\nOBJCOPY_CMD = rust-objcopy \\\n    --strip-all            \\\n    -O binary\n\nEXEC_QEMU          = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE)\nEXEC_TEST_MINIPUSH = ruby tests/chainboot_test.rb\nEXEC_MINIPUSH      = ruby ../common/serial/minipush.rb\n\n##------------------------------------------------------------------------------\n## Dockerization\n##------------------------------------------------------------------------------\nDOCKER_CMD            = docker run -t --rm -v $(shell pwd):/work/tutorial -w /work/tutorial\nDOCKER_CMD_INTERACT   = $(DOCKER_CMD) -i\nDOCKER_ARG_DIR_COMMON = -v $(shell pwd)/../common:/work/common\nDOCKER_ARG_DEV        = --privileged -v /dev:/dev\n\n# DOCKER_IMAGE defined in include file (see top of this file).\nDOCKER_QEMU  = $(DOCKER_CMD_INTERACT) $(DOCKER_IMAGE)\nDOCKER_TOOLS = $(DOCKER_CMD) $(DOCKER_IMAGE)\nDOCKER_TEST  = $(DOCKER_CMD) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\n\n# Dockerize commands, which require USB device passthrough, only on Linux.\nifeq ($(shell uname -s),Linux)\n    DOCKER_CMD_DEV = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_DEV)\n\n    DOCKER_CHAINBOOT = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\nendif\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: all doc qemu chainboot clippy clean readelf objdump nm check\n\nall: $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Save the configuration as a file, so make understands if it changed.\n##------------------------------------------------------------------------------\n$(LAST_BUILD_CONFIG):\n\t@rm -f target/*.build_config\n\t@mkdir -p target\n\t@touch $(LAST_BUILD_CONFIG)\n\n##------------------------------------------------------------------------------\n## Compile the kernel ELF\n##------------------------------------------------------------------------------\n$(KERNEL_ELF): $(KERNEL_ELF_DEPS)\n\t$(call color_header, \"Compiling kernel ELF - $(BSP)\")\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(RUSTC_CMD)\n\n##------------------------------------------------------------------------------\n## Generate the stripped kernel binary\n##------------------------------------------------------------------------------\n$(KERNEL_BIN): $(KERNEL_ELF)\n\t$(call color_header, \"Generating stripped binary\")\n\t@$(OBJCOPY_CMD) $(KERNEL_ELF) $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Name\")\n\t@echo $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Size\")\n\t$(call disk_usage_KiB, $(KERNEL_BIN))\n\n##------------------------------------------------------------------------------\n## Generate the documentation\n##------------------------------------------------------------------------------\ndoc:\n\t$(call color_header, \"Generating docs\")\n\t@$(DOC_CMD) --document-private-items --open\n\n##------------------------------------------------------------------------------\n## Run the kernel in QEMU\n##------------------------------------------------------------------------------\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\nqemu qemuasm:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\nqemu: $(KERNEL_BIN)\n\t$(call color_header, \"Launching QEMU\")\n\t@$(DOCKER_QEMU) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\nqemuasm: $(KERNEL_BIN)\n\t$(call color_header, \"Launching QEMU with ASM output\")\n\t@$(DOCKER_QEMU) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN) -d in_asm\n\nendif\n\n##------------------------------------------------------------------------------\n## Push the kernel to the real HW target\n##------------------------------------------------------------------------------\nchainboot: $(KERNEL_BIN)\n\t@$(DOCKER_CHAINBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(CHAINBOOT_DEMO_PAYLOAD)\n\n##------------------------------------------------------------------------------\n## Run clippy\n##------------------------------------------------------------------------------\nclippy:\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(CLIPPY_CMD)\n\n##------------------------------------------------------------------------------\n## Clean\n##------------------------------------------------------------------------------\nclean:\n\trm -rf target $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run readelf\n##------------------------------------------------------------------------------\nreadelf: $(KERNEL_ELF)\n\t$(call color_header, \"Launching readelf\")\n\t@$(DOCKER_TOOLS) $(READELF_BINARY) --headers $(KERNEL_ELF)\n\n##------------------------------------------------------------------------------\n## Run objdump\n##------------------------------------------------------------------------------\nobjdump: $(KERNEL_ELF)\n\t$(call color_header, \"Launching objdump\")\n\t@$(DOCKER_TOOLS) $(OBJDUMP_BINARY) --disassemble --demangle \\\n                --section .text   \\\n                --section .rodata \\\n                $(KERNEL_ELF) | rustfilt\n\n##------------------------------------------------------------------------------\n## Run nm\n##------------------------------------------------------------------------------\nnm: $(KERNEL_ELF)\n\t$(call color_header, \"Launching nm\")\n\t@$(DOCKER_TOOLS) $(NM_BINARY) --demangle --print-size $(KERNEL_ELF) | sort | rustfilt\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Testing targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: test test_boot\n\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\ntest_boot test:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\n##------------------------------------------------------------------------------\n## Run boot test\n##------------------------------------------------------------------------------\ntest_boot: $(KERNEL_BIN)\n\t$(call color_header, \"Boot test - $(BSP)\")\n\t@$(DOCKER_TEST) $(EXEC_TEST_MINIPUSH) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) \\\n\t\t-kernel $(KERNEL_BIN) $(CHAINBOOT_DEMO_PAYLOAD)\n\ntest: test_boot\n\nendif\n"
  },
  {
    "path": "06_uart_chainloader/README.CN.md",
    "content": "# 教程06 - UART链加载器\r\n\r\n## tl;dr\r\n\r\n- 从SD卡上运行是一次不错的体验，但是每次都为每个新的二进制文件这样做将非常繁琐。\r\n  因此，让我们编写一个[chainloader]。\r\n- 这将是您需要放在SD卡上的最后一个二进制文件。\r\n  每个后续的教程都将在`Makefile`中提供一个`chainboot`，让您方便地通过`UART`加载内核。\r\n\r\n[chainloader]: https://en.wikipedia.org/wiki/Chain_loading\r\n\r\n\r\n## 注意\r\n\r\n请注意，这个教程中有一些内容仅通过查看源代码很难理解。\r\n\r\n大致的意思是，在`boot.s`中，我们编写了一段[position independent code]代码，\r\n它会自动确定固件加载二进制文件的位置（`0x8_0000`），以及链接到的位置（`0x200_0000`，参见 `kernel.ld`）。\r\n然后，二进制文件将自身从加载地址复制到链接地址（也就是\"重定位\"自身），然后跳转到`_start_rust()`的重定位版本。\r\n\r\n由于链加载程序现在已经\"脱离了路径\"，它现在可以从`UART`接收另一个内核二进制文件，并将其复制到RPi固件的标准加载地址`0x8_0000`。\r\n最后，它跳转到`0x8_0000`，新加载的二进制文件会透明地执行，就好像它一直从SD卡加载一样。\r\n\r\n在我有时间详细写下这些内容之前，请耐心等待。目前，请将这个教程视为一种便利功能的启用程序，它允许快速启动以下教程。\r\n_对于那些渴望深入了解的人，可以直接跳到第[15章](../15_virtual_mem_part3_precomputed_tables)，阅读README的前半部分，\r\n其中讨论了`Load Address != Link Address`的问题_。\r\n\r\n[position independent code]: https://en.wikipedia.org/wiki/Position-independent_code\r\n\r\n## 安装并测试它\r\n\r\n我们的链加载程序称为`MiniLoad`，受到了[raspbootin]的启发。\r\n\r\n您可以按照以下教程尝试它：\r\n1. 根据您的目标硬件运行命令：`make`或`BSP=rpi4 make`。\r\n1. 将`kernel8.img`复制到SD卡中，并将SD卡重新插入您的RPi。\r\n1. 运行命令`make chainboot`或`BSP=rpi4 make chainboot`。\r\n1. 将USB串口连接到您的主机PC上。\r\n    - 请参考[top-level README](../README.md#-usb-serial-output)中的接线图。\r\n    - 确保您**没有**连接USB串口的电源引脚，只连接RX/TX和GND。\r\n1. 将RPi连接到（USB）电源线。\r\n1. 观察加载程序通过`UART`获取内核：\r\n\r\n> ❗ **注意**: `make chainboot`假设默认的串行设备名称为`/dev/ttyUSB0`。根据您的主机操作系统，设备名称可能会有所不同。\r\n> 例如，在`macOS`上，它可能是类似于`/dev/tty.usbserial-0001`的名称。\r\n> 在这种情况下，请明确给出设备名称：\r\n\r\n\r\n```console\r\n$ DEV_SERIAL=/dev/tty.usbserial-0001 make chainboot\r\n```\r\n\r\n[raspbootin]: https://github.com/mrvn/raspbootin\r\n\r\n```console\r\n$ make chainboot\r\n[...]\r\nMinipush 1.0\r\n\r\n[MP] ⏳ Waiting for /dev/ttyUSB0\r\n[MP] ✅ Serial connected\r\n[MP] 🔌 Please power the target now\r\n\r\n __  __ _      _ _                 _\r\n|  \\/  (_)_ _ (_) |   ___  __ _ __| |\r\n| |\\/| | | ' \\| | |__/ _ \\/ _` / _` |\r\n|_|  |_|_|_||_|_|____\\___/\\__,_\\__,_|\r\n\r\n           Raspberry Pi 3\r\n\r\n[ML] Requesting binary\r\n[MP] ⏩ Pushing 7 KiB ==========================================🦀 100% 0 KiB/s Time: 00:00:00\r\n[ML] Loaded! Executing the payload now\r\n\r\n[0] mingo version 0.5.0\r\n[1] Booting on: Raspberry Pi 3\r\n[2] Drivers loaded:\r\n      1. BCM PL011 UART\r\n      2. BCM GPIO\r\n[3] Chars written: 117\r\n[4] Echoing input now\r\n```\r\n\r\n在这个教程中，为了演示目的，加载了上一个教程中的内核版本。在后续的教程中，将使用工作目录的内核。\r\n\r\n## 测试它\r\n\r\n这个教程中的`Makefile`有一个额外的目标`qemuasm`，它可以让你很好地观察到内核在重新定位后如何从加载地址区域（`0x80_XXX`）\r\n跳转到重新定位的代码（`0x0200_0XXX`）：\r\n\r\n```console\r\n$ make qemuasm\r\n[...]\r\nN:\r\n0x00080030:  58000140  ldr      x0, #0x80058\r\n0x00080034:  9100001f  mov      sp, x0\r\n0x00080038:  58000141  ldr      x1, #0x80060\r\n0x0008003c:  d61f0020  br       x1\r\n\r\n----------------\r\nIN:\r\n0x02000070:  9400044c  bl       #0x20011a0\r\n\r\n----------------\r\nIN:\r\n0x020011a0:  90000008  adrp     x8, #0x2001000\r\n0x020011a4:  90000009  adrp     x9, #0x2001000\r\n0x020011a8:  f9446508  ldr      x8, [x8, #0x8c8]\r\n0x020011ac:  f9446929  ldr      x9, [x9, #0x8d0]\r\n0x020011b0:  eb08013f  cmp      x9, x8\r\n0x020011b4:  54000109  b.ls     #0x20011d4\r\n[...]\r\n```\r\n\r\n## 相比之前的变化（diff）\r\n请检查[英文版本](README.md#diff-to-previous)，这是最新的。"
  },
  {
    "path": "06_uart_chainloader/README.md",
    "content": "# Tutorial 06 - UART Chainloader\n\n## tl;dr\n\n- Running from an SD card was a nice experience, but it would be extremely tedious to do it for\n  every new binary. So let's write a [chainloader].\n- This will be the last binary you need to put on the SD card. Each following tutorial will provide\n  a `chainboot` target in the `Makefile` that lets you conveniently load the kernel over `UART`.\n\n[chainloader]: https://en.wikipedia.org/wiki/Chain_loading\n\n\n## Note\n\nPlease note that there is stuff going on in this tutorial that is very hard to grasp by only looking\nat the source code changes.\n\nThe gist of it is that in `boot.s`, we are writing a piece of [position independent code] which\nautomatically determines where the firmware has loaded the binary (`0x8_0000`), and where it was\nlinked to (`0x200_0000`, see `kernel.ld`). The binary then copies itself from loaded to linked\naddress (aka  \"relocating\" itself), and then jumps to the relocated version of `_start_rust()`.\n\nSince the chainloader has put itself \"out of the way\" now, it can now receive another kernel binary\nfrom the `UART` and copy it to the standard load address of the RPi firmware at `0x8_0000`. Finally,\nit jumps to `0x8_0000` and the newly loaded binary transparently executes as if it had been loaded\nfrom SD card all along.\n\nPlease bear with me until I find the time to write it all down here elaborately. For the time being,\nplease see this tutorial as an enabler for a convenience feature that allows booting the following\ntutorials in a quick manner. _For those keen to get a deeper understanding, it could make sense to\nskip forward to [Chapter 15](../15_virtual_mem_part3_precomputed_tables) and read the first half of\nthe README, where `Load Address != Link Address` is discussed_.\n\n[position independent code]: https://en.wikipedia.org/wiki/Position-independent_code\n\n## Install and test it\n\nOur chainloader is called `MiniLoad` and is inspired by [raspbootin].\n\nYou can try it with this tutorial already:\n1. Depending on your target hardware, run:`make` or `BSP=rpi4 make`.\n1. Copy `kernel8.img` to the SD card and put the SD card back into your RPi.\n1. Run `make chainboot` or `BSP=rpi4 make chainboot`.\n1. Connect the USB serial to your host PC.\n    - Wiring diagram at [top-level README](../README.md#-usb-serial-output).\n    - Make sure that you **DID NOT** connect the power pin of the USB serial. Only RX/TX and GND.\n1. Connect the RPi to the (USB) power cable.\n1. Observe the loader fetching a kernel over `UART`:\n\n> ❗ **NOTE**: `make chainboot` assumes a default serial device name of `/dev/ttyUSB0`. Depending on\n> your host operating system, the device name might differ. For example, on `macOS`, it might be\n> something like `/dev/tty.usbserial-0001`. In this case, please give the name explicitly:\n\n\n```console\n$ DEV_SERIAL=/dev/tty.usbserial-0001 make chainboot\n```\n\n[raspbootin]: https://github.com/mrvn/raspbootin\n\n```console\n$ make chainboot\n[...]\nMinipush 1.0\n\n[MP] ⏳ Waiting for /dev/ttyUSB0\n[MP] ✅ Serial connected\n[MP] 🔌 Please power the target now\n\n __  __ _      _ _                 _\n|  \\/  (_)_ _ (_) |   ___  __ _ __| |\n| |\\/| | | ' \\| | |__/ _ \\/ _` / _` |\n|_|  |_|_|_||_|_|____\\___/\\__,_\\__,_|\n\n           Raspberry Pi 3\n\n[ML] Requesting binary\n[MP] ⏩ Pushing 7 KiB ==========================================🦀 100% 0 KiB/s Time: 00:00:00\n[ML] Loaded! Executing the payload now\n\n[0] mingo version 0.5.0\n[1] Booting on: Raspberry Pi 3\n[2] Drivers loaded:\n      1. BCM PL011 UART\n      2. BCM GPIO\n[3] Chars written: 117\n[4] Echoing input now\n```\n\nIn this tutorial, a version of the kernel from the previous tutorial is loaded for demo purposes. In\nsubsequent tutorials, it will be the working directory's kernel.\n\n## Test it\n\nThe `Makefile` in this tutorial has an additional target, `qemuasm`, that lets you nicely observe\nhow the kernel, after relocating itself, jumps the load address region (`0x80_XXX`) to the relocated\ncode at (`0x0200_0XXX`):\n\n```console\n$ make qemuasm\n[...]\nN:\n0x00080030:  58000140  ldr      x0, #0x80058\n0x00080034:  9100001f  mov      sp, x0\n0x00080038:  58000141  ldr      x1, #0x80060\n0x0008003c:  d61f0020  br       x1\n\n----------------\nIN:\n0x02000070:  9400044c  bl       #0x20011a0\n\n----------------\nIN:\n0x020011a0:  90000008  adrp     x8, #0x2001000\n0x020011a4:  90000009  adrp     x9, #0x2001000\n0x020011a8:  f9446508  ldr      x8, [x8, #0x8c8]\n0x020011ac:  f9446929  ldr      x9, [x9, #0x8d0]\n0x020011b0:  eb08013f  cmp      x9, x8\n0x020011b4:  54000109  b.ls     #0x20011d4\n[...]\n```\n\n## Diff to previous\n```diff\n\ndiff -uNr 05_drivers_gpio_uart/Cargo.toml 06_uart_chainloader/Cargo.toml\n--- 05_drivers_gpio_uart/Cargo.toml\n+++ 06_uart_chainloader/Cargo.toml\n@@ -1,6 +1,6 @@\n [package]\n name = \"mingo\"\n-version = \"0.5.0\"\n+version = \"0.6.0\"\n authors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\n edition = \"2021\"\n\nBinary files 05_drivers_gpio_uart/demo_payload_rpi3.img and 06_uart_chainloader/demo_payload_rpi3.img differ\nBinary files 05_drivers_gpio_uart/demo_payload_rpi4.img and 06_uart_chainloader/demo_payload_rpi4.img differ\n\ndiff -uNr 05_drivers_gpio_uart/Makefile 06_uart_chainloader/Makefile\n--- 05_drivers_gpio_uart/Makefile\n+++ 06_uart_chainloader/Makefile\n@@ -24,27 +24,29 @@\n QEMU_MISSING_STRING = \"This board is not yet supported for QEMU.\"\n\n ifeq ($(BSP),rpi3)\n-    TARGET            = aarch64-unknown-none-softfloat\n-    KERNEL_BIN        = kernel8.img\n-    QEMU_BINARY       = qemu-system-aarch64\n-    QEMU_MACHINE_TYPE = raspi3\n-    QEMU_RELEASE_ARGS = -serial stdio -display none\n-    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n-    NM_BINARY         = aarch64-none-elf-nm\n-    READELF_BINARY    = aarch64-none-elf-readelf\n-    LD_SCRIPT_PATH    = $(shell pwd)/src/bsp/raspberrypi\n-    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a53\n+    TARGET                 = aarch64-unknown-none-softfloat\n+    KERNEL_BIN             = kernel8.img\n+    QEMU_BINARY            = qemu-system-aarch64\n+    QEMU_MACHINE_TYPE      = raspi3\n+    QEMU_RELEASE_ARGS      = -serial stdio -display none\n+    OBJDUMP_BINARY         = aarch64-none-elf-objdump\n+    NM_BINARY              = aarch64-none-elf-nm\n+    READELF_BINARY         = aarch64-none-elf-readelf\n+    LD_SCRIPT_PATH         = $(shell pwd)/src/bsp/raspberrypi\n+    RUSTC_MISC_ARGS        = -C target-cpu=cortex-a53\n+    CHAINBOOT_DEMO_PAYLOAD = demo_payload_rpi3.img\n else ifeq ($(BSP),rpi4)\n-    TARGET            = aarch64-unknown-none-softfloat\n-    KERNEL_BIN        = kernel8.img\n-    QEMU_BINARY       = qemu-system-aarch64\n-    QEMU_MACHINE_TYPE =\n-    QEMU_RELEASE_ARGS = -serial stdio -display none\n-    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n-    NM_BINARY         = aarch64-none-elf-nm\n-    READELF_BINARY    = aarch64-none-elf-readelf\n-    LD_SCRIPT_PATH    = $(shell pwd)/src/bsp/raspberrypi\n-    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a72\n+    TARGET                 = aarch64-unknown-none-softfloat\n+    KERNEL_BIN             = kernel8.img\n+    QEMU_BINARY            = qemu-system-aarch64\n+    QEMU_MACHINE_TYPE      =\n+    QEMU_RELEASE_ARGS      = -serial stdio -display none\n+    OBJDUMP_BINARY         = aarch64-none-elf-objdump\n+    NM_BINARY              = aarch64-none-elf-nm\n+    READELF_BINARY         = aarch64-none-elf-readelf\n+    LD_SCRIPT_PATH         = $(shell pwd)/src/bsp/raspberrypi\n+    RUSTC_MISC_ARGS        = -C target-cpu=cortex-a72\n+    CHAINBOOT_DEMO_PAYLOAD = demo_payload_rpi4.img\n endif\n\n # Export for build.rs.\n@@ -90,8 +92,8 @@\n     -O binary\n\n EXEC_QEMU          = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE)\n-EXEC_TEST_DISPATCH = ruby ../common/tests/dispatch.rb\n-EXEC_MINITERM      = ruby ../common/serial/miniterm.rb\n+EXEC_TEST_MINIPUSH = ruby tests/chainboot_test.rb\n+EXEC_MINIPUSH      = ruby ../common/serial/minipush.rb\n\n ##------------------------------------------------------------------------------\n ## Dockerization\n@@ -110,7 +112,7 @@\n ifeq ($(shell uname -s),Linux)\n     DOCKER_CMD_DEV = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_DEV)\n\n-    DOCKER_MINITERM = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\n+    DOCKER_CHAINBOOT = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\n endif\n\n\n@@ -118,7 +120,7 @@\n ##--------------------------------------------------------------------------------------------------\n ## Targets\n ##--------------------------------------------------------------------------------------------------\n-.PHONY: all doc qemu miniterm clippy clean readelf objdump nm check\n+.PHONY: all doc qemu chainboot clippy clean readelf objdump nm check\n\n all: $(KERNEL_BIN)\n\n@@ -160,7 +162,7 @@\n ##------------------------------------------------------------------------------\n ifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\n-qemu:\n+qemu qemuasm:\n \t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\n else # QEMU is supported.\n@@ -169,13 +171,17 @@\n \t$(call color_header, \"Launching QEMU\")\n \t@$(DOCKER_QEMU) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\n+qemuasm: $(KERNEL_BIN)\n+\t$(call color_header, \"Launching QEMU with ASM output\")\n+\t@$(DOCKER_QEMU) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN) -d in_asm\n+\n endif\n ##------------------------------------------------------------------------------\n-## Connect to the target's serial\n+## Push the kernel to the real HW target\n ##------------------------------------------------------------------------------\n-miniterm:\n-\t@$(DOCKER_MINITERM) $(EXEC_MINITERM) $(DEV_SERIAL)\n+chainboot: $(KERNEL_BIN)\n+\t@$(DOCKER_CHAINBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(CHAINBOOT_DEMO_PAYLOAD)\n\n ##------------------------------------------------------------------------------\n ## Run clippy\n@@ -232,7 +238,8 @@\n ##------------------------------------------------------------------------------\n test_boot: $(KERNEL_BIN)\n \t$(call color_header, \"Boot test - $(BSP)\")\n-\t@$(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n+\t@$(DOCKER_TEST) $(EXEC_TEST_MINIPUSH) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) \\\n+\t\t-kernel $(KERNEL_BIN) $(CHAINBOOT_DEMO_PAYLOAD)\n\n test: test_boot\n\n\ndiff -uNr 05_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.s 06_uart_chainloader/src/_arch/aarch64/cpu/boot.s\n--- 05_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.s\n+++ 06_uart_chainloader/src/_arch/aarch64/cpu/boot.s\n@@ -18,6 +18,17 @@\n \tadd\t\\register, \\register, #:lo12:\\symbol\n .endm\n\n+// Load the address of a symbol into a register, absolute.\n+//\n+// # Resources\n+//\n+// - https://sourceware.org/binutils/docs-2.36/as/AArch64_002dRelocations.html\n+.macro ADR_ABS register, symbol\n+\tmovz\t\\register, #:abs_g2:\\symbol\n+\tmovk\t\\register, #:abs_g1_nc:\\symbol\n+\tmovk\t\\register, #:abs_g0_nc:\\symbol\n+.endm\n+\n //--------------------------------------------------------------------------------------------------\n // Public Code\n //--------------------------------------------------------------------------------------------------\n@@ -37,23 +48,35 @@\n \t// If execution reaches here, it is the boot core.\n\n \t// Initialize DRAM.\n-\tADR_REL\tx0, __bss_start\n-\tADR_REL x1, __bss_end_exclusive\n+\tADR_ABS\tx0, __bss_start\n+\tADR_ABS x1, __bss_end_exclusive\n\n .L_bss_init_loop:\n \tcmp\tx0, x1\n-\tb.eq\t.L_prepare_rust\n+\tb.eq\t.L_relocate_binary\n \tstp\txzr, xzr, [x0], #16\n \tb\t.L_bss_init_loop\n\n+\t// Next, relocate the binary.\n+.L_relocate_binary:\n+\tADR_REL\tx0, __binary_nonzero_start         // The address the binary got loaded to.\n+\tADR_ABS\tx1, __binary_nonzero_start         // The address the binary was linked to.\n+\tADR_ABS\tx2, __binary_nonzero_end_exclusive\n+\n+.L_copy_loop:\n+\tldr\tx3, [x0], #8\n+\tstr\tx3, [x1], #8\n+\tcmp\tx1, x2\n+\tb.lo\t.L_copy_loop\n+\n \t// Prepare the jump to Rust code.\n-.L_prepare_rust:\n \t// Set the stack pointer.\n-\tADR_REL\tx0, __boot_core_stack_end_exclusive\n+\tADR_ABS\tx0, __boot_core_stack_end_exclusive\n \tmov\tsp, x0\n\n-\t// Jump to Rust code.\n-\tb\t_start_rust\n+\t// Jump to the relocated Rust code.\n+\tADR_ABS\tx1, _start_rust\n+\tbr\tx1\n\n \t// Infinitely wait for events (aka \"park the core\").\n .L_parking_loop:\n\ndiff -uNr 05_drivers_gpio_uart/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs 06_uart_chainloader/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs\n--- 05_drivers_gpio_uart/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs\n+++ 06_uart_chainloader/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs\n@@ -275,7 +275,7 @@\n     }\n\n     /// Retrieve a character.\n-    fn read_char_converting(&mut self, blocking_mode: BlockingMode) -> Option<char> {\n+    fn read_char(&mut self, blocking_mode: BlockingMode) -> Option<char> {\n         // If RX FIFO is empty,\n         if self.registers.FR.matches_all(FR::RXFE::SET) {\n             // immediately return in non-blocking mode.\n@@ -290,12 +290,7 @@\n         }\n\n         // Read one character.\n-        let mut ret = self.registers.DR.get() as u8 as char;\n-\n-        // Convert carrige return to newline.\n-        if ret == '\\r' {\n-            ret = '\\n'\n-        }\n+        let ret = self.registers.DR.get() as u8 as char;\n\n         // Update statistics.\n         self.chars_read += 1;\n@@ -381,14 +376,14 @@\n impl console::interface::Read for PL011Uart {\n     fn read_char(&self) -> char {\n         self.inner\n-            .lock(|inner| inner.read_char_converting(BlockingMode::Blocking).unwrap())\n+            .lock(|inner| inner.read_char(BlockingMode::Blocking).unwrap())\n     }\n\n     fn clear_rx(&self) {\n         // Read from the RX FIFO until it is indicating empty.\n         while self\n             .inner\n-            .lock(|inner| inner.read_char_converting(BlockingMode::NonBlocking))\n+            .lock(|inner| inner.read_char(BlockingMode::NonBlocking))\n             .is_some()\n         {}\n     }\n\ndiff -uNr 05_drivers_gpio_uart/src/bsp/raspberrypi/console.rs 06_uart_chainloader/src/bsp/raspberrypi/console.rs\n--- 05_drivers_gpio_uart/src/bsp/raspberrypi/console.rs\n+++ 06_uart_chainloader/src/bsp/raspberrypi/console.rs\n@@ -1,16 +0,0 @@\n-// SPDX-License-Identifier: MIT OR Apache-2.0\n-//\n-// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n-\n-//! BSP console facilities.\n-\n-use crate::console;\n-\n-//--------------------------------------------------------------------------------------------------\n-// Public Code\n-//--------------------------------------------------------------------------------------------------\n-\n-/// Return a reference to the console.\n-pub fn console() -> &'static dyn console::interface::All {\n-    &super::driver::PL011_UART\n-}\n\ndiff -uNr 05_drivers_gpio_uart/src/bsp/raspberrypi/kernel.ld 06_uart_chainloader/src/bsp/raspberrypi/kernel.ld\n--- 05_drivers_gpio_uart/src/bsp/raspberrypi/kernel.ld\n+++ 06_uart_chainloader/src/bsp/raspberrypi/kernel.ld\n@@ -3,8 +3,6 @@\n  * Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n  */\n\n-__rpi_phys_dram_start_addr = 0;\n-\n /* The physical address at which the the kernel binary will be loaded by the Raspberry's firmware */\n __rpi_phys_binary_load_addr = 0x80000;\n\n@@ -28,7 +26,8 @@\n\n SECTIONS\n {\n-    . =  __rpi_phys_dram_start_addr;\n+    /* Set the link address to 32 MiB */\n+    . = 0x2000000;\n\n     /***********************************************************************************************\n     * Boot Core Stack\n@@ -45,6 +44,7 @@\n     /***********************************************************************************************\n     * Code + RO Data + Global Offset Table\n     ***********************************************************************************************/\n+    __binary_nonzero_start = .;\n     .text :\n     {\n         KEEP(*(.text._start))\n@@ -60,6 +60,10 @@\n     ***********************************************************************************************/\n     .data : { *(.data*) } :segment_data\n\n+    /* Fill up to 8 byte, b/c relocating the binary is done in u64 chunks */\n+    . = ALIGN(8);\n+    __binary_nonzero_end_exclusive = .;\n+\n     /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */\n     .bss (NOLOAD) : ALIGN(16)\n     {\n\ndiff -uNr 05_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs 06_uart_chainloader/src/bsp/raspberrypi/memory.rs\n--- 05_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs\n+++ 06_uart_chainloader/src/bsp/raspberrypi/memory.rs\n@@ -11,6 +11,7 @@\n /// The board's physical memory map.\n #[rustfmt::skip]\n pub(super) mod map {\n+    pub const BOARD_DEFAULT_LOAD_ADDRESS: usize =        0x8_0000;\n\n     pub const GPIO_OFFSET:         usize = 0x0020_0000;\n     pub const UART_OFFSET:         usize = 0x0020_1000;\n@@ -35,3 +36,13 @@\n         pub const PL011_UART_START: usize = START + UART_OFFSET;\n     }\n }\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+/// The address on which the Raspberry firmware loads every binary by default.\n+#[inline(always)]\n+pub fn board_default_load_addr() -> *const u64 {\n+    map::BOARD_DEFAULT_LOAD_ADDRESS as _\n+}\n\ndiff -uNr 05_drivers_gpio_uart/src/driver.rs 06_uart_chainloader/src/driver.rs\n--- 05_drivers_gpio_uart/src/driver.rs\n+++ 06_uart_chainloader/src/driver.rs\n@@ -4,10 +4,7 @@\n\n //! Driver support.\n\n-use crate::{\n-    println,\n-    synchronization::{interface::Mutex, NullLock},\n-};\n+use crate::synchronization::{interface::Mutex, NullLock};\n\n //--------------------------------------------------------------------------------------------------\n // Private Definitions\n@@ -154,14 +151,4 @@\n             }\n         });\n     }\n-\n-    /// Enumerate all registered device drivers.\n-    pub fn enumerate(&self) {\n-        let mut i: usize = 1;\n-        self.for_each_descriptor(|descriptor| {\n-            println!(\"      {}. {}\", i, descriptor.device_driver.compatible());\n-\n-            i += 1;\n-        });\n-    }\n }\n\ndiff -uNr 05_drivers_gpio_uart/src/main.rs 06_uart_chainloader/src/main.rs\n--- 05_drivers_gpio_uart/src/main.rs\n+++ 06_uart_chainloader/src/main.rs\n@@ -142,27 +142,55 @@\n     kernel_main()\n }\n\n+const MINILOAD_LOGO: &str = r#\"\n+ __  __ _      _ _                 _\n+|  \\/  (_)_ _ (_) |   ___  __ _ __| |\n+| |\\/| | | ' \\| | |__/ _ \\/ _` / _` |\n+|_|  |_|_|_||_|_|____\\___/\\__,_\\__,_|\n+\"#;\n+\n /// The main function running after the early init.\n fn kernel_main() -> ! {\n     use console::console;\n\n-    println!(\n-        \"[0] {} version {}\",\n-        env!(\"CARGO_PKG_NAME\"),\n-        env!(\"CARGO_PKG_VERSION\")\n-    );\n-    println!(\"[1] Booting on: {}\", bsp::board_name());\n+    println!(\"{}\", MINILOAD_LOGO);\n+    println!(\"{:^37}\", bsp::board_name());\n+    println!();\n+    println!(\"[ML] Requesting binary\");\n+    console().flush();\n\n-    println!(\"[2] Drivers loaded:\");\n-    driver::driver_manager().enumerate();\n+    // Discard any spurious received characters before starting with the loader protocol.\n+    console().clear_rx();\n\n-    println!(\"[3] Chars written: {}\", console().chars_written());\n-    println!(\"[4] Echoing input now\");\n+    // Notify `Minipush` to send the binary.\n+    for _ in 0..3 {\n+        console().write_char(3 as char);\n+    }\n\n-    // Discard any spurious received characters before going into echo mode.\n-    console().clear_rx();\n-    loop {\n-        let c = console().read_char();\n-        console().write_char(c);\n+    // Read the binary's size.\n+    let mut size: u32 = u32::from(console().read_char() as u8);\n+    size |= u32::from(console().read_char() as u8) << 8;\n+    size |= u32::from(console().read_char() as u8) << 16;\n+    size |= u32::from(console().read_char() as u8) << 24;\n+\n+    // Trust it's not too big.\n+    console().write_char('O');\n+    console().write_char('K');\n+\n+    let kernel_addr: *mut u8 = bsp::memory::board_default_load_addr() as *mut u8;\n+    unsafe {\n+        // Read the kernel byte by byte.\n+        for i in 0..size {\n+            core::ptr::write_volatile(kernel_addr.offset(i as isize), console().read_char() as u8)\n+        }\n     }\n+\n+    println!(\"[ML] Loaded! Executing the payload now\\n\");\n+    console().flush();\n+\n+    // Use black magic to create a function pointer.\n+    let kernel: fn() -> ! = unsafe { core::mem::transmute(kernel_addr) };\n+\n+    // Jump to loaded kernel!\n+    kernel()\n }\n\ndiff -uNr 05_drivers_gpio_uart/tests/boot_test_string.rb 06_uart_chainloader/tests/boot_test_string.rb\n--- 05_drivers_gpio_uart/tests/boot_test_string.rb\n+++ 06_uart_chainloader/tests/boot_test_string.rb\n@@ -1,3 +0,0 @@\n-# frozen_string_literal: true\n-\n-EXPECTED_PRINT = 'Echoing input now'\n\ndiff -uNr 05_drivers_gpio_uart/tests/chainboot_test.rb 06_uart_chainloader/tests/chainboot_test.rb\n--- 05_drivers_gpio_uart/tests/chainboot_test.rb\n+++ 06_uart_chainloader/tests/chainboot_test.rb\n@@ -0,0 +1,78 @@\n+# frozen_string_literal: true\n+\n+# SPDX-License-Identifier: MIT OR Apache-2.0\n+#\n+# Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+require_relative '../../common/serial/minipush'\n+require_relative '../../common/tests/boot_test'\n+require 'pty'\n+\n+# Match for the last print that 'demo_payload_rpiX.img' produces.\n+EXPECTED_PRINT = 'Echoing input now'\n+\n+# Wait for request to power the target.\n+class PowerTargetRequestTest < SubtestBase\n+    MINIPUSH_POWER_TARGET_REQUEST = 'Please power the target now'\n+\n+    def initialize(qemu_cmd, pty_main)\n+        super()\n+        @qemu_cmd = qemu_cmd\n+        @pty_main = pty_main\n+    end\n+\n+    def name\n+        'Waiting for request to power target'\n+    end\n+\n+    def run(qemu_out, _qemu_in)\n+        expect_or_raise(qemu_out, MINIPUSH_POWER_TARGET_REQUEST)\n+\n+        # Now is the time to start QEMU with the chainloader binary. QEMU's virtual tty connects to\n+        # the MiniPush instance spawned on pty_main, so that the two processes talk to each other.\n+        Process.spawn(@qemu_cmd, in: @pty_main, out: @pty_main, err: '/dev/null')\n+    end\n+end\n+\n+# Extend BootTest so that it listens on the output of a MiniPush instance, which is itself connected\n+# to a QEMU instance instead of a real HW.\n+class ChainbootTest < BootTest\n+    MINIPUSH = '../common/serial/minipush.rb'\n+\n+    def initialize(qemu_cmd, payload_path)\n+        super(qemu_cmd, EXPECTED_PRINT)\n+\n+        @test_name = 'Boot test using Minipush'\n+\n+        @payload_path = payload_path\n+    end\n+\n+    private\n+\n+    # override\n+    def setup\n+        pty_main, pty_secondary = PTY.open\n+        mp_out, _mp_in = PTY.spawn(\"ruby #{MINIPUSH} #{pty_secondary.path} #{@payload_path}\")\n+\n+        # The subtests (from this class and the parents) listen on @qemu_out_wrapped. Hence, point\n+        # it to MiniPush's output.\n+        @qemu_out_wrapped = PTYLoggerWrapper.new(mp_out, \"\\r\\n\")\n+\n+        # Important: Run this subtest before the one in the parent class.\n+        @console_subtests.prepend(PowerTargetRequestTest.new(@qemu_cmd, pty_main))\n+    end\n+\n+    # override\n+    def finish\n+        super()\n+        @test_output.map! { |x| x.gsub(/.*\\r/, '  ') }\n+    end\n+end\n+\n+##--------------------------------------------------------------------------------------------------\n+## Execution starts here\n+##--------------------------------------------------------------------------------------------------\n+payload_path = ARGV.pop\n+qemu_cmd = ARGV.join(' ')\n+\n+ChainbootTest.new(qemu_cmd, payload_path).run\n\ndiff -uNr 05_drivers_gpio_uart/update.sh 06_uart_chainloader/update.sh\n--- 05_drivers_gpio_uart/update.sh\n+++ 06_uart_chainloader/update.sh\n@@ -0,0 +1,8 @@\n+#!/usr/bin/env bash\n+\n+cd ../05_drivers_gpio_uart\n+BSP=rpi4 make\n+cp kernel8.img ../06_uart_chainloader/demo_payload_rpi4.img\n+make\n+cp kernel8.img ../06_uart_chainloader/demo_payload_rpi3.img\n+rm kernel8.img\n\n```\n"
  },
  {
    "path": "06_uart_chainloader/build.rs",
    "content": "use std::{env, fs, process};\n\nfn main() {\n    let ld_script_path = match env::var(\"LD_SCRIPT_PATH\") {\n        Ok(var) => var,\n        _ => process::exit(0),\n    };\n\n    let files = fs::read_dir(ld_script_path).unwrap();\n    files\n        .filter_map(Result::ok)\n        .filter(|d| {\n            if let Some(e) = d.path().extension() {\n                e == \"ld\"\n            } else {\n                false\n            }\n        })\n        .for_each(|f| println!(\"cargo:rerun-if-changed={}\", f.path().display()));\n}\n"
  },
  {
    "path": "06_uart_chainloader/src/_arch/aarch64/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural boot code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::boot::arch_boot\n\nuse core::arch::global_asm;\n\n// Assembly counterpart to this file.\nglobal_asm!(\n    include_str!(\"boot.s\"),\n    CONST_CORE_ID_MASK = const 0b11\n);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The Rust entry of the `kernel` binary.\n///\n/// The function is called from the assembly `_start` function.\n#[no_mangle]\npub unsafe fn _start_rust() -> ! {\n    crate::kernel_init()\n}\n"
  },
  {
    "path": "06_uart_chainloader/src/_arch/aarch64/cpu/boot.s",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2022 Andre Richter <andre.o.richter@gmail.com>\n\n//--------------------------------------------------------------------------------------------------\n// Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Load the address of a symbol into a register, PC-relative.\n//\n// The symbol must lie within +/- 4 GiB of the Program Counter.\n//\n// # Resources\n//\n// - https://sourceware.org/binutils/docs-2.36/as/AArch64_002dRelocations.html\n.macro ADR_REL register, symbol\n\tadrp\t\\register, \\symbol\n\tadd\t\\register, \\register, #:lo12:\\symbol\n.endm\n\n// Load the address of a symbol into a register, absolute.\n//\n// # Resources\n//\n// - https://sourceware.org/binutils/docs-2.36/as/AArch64_002dRelocations.html\n.macro ADR_ABS register, symbol\n\tmovz\t\\register, #:abs_g2:\\symbol\n\tmovk\t\\register, #:abs_g1_nc:\\symbol\n\tmovk\t\\register, #:abs_g0_nc:\\symbol\n.endm\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n.section .text._start\n\n//------------------------------------------------------------------------------\n// fn _start()\n//------------------------------------------------------------------------------\n_start:\n\t// Only proceed on the boot core. Park it otherwise.\n\tmrs\tx0, MPIDR_EL1\n\tand\tx0, x0, {CONST_CORE_ID_MASK}\n\tldr\tx1, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs\n\tcmp\tx0, x1\n\tb.ne\t.L_parking_loop\n\n\t// If execution reaches here, it is the boot core.\n\n\t// Initialize DRAM.\n\tADR_ABS\tx0, __bss_start\n\tADR_ABS x1, __bss_end_exclusive\n\n.L_bss_init_loop:\n\tcmp\tx0, x1\n\tb.eq\t.L_relocate_binary\n\tstp\txzr, xzr, [x0], #16\n\tb\t.L_bss_init_loop\n\n\t// Next, relocate the binary.\n.L_relocate_binary:\n\tADR_REL\tx0, __binary_nonzero_start         // The address the binary got loaded to.\n\tADR_ABS\tx1, __binary_nonzero_start         // The address the binary was linked to.\n\tADR_ABS\tx2, __binary_nonzero_end_exclusive\n\n.L_copy_loop:\n\tldr\tx3, [x0], #8\n\tstr\tx3, [x1], #8\n\tcmp\tx1, x2\n\tb.lo\t.L_copy_loop\n\n\t// Prepare the jump to Rust code.\n\t// Set the stack pointer.\n\tADR_ABS\tx0, __boot_core_stack_end_exclusive\n\tmov\tsp, x0\n\n\t// Jump to the relocated Rust code.\n\tADR_ABS\tx1, _start_rust\n\tbr\tx1\n\n\t// Infinitely wait for events (aka \"park the core\").\n.L_parking_loop:\n\twfe\n\tb\t.L_parking_loop\n\n.size\t_start, . - _start\n.type\t_start, function\n.global\t_start\n"
  },
  {
    "path": "06_uart_chainloader/src/_arch/aarch64/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural processor code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::arch_cpu\n\nuse aarch64_cpu::asm;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\npub use asm::nop;\n\n/// Spin for `n` cycles.\n#[cfg(feature = \"bsp_rpi3\")]\n#[inline(always)]\npub fn spin_for_cycles(n: usize) {\n    for _ in 0..n {\n        asm::nop();\n    }\n}\n\n/// Pause execution on the core.\n#[inline(always)]\npub fn wait_forever() -> ! {\n    loop {\n        asm::wfe()\n    }\n}\n"
  },
  {
    "path": "06_uart_chainloader/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GPIO Driver.\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper, driver, synchronization,\n    synchronization::NullLock,\n};\nuse tock_registers::{\n    interfaces::{ReadWriteable, Writeable},\n    register_bitfields, register_structs,\n    registers::ReadWrite,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// GPIO registers.\n//\n// Descriptions taken from\n// - https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf\n// - https://datasheets.raspberrypi.org/bcm2711/bcm2711-peripherals.pdf\nregister_bitfields! {\n    u32,\n\n    /// GPIO Function Select 1\n    GPFSEL1 [\n        /// Pin 15\n        FSEL15 OFFSET(15) NUMBITS(3) [\n            Input = 0b000,\n            Output = 0b001,\n            AltFunc0 = 0b100  // PL011 UART RX\n\n        ],\n\n        /// Pin 14\n        FSEL14 OFFSET(12) NUMBITS(3) [\n            Input = 0b000,\n            Output = 0b001,\n            AltFunc0 = 0b100  // PL011 UART TX\n        ]\n    ],\n\n    /// GPIO Pull-up/down Register\n    ///\n    /// BCM2837 only.\n    GPPUD [\n        /// Controls the actuation of the internal pull-up/down control line to ALL the GPIO pins.\n        PUD OFFSET(0) NUMBITS(2) [\n            Off = 0b00,\n            PullDown = 0b01,\n            PullUp = 0b10\n        ]\n    ],\n\n    /// GPIO Pull-up/down Clock Register 0\n    ///\n    /// BCM2837 only.\n    GPPUDCLK0 [\n        /// Pin 15\n        PUDCLK15 OFFSET(15) NUMBITS(1) [\n            NoEffect = 0,\n            AssertClock = 1\n        ],\n\n        /// Pin 14\n        PUDCLK14 OFFSET(14) NUMBITS(1) [\n            NoEffect = 0,\n            AssertClock = 1\n        ]\n    ],\n\n    /// GPIO Pull-up / Pull-down Register 0\n    ///\n    /// BCM2711 only.\n    GPIO_PUP_PDN_CNTRL_REG0 [\n        /// Pin 15\n        GPIO_PUP_PDN_CNTRL15 OFFSET(30) NUMBITS(2) [\n            NoResistor = 0b00,\n            PullUp = 0b01\n        ],\n\n        /// Pin 14\n        GPIO_PUP_PDN_CNTRL14 OFFSET(28) NUMBITS(2) [\n            NoResistor = 0b00,\n            PullUp = 0b01\n        ]\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    RegisterBlock {\n        (0x00 => _reserved1),\n        (0x04 => GPFSEL1: ReadWrite<u32, GPFSEL1::Register>),\n        (0x08 => _reserved2),\n        (0x94 => GPPUD: ReadWrite<u32, GPPUD::Register>),\n        (0x98 => GPPUDCLK0: ReadWrite<u32, GPPUDCLK0::Register>),\n        (0x9C => _reserved3),\n        (0xE4 => GPIO_PUP_PDN_CNTRL_REG0: ReadWrite<u32, GPIO_PUP_PDN_CNTRL_REG0::Register>),\n        (0xE8 => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\nstruct GPIOInner {\n    registers: Registers,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the GPIO HW.\npub struct GPIO {\n    inner: NullLock<GPIOInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GPIOInner {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n        }\n    }\n\n    /// Disable pull-up/down on pins 14 and 15.\n    #[cfg(feature = \"bsp_rpi3\")]\n    fn disable_pud_14_15_bcm2837(&mut self) {\n        use crate::cpu;\n\n        // Make an educated guess for a good delay value (Sequence described in the BCM2837\n        // peripherals PDF).\n        //\n        // - According to Wikipedia, the fastest RPi4 clocks around 1.5 GHz.\n        // - The Linux 2837 GPIO driver waits 1 µs between the steps.\n        //\n        // So lets try to be on the safe side and default to 2000 cycles, which would equal 1 µs\n        // would the CPU be clocked at 2 GHz.\n        const DELAY: usize = 2000;\n\n        self.registers.GPPUD.write(GPPUD::PUD::Off);\n        cpu::spin_for_cycles(DELAY);\n\n        self.registers\n            .GPPUDCLK0\n            .write(GPPUDCLK0::PUDCLK15::AssertClock + GPPUDCLK0::PUDCLK14::AssertClock);\n        cpu::spin_for_cycles(DELAY);\n\n        self.registers.GPPUD.write(GPPUD::PUD::Off);\n        self.registers.GPPUDCLK0.set(0);\n    }\n\n    /// Disable pull-up/down on pins 14 and 15.\n    #[cfg(feature = \"bsp_rpi4\")]\n    fn disable_pud_14_15_bcm2711(&mut self) {\n        self.registers.GPIO_PUP_PDN_CNTRL_REG0.write(\n            GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL15::PullUp\n                + GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL14::PullUp,\n        );\n    }\n\n    /// Map PL011 UART as standard output.\n    ///\n    /// TX to pin 14\n    /// RX to pin 15\n    pub fn map_pl011_uart(&mut self) {\n        // Select the UART on pins 14 and 15.\n        self.registers\n            .GPFSEL1\n            .modify(GPFSEL1::FSEL15::AltFunc0 + GPFSEL1::FSEL14::AltFunc0);\n\n        // Disable pull-up/down on pins 14 and 15.\n        #[cfg(feature = \"bsp_rpi3\")]\n        self.disable_pud_14_15_bcm2837();\n\n        #[cfg(feature = \"bsp_rpi4\")]\n        self.disable_pud_14_15_bcm2711();\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GPIO {\n    pub const COMPATIBLE: &'static str = \"BCM GPIO\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            inner: NullLock::new(GPIOInner::new(mmio_start_addr)),\n        }\n    }\n\n    /// Concurrency safe version of `GPIOInner.map_pl011_uart()`\n    pub fn map_pl011_uart(&self) {\n        self.inner.lock(|inner| inner.map_pl011_uart())\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl driver::interface::DeviceDriver for GPIO {\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n}\n"
  },
  {
    "path": "06_uart_chainloader/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! PL011 UART driver.\n//!\n//! # Resources\n//!\n//! - <https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf>\n//! - <https://developer.arm.com/documentation/ddi0183/latest>\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper, console, cpu, driver, synchronization,\n    synchronization::NullLock,\n};\nuse core::fmt;\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields, register_structs,\n    registers::{ReadOnly, ReadWrite, WriteOnly},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// PL011 UART registers.\n//\n// Descriptions taken from \"PrimeCell UART (PL011) Technical Reference Manual\" r1p5.\nregister_bitfields! {\n    u32,\n\n    /// Flag Register.\n    FR [\n        /// Transmit FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n        /// Line Control Register, LCR_H.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the transmit holding register is empty.\n        /// - If the FIFO is enabled, the TXFE bit is set when the transmit FIFO is empty.\n        /// - This bit does not indicate if there is data in the transmit shift register.\n        TXFE OFFSET(7) NUMBITS(1) [],\n\n        /// Transmit FIFO full. The meaning of this bit depends on the state of the FEN bit in the\n        /// LCR_H Register.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the transmit holding register is full.\n        /// - If the FIFO is enabled, the TXFF bit is set when the transmit FIFO is full.\n        TXFF OFFSET(5) NUMBITS(1) [],\n\n        /// Receive FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n        /// LCR_H Register.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the receive holding register is empty.\n        /// - If the FIFO is enabled, the RXFE bit is set when the receive FIFO is empty.\n        RXFE OFFSET(4) NUMBITS(1) [],\n\n        /// UART busy. If this bit is set to 1, the UART is busy transmitting data. This bit remains\n        /// set until the complete byte, including all the stop bits, has been sent from the shift\n        /// register.\n        ///\n        /// This bit is set as soon as the transmit FIFO becomes non-empty, regardless of whether\n        /// the UART is enabled or not.\n        BUSY OFFSET(3) NUMBITS(1) []\n    ],\n\n    /// Integer Baud Rate Divisor.\n    IBRD [\n        /// The integer baud rate divisor.\n        BAUD_DIVINT OFFSET(0) NUMBITS(16) []\n    ],\n\n    /// Fractional Baud Rate Divisor.\n    FBRD [\n        ///  The fractional baud rate divisor.\n        BAUD_DIVFRAC OFFSET(0) NUMBITS(6) []\n    ],\n\n    /// Line Control Register.\n    LCR_H [\n        /// Word length. These bits indicate the number of data bits transmitted or received in a\n        /// frame.\n        #[allow(clippy::enum_variant_names)]\n        WLEN OFFSET(5) NUMBITS(2) [\n            FiveBit = 0b00,\n            SixBit = 0b01,\n            SevenBit = 0b10,\n            EightBit = 0b11\n        ],\n\n        /// Enable FIFOs:\n        ///\n        /// 0 = FIFOs are disabled (character mode) that is, the FIFOs become 1-byte-deep holding\n        /// registers.\n        ///\n        /// 1 = Transmit and receive FIFO buffers are enabled (FIFO mode).\n        FEN  OFFSET(4) NUMBITS(1) [\n            FifosDisabled = 0,\n            FifosEnabled = 1\n        ]\n    ],\n\n    /// Control Register.\n    CR [\n        /// Receive enable. If this bit is set to 1, the receive section of the UART is enabled.\n        /// Data reception occurs for either UART signals or SIR signals depending on the setting of\n        /// the SIREN bit. When the UART is disabled in the middle of reception, it completes the\n        /// current character before stopping.\n        RXE OFFSET(9) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// Transmit enable. If this bit is set to 1, the transmit section of the UART is enabled.\n        /// Data transmission occurs for either UART signals, or SIR signals depending on the\n        /// setting of the SIREN bit. When the UART is disabled in the middle of transmission, it\n        /// completes the current character before stopping.\n        TXE OFFSET(8) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// UART enable:\n        ///\n        /// 0 = UART is disabled. If the UART is disabled in the middle of transmission or\n        /// reception, it completes the current character before stopping.\n        ///\n        /// 1 = The UART is enabled. Data transmission and reception occurs for either UART signals\n        /// or SIR signals depending on the setting of the SIREN bit\n        UARTEN OFFSET(0) NUMBITS(1) [\n            /// If the UART is disabled in the middle of transmission or reception, it completes the\n            /// current character before stopping.\n            Disabled = 0,\n            Enabled = 1\n        ]\n    ],\n\n    /// Interrupt Clear Register.\n    ICR [\n        /// Meta field for all pending interrupts.\n        ALL OFFSET(0) NUMBITS(11) []\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    pub RegisterBlock {\n        (0x00 => DR: ReadWrite<u32>),\n        (0x04 => _reserved1),\n        (0x18 => FR: ReadOnly<u32, FR::Register>),\n        (0x1c => _reserved2),\n        (0x24 => IBRD: WriteOnly<u32, IBRD::Register>),\n        (0x28 => FBRD: WriteOnly<u32, FBRD::Register>),\n        (0x2c => LCR_H: WriteOnly<u32, LCR_H::Register>),\n        (0x30 => CR: WriteOnly<u32, CR::Register>),\n        (0x34 => _reserved3),\n        (0x44 => ICR: WriteOnly<u32, ICR::Register>),\n        (0x48 => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\n#[derive(PartialEq)]\nenum BlockingMode {\n    Blocking,\n    NonBlocking,\n}\n\nstruct PL011UartInner {\n    registers: Registers,\n    chars_written: usize,\n    chars_read: usize,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the UART.\npub struct PL011Uart {\n    inner: NullLock<PL011UartInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PL011UartInner {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n            chars_written: 0,\n            chars_read: 0,\n        }\n    }\n\n    /// Set up baud rate and characteristics.\n    ///\n    /// This results in 8N1 and 921_600 baud.\n    ///\n    /// The calculation for the BRD is (we set the clock to 48 MHz in config.txt):\n    /// `(48_000_000 / 16) / 921_600 = 3.2552083`.\n    ///\n    /// This means the integer part is `3` and goes into the `IBRD`.\n    /// The fractional part is `0.2552083`.\n    ///\n    /// `FBRD` calculation according to the PL011 Technical Reference Manual:\n    /// `INTEGER((0.2552083 * 64) + 0.5) = 16`.\n    ///\n    /// Therefore, the generated baud rate divider is: `3 + 16/64 = 3.25`. Which results in a\n    /// genrated baud rate of `48_000_000 / (16 * 3.25) = 923_077`.\n    ///\n    /// Error = `((923_077 - 921_600) / 921_600) * 100 = 0.16%`.\n    pub fn init(&mut self) {\n        // Execution can arrive here while there are still characters queued in the TX FIFO and\n        // actively being sent out by the UART hardware. If the UART is turned off in this case,\n        // those queued characters would be lost.\n        //\n        // For example, this can happen during runtime on a call to panic!(), because panic!()\n        // initializes its own UART instance and calls init().\n        //\n        // Hence, flush first to ensure all pending characters are transmitted.\n        self.flush();\n\n        // Turn the UART off temporarily.\n        self.registers.CR.set(0);\n\n        // Clear all pending interrupts.\n        self.registers.ICR.write(ICR::ALL::CLEAR);\n\n        // From the PL011 Technical Reference Manual:\n        //\n        // The LCR_H, IBRD, and FBRD registers form the single 30-bit wide LCR Register that is\n        // updated on a single write strobe generated by a LCR_H write. So, to internally update the\n        // contents of IBRD or FBRD, a LCR_H write must always be performed at the end.\n        //\n        // Set the baud rate, 8N1 and FIFO enabled.\n        self.registers.IBRD.write(IBRD::BAUD_DIVINT.val(3));\n        self.registers.FBRD.write(FBRD::BAUD_DIVFRAC.val(16));\n        self.registers\n            .LCR_H\n            .write(LCR_H::WLEN::EightBit + LCR_H::FEN::FifosEnabled);\n\n        // Turn the UART on.\n        self.registers\n            .CR\n            .write(CR::UARTEN::Enabled + CR::TXE::Enabled + CR::RXE::Enabled);\n    }\n\n    /// Send a character.\n    fn write_char(&mut self, c: char) {\n        // Spin while TX FIFO full is set, waiting for an empty slot.\n        while self.registers.FR.matches_all(FR::TXFF::SET) {\n            cpu::nop();\n        }\n\n        // Write the character to the buffer.\n        self.registers.DR.set(c as u32);\n\n        self.chars_written += 1;\n    }\n\n    /// Block execution until the last buffered character has been physically put on the TX wire.\n    fn flush(&self) {\n        // Spin until the busy bit is cleared.\n        while self.registers.FR.matches_all(FR::BUSY::SET) {\n            cpu::nop();\n        }\n    }\n\n    /// Retrieve a character.\n    fn read_char(&mut self, blocking_mode: BlockingMode) -> Option<char> {\n        // If RX FIFO is empty,\n        if self.registers.FR.matches_all(FR::RXFE::SET) {\n            // immediately return in non-blocking mode.\n            if blocking_mode == BlockingMode::NonBlocking {\n                return None;\n            }\n\n            // Otherwise, wait until a char was received.\n            while self.registers.FR.matches_all(FR::RXFE::SET) {\n                cpu::nop();\n            }\n        }\n\n        // Read one character.\n        let ret = self.registers.DR.get() as u8 as char;\n\n        // Update statistics.\n        self.chars_read += 1;\n\n        Some(ret)\n    }\n}\n\n/// Implementing `core::fmt::Write` enables usage of the `format_args!` macros, which in turn are\n/// used to implement the `kernel`'s `print!` and `println!` macros. By implementing `write_str()`,\n/// we get `write_fmt()` automatically.\n///\n/// The function takes an `&mut self`, so it must be implemented for the inner struct.\n///\n/// See [`src/print.rs`].\n///\n/// [`src/print.rs`]: ../../print/index.html\nimpl fmt::Write for PL011UartInner {\n    fn write_str(&mut self, s: &str) -> fmt::Result {\n        for c in s.chars() {\n            self.write_char(c);\n        }\n\n        Ok(())\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PL011Uart {\n    pub const COMPATIBLE: &'static str = \"BCM PL011 UART\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            inner: NullLock::new(PL011UartInner::new(mmio_start_addr)),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl driver::interface::DeviceDriver for PL011Uart {\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n\n    unsafe fn init(&self) -> Result<(), &'static str> {\n        self.inner.lock(|inner| inner.init());\n\n        Ok(())\n    }\n}\n\nimpl console::interface::Write for PL011Uart {\n    /// Passthrough of `args` to the `core::fmt::Write` implementation, but guarded by a Mutex to\n    /// serialize access.\n    fn write_char(&self, c: char) {\n        self.inner.lock(|inner| inner.write_char(c));\n    }\n\n    fn write_fmt(&self, args: core::fmt::Arguments) -> fmt::Result {\n        // Fully qualified syntax for the call to `core::fmt::Write::write_fmt()` to increase\n        // readability.\n        self.inner.lock(|inner| fmt::Write::write_fmt(inner, args))\n    }\n\n    fn flush(&self) {\n        // Spin until TX FIFO empty is set.\n        self.inner.lock(|inner| inner.flush());\n    }\n}\n\nimpl console::interface::Read for PL011Uart {\n    fn read_char(&self) -> char {\n        self.inner\n            .lock(|inner| inner.read_char(BlockingMode::Blocking).unwrap())\n    }\n\n    fn clear_rx(&self) {\n        // Read from the RX FIFO until it is indicating empty.\n        while self\n            .inner\n            .lock(|inner| inner.read_char(BlockingMode::NonBlocking))\n            .is_some()\n        {}\n    }\n}\n\nimpl console::interface::Statistics for PL011Uart {\n    fn chars_written(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_written)\n    }\n\n    fn chars_read(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_read)\n    }\n}\n\nimpl console::interface::All for PL011Uart {}\n"
  },
  {
    "path": "06_uart_chainloader/src/bsp/device_driver/bcm.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BCM driver top level.\n\nmod bcm2xxx_gpio;\nmod bcm2xxx_pl011_uart;\n\npub use bcm2xxx_gpio::*;\npub use bcm2xxx_pl011_uart::*;\n"
  },
  {
    "path": "06_uart_chainloader/src/bsp/device_driver/common.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Common device driver code.\n\nuse core::{marker::PhantomData, ops};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct MMIODerefWrapper<T> {\n    start_addr: usize,\n    phantom: PhantomData<fn() -> T>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T> MMIODerefWrapper<T> {\n    /// Create an instance.\n    pub const unsafe fn new(start_addr: usize) -> Self {\n        Self {\n            start_addr,\n            phantom: PhantomData,\n        }\n    }\n}\n\nimpl<T> ops::Deref for MMIODerefWrapper<T> {\n    type Target = T;\n\n    fn deref(&self) -> &Self::Target {\n        unsafe { &*(self.start_addr as *const _) }\n    }\n}\n"
  },
  {
    "path": "06_uart_chainloader/src/bsp/device_driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Device driver.\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod bcm;\nmod common;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use bcm::*;\n"
  },
  {
    "path": "06_uart_chainloader/src/bsp/raspberrypi/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Processor code.\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Used by `arch` code to find the early boot core.\n#[no_mangle]\n#[link_section = \".text._start_arguments\"]\npub static BOOT_CORE_ID: u64 = 0;\n"
  },
  {
    "path": "06_uart_chainloader/src/bsp/raspberrypi/driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP driver support.\n\nuse super::memory::map::mmio;\nuse crate::{bsp::device_driver, console, driver as generic_driver};\nuse core::sync::atomic::{AtomicBool, Ordering};\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic PL011_UART: device_driver::PL011Uart =\n    unsafe { device_driver::PL011Uart::new(mmio::PL011_UART_START) };\nstatic GPIO: device_driver::GPIO = unsafe { device_driver::GPIO::new(mmio::GPIO_START) };\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// This must be called only after successful init of the UART driver.\nfn post_init_uart() -> Result<(), &'static str> {\n    console::register_console(&PL011_UART);\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the GPIO driver.\nfn post_init_gpio() -> Result<(), &'static str> {\n    GPIO.map_pl011_uart();\n    Ok(())\n}\n\nfn driver_uart() -> Result<(), &'static str> {\n    let uart_descriptor =\n        generic_driver::DeviceDriverDescriptor::new(&PL011_UART, Some(post_init_uart));\n    generic_driver::driver_manager().register_driver(uart_descriptor);\n\n    Ok(())\n}\n\nfn driver_gpio() -> Result<(), &'static str> {\n    let gpio_descriptor = generic_driver::DeviceDriverDescriptor::new(&GPIO, Some(post_init_gpio));\n    generic_driver::driver_manager().register_driver(gpio_descriptor);\n\n    Ok(())\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Initialize the driver subsystem.\n///\n/// # Safety\n///\n/// See child function calls.\npub unsafe fn init() -> Result<(), &'static str> {\n    static INIT_DONE: AtomicBool = AtomicBool::new(false);\n    if INIT_DONE.load(Ordering::Relaxed) {\n        return Err(\"Init already done\");\n    }\n\n    driver_uart()?;\n    driver_gpio()?;\n\n    INIT_DONE.store(true, Ordering::Relaxed);\n    Ok(())\n}\n"
  },
  {
    "path": "06_uart_chainloader/src/bsp/raspberrypi/kernel.ld",
    "content": "/* SPDX-License-Identifier: MIT OR Apache-2.0\n *\n * Copyright (c) 2018-2022 Andre Richter <andre.o.richter@gmail.com>\n */\n\n/* The physical address at which the the kernel binary will be loaded by the Raspberry's firmware */\n__rpi_phys_binary_load_addr = 0x80000;\n\n\nENTRY(__rpi_phys_binary_load_addr)\n\n/* Flags:\n *     4 == R\n *     5 == RX\n *     6 == RW\n *\n * Segments are marked PT_LOAD below so that the ELF file provides virtual and physical addresses.\n * It doesn't mean all of them need actually be loaded.\n */\nPHDRS\n{\n    segment_boot_core_stack PT_LOAD FLAGS(6);\n    segment_code            PT_LOAD FLAGS(5);\n    segment_data            PT_LOAD FLAGS(6);\n}\n\nSECTIONS\n{\n    /* Set the link address to 32 MiB */\n    . = 0x2000000;\n\n    /***********************************************************************************************\n    * Boot Core Stack\n    ***********************************************************************************************/\n    .boot_core_stack (NOLOAD) :\n    {\n                                             /*   ^             */\n                                             /*   | stack       */\n        . += __rpi_phys_binary_load_addr;    /*   | growth      */\n                                             /*   | direction   */\n        __boot_core_stack_end_exclusive = .; /*   |             */\n    } :segment_boot_core_stack\n\n    /***********************************************************************************************\n    * Code + RO Data + Global Offset Table\n    ***********************************************************************************************/\n    __binary_nonzero_start = .;\n    .text :\n    {\n        KEEP(*(.text._start))\n        *(.text._start_arguments) /* Constants (or statics in Rust speak) read by _start(). */\n        *(.text._start_rust)      /* The Rust entry point */\n        *(.text*)                 /* Everything else */\n    } :segment_code\n\n    .rodata : ALIGN(8) { *(.rodata*) } :segment_code\n\n    /***********************************************************************************************\n    * Data + BSS\n    ***********************************************************************************************/\n    .data : { *(.data*) } :segment_data\n\n    /* Fill up to 8 byte, b/c relocating the binary is done in u64 chunks */\n    . = ALIGN(8);\n    __binary_nonzero_end_exclusive = .;\n\n    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */\n    .bss (NOLOAD) : ALIGN(16)\n    {\n        __bss_start = .;\n        *(.bss*);\n        . = ALIGN(16);\n        __bss_end_exclusive = .;\n    } :segment_data\n\n    /***********************************************************************************************\n    * Misc\n    ***********************************************************************************************/\n    .got : { *(.got*) }\n    ASSERT(SIZEOF(.got) == 0, \"Relocation support not expected\")\n\n    /DISCARD/ : { *(.comment*) }\n}\n"
  },
  {
    "path": "06_uart_chainloader/src/bsp/raspberrypi/memory.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Memory Management.\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// The board's physical memory map.\n#[rustfmt::skip]\npub(super) mod map {\n    pub const BOARD_DEFAULT_LOAD_ADDRESS: usize =        0x8_0000;\n\n    pub const GPIO_OFFSET:         usize = 0x0020_0000;\n    pub const UART_OFFSET:         usize = 0x0020_1000;\n\n    /// Physical devices.\n    #[cfg(feature = \"bsp_rpi3\")]\n    pub mod mmio {\n        use super::*;\n\n        pub const START:            usize =         0x3F00_0000;\n        pub const GPIO_START:       usize = START + GPIO_OFFSET;\n        pub const PL011_UART_START: usize = START + UART_OFFSET;\n    }\n\n    /// Physical devices.\n    #[cfg(feature = \"bsp_rpi4\")]\n    pub mod mmio {\n        use super::*;\n\n        pub const START:            usize =         0xFE00_0000;\n        pub const GPIO_START:       usize = START + GPIO_OFFSET;\n        pub const PL011_UART_START: usize = START + UART_OFFSET;\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The address on which the Raspberry firmware loads every binary by default.\n#[inline(always)]\npub fn board_default_load_addr() -> *const u64 {\n    map::BOARD_DEFAULT_LOAD_ADDRESS as _\n}\n"
  },
  {
    "path": "06_uart_chainloader/src/bsp/raspberrypi.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Top-level BSP file for the Raspberry Pi 3 and 4.\n\npub mod cpu;\npub mod driver;\npub mod memory;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Board identification.\npub fn board_name() -> &'static str {\n    #[cfg(feature = \"bsp_rpi3\")]\n    {\n        \"Raspberry Pi 3\"\n    }\n\n    #[cfg(feature = \"bsp_rpi4\")]\n    {\n        \"Raspberry Pi 4\"\n    }\n}\n"
  },
  {
    "path": "06_uart_chainloader/src/bsp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Conditional reexporting of Board Support Packages.\n\nmod device_driver;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod raspberrypi;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use raspberrypi::*;\n"
  },
  {
    "path": "06_uart_chainloader/src/console/null_console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Null console.\n\nuse super::interface;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct NullConsole;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\npub static NULL_CONSOLE: NullConsole = NullConsole {};\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl interface::Write for NullConsole {\n    fn write_char(&self, _c: char) {}\n\n    fn write_fmt(&self, _args: fmt::Arguments) -> fmt::Result {\n        fmt::Result::Ok(())\n    }\n\n    fn flush(&self) {}\n}\n\nimpl interface::Read for NullConsole {\n    fn clear_rx(&self) {}\n}\n\nimpl interface::Statistics for NullConsole {}\nimpl interface::All for NullConsole {}\n"
  },
  {
    "path": "06_uart_chainloader/src/console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! System console.\n\nmod null_console;\n\nuse crate::synchronization::{self, NullLock};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Console interfaces.\npub mod interface {\n    use core::fmt;\n\n    /// Console write functions.\n    pub trait Write {\n        /// Write a single character.\n        fn write_char(&self, c: char);\n\n        /// Write a Rust format string.\n        fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result;\n\n        /// Block until the last buffered character has been physically put on the TX wire.\n        fn flush(&self);\n    }\n\n    /// Console read functions.\n    pub trait Read {\n        /// Read a single character.\n        fn read_char(&self) -> char {\n            ' '\n        }\n\n        /// Clear RX buffers, if any.\n        fn clear_rx(&self);\n    }\n\n    /// Console statistics.\n    pub trait Statistics {\n        /// Return the number of characters written.\n        fn chars_written(&self) -> usize {\n            0\n        }\n\n        /// Return the number of characters read.\n        fn chars_read(&self) -> usize {\n            0\n        }\n    }\n\n    /// Trait alias for a full-fledged console.\n    pub trait All: Write + Read + Statistics {}\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic CUR_CONSOLE: NullLock<&'static (dyn interface::All + Sync)> =\n    NullLock::new(&null_console::NULL_CONSOLE);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\n/// Register a new console.\npub fn register_console(new_console: &'static (dyn interface::All + Sync)) {\n    CUR_CONSOLE.lock(|con| *con = new_console);\n}\n\n/// Return a reference to the currently registered console.\n///\n/// This is the global console used by all printing macros.\npub fn console() -> &'static dyn interface::All {\n    CUR_CONSOLE.lock(|con| *con)\n}\n"
  },
  {
    "path": "06_uart_chainloader/src/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Boot code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/cpu/boot.rs\"]\nmod arch_boot;\n"
  },
  {
    "path": "06_uart_chainloader/src/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Processor code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/cpu.rs\"]\nmod arch_cpu;\n\nmod boot;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_cpu::{nop, wait_forever};\n\n#[cfg(feature = \"bsp_rpi3\")]\npub use arch_cpu::spin_for_cycles;\n"
  },
  {
    "path": "06_uart_chainloader/src/driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Driver support.\n\nuse crate::synchronization::{interface::Mutex, NullLock};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nconst NUM_DRIVERS: usize = 5;\n\nstruct DriverManagerInner {\n    next_index: usize,\n    descriptors: [Option<DeviceDriverDescriptor>; NUM_DRIVERS],\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Driver interfaces.\npub mod interface {\n    /// Device Driver functions.\n    pub trait DeviceDriver {\n        /// Return a compatibility string for identifying the driver.\n        fn compatible(&self) -> &'static str;\n\n        /// Called by the kernel to bring up the device.\n        ///\n        /// # Safety\n        ///\n        /// - During init, drivers might do stuff with system-wide impact.\n        unsafe fn init(&self) -> Result<(), &'static str> {\n            Ok(())\n        }\n    }\n}\n\n/// Tpye to be used as an optional callback after a driver's init() has run.\npub type DeviceDriverPostInitCallback = unsafe fn() -> Result<(), &'static str>;\n\n/// A descriptor for device drivers.\n#[derive(Copy, Clone)]\npub struct DeviceDriverDescriptor {\n    device_driver: &'static (dyn interface::DeviceDriver + Sync),\n    post_init_callback: Option<DeviceDriverPostInitCallback>,\n}\n\n/// Provides device driver management functions.\npub struct DriverManager {\n    inner: NullLock<DriverManagerInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic DRIVER_MANAGER: DriverManager = DriverManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl DriverManagerInner {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            next_index: 0,\n            descriptors: [None; NUM_DRIVERS],\n        }\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl DeviceDriverDescriptor {\n    /// Create an instance.\n    pub fn new(\n        device_driver: &'static (dyn interface::DeviceDriver + Sync),\n        post_init_callback: Option<DeviceDriverPostInitCallback>,\n    ) -> Self {\n        Self {\n            device_driver,\n            post_init_callback,\n        }\n    }\n}\n\n/// Return a reference to the global DriverManager.\npub fn driver_manager() -> &'static DriverManager {\n    &DRIVER_MANAGER\n}\n\nimpl DriverManager {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            inner: NullLock::new(DriverManagerInner::new()),\n        }\n    }\n\n    /// Register a device driver with the kernel.\n    pub fn register_driver(&self, descriptor: DeviceDriverDescriptor) {\n        self.inner.lock(|inner| {\n            inner.descriptors[inner.next_index] = Some(descriptor);\n            inner.next_index += 1;\n        })\n    }\n\n    /// Helper for iterating over registered drivers.\n    fn for_each_descriptor<'a>(&'a self, f: impl FnMut(&'a DeviceDriverDescriptor)) {\n        self.inner.lock(|inner| {\n            inner\n                .descriptors\n                .iter()\n                .filter_map(|x| x.as_ref())\n                .for_each(f)\n        })\n    }\n\n    /// Fully initialize all drivers.\n    ///\n    /// # Safety\n    ///\n    /// - During init, drivers might do stuff with system-wide impact.\n    pub unsafe fn init_drivers(&self) {\n        self.for_each_descriptor(|descriptor| {\n            // 1. Initialize driver.\n            if let Err(x) = descriptor.device_driver.init() {\n                panic!(\n                    \"Error initializing driver: {}: {}\",\n                    descriptor.device_driver.compatible(),\n                    x\n                );\n            }\n\n            // 2. Call corresponding post init callback.\n            if let Some(callback) = &descriptor.post_init_callback {\n                if let Err(x) = callback() {\n                    panic!(\n                        \"Error during driver post-init callback: {}: {}\",\n                        descriptor.device_driver.compatible(),\n                        x\n                    );\n                }\n            }\n        });\n    }\n}\n"
  },
  {
    "path": "06_uart_chainloader/src/main.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n// Rust embedded logo for `make doc`.\n#![doc(\n    html_logo_url = \"https://raw.githubusercontent.com/rust-embedded/wg/master/assets/logo/ewg-logo-blue-white-on-transparent.png\"\n)]\n\n//! The `kernel` binary.\n//!\n//! # Code organization and architecture\n//!\n//! The code is divided into different *modules*, each representing a typical **subsystem** of the\n//! `kernel`. Top-level module files of subsystems reside directly in the `src` folder. For example,\n//! `src/memory.rs` contains code that is concerned with all things memory management.\n//!\n//! ## Visibility of processor architecture code\n//!\n//! Some of the `kernel`'s subsystems depend on low-level code that is specific to the target\n//! processor architecture. For each supported processor architecture, there exists a subfolder in\n//! `src/_arch`, for example, `src/_arch/aarch64`.\n//!\n//! The architecture folders mirror the subsystem modules laid out in `src`. For example,\n//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go\n//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in\n//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic\n//! module's name prefixed with `arch_`.\n//!\n//! For example, this is the top of `src/memory/mmu.rs`:\n//!\n//! ```\n//! #[cfg(target_arch = \"aarch64\")]\n//! #[path = \"../_arch/aarch64/memory/mmu.rs\"]\n//! mod arch_mmu;\n//! ```\n//!\n//! Often times, items from the `arch_ module` will be publicly reexported by the parent module.\n//! This way, each architecture specific module can provide its implementation of an item, while the\n//! caller must not be concerned which architecture has been conditionally compiled.\n//!\n//! ## BSP code\n//!\n//! `BSP` stands for Board Support Package. `BSP` code is organized under `src/bsp.rs` and contains\n//! target board specific definitions and functions. These are things such as the board's memory map\n//! or instances of drivers for devices that are featured on the respective board.\n//!\n//! Just like processor architecture code, the `BSP` code's module structure tries to mirror the\n//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is\n//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`.\n//!\n//! ## Kernel interfaces\n//!\n//! Both `arch` and `bsp` contain code that is conditionally compiled depending on the actual target\n//! and board for which the kernel is compiled. For example, the `interrupt controller` hardware of\n//! the `Raspberry Pi 3` and the `Raspberry Pi 4` is different, but we want the rest of the `kernel`\n//! code to play nicely with any of the two without much hassle.\n//!\n//! In order to provide a clean abstraction between `arch`, `bsp` and `generic kernel code`,\n//! `interface` traits are provided *whenever possible* and *where it makes sense*. They are defined\n//! in the respective subsystem module and help to enforce the idiom of *program to an interface,\n//! not an implementation*. For example, there will be a common IRQ handling interface which the two\n//! different interrupt controller `drivers` of both Raspberrys will implement, and only export the\n//! interface to the rest of the `kernel`.\n//!\n//! ```\n//!         +-------------------+\n//!         | Interface (Trait) |\n//!         |                   |\n//!         +--+-------------+--+\n//!            ^             ^\n//!            |             |\n//!            |             |\n//! +----------+--+       +--+----------+\n//! | kernel code |       |  bsp code   |\n//! |             |       |  arch code  |\n//! +-------------+       +-------------+\n//! ```\n//!\n//! # Summary\n//!\n//! For a logical `kernel` subsystem, corresponding code can be distributed over several physical\n//! locations. Here is an example for the **memory** subsystem:\n//!\n//! - `src/memory.rs` and `src/memory/**/*`\n//!   - Common code that is agnostic of target processor architecture and `BSP` characteristics.\n//!     - Example: A function to zero a chunk of memory.\n//!   - Interfaces for the memory subsystem that are implemented by `arch` or `BSP` code.\n//!     - Example: An `MMU` interface that defines `MMU` function prototypes.\n//! - `src/bsp/__board_name__/memory.rs` and `src/bsp/__board_name__/memory/**/*`\n//!   - `BSP` specific code.\n//!   - Example: The board's memory map (physical addresses of DRAM and MMIO devices).\n//! - `src/_arch/__arch_name__/memory.rs` and `src/_arch/__arch_name__/memory/**/*`\n//!   - Processor architecture specific code.\n//!   - Example: Implementation of the `MMU` interface for the `__arch_name__` processor\n//!     architecture.\n//!\n//! From a namespace perspective, **memory** subsystem code lives in:\n//!\n//! - `crate::memory::*`\n//! - `crate::bsp::memory::*`\n//!\n//! # Boot flow\n//!\n//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.\n//!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.\n//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.\n\n#![allow(clippy::upper_case_acronyms)]\n#![feature(asm_const)]\n#![feature(format_args_nl)]\n#![feature(panic_info_message)]\n#![feature(trait_alias)]\n#![no_main]\n#![no_std]\n\nmod bsp;\nmod console;\nmod cpu;\nmod driver;\nmod panic_wait;\nmod print;\nmod synchronization;\n\n/// Early init code.\n///\n/// # Safety\n///\n/// - Only a single core must be active and running this function.\n/// - The init calls in this function must appear in the correct order.\nunsafe fn kernel_init() -> ! {\n    // Initialize the BSP driver subsystem.\n    if let Err(x) = bsp::driver::init() {\n        panic!(\"Error initializing BSP driver subsystem: {}\", x);\n    }\n\n    // Initialize all device drivers.\n    driver::driver_manager().init_drivers();\n    // println! is usable from here on.\n\n    // Transition from unsafe to safe.\n    kernel_main()\n}\n\nconst MINILOAD_LOGO: &str = r#\"\n __  __ _      _ _                 _\n|  \\/  (_)_ _ (_) |   ___  __ _ __| |\n| |\\/| | | ' \\| | |__/ _ \\/ _` / _` |\n|_|  |_|_|_||_|_|____\\___/\\__,_\\__,_|\n\"#;\n\n/// The main function running after the early init.\nfn kernel_main() -> ! {\n    use console::console;\n\n    println!(\"{}\", MINILOAD_LOGO);\n    println!(\"{:^37}\", bsp::board_name());\n    println!();\n    println!(\"[ML] Requesting binary\");\n    console().flush();\n\n    // Discard any spurious received characters before starting with the loader protocol.\n    console().clear_rx();\n\n    // Notify `Minipush` to send the binary.\n    for _ in 0..3 {\n        console().write_char(3 as char);\n    }\n\n    // Read the binary's size.\n    let mut size: u32 = u32::from(console().read_char() as u8);\n    size |= u32::from(console().read_char() as u8) << 8;\n    size |= u32::from(console().read_char() as u8) << 16;\n    size |= u32::from(console().read_char() as u8) << 24;\n\n    // Trust it's not too big.\n    console().write_char('O');\n    console().write_char('K');\n\n    let kernel_addr: *mut u8 = bsp::memory::board_default_load_addr() as *mut u8;\n    unsafe {\n        // Read the kernel byte by byte.\n        for i in 0..size {\n            core::ptr::write_volatile(kernel_addr.offset(i as isize), console().read_char() as u8)\n        }\n    }\n\n    println!(\"[ML] Loaded! Executing the payload now\\n\");\n    console().flush();\n\n    // Use black magic to create a function pointer.\n    let kernel: fn() -> ! = unsafe { core::mem::transmute(kernel_addr) };\n\n    // Jump to loaded kernel!\n    kernel()\n}\n"
  },
  {
    "path": "06_uart_chainloader/src/panic_wait.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A panic handler that infinitely waits.\n\nuse crate::{cpu, println};\nuse core::panic::PanicInfo;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Stop immediately if called a second time.\n///\n/// # Note\n///\n/// Using atomics here relieves us from needing to use `unsafe` for the static variable.\n///\n/// On `AArch64`, which is the only implemented architecture at the time of writing this,\n/// [`AtomicBool::load`] and [`AtomicBool::store`] are lowered to ordinary load and store\n/// instructions. They are therefore safe to use even with MMU + caching deactivated.\n///\n/// [`AtomicBool::load`]: core::sync::atomic::AtomicBool::load\n/// [`AtomicBool::store`]: core::sync::atomic::AtomicBool::store\nfn panic_prevent_reenter() {\n    use core::sync::atomic::{AtomicBool, Ordering};\n\n    #[cfg(not(target_arch = \"aarch64\"))]\n    compile_error!(\"Add the target_arch to above's check if the following code is safe to use\");\n\n    static PANIC_IN_PROGRESS: AtomicBool = AtomicBool::new(false);\n\n    if !PANIC_IN_PROGRESS.load(Ordering::Relaxed) {\n        PANIC_IN_PROGRESS.store(true, Ordering::Relaxed);\n\n        return;\n    }\n\n    cpu::wait_forever()\n}\n\n#[panic_handler]\nfn panic(info: &PanicInfo) -> ! {\n    // Protect against panic infinite loops if any of the following code panics itself.\n    panic_prevent_reenter();\n\n    let (location, line, column) = match info.location() {\n        Some(loc) => (loc.file(), loc.line(), loc.column()),\n        _ => (\"???\", 0, 0),\n    };\n\n    println!(\n        \"Kernel panic!\\n\\n\\\n        Panic location:\\n      File '{}', line {}, column {}\\n\\n\\\n        {}\",\n        location,\n        line,\n        column,\n        info.message().unwrap_or(&format_args!(\"\")),\n    );\n\n    cpu::wait_forever()\n}\n"
  },
  {
    "path": "06_uart_chainloader/src/print.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Printing.\n\nuse crate::console;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n#[doc(hidden)]\npub fn _print(args: fmt::Arguments) {\n    console::console().write_fmt(args).unwrap();\n}\n\n/// Prints without a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! print {\n    ($($arg:tt)*) => ($crate::print::_print(format_args!($($arg)*)));\n}\n\n/// Prints with a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! println {\n    () => ($crate::print!(\"\\n\"));\n    ($($arg:tt)*) => ({\n        $crate::print::_print(format_args_nl!($($arg)*));\n    })\n}\n"
  },
  {
    "path": "06_uart_chainloader/src/synchronization.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Synchronization primitives.\n//!\n//! # Resources\n//!\n//!   - <https://doc.rust-lang.org/book/ch16-04-extensible-concurrency-sync-and-send.html>\n//!   - <https://stackoverflow.com/questions/59428096/understanding-the-send-trait>\n//!   - <https://doc.rust-lang.org/std/cell/index.html>\n\nuse core::cell::UnsafeCell;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Synchronization interfaces.\npub mod interface {\n\n    /// Any object implementing this trait guarantees exclusive access to the data wrapped within\n    /// the Mutex for the duration of the provided closure.\n    pub trait Mutex {\n        /// The type of the data that is wrapped by this mutex.\n        type Data;\n\n        /// Locks the mutex and grants the closure temporary mutable access to the wrapped data.\n        fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R;\n    }\n}\n\n/// A pseudo-lock for teaching purposes.\n///\n/// In contrast to a real Mutex implementation, does not protect against concurrent access from\n/// other cores to the contained data. This part is preserved for later lessons.\n///\n/// The lock will only be used as long as it is safe to do so, i.e. as long as the kernel is\n/// executing single-threaded, aka only running on a single core with interrupts disabled.\npub struct NullLock<T>\nwhere\n    T: ?Sized,\n{\n    data: UnsafeCell<T>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nunsafe impl<T> Send for NullLock<T> where T: ?Sized + Send {}\nunsafe impl<T> Sync for NullLock<T> where T: ?Sized + Send {}\n\nimpl<T> NullLock<T> {\n    /// Create an instance.\n    pub const fn new(data: T) -> Self {\n        Self {\n            data: UnsafeCell::new(data),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\n\nimpl<T> interface::Mutex for NullLock<T> {\n    type Data = T;\n\n    fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R {\n        // In a real lock, there would be code encapsulating this line that ensures that this\n        // mutable reference will ever only be given out once at a time.\n        let data = unsafe { &mut *self.data.get() };\n\n        f(data)\n    }\n}\n"
  },
  {
    "path": "06_uart_chainloader/tests/chainboot_test.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire_relative '../../common/serial/minipush'\nrequire_relative '../../common/tests/boot_test'\nrequire 'pty'\n\n# Match for the last print that 'demo_payload_rpiX.img' produces.\nEXPECTED_PRINT = 'Echoing input now'\n\n# Wait for request to power the target.\nclass PowerTargetRequestTest < SubtestBase\n    MINIPUSH_POWER_TARGET_REQUEST = 'Please power the target now'\n\n    def initialize(qemu_cmd, pty_main)\n        super()\n        @qemu_cmd = qemu_cmd\n        @pty_main = pty_main\n    end\n\n    def name\n        'Waiting for request to power target'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, MINIPUSH_POWER_TARGET_REQUEST)\n\n        # Now is the time to start QEMU with the chainloader binary. QEMU's virtual tty connects to\n        # the MiniPush instance spawned on pty_main, so that the two processes talk to each other.\n        Process.spawn(@qemu_cmd, in: @pty_main, out: @pty_main, err: '/dev/null')\n    end\nend\n\n# Extend BootTest so that it listens on the output of a MiniPush instance, which is itself connected\n# to a QEMU instance instead of a real HW.\nclass ChainbootTest < BootTest\n    MINIPUSH = '../common/serial/minipush.rb'\n\n    def initialize(qemu_cmd, payload_path)\n        super(qemu_cmd, EXPECTED_PRINT)\n\n        @test_name = 'Boot test using Minipush'\n\n        @payload_path = payload_path\n    end\n\n    private\n\n    # override\n    def setup\n        pty_main, pty_secondary = PTY.open\n        mp_out, _mp_in = PTY.spawn(\"ruby #{MINIPUSH} #{pty_secondary.path} #{@payload_path}\")\n\n        # The subtests (from this class and the parents) listen on @qemu_out_wrapped. Hence, point\n        # it to MiniPush's output.\n        @qemu_out_wrapped = PTYLoggerWrapper.new(mp_out, \"\\r\\n\")\n\n        # Important: Run this subtest before the one in the parent class.\n        @console_subtests.prepend(PowerTargetRequestTest.new(@qemu_cmd, pty_main))\n    end\n\n    # override\n    def finish\n        super()\n        @test_output.map! { |x| x.gsub(/.*\\r/, '  ') }\n    end\nend\n\n## -------------------------------------------------------------------------------------------------\n## Execution starts here\n## -------------------------------------------------------------------------------------------------\npayload_path = ARGV.pop\nqemu_cmd = ARGV.join(' ')\n\nChainbootTest.new(qemu_cmd, payload_path).run\n"
  },
  {
    "path": "06_uart_chainloader/update.sh",
    "content": "#!/usr/bin/env bash\n\ncd ../05_drivers_gpio_uart\nBSP=rpi4 make\ncp kernel8.img ../06_uart_chainloader/demo_payload_rpi4.img\nmake\ncp kernel8.img ../06_uart_chainloader/demo_payload_rpi3.img\nrm kernel8.img\n"
  },
  {
    "path": "07_timestamps/.vscode/settings.json",
    "content": "{\n    \"editor.formatOnSave\": true,\n    \"editor.rulers\": [100],\n    \"rust-analyzer.cargo.target\": \"aarch64-unknown-none-softfloat\",\n    \"rust-analyzer.cargo.features\": [\"bsp_rpi3\"],\n    \"rust-analyzer.checkOnSave.allTargets\": false,\n    \"rust-analyzer.checkOnSave.extraArgs\": [\"--bins\"],\n    \"rust-analyzer.lens.debug\": false,\n    \"rust-analyzer.lens.run\": false\n}\n"
  },
  {
    "path": "07_timestamps/Cargo.toml",
    "content": "[package]\nname = \"mingo\"\nversion = \"0.7.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n\n[profile.release]\nlto = true\n\n[features]\ndefault = []\nbsp_rpi3 = [\"tock-registers\"]\nbsp_rpi4 = [\"tock-registers\"]\n\n[[bin]]\nname = \"kernel\"\npath = \"src/main.rs\"\n\n##--------------------------------------------------------------------------------------------------\n## Dependencies\n##--------------------------------------------------------------------------------------------------\n\n[dependencies]\n\n# Optional dependencies\ntock-registers = { version = \"0.8.x\", default-features = false, features = [\"register_types\"], optional = true }\n\n# Platform specific dependencies\n[target.'cfg(target_arch = \"aarch64\")'.dependencies]\naarch64-cpu = { version = \"9.x.x\" }\n"
  },
  {
    "path": "07_timestamps/Makefile",
    "content": "## SPDX-License-Identifier: MIT OR Apache-2.0\n##\n## Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\ninclude ../common/docker.mk\ninclude ../common/format.mk\ninclude ../common/operating_system.mk\n\n##--------------------------------------------------------------------------------------------------\n## Optional, user-provided configuration values\n##--------------------------------------------------------------------------------------------------\n\n# Default to the RPi3.\nBSP ?= rpi3\n\n# Default to a serial device name that is common in Linux.\nDEV_SERIAL ?= /dev/ttyUSB0\n\n\n\n##--------------------------------------------------------------------------------------------------\n## BSP-specific configuration values\n##--------------------------------------------------------------------------------------------------\nQEMU_MISSING_STRING = \"This board is not yet supported for QEMU.\"\n\nifeq ($(BSP),rpi3)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE = raspi3\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    LD_SCRIPT_PATH    = $(shell pwd)/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a53\nelse ifeq ($(BSP),rpi4)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE =\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    LD_SCRIPT_PATH    = $(shell pwd)/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a72\nendif\n\n# Export for build.rs.\nexport LD_SCRIPT_PATH\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets and Prerequisites\n##--------------------------------------------------------------------------------------------------\nKERNEL_MANIFEST      = Cargo.toml\nKERNEL_LINKER_SCRIPT = kernel.ld\nLAST_BUILD_CONFIG    = target/$(BSP).build_config\n\nKERNEL_ELF      = target/$(TARGET)/release/kernel\n# This parses cargo's dep-info file.\n# https://doc.rust-lang.org/cargo/guide/build-cache.html#dep-info-files\nKERNEL_ELF_DEPS = $(filter-out %: ,$(file < $(KERNEL_ELF).d)) $(KERNEL_MANIFEST) $(LAST_BUILD_CONFIG)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Command building blocks\n##--------------------------------------------------------------------------------------------------\nRUSTFLAGS = $(RUSTC_MISC_ARGS)                   \\\n    -C link-arg=--library-path=$(LD_SCRIPT_PATH) \\\n    -C link-arg=--script=$(KERNEL_LINKER_SCRIPT)\n\nRUSTFLAGS_PEDANTIC = $(RUSTFLAGS) \\\n    -D warnings                   \\\n    -D missing_docs\n\nFEATURES      = --features bsp_$(BSP)\nCOMPILER_ARGS = --target=$(TARGET) \\\n    $(FEATURES)                    \\\n    --release\n\nRUSTC_CMD   = cargo rustc $(COMPILER_ARGS)\nDOC_CMD     = cargo doc $(COMPILER_ARGS)\nCLIPPY_CMD  = cargo clippy $(COMPILER_ARGS)\nOBJCOPY_CMD = rust-objcopy \\\n    --strip-all            \\\n    -O binary\n\nEXEC_QEMU          = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE)\nEXEC_TEST_DISPATCH = ruby ../common/tests/dispatch.rb\nEXEC_MINIPUSH      = ruby ../common/serial/minipush.rb\n\n##------------------------------------------------------------------------------\n## Dockerization\n##------------------------------------------------------------------------------\nDOCKER_CMD            = docker run -t --rm -v $(shell pwd):/work/tutorial -w /work/tutorial\nDOCKER_CMD_INTERACT   = $(DOCKER_CMD) -i\nDOCKER_ARG_DIR_COMMON = -v $(shell pwd)/../common:/work/common\nDOCKER_ARG_DEV        = --privileged -v /dev:/dev\n\n# DOCKER_IMAGE defined in include file (see top of this file).\nDOCKER_QEMU  = $(DOCKER_CMD_INTERACT) $(DOCKER_IMAGE)\nDOCKER_TOOLS = $(DOCKER_CMD) $(DOCKER_IMAGE)\nDOCKER_TEST  = $(DOCKER_CMD) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\n\n# Dockerize commands, which require USB device passthrough, only on Linux.\nifeq ($(shell uname -s),Linux)\n    DOCKER_CMD_DEV = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_DEV)\n\n    DOCKER_CHAINBOOT = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\nendif\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: all doc qemu chainboot clippy clean readelf objdump nm check\n\nall: $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Save the configuration as a file, so make understands if it changed.\n##------------------------------------------------------------------------------\n$(LAST_BUILD_CONFIG):\n\t@rm -f target/*.build_config\n\t@mkdir -p target\n\t@touch $(LAST_BUILD_CONFIG)\n\n##------------------------------------------------------------------------------\n## Compile the kernel ELF\n##------------------------------------------------------------------------------\n$(KERNEL_ELF): $(KERNEL_ELF_DEPS)\n\t$(call color_header, \"Compiling kernel ELF - $(BSP)\")\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(RUSTC_CMD)\n\n##------------------------------------------------------------------------------\n## Generate the stripped kernel binary\n##------------------------------------------------------------------------------\n$(KERNEL_BIN): $(KERNEL_ELF)\n\t$(call color_header, \"Generating stripped binary\")\n\t@$(OBJCOPY_CMD) $(KERNEL_ELF) $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Name\")\n\t@echo $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Size\")\n\t$(call disk_usage_KiB, $(KERNEL_BIN))\n\n##------------------------------------------------------------------------------\n## Generate the documentation\n##------------------------------------------------------------------------------\ndoc:\n\t$(call color_header, \"Generating docs\")\n\t@$(DOC_CMD) --document-private-items --open\n\n##------------------------------------------------------------------------------\n## Run the kernel in QEMU\n##------------------------------------------------------------------------------\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\nqemu:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\nqemu: $(KERNEL_BIN)\n\t$(call color_header, \"Launching QEMU\")\n\t@$(DOCKER_QEMU) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\nendif\n\n##------------------------------------------------------------------------------\n## Push the kernel to the real HW target\n##------------------------------------------------------------------------------\nchainboot: $(KERNEL_BIN)\n\t@$(DOCKER_CHAINBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run clippy\n##------------------------------------------------------------------------------\nclippy:\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(CLIPPY_CMD)\n\n##------------------------------------------------------------------------------\n## Clean\n##------------------------------------------------------------------------------\nclean:\n\trm -rf target $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run readelf\n##------------------------------------------------------------------------------\nreadelf: $(KERNEL_ELF)\n\t$(call color_header, \"Launching readelf\")\n\t@$(DOCKER_TOOLS) $(READELF_BINARY) --headers $(KERNEL_ELF)\n\n##------------------------------------------------------------------------------\n## Run objdump\n##------------------------------------------------------------------------------\nobjdump: $(KERNEL_ELF)\n\t$(call color_header, \"Launching objdump\")\n\t@$(DOCKER_TOOLS) $(OBJDUMP_BINARY) --disassemble --demangle \\\n                --section .text   \\\n                --section .rodata \\\n                $(KERNEL_ELF) | rustfilt\n\n##------------------------------------------------------------------------------\n## Run nm\n##------------------------------------------------------------------------------\nnm: $(KERNEL_ELF)\n\t$(call color_header, \"Launching nm\")\n\t@$(DOCKER_TOOLS) $(NM_BINARY) --demangle --print-size $(KERNEL_ELF) | sort | rustfilt\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Testing targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: test test_boot\n\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\ntest_boot test:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\n##------------------------------------------------------------------------------\n## Run boot test\n##------------------------------------------------------------------------------\ntest_boot: $(KERNEL_BIN)\n\t$(call color_header, \"Boot test - $(BSP)\")\n\t@$(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\ntest: test_boot\n\nendif\n"
  },
  {
    "path": "07_timestamps/README.CN.md",
    "content": "# 教程 07 - 时间戳\n\n## tl;dr\n\n- 我们为计时器硬件添加了抽象，并在`_arch/aarch64`中实现了ARM架构计时器。\n- 新的计时器函数用于给UART打印添加时间戳，并且用于消除`GPIO`设备驱动中基于周期的延迟，从而提高准确性。\n- 添加了`warn!()`宏。\n\n## 测试它\n\n请通过 chainboot 进行检查（在上一个教程中添加）。\n```console\n$ make chainboot\n[...]\nMinipush 1.0\n\n[MP] ⏳ Waiting for /dev/ttyUSB0\n[MP] ✅ Serial connected\n[MP] 🔌 Please power the target now\n\n __  __ _      _ _                 _\n|  \\/  (_)_ _ (_) |   ___  __ _ __| |\n| |\\/| | | ' \\| | |__/ _ \\/ _` / _` |\n|_|  |_|_|_||_|_|____\\___/\\__,_\\__,_|\n\n           Raspberry Pi 3\n\n[ML] Requesting binary\n[MP] ⏩ Pushing 12 KiB =========================================🦀 100% 0 KiB/s Time: 00:00:00\n[ML] Loaded! Executing the payload now\n\n[    0.143123] mingo version 0.7.0\n[    0.143323] Booting on: Raspberry Pi 3\n[    0.143778] Architectural timer resolution: 52 ns\n[    0.144352] Drivers loaded:\n[    0.144688]       1. BCM PL011 UART\n[    0.145110]       2. BCM GPIO\n[W   0.145469] Spin duration smaller than architecturally supported, skipping\n[    0.146313] Spinning for 1 second\n[    1.146715] Spinning for 1 second\n[    2.146938] Spinning for 1 second\n```\n\n## 相比之前的变化（diff）\n请检查[英文版本](README.md#diff-to-previous)，这是最新的。\n"
  },
  {
    "path": "07_timestamps/README.md",
    "content": "# Tutorial 07 - Timestamps\n\n## tl;dr\n\n- We add abstractions for timer hardware, and implement them for the ARM architectural timer in\n  `_arch/aarch64`.\n- The new timer functions are used to annotate UART prints with timestamps, and to get rid of the\n  cycle-based delays in the `GPIO` device driver, which boosts accuracy.\n- A `warn!()` macro is added.\n\n## Test it\n\nCheck it out via chainboot (added in previous tutorial):\n```console\n$ make chainboot\n[...]\nMinipush 1.0\n\n[MP] ⏳ Waiting for /dev/ttyUSB0\n[MP] ✅ Serial connected\n[MP] 🔌 Please power the target now\n\n __  __ _      _ _                 _\n|  \\/  (_)_ _ (_) |   ___  __ _ __| |\n| |\\/| | | ' \\| | |__/ _ \\/ _` / _` |\n|_|  |_|_|_||_|_|____\\___/\\__,_\\__,_|\n\n           Raspberry Pi 3\n\n[ML] Requesting binary\n[MP] ⏩ Pushing 12 KiB =========================================🦀 100% 0 KiB/s Time: 00:00:00\n[ML] Loaded! Executing the payload now\n\n[    0.143123] mingo version 0.7.0\n[    0.143323] Booting on: Raspberry Pi 3\n[    0.143778] Architectural timer resolution: 52 ns\n[    0.144352] Drivers loaded:\n[    0.144688]       1. BCM PL011 UART\n[    0.145110]       2. BCM GPIO\n[W   0.145469] Spin duration smaller than architecturally supported, skipping\n[    0.146313] Spinning for 1 second\n[    1.146715] Spinning for 1 second\n[    2.146938] Spinning for 1 second\n```\n\n## Diff to previous\n```diff\n\ndiff -uNr 06_uart_chainloader/Cargo.toml 07_timestamps/Cargo.toml\n--- 06_uart_chainloader/Cargo.toml\n+++ 07_timestamps/Cargo.toml\n@@ -1,6 +1,6 @@\n [package]\n name = \"mingo\"\n-version = \"0.6.0\"\n+version = \"0.7.0\"\n authors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\n edition = \"2021\"\n\nBinary files 06_uart_chainloader/demo_payload_rpi3.img and 07_timestamps/demo_payload_rpi3.img differ\nBinary files 06_uart_chainloader/demo_payload_rpi4.img and 07_timestamps/demo_payload_rpi4.img differ\n\ndiff -uNr 06_uart_chainloader/Makefile 07_timestamps/Makefile\n--- 06_uart_chainloader/Makefile\n+++ 07_timestamps/Makefile\n@@ -24,29 +24,27 @@\n QEMU_MISSING_STRING = \"This board is not yet supported for QEMU.\"\n\n ifeq ($(BSP),rpi3)\n-    TARGET                 = aarch64-unknown-none-softfloat\n-    KERNEL_BIN             = kernel8.img\n-    QEMU_BINARY            = qemu-system-aarch64\n-    QEMU_MACHINE_TYPE      = raspi3\n-    QEMU_RELEASE_ARGS      = -serial stdio -display none\n-    OBJDUMP_BINARY         = aarch64-none-elf-objdump\n-    NM_BINARY              = aarch64-none-elf-nm\n-    READELF_BINARY         = aarch64-none-elf-readelf\n-    LD_SCRIPT_PATH         = $(shell pwd)/src/bsp/raspberrypi\n-    RUSTC_MISC_ARGS        = -C target-cpu=cortex-a53\n-    CHAINBOOT_DEMO_PAYLOAD = demo_payload_rpi3.img\n+    TARGET            = aarch64-unknown-none-softfloat\n+    KERNEL_BIN        = kernel8.img\n+    QEMU_BINARY       = qemu-system-aarch64\n+    QEMU_MACHINE_TYPE = raspi3\n+    QEMU_RELEASE_ARGS = -serial stdio -display none\n+    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n+    NM_BINARY         = aarch64-none-elf-nm\n+    READELF_BINARY    = aarch64-none-elf-readelf\n+    LD_SCRIPT_PATH    = $(shell pwd)/src/bsp/raspberrypi\n+    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a53\n else ifeq ($(BSP),rpi4)\n-    TARGET                 = aarch64-unknown-none-softfloat\n-    KERNEL_BIN             = kernel8.img\n-    QEMU_BINARY            = qemu-system-aarch64\n-    QEMU_MACHINE_TYPE      =\n-    QEMU_RELEASE_ARGS      = -serial stdio -display none\n-    OBJDUMP_BINARY         = aarch64-none-elf-objdump\n-    NM_BINARY              = aarch64-none-elf-nm\n-    READELF_BINARY         = aarch64-none-elf-readelf\n-    LD_SCRIPT_PATH         = $(shell pwd)/src/bsp/raspberrypi\n-    RUSTC_MISC_ARGS        = -C target-cpu=cortex-a72\n-    CHAINBOOT_DEMO_PAYLOAD = demo_payload_rpi4.img\n+    TARGET            = aarch64-unknown-none-softfloat\n+    KERNEL_BIN        = kernel8.img\n+    QEMU_BINARY       = qemu-system-aarch64\n+    QEMU_MACHINE_TYPE =\n+    QEMU_RELEASE_ARGS = -serial stdio -display none\n+    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n+    NM_BINARY         = aarch64-none-elf-nm\n+    READELF_BINARY    = aarch64-none-elf-readelf\n+    LD_SCRIPT_PATH    = $(shell pwd)/src/bsp/raspberrypi\n+    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a72\n endif\n\n # Export for build.rs.\n@@ -92,7 +90,7 @@\n     -O binary\n\n EXEC_QEMU          = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE)\n-EXEC_TEST_MINIPUSH = ruby tests/chainboot_test.rb\n+EXEC_TEST_DISPATCH = ruby ../common/tests/dispatch.rb\n EXEC_MINIPUSH      = ruby ../common/serial/minipush.rb\n\n ##------------------------------------------------------------------------------\n@@ -162,7 +160,7 @@\n ##------------------------------------------------------------------------------\n ifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\n-qemu qemuasm:\n+qemu:\n \t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\n else # QEMU is supported.\n@@ -171,17 +169,13 @@\n \t$(call color_header, \"Launching QEMU\")\n \t@$(DOCKER_QEMU) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\n-qemuasm: $(KERNEL_BIN)\n-\t$(call color_header, \"Launching QEMU with ASM output\")\n-\t@$(DOCKER_QEMU) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN) -d in_asm\n-\n endif\n\n ##------------------------------------------------------------------------------\n ## Push the kernel to the real HW target\n ##------------------------------------------------------------------------------\n chainboot: $(KERNEL_BIN)\n-\t@$(DOCKER_CHAINBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(CHAINBOOT_DEMO_PAYLOAD)\n+\t@$(DOCKER_CHAINBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(KERNEL_BIN)\n\n ##------------------------------------------------------------------------------\n ## Run clippy\n@@ -238,8 +232,7 @@\n ##------------------------------------------------------------------------------\n test_boot: $(KERNEL_BIN)\n \t$(call color_header, \"Boot test - $(BSP)\")\n-\t@$(DOCKER_TEST) $(EXEC_TEST_MINIPUSH) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) \\\n-\t\t-kernel $(KERNEL_BIN) $(CHAINBOOT_DEMO_PAYLOAD)\n+\t@$(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\n test: test_boot\n\n\ndiff -uNr 06_uart_chainloader/src/_arch/aarch64/cpu/boot.s 07_timestamps/src/_arch/aarch64/cpu/boot.s\n--- 06_uart_chainloader/src/_arch/aarch64/cpu/boot.s\n+++ 07_timestamps/src/_arch/aarch64/cpu/boot.s\n@@ -18,17 +18,6 @@\n \tadd\t\\register, \\register, #:lo12:\\symbol\n .endm\n\n-// Load the address of a symbol into a register, absolute.\n-//\n-// # Resources\n-//\n-// - https://sourceware.org/binutils/docs-2.36/as/AArch64_002dRelocations.html\n-.macro ADR_ABS register, symbol\n-\tmovz\t\\register, #:abs_g2:\\symbol\n-\tmovk\t\\register, #:abs_g1_nc:\\symbol\n-\tmovk\t\\register, #:abs_g0_nc:\\symbol\n-.endm\n-\n //--------------------------------------------------------------------------------------------------\n // Public Code\n //--------------------------------------------------------------------------------------------------\n@@ -48,35 +37,31 @@\n \t// If execution reaches here, it is the boot core.\n\n \t// Initialize DRAM.\n-\tADR_ABS\tx0, __bss_start\n-\tADR_ABS x1, __bss_end_exclusive\n+\tADR_REL\tx0, __bss_start\n+\tADR_REL x1, __bss_end_exclusive\n\n .L_bss_init_loop:\n \tcmp\tx0, x1\n-\tb.eq\t.L_relocate_binary\n+\tb.eq\t.L_prepare_rust\n \tstp\txzr, xzr, [x0], #16\n \tb\t.L_bss_init_loop\n\n-\t// Next, relocate the binary.\n-.L_relocate_binary:\n-\tADR_REL\tx0, __binary_nonzero_start         // The address the binary got loaded to.\n-\tADR_ABS\tx1, __binary_nonzero_start         // The address the binary was linked to.\n-\tADR_ABS\tx2, __binary_nonzero_end_exclusive\n-\n-.L_copy_loop:\n-\tldr\tx3, [x0], #8\n-\tstr\tx3, [x1], #8\n-\tcmp\tx1, x2\n-\tb.lo\t.L_copy_loop\n-\n \t// Prepare the jump to Rust code.\n+.L_prepare_rust:\n \t// Set the stack pointer.\n-\tADR_ABS\tx0, __boot_core_stack_end_exclusive\n+\tADR_REL\tx0, __boot_core_stack_end_exclusive\n \tmov\tsp, x0\n\n-\t// Jump to the relocated Rust code.\n-\tADR_ABS\tx1, _start_rust\n-\tbr\tx1\n+\t// Read the CPU's timer counter frequency and store it in ARCH_TIMER_COUNTER_FREQUENCY.\n+\t// Abort if the frequency read back as 0.\n+\tADR_REL\tx1, ARCH_TIMER_COUNTER_FREQUENCY // provided by aarch64/time.rs\n+\tmrs\tx2, CNTFRQ_EL0\n+\tcmp\tx2, xzr\n+\tb.eq\t.L_parking_loop\n+\tstr\tw2, [x1]\n+\n+\t// Jump to Rust code.\n+\tb\t_start_rust\n\n \t// Infinitely wait for events (aka \"park the core\").\n .L_parking_loop:\n\ndiff -uNr 06_uart_chainloader/src/_arch/aarch64/cpu.rs 07_timestamps/src/_arch/aarch64/cpu.rs\n--- 06_uart_chainloader/src/_arch/aarch64/cpu.rs\n+++ 07_timestamps/src/_arch/aarch64/cpu.rs\n@@ -19,15 +19,6 @@\n\n pub use asm::nop;\n\n-/// Spin for `n` cycles.\n-#[cfg(feature = \"bsp_rpi3\")]\n-#[inline(always)]\n-pub fn spin_for_cycles(n: usize) {\n-    for _ in 0..n {\n-        asm::nop();\n-    }\n-}\n-\n /// Pause execution on the core.\n #[inline(always)]\n pub fn wait_forever() -> ! {\n\ndiff -uNr 06_uart_chainloader/src/_arch/aarch64/time.rs 07_timestamps/src/_arch/aarch64/time.rs\n--- 06_uart_chainloader/src/_arch/aarch64/time.rs\n+++ 07_timestamps/src/_arch/aarch64/time.rs\n@@ -0,0 +1,162 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Architectural timer primitives.\n+//!\n+//! # Orientation\n+//!\n+//! Since arch modules are imported into generic modules using the path attribute, the path of this\n+//! file is:\n+//!\n+//! crate::time::arch_time\n+\n+use crate::warn;\n+use aarch64_cpu::{asm::barrier, registers::*};\n+use core::{\n+    num::{NonZeroU128, NonZeroU32, NonZeroU64},\n+    ops::{Add, Div},\n+    time::Duration,\n+};\n+use tock_registers::interfaces::Readable;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+const NANOSEC_PER_SEC: NonZeroU64 = NonZeroU64::new(1_000_000_000).unwrap();\n+\n+#[derive(Copy, Clone, PartialOrd, PartialEq)]\n+struct GenericTimerCounterValue(u64);\n+\n+//--------------------------------------------------------------------------------------------------\n+// Global instances\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Boot assembly code overwrites this value with the value of CNTFRQ_EL0 before any Rust code is\n+/// executed. This given value here is just a (safe) dummy.\n+#[no_mangle]\n+static ARCH_TIMER_COUNTER_FREQUENCY: NonZeroU32 = NonZeroU32::MIN;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Code\n+//--------------------------------------------------------------------------------------------------\n+\n+fn arch_timer_counter_frequency() -> NonZeroU32 {\n+    // Read volatile is needed here to prevent the compiler from optimizing\n+    // ARCH_TIMER_COUNTER_FREQUENCY away.\n+    //\n+    // This is safe, because all the safety requirements as stated in read_volatile()'s\n+    // documentation are fulfilled.\n+    unsafe { core::ptr::read_volatile(&ARCH_TIMER_COUNTER_FREQUENCY) }\n+}\n+\n+impl GenericTimerCounterValue {\n+    pub const MAX: Self = GenericTimerCounterValue(u64::MAX);\n+}\n+\n+impl Add for GenericTimerCounterValue {\n+    type Output = Self;\n+\n+    fn add(self, other: Self) -> Self {\n+        GenericTimerCounterValue(self.0.wrapping_add(other.0))\n+    }\n+}\n+\n+impl From<GenericTimerCounterValue> for Duration {\n+    fn from(counter_value: GenericTimerCounterValue) -> Self {\n+        if counter_value.0 == 0 {\n+            return Duration::ZERO;\n+        }\n+\n+        let frequency: NonZeroU64 = arch_timer_counter_frequency().into();\n+\n+        // Div<NonZeroU64> implementation for u64 cannot panic.\n+        let secs = counter_value.0.div(frequency);\n+\n+        // This is safe, because frequency can never be greater than u32::MAX, which means the\n+        // largest theoretical value for sub_second_counter_value is (u32::MAX - 1). Therefore,\n+        // (sub_second_counter_value * NANOSEC_PER_SEC) cannot overflow an u64.\n+        //\n+        // The subsequent division ensures the result fits into u32, since the max result is smaller\n+        // than NANOSEC_PER_SEC. Therefore, just cast it to u32 using `as`.\n+        let sub_second_counter_value = counter_value.0 modulo frequency;\n+        let nanos = unsafe { sub_second_counter_value.unchecked_mul(u64::from(NANOSEC_PER_SEC)) }\n+            .div(frequency) as u32;\n+\n+        Duration::new(secs, nanos)\n+    }\n+}\n+\n+fn max_duration() -> Duration {\n+    Duration::from(GenericTimerCounterValue::MAX)\n+}\n+\n+impl TryFrom<Duration> for GenericTimerCounterValue {\n+    type Error = &'static str;\n+\n+    fn try_from(duration: Duration) -> Result<Self, Self::Error> {\n+        if duration < resolution() {\n+            return Ok(GenericTimerCounterValue(0));\n+        }\n+\n+        if duration > max_duration() {\n+            return Err(\"Conversion error. Duration too big\");\n+        }\n+\n+        let frequency: u128 = u32::from(arch_timer_counter_frequency()) as u128;\n+        let duration: u128 = duration.as_nanos();\n+\n+        // This is safe, because frequency can never be greater than u32::MAX, and\n+        // (Duration::MAX.as_nanos() * u32::MAX) < u128::MAX.\n+        let counter_value =\n+            unsafe { duration.unchecked_mul(frequency) }.div(NonZeroU128::from(NANOSEC_PER_SEC));\n+\n+        // Since we checked above that we are <= max_duration(), just cast to u64.\n+        Ok(GenericTimerCounterValue(counter_value as u64))\n+    }\n+}\n+\n+#[inline(always)]\n+fn read_cntpct() -> GenericTimerCounterValue {\n+    // Prevent that the counter is read ahead of time due to out-of-order execution.\n+    barrier::isb(barrier::SY);\n+    let cnt = CNTPCT_EL0.get();\n+\n+    GenericTimerCounterValue(cnt)\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+/// The timer's resolution.\n+pub fn resolution() -> Duration {\n+    Duration::from(GenericTimerCounterValue(1))\n+}\n+\n+/// The uptime since power-on of the device.\n+///\n+/// This includes time consumed by firmware and bootloaders.\n+pub fn uptime() -> Duration {\n+    read_cntpct().into()\n+}\n+\n+/// Spin for a given duration.\n+pub fn spin_for(duration: Duration) {\n+    let curr_counter_value = read_cntpct();\n+\n+    let counter_value_delta: GenericTimerCounterValue = match duration.try_into() {\n+        Err(msg) => {\n+            warn!(\"spin_for: {}. Skipping\", msg);\n+            return;\n+        }\n+        Ok(val) => val,\n+    };\n+    let counter_value_target = curr_counter_value + counter_value_delta;\n+\n+    // Busy wait.\n+    //\n+    // Read CNTPCT_EL0 directly to avoid the ISB that is part of [`read_cntpct`].\n+    while GenericTimerCounterValue(CNTPCT_EL0.get()) < counter_value_target {}\n+}\n\ndiff -uNr 06_uart_chainloader/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs 07_timestamps/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs\n--- 06_uart_chainloader/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs\n+++ 07_timestamps/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs\n@@ -140,25 +140,19 @@\n     /// Disable pull-up/down on pins 14 and 15.\n     #[cfg(feature = \"bsp_rpi3\")]\n     fn disable_pud_14_15_bcm2837(&mut self) {\n-        use crate::cpu;\n+        use crate::time;\n+        use core::time::Duration;\n\n-        // Make an educated guess for a good delay value (Sequence described in the BCM2837\n-        // peripherals PDF).\n-        //\n-        // - According to Wikipedia, the fastest RPi4 clocks around 1.5 GHz.\n-        // - The Linux 2837 GPIO driver waits 1 µs between the steps.\n-        //\n-        // So lets try to be on the safe side and default to 2000 cycles, which would equal 1 µs\n-        // would the CPU be clocked at 2 GHz.\n-        const DELAY: usize = 2000;\n+        // The Linux 2837 GPIO driver waits 1 µs between the steps.\n+        const DELAY: Duration = Duration::from_micros(1);\n\n         self.registers.GPPUD.write(GPPUD::PUD::Off);\n-        cpu::spin_for_cycles(DELAY);\n+        time::time_manager().spin_for(DELAY);\n\n         self.registers\n             .GPPUDCLK0\n             .write(GPPUDCLK0::PUDCLK15::AssertClock + GPPUDCLK0::PUDCLK14::AssertClock);\n-        cpu::spin_for_cycles(DELAY);\n+        time::time_manager().spin_for(DELAY);\n\n         self.registers.GPPUD.write(GPPUD::PUD::Off);\n         self.registers.GPPUDCLK0.set(0);\n\ndiff -uNr 06_uart_chainloader/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs 07_timestamps/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs\n--- 06_uart_chainloader/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs\n+++ 07_timestamps/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs\n@@ -275,7 +275,7 @@\n     }\n\n     /// Retrieve a character.\n-    fn read_char(&mut self, blocking_mode: BlockingMode) -> Option<char> {\n+    fn read_char_converting(&mut self, blocking_mode: BlockingMode) -> Option<char> {\n         // If RX FIFO is empty,\n         if self.registers.FR.matches_all(FR::RXFE::SET) {\n             // immediately return in non-blocking mode.\n@@ -290,7 +290,12 @@\n         }\n\n         // Read one character.\n-        let ret = self.registers.DR.get() as u8 as char;\n+        let mut ret = self.registers.DR.get() as u8 as char;\n+\n+        // Convert carrige return to newline.\n+        if ret == '\\r' {\n+            ret = '\\n'\n+        }\n\n         // Update statistics.\n         self.chars_read += 1;\n@@ -376,14 +381,14 @@\n impl console::interface::Read for PL011Uart {\n     fn read_char(&self) -> char {\n         self.inner\n-            .lock(|inner| inner.read_char(BlockingMode::Blocking).unwrap())\n+            .lock(|inner| inner.read_char_converting(BlockingMode::Blocking).unwrap())\n     }\n\n     fn clear_rx(&self) {\n         // Read from the RX FIFO until it is indicating empty.\n         while self\n             .inner\n-            .lock(|inner| inner.read_char(BlockingMode::NonBlocking))\n+            .lock(|inner| inner.read_char_converting(BlockingMode::NonBlocking))\n             .is_some()\n         {}\n     }\n\ndiff -uNr 06_uart_chainloader/src/bsp/raspberrypi/driver.rs 07_timestamps/src/bsp/raspberrypi/driver.rs\n--- 06_uart_chainloader/src/bsp/raspberrypi/driver.rs\n+++ 07_timestamps/src/bsp/raspberrypi/driver.rs\n@@ -57,6 +57,17 @@\n /// # Safety\n ///\n /// See child function calls.\n+///\n+/// # Note\n+///\n+/// Using atomics here relieves us from needing to use `unsafe` for the static variable.\n+///\n+/// On `AArch64`, which is the only implemented architecture at the time of writing this,\n+/// [`AtomicBool::load`] and [`AtomicBool::store`] are lowered to ordinary load and store\n+/// instructions. They are therefore safe to use even with MMU + caching deactivated.\n+///\n+/// [`AtomicBool::load`]: core::sync::atomic::AtomicBool::load\n+/// [`AtomicBool::store`]: core::sync::atomic::AtomicBool::store\n pub unsafe fn init() -> Result<(), &'static str> {\n     static INIT_DONE: AtomicBool = AtomicBool::new(false);\n     if INIT_DONE.load(Ordering::Relaxed) {\n\ndiff -uNr 06_uart_chainloader/src/bsp/raspberrypi/kernel.ld 07_timestamps/src/bsp/raspberrypi/kernel.ld\n--- 06_uart_chainloader/src/bsp/raspberrypi/kernel.ld\n+++ 07_timestamps/src/bsp/raspberrypi/kernel.ld\n@@ -3,6 +3,8 @@\n  * Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n  */\n\n+__rpi_phys_dram_start_addr = 0;\n+\n /* The physical address at which the the kernel binary will be loaded by the Raspberry's firmware */\n __rpi_phys_binary_load_addr = 0x80000;\n\n@@ -26,8 +28,7 @@\n\n SECTIONS\n {\n-    /* Set the link address to 32 MiB */\n-    . = 0x2000000;\n+    . =  __rpi_phys_dram_start_addr;\n\n     /***********************************************************************************************\n     * Boot Core Stack\n@@ -44,7 +45,6 @@\n     /***********************************************************************************************\n     * Code + RO Data + Global Offset Table\n     ***********************************************************************************************/\n-    __binary_nonzero_start = .;\n     .text :\n     {\n         KEEP(*(.text._start))\n@@ -60,10 +60,6 @@\n     ***********************************************************************************************/\n     .data : { *(.data*) } :segment_data\n\n-    /* Fill up to 8 byte, b/c relocating the binary is done in u64 chunks */\n-    . = ALIGN(8);\n-    __binary_nonzero_end_exclusive = .;\n-\n     /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */\n     .bss (NOLOAD) : ALIGN(16)\n     {\n\ndiff -uNr 06_uart_chainloader/src/bsp/raspberrypi/memory.rs 07_timestamps/src/bsp/raspberrypi/memory.rs\n--- 06_uart_chainloader/src/bsp/raspberrypi/memory.rs\n+++ 07_timestamps/src/bsp/raspberrypi/memory.rs\n@@ -11,7 +11,6 @@\n /// The board's physical memory map.\n #[rustfmt::skip]\n pub(super) mod map {\n-    pub const BOARD_DEFAULT_LOAD_ADDRESS: usize =        0x8_0000;\n\n     pub const GPIO_OFFSET:         usize = 0x0020_0000;\n     pub const UART_OFFSET:         usize = 0x0020_1000;\n@@ -36,13 +35,3 @@\n         pub const PL011_UART_START: usize = START + UART_OFFSET;\n     }\n }\n-\n-//--------------------------------------------------------------------------------------------------\n-// Public Code\n-//--------------------------------------------------------------------------------------------------\n-\n-/// The address on which the Raspberry firmware loads every binary by default.\n-#[inline(always)]\n-pub fn board_default_load_addr() -> *const u64 {\n-    map::BOARD_DEFAULT_LOAD_ADDRESS as _\n-}\n\ndiff -uNr 06_uart_chainloader/src/cpu.rs 07_timestamps/src/cpu.rs\n--- 06_uart_chainloader/src/cpu.rs\n+++ 07_timestamps/src/cpu.rs\n@@ -14,6 +14,3 @@\n // Architectural Public Reexports\n //--------------------------------------------------------------------------------------------------\n pub use arch_cpu::{nop, wait_forever};\n-\n-#[cfg(feature = \"bsp_rpi3\")]\n-pub use arch_cpu::spin_for_cycles;\n\ndiff -uNr 06_uart_chainloader/src/driver.rs 07_timestamps/src/driver.rs\n--- 06_uart_chainloader/src/driver.rs\n+++ 07_timestamps/src/driver.rs\n@@ -4,7 +4,10 @@\n\n //! Driver support.\n\n-use crate::synchronization::{interface::Mutex, NullLock};\n+use crate::{\n+    info,\n+    synchronization::{interface::Mutex, NullLock},\n+};\n\n //--------------------------------------------------------------------------------------------------\n // Private Definitions\n@@ -151,4 +154,14 @@\n             }\n         });\n     }\n+\n+    /// Enumerate all registered device drivers.\n+    pub fn enumerate(&self) {\n+        let mut i: usize = 1;\n+        self.for_each_descriptor(|descriptor| {\n+            info!(\"      {}. {}\", i, descriptor.device_driver.compatible());\n+\n+            i += 1;\n+        });\n+    }\n }\n\ndiff -uNr 06_uart_chainloader/src/main.rs 07_timestamps/src/main.rs\n--- 06_uart_chainloader/src/main.rs\n+++ 07_timestamps/src/main.rs\n@@ -108,9 +108,12 @@\n\n #![allow(clippy::upper_case_acronyms)]\n #![feature(asm_const)]\n+#![feature(const_option)]\n #![feature(format_args_nl)]\n+#![feature(nonzero_min_max)]\n #![feature(panic_info_message)]\n #![feature(trait_alias)]\n+#![feature(unchecked_math)]\n #![no_main]\n #![no_std]\n\n@@ -121,6 +124,7 @@\n mod panic_wait;\n mod print;\n mod synchronization;\n+mod time;\n\n /// Early init code.\n ///\n@@ -142,55 +146,30 @@\n     kernel_main()\n }\n\n-const MINILOAD_LOGO: &str = r#\"\n- __  __ _      _ _                 _\n-|  \\/  (_)_ _ (_) |   ___  __ _ __| |\n-| |\\/| | | ' \\| | |__/ _ \\/ _` / _` |\n-|_|  |_|_|_||_|_|____\\___/\\__,_\\__,_|\n-\"#;\n-\n /// The main function running after the early init.\n fn kernel_main() -> ! {\n-    use console::console;\n-\n-    println!(\"{}\", MINILOAD_LOGO);\n-    println!(\"{:^37}\", bsp::board_name());\n-    println!();\n-    println!(\"[ML] Requesting binary\");\n-    console().flush();\n-\n-    // Discard any spurious received characters before starting with the loader protocol.\n-    console().clear_rx();\n-\n-    // Notify `Minipush` to send the binary.\n-    for _ in 0..3 {\n-        console().write_char(3 as char);\n-    }\n+    use core::time::Duration;\n\n-    // Read the binary's size.\n-    let mut size: u32 = u32::from(console().read_char() as u8);\n-    size |= u32::from(console().read_char() as u8) << 8;\n-    size |= u32::from(console().read_char() as u8) << 16;\n-    size |= u32::from(console().read_char() as u8) << 24;\n-\n-    // Trust it's not too big.\n-    console().write_char('O');\n-    console().write_char('K');\n-\n-    let kernel_addr: *mut u8 = bsp::memory::board_default_load_addr() as *mut u8;\n-    unsafe {\n-        // Read the kernel byte by byte.\n-        for i in 0..size {\n-            core::ptr::write_volatile(kernel_addr.offset(i as isize), console().read_char() as u8)\n-        }\n+    info!(\n+        \"{} version {}\",\n+        env!(\"CARGO_PKG_NAME\"),\n+        env!(\"CARGO_PKG_VERSION\")\n+    );\n+    info!(\"Booting on: {}\", bsp::board_name());\n+\n+    info!(\n+        \"Architectural timer resolution: {} ns\",\n+        time::time_manager().resolution().as_nanos()\n+    );\n+\n+    info!(\"Drivers loaded:\");\n+    driver::driver_manager().enumerate();\n+\n+    // Test a failing timer case.\n+    time::time_manager().spin_for(Duration::from_nanos(1));\n+\n+    loop {\n+        info!(\"Spinning for 1 second\");\n+        time::time_manager().spin_for(Duration::from_secs(1));\n     }\n-\n-    println!(\"[ML] Loaded! Executing the payload now\\n\");\n-    console().flush();\n-\n-    // Use black magic to create a function pointer.\n-    let kernel: fn() -> ! = unsafe { core::mem::transmute(kernel_addr) };\n-\n-    // Jump to loaded kernel!\n-    kernel()\n }\n\ndiff -uNr 06_uart_chainloader/src/panic_wait.rs 07_timestamps/src/panic_wait.rs\n--- 06_uart_chainloader/src/panic_wait.rs\n+++ 07_timestamps/src/panic_wait.rs\n@@ -45,15 +45,18 @@\n     // Protect against panic infinite loops if any of the following code panics itself.\n     panic_prevent_reenter();\n\n+    let timestamp = crate::time::time_manager().uptime();\n     let (location, line, column) = match info.location() {\n         Some(loc) => (loc.file(), loc.line(), loc.column()),\n         _ => (\"???\", 0, 0),\n     };\n\n     println!(\n-        \"Kernel panic!\\n\\n\\\n+        \"[  {:>3}.{:06}] Kernel panic!\\n\\n\\\n         Panic location:\\n      File '{}', line {}, column {}\\n\\n\\\n         {}\",\n+        timestamp.as_secs(),\n+        timestamp.subsec_micros(),\n         location,\n         line,\n         column,\n\ndiff -uNr 06_uart_chainloader/src/print.rs 07_timestamps/src/print.rs\n--- 06_uart_chainloader/src/print.rs\n+++ 07_timestamps/src/print.rs\n@@ -34,3 +34,51 @@\n         $crate::print::_print(format_args_nl!($($arg)*));\n     })\n }\n+\n+/// Prints an info, with a newline.\n+#[macro_export]\n+macro_rules! info {\n+    ($string:expr) => ({\n+        let timestamp = $crate::time::time_manager().uptime();\n+\n+        $crate::print::_print(format_args_nl!(\n+            concat!(\"[  {:>3}.{:06}] \", $string),\n+            timestamp.as_secs(),\n+            timestamp.subsec_micros(),\n+        ));\n+    });\n+    ($format_string:expr, $($arg:tt)*) => ({\n+        let timestamp = $crate::time::time_manager().uptime();\n+\n+        $crate::print::_print(format_args_nl!(\n+            concat!(\"[  {:>3}.{:06}] \", $format_string),\n+            timestamp.as_secs(),\n+            timestamp.subsec_micros(),\n+            $($arg)*\n+        ));\n+    })\n+}\n+\n+/// Prints a warning, with a newline.\n+#[macro_export]\n+macro_rules! warn {\n+    ($string:expr) => ({\n+        let timestamp = $crate::time::time_manager().uptime();\n+\n+        $crate::print::_print(format_args_nl!(\n+            concat!(\"[W {:>3}.{:06}] \", $string),\n+            timestamp.as_secs(),\n+            timestamp.subsec_micros(),\n+        ));\n+    });\n+    ($format_string:expr, $($arg:tt)*) => ({\n+        let timestamp = $crate::time::time_manager().uptime();\n+\n+        $crate::print::_print(format_args_nl!(\n+            concat!(\"[W {:>3}.{:06}] \", $format_string),\n+            timestamp.as_secs(),\n+            timestamp.subsec_micros(),\n+            $($arg)*\n+        ));\n+    })\n+}\n\ndiff -uNr 06_uart_chainloader/src/time.rs 07_timestamps/src/time.rs\n--- 06_uart_chainloader/src/time.rs\n+++ 07_timestamps/src/time.rs\n@@ -0,0 +1,57 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Timer primitives.\n+\n+#[cfg(target_arch = \"aarch64\")]\n+#[path = \"_arch/aarch64/time.rs\"]\n+mod arch_time;\n+\n+use core::time::Duration;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Provides time management functions.\n+pub struct TimeManager;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Global instances\n+//--------------------------------------------------------------------------------------------------\n+\n+static TIME_MANAGER: TimeManager = TimeManager::new();\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Return a reference to the global TimeManager.\n+pub fn time_manager() -> &'static TimeManager {\n+    &TIME_MANAGER\n+}\n+\n+impl TimeManager {\n+    /// Create an instance.\n+    pub const fn new() -> Self {\n+        Self\n+    }\n+\n+    /// The timer's resolution.\n+    pub fn resolution(&self) -> Duration {\n+        arch_time::resolution()\n+    }\n+\n+    /// The uptime since power-on of the device.\n+    ///\n+    /// This includes time consumed by firmware and bootloaders.\n+    pub fn uptime(&self) -> Duration {\n+        arch_time::uptime()\n+    }\n+\n+    /// Spin for a given duration.\n+    pub fn spin_for(&self, duration: Duration) {\n+        arch_time::spin_for(duration)\n+    }\n+}\n\ndiff -uNr 06_uart_chainloader/tests/boot_test_string.rb 07_timestamps/tests/boot_test_string.rb\n--- 06_uart_chainloader/tests/boot_test_string.rb\n+++ 07_timestamps/tests/boot_test_string.rb\n@@ -0,0 +1,3 @@\n+# frozen_string_literal: true\n+\n+EXPECTED_PRINT = 'Spinning for 1 second'\n\ndiff -uNr 06_uart_chainloader/tests/chainboot_test.rb 07_timestamps/tests/chainboot_test.rb\n--- 06_uart_chainloader/tests/chainboot_test.rb\n+++ 07_timestamps/tests/chainboot_test.rb\n@@ -1,78 +0,0 @@\n-# frozen_string_literal: true\n-\n-# SPDX-License-Identifier: MIT OR Apache-2.0\n-#\n-# Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n-\n-require_relative '../../common/serial/minipush'\n-require_relative '../../common/tests/boot_test'\n-require 'pty'\n-\n-# Match for the last print that 'demo_payload_rpiX.img' produces.\n-EXPECTED_PRINT = 'Echoing input now'\n-\n-# Wait for request to power the target.\n-class PowerTargetRequestTest < SubtestBase\n-    MINIPUSH_POWER_TARGET_REQUEST = 'Please power the target now'\n-\n-    def initialize(qemu_cmd, pty_main)\n-        super()\n-        @qemu_cmd = qemu_cmd\n-        @pty_main = pty_main\n-    end\n-\n-    def name\n-        'Waiting for request to power target'\n-    end\n-\n-    def run(qemu_out, _qemu_in)\n-        expect_or_raise(qemu_out, MINIPUSH_POWER_TARGET_REQUEST)\n-\n-        # Now is the time to start QEMU with the chainloader binary. QEMU's virtual tty connects to\n-        # the MiniPush instance spawned on pty_main, so that the two processes talk to each other.\n-        Process.spawn(@qemu_cmd, in: @pty_main, out: @pty_main, err: '/dev/null')\n-    end\n-end\n-\n-# Extend BootTest so that it listens on the output of a MiniPush instance, which is itself connected\n-# to a QEMU instance instead of a real HW.\n-class ChainbootTest < BootTest\n-    MINIPUSH = '../common/serial/minipush.rb'\n-\n-    def initialize(qemu_cmd, payload_path)\n-        super(qemu_cmd, EXPECTED_PRINT)\n-\n-        @test_name = 'Boot test using Minipush'\n-\n-        @payload_path = payload_path\n-    end\n-\n-    private\n-\n-    # override\n-    def setup\n-        pty_main, pty_secondary = PTY.open\n-        mp_out, _mp_in = PTY.spawn(\"ruby #{MINIPUSH} #{pty_secondary.path} #{@payload_path}\")\n-\n-        # The subtests (from this class and the parents) listen on @qemu_out_wrapped. Hence, point\n-        # it to MiniPush's output.\n-        @qemu_out_wrapped = PTYLoggerWrapper.new(mp_out, \"\\r\\n\")\n-\n-        # Important: Run this subtest before the one in the parent class.\n-        @console_subtests.prepend(PowerTargetRequestTest.new(@qemu_cmd, pty_main))\n-    end\n-\n-    # override\n-    def finish\n-        super()\n-        @test_output.map! { |x| x.gsub(/.*\\r/, '  ') }\n-    end\n-end\n-\n-##--------------------------------------------------------------------------------------------------\n-## Execution starts here\n-##--------------------------------------------------------------------------------------------------\n-payload_path = ARGV.pop\n-qemu_cmd = ARGV.join(' ')\n-\n-ChainbootTest.new(qemu_cmd, payload_path).run\n\ndiff -uNr 06_uart_chainloader/update.sh 07_timestamps/update.sh\n--- 06_uart_chainloader/update.sh\n+++ 07_timestamps/update.sh\n@@ -1,8 +0,0 @@\n-#!/usr/bin/env bash\n-\n-cd ../05_drivers_gpio_uart\n-BSP=rpi4 make\n-cp kernel8.img ../06_uart_chainloader/demo_payload_rpi4.img\n-make\n-cp kernel8.img ../06_uart_chainloader/demo_payload_rpi3.img\n-rm kernel8.img\n\n```\n"
  },
  {
    "path": "07_timestamps/build.rs",
    "content": "use std::{env, fs, process};\n\nfn main() {\n    let ld_script_path = match env::var(\"LD_SCRIPT_PATH\") {\n        Ok(var) => var,\n        _ => process::exit(0),\n    };\n\n    let files = fs::read_dir(ld_script_path).unwrap();\n    files\n        .filter_map(Result::ok)\n        .filter(|d| {\n            if let Some(e) = d.path().extension() {\n                e == \"ld\"\n            } else {\n                false\n            }\n        })\n        .for_each(|f| println!(\"cargo:rerun-if-changed={}\", f.path().display()));\n}\n"
  },
  {
    "path": "07_timestamps/src/_arch/aarch64/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural boot code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::boot::arch_boot\n\nuse core::arch::global_asm;\n\n// Assembly counterpart to this file.\nglobal_asm!(\n    include_str!(\"boot.s\"),\n    CONST_CORE_ID_MASK = const 0b11\n);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The Rust entry of the `kernel` binary.\n///\n/// The function is called from the assembly `_start` function.\n#[no_mangle]\npub unsafe fn _start_rust() -> ! {\n    crate::kernel_init()\n}\n"
  },
  {
    "path": "07_timestamps/src/_arch/aarch64/cpu/boot.s",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2022 Andre Richter <andre.o.richter@gmail.com>\n\n//--------------------------------------------------------------------------------------------------\n// Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Load the address of a symbol into a register, PC-relative.\n//\n// The symbol must lie within +/- 4 GiB of the Program Counter.\n//\n// # Resources\n//\n// - https://sourceware.org/binutils/docs-2.36/as/AArch64_002dRelocations.html\n.macro ADR_REL register, symbol\n\tadrp\t\\register, \\symbol\n\tadd\t\\register, \\register, #:lo12:\\symbol\n.endm\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n.section .text._start\n\n//------------------------------------------------------------------------------\n// fn _start()\n//------------------------------------------------------------------------------\n_start:\n\t// Only proceed on the boot core. Park it otherwise.\n\tmrs\tx0, MPIDR_EL1\n\tand\tx0, x0, {CONST_CORE_ID_MASK}\n\tldr\tx1, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs\n\tcmp\tx0, x1\n\tb.ne\t.L_parking_loop\n\n\t// If execution reaches here, it is the boot core.\n\n\t// Initialize DRAM.\n\tADR_REL\tx0, __bss_start\n\tADR_REL x1, __bss_end_exclusive\n\n.L_bss_init_loop:\n\tcmp\tx0, x1\n\tb.eq\t.L_prepare_rust\n\tstp\txzr, xzr, [x0], #16\n\tb\t.L_bss_init_loop\n\n\t// Prepare the jump to Rust code.\n.L_prepare_rust:\n\t// Set the stack pointer.\n\tADR_REL\tx0, __boot_core_stack_end_exclusive\n\tmov\tsp, x0\n\n\t// Read the CPU's timer counter frequency and store it in ARCH_TIMER_COUNTER_FREQUENCY.\n\t// Abort if the frequency read back as 0.\n\tADR_REL\tx1, ARCH_TIMER_COUNTER_FREQUENCY // provided by aarch64/time.rs\n\tmrs\tx2, CNTFRQ_EL0\n\tcmp\tx2, xzr\n\tb.eq\t.L_parking_loop\n\tstr\tw2, [x1]\n\n\t// Jump to Rust code.\n\tb\t_start_rust\n\n\t// Infinitely wait for events (aka \"park the core\").\n.L_parking_loop:\n\twfe\n\tb\t.L_parking_loop\n\n.size\t_start, . - _start\n.type\t_start, function\n.global\t_start\n"
  },
  {
    "path": "07_timestamps/src/_arch/aarch64/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural processor code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::arch_cpu\n\nuse aarch64_cpu::asm;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\npub use asm::nop;\n\n/// Pause execution on the core.\n#[inline(always)]\npub fn wait_forever() -> ! {\n    loop {\n        asm::wfe()\n    }\n}\n"
  },
  {
    "path": "07_timestamps/src/_arch/aarch64/time.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural timer primitives.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::time::arch_time\n\nuse crate::warn;\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::{\n    num::{NonZeroU128, NonZeroU32, NonZeroU64},\n    ops::{Add, Div},\n    time::Duration,\n};\nuse tock_registers::interfaces::Readable;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nconst NANOSEC_PER_SEC: NonZeroU64 = NonZeroU64::new(1_000_000_000).unwrap();\n\n#[derive(Copy, Clone, PartialOrd, PartialEq)]\nstruct GenericTimerCounterValue(u64);\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n/// Boot assembly code overwrites this value with the value of CNTFRQ_EL0 before any Rust code is\n/// executed. This given value here is just a (safe) dummy.\n#[no_mangle]\nstatic ARCH_TIMER_COUNTER_FREQUENCY: NonZeroU32 = NonZeroU32::MIN;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nfn arch_timer_counter_frequency() -> NonZeroU32 {\n    // Read volatile is needed here to prevent the compiler from optimizing\n    // ARCH_TIMER_COUNTER_FREQUENCY away.\n    //\n    // This is safe, because all the safety requirements as stated in read_volatile()'s\n    // documentation are fulfilled.\n    unsafe { core::ptr::read_volatile(&ARCH_TIMER_COUNTER_FREQUENCY) }\n}\n\nimpl GenericTimerCounterValue {\n    pub const MAX: Self = GenericTimerCounterValue(u64::MAX);\n}\n\nimpl Add for GenericTimerCounterValue {\n    type Output = Self;\n\n    fn add(self, other: Self) -> Self {\n        GenericTimerCounterValue(self.0.wrapping_add(other.0))\n    }\n}\n\nimpl From<GenericTimerCounterValue> for Duration {\n    fn from(counter_value: GenericTimerCounterValue) -> Self {\n        if counter_value.0 == 0 {\n            return Duration::ZERO;\n        }\n\n        let frequency: NonZeroU64 = arch_timer_counter_frequency().into();\n\n        // Div<NonZeroU64> implementation for u64 cannot panic.\n        let secs = counter_value.0.div(frequency);\n\n        // This is safe, because frequency can never be greater than u32::MAX, which means the\n        // largest theoretical value for sub_second_counter_value is (u32::MAX - 1). Therefore,\n        // (sub_second_counter_value * NANOSEC_PER_SEC) cannot overflow an u64.\n        //\n        // The subsequent division ensures the result fits into u32, since the max result is smaller\n        // than NANOSEC_PER_SEC. Therefore, just cast it to u32 using `as`.\n        let sub_second_counter_value = counter_value.0 % frequency;\n        let nanos = unsafe { sub_second_counter_value.unchecked_mul(u64::from(NANOSEC_PER_SEC)) }\n            .div(frequency) as u32;\n\n        Duration::new(secs, nanos)\n    }\n}\n\nfn max_duration() -> Duration {\n    Duration::from(GenericTimerCounterValue::MAX)\n}\n\nimpl TryFrom<Duration> for GenericTimerCounterValue {\n    type Error = &'static str;\n\n    fn try_from(duration: Duration) -> Result<Self, Self::Error> {\n        if duration < resolution() {\n            return Ok(GenericTimerCounterValue(0));\n        }\n\n        if duration > max_duration() {\n            return Err(\"Conversion error. Duration too big\");\n        }\n\n        let frequency: u128 = u32::from(arch_timer_counter_frequency()) as u128;\n        let duration: u128 = duration.as_nanos();\n\n        // This is safe, because frequency can never be greater than u32::MAX, and\n        // (Duration::MAX.as_nanos() * u32::MAX) < u128::MAX.\n        let counter_value =\n            unsafe { duration.unchecked_mul(frequency) }.div(NonZeroU128::from(NANOSEC_PER_SEC));\n\n        // Since we checked above that we are <= max_duration(), just cast to u64.\n        Ok(GenericTimerCounterValue(counter_value as u64))\n    }\n}\n\n#[inline(always)]\nfn read_cntpct() -> GenericTimerCounterValue {\n    // Prevent that the counter is read ahead of time due to out-of-order execution.\n    barrier::isb(barrier::SY);\n    let cnt = CNTPCT_EL0.get();\n\n    GenericTimerCounterValue(cnt)\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The timer's resolution.\npub fn resolution() -> Duration {\n    Duration::from(GenericTimerCounterValue(1))\n}\n\n/// The uptime since power-on of the device.\n///\n/// This includes time consumed by firmware and bootloaders.\npub fn uptime() -> Duration {\n    read_cntpct().into()\n}\n\n/// Spin for a given duration.\npub fn spin_for(duration: Duration) {\n    let curr_counter_value = read_cntpct();\n\n    let counter_value_delta: GenericTimerCounterValue = match duration.try_into() {\n        Err(msg) => {\n            warn!(\"spin_for: {}. Skipping\", msg);\n            return;\n        }\n        Ok(val) => val,\n    };\n    let counter_value_target = curr_counter_value + counter_value_delta;\n\n    // Busy wait.\n    //\n    // Read CNTPCT_EL0 directly to avoid the ISB that is part of [`read_cntpct`].\n    while GenericTimerCounterValue(CNTPCT_EL0.get()) < counter_value_target {}\n}\n"
  },
  {
    "path": "07_timestamps/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GPIO Driver.\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper, driver, synchronization,\n    synchronization::NullLock,\n};\nuse tock_registers::{\n    interfaces::{ReadWriteable, Writeable},\n    register_bitfields, register_structs,\n    registers::ReadWrite,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// GPIO registers.\n//\n// Descriptions taken from\n// - https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf\n// - https://datasheets.raspberrypi.org/bcm2711/bcm2711-peripherals.pdf\nregister_bitfields! {\n    u32,\n\n    /// GPIO Function Select 1\n    GPFSEL1 [\n        /// Pin 15\n        FSEL15 OFFSET(15) NUMBITS(3) [\n            Input = 0b000,\n            Output = 0b001,\n            AltFunc0 = 0b100  // PL011 UART RX\n\n        ],\n\n        /// Pin 14\n        FSEL14 OFFSET(12) NUMBITS(3) [\n            Input = 0b000,\n            Output = 0b001,\n            AltFunc0 = 0b100  // PL011 UART TX\n        ]\n    ],\n\n    /// GPIO Pull-up/down Register\n    ///\n    /// BCM2837 only.\n    GPPUD [\n        /// Controls the actuation of the internal pull-up/down control line to ALL the GPIO pins.\n        PUD OFFSET(0) NUMBITS(2) [\n            Off = 0b00,\n            PullDown = 0b01,\n            PullUp = 0b10\n        ]\n    ],\n\n    /// GPIO Pull-up/down Clock Register 0\n    ///\n    /// BCM2837 only.\n    GPPUDCLK0 [\n        /// Pin 15\n        PUDCLK15 OFFSET(15) NUMBITS(1) [\n            NoEffect = 0,\n            AssertClock = 1\n        ],\n\n        /// Pin 14\n        PUDCLK14 OFFSET(14) NUMBITS(1) [\n            NoEffect = 0,\n            AssertClock = 1\n        ]\n    ],\n\n    /// GPIO Pull-up / Pull-down Register 0\n    ///\n    /// BCM2711 only.\n    GPIO_PUP_PDN_CNTRL_REG0 [\n        /// Pin 15\n        GPIO_PUP_PDN_CNTRL15 OFFSET(30) NUMBITS(2) [\n            NoResistor = 0b00,\n            PullUp = 0b01\n        ],\n\n        /// Pin 14\n        GPIO_PUP_PDN_CNTRL14 OFFSET(28) NUMBITS(2) [\n            NoResistor = 0b00,\n            PullUp = 0b01\n        ]\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    RegisterBlock {\n        (0x00 => _reserved1),\n        (0x04 => GPFSEL1: ReadWrite<u32, GPFSEL1::Register>),\n        (0x08 => _reserved2),\n        (0x94 => GPPUD: ReadWrite<u32, GPPUD::Register>),\n        (0x98 => GPPUDCLK0: ReadWrite<u32, GPPUDCLK0::Register>),\n        (0x9C => _reserved3),\n        (0xE4 => GPIO_PUP_PDN_CNTRL_REG0: ReadWrite<u32, GPIO_PUP_PDN_CNTRL_REG0::Register>),\n        (0xE8 => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\nstruct GPIOInner {\n    registers: Registers,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the GPIO HW.\npub struct GPIO {\n    inner: NullLock<GPIOInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GPIOInner {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n        }\n    }\n\n    /// Disable pull-up/down on pins 14 and 15.\n    #[cfg(feature = \"bsp_rpi3\")]\n    fn disable_pud_14_15_bcm2837(&mut self) {\n        use crate::time;\n        use core::time::Duration;\n\n        // The Linux 2837 GPIO driver waits 1 µs between the steps.\n        const DELAY: Duration = Duration::from_micros(1);\n\n        self.registers.GPPUD.write(GPPUD::PUD::Off);\n        time::time_manager().spin_for(DELAY);\n\n        self.registers\n            .GPPUDCLK0\n            .write(GPPUDCLK0::PUDCLK15::AssertClock + GPPUDCLK0::PUDCLK14::AssertClock);\n        time::time_manager().spin_for(DELAY);\n\n        self.registers.GPPUD.write(GPPUD::PUD::Off);\n        self.registers.GPPUDCLK0.set(0);\n    }\n\n    /// Disable pull-up/down on pins 14 and 15.\n    #[cfg(feature = \"bsp_rpi4\")]\n    fn disable_pud_14_15_bcm2711(&mut self) {\n        self.registers.GPIO_PUP_PDN_CNTRL_REG0.write(\n            GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL15::PullUp\n                + GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL14::PullUp,\n        );\n    }\n\n    /// Map PL011 UART as standard output.\n    ///\n    /// TX to pin 14\n    /// RX to pin 15\n    pub fn map_pl011_uart(&mut self) {\n        // Select the UART on pins 14 and 15.\n        self.registers\n            .GPFSEL1\n            .modify(GPFSEL1::FSEL15::AltFunc0 + GPFSEL1::FSEL14::AltFunc0);\n\n        // Disable pull-up/down on pins 14 and 15.\n        #[cfg(feature = \"bsp_rpi3\")]\n        self.disable_pud_14_15_bcm2837();\n\n        #[cfg(feature = \"bsp_rpi4\")]\n        self.disable_pud_14_15_bcm2711();\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GPIO {\n    pub const COMPATIBLE: &'static str = \"BCM GPIO\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            inner: NullLock::new(GPIOInner::new(mmio_start_addr)),\n        }\n    }\n\n    /// Concurrency safe version of `GPIOInner.map_pl011_uart()`\n    pub fn map_pl011_uart(&self) {\n        self.inner.lock(|inner| inner.map_pl011_uart())\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl driver::interface::DeviceDriver for GPIO {\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n}\n"
  },
  {
    "path": "07_timestamps/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! PL011 UART driver.\n//!\n//! # Resources\n//!\n//! - <https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf>\n//! - <https://developer.arm.com/documentation/ddi0183/latest>\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper, console, cpu, driver, synchronization,\n    synchronization::NullLock,\n};\nuse core::fmt;\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields, register_structs,\n    registers::{ReadOnly, ReadWrite, WriteOnly},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// PL011 UART registers.\n//\n// Descriptions taken from \"PrimeCell UART (PL011) Technical Reference Manual\" r1p5.\nregister_bitfields! {\n    u32,\n\n    /// Flag Register.\n    FR [\n        /// Transmit FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n        /// Line Control Register, LCR_H.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the transmit holding register is empty.\n        /// - If the FIFO is enabled, the TXFE bit is set when the transmit FIFO is empty.\n        /// - This bit does not indicate if there is data in the transmit shift register.\n        TXFE OFFSET(7) NUMBITS(1) [],\n\n        /// Transmit FIFO full. The meaning of this bit depends on the state of the FEN bit in the\n        /// LCR_H Register.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the transmit holding register is full.\n        /// - If the FIFO is enabled, the TXFF bit is set when the transmit FIFO is full.\n        TXFF OFFSET(5) NUMBITS(1) [],\n\n        /// Receive FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n        /// LCR_H Register.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the receive holding register is empty.\n        /// - If the FIFO is enabled, the RXFE bit is set when the receive FIFO is empty.\n        RXFE OFFSET(4) NUMBITS(1) [],\n\n        /// UART busy. If this bit is set to 1, the UART is busy transmitting data. This bit remains\n        /// set until the complete byte, including all the stop bits, has been sent from the shift\n        /// register.\n        ///\n        /// This bit is set as soon as the transmit FIFO becomes non-empty, regardless of whether\n        /// the UART is enabled or not.\n        BUSY OFFSET(3) NUMBITS(1) []\n    ],\n\n    /// Integer Baud Rate Divisor.\n    IBRD [\n        /// The integer baud rate divisor.\n        BAUD_DIVINT OFFSET(0) NUMBITS(16) []\n    ],\n\n    /// Fractional Baud Rate Divisor.\n    FBRD [\n        ///  The fractional baud rate divisor.\n        BAUD_DIVFRAC OFFSET(0) NUMBITS(6) []\n    ],\n\n    /// Line Control Register.\n    LCR_H [\n        /// Word length. These bits indicate the number of data bits transmitted or received in a\n        /// frame.\n        #[allow(clippy::enum_variant_names)]\n        WLEN OFFSET(5) NUMBITS(2) [\n            FiveBit = 0b00,\n            SixBit = 0b01,\n            SevenBit = 0b10,\n            EightBit = 0b11\n        ],\n\n        /// Enable FIFOs:\n        ///\n        /// 0 = FIFOs are disabled (character mode) that is, the FIFOs become 1-byte-deep holding\n        /// registers.\n        ///\n        /// 1 = Transmit and receive FIFO buffers are enabled (FIFO mode).\n        FEN  OFFSET(4) NUMBITS(1) [\n            FifosDisabled = 0,\n            FifosEnabled = 1\n        ]\n    ],\n\n    /// Control Register.\n    CR [\n        /// Receive enable. If this bit is set to 1, the receive section of the UART is enabled.\n        /// Data reception occurs for either UART signals or SIR signals depending on the setting of\n        /// the SIREN bit. When the UART is disabled in the middle of reception, it completes the\n        /// current character before stopping.\n        RXE OFFSET(9) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// Transmit enable. If this bit is set to 1, the transmit section of the UART is enabled.\n        /// Data transmission occurs for either UART signals, or SIR signals depending on the\n        /// setting of the SIREN bit. When the UART is disabled in the middle of transmission, it\n        /// completes the current character before stopping.\n        TXE OFFSET(8) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// UART enable:\n        ///\n        /// 0 = UART is disabled. If the UART is disabled in the middle of transmission or\n        /// reception, it completes the current character before stopping.\n        ///\n        /// 1 = The UART is enabled. Data transmission and reception occurs for either UART signals\n        /// or SIR signals depending on the setting of the SIREN bit\n        UARTEN OFFSET(0) NUMBITS(1) [\n            /// If the UART is disabled in the middle of transmission or reception, it completes the\n            /// current character before stopping.\n            Disabled = 0,\n            Enabled = 1\n        ]\n    ],\n\n    /// Interrupt Clear Register.\n    ICR [\n        /// Meta field for all pending interrupts.\n        ALL OFFSET(0) NUMBITS(11) []\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    pub RegisterBlock {\n        (0x00 => DR: ReadWrite<u32>),\n        (0x04 => _reserved1),\n        (0x18 => FR: ReadOnly<u32, FR::Register>),\n        (0x1c => _reserved2),\n        (0x24 => IBRD: WriteOnly<u32, IBRD::Register>),\n        (0x28 => FBRD: WriteOnly<u32, FBRD::Register>),\n        (0x2c => LCR_H: WriteOnly<u32, LCR_H::Register>),\n        (0x30 => CR: WriteOnly<u32, CR::Register>),\n        (0x34 => _reserved3),\n        (0x44 => ICR: WriteOnly<u32, ICR::Register>),\n        (0x48 => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\n#[derive(PartialEq)]\nenum BlockingMode {\n    Blocking,\n    NonBlocking,\n}\n\nstruct PL011UartInner {\n    registers: Registers,\n    chars_written: usize,\n    chars_read: usize,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the UART.\npub struct PL011Uart {\n    inner: NullLock<PL011UartInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PL011UartInner {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n            chars_written: 0,\n            chars_read: 0,\n        }\n    }\n\n    /// Set up baud rate and characteristics.\n    ///\n    /// This results in 8N1 and 921_600 baud.\n    ///\n    /// The calculation for the BRD is (we set the clock to 48 MHz in config.txt):\n    /// `(48_000_000 / 16) / 921_600 = 3.2552083`.\n    ///\n    /// This means the integer part is `3` and goes into the `IBRD`.\n    /// The fractional part is `0.2552083`.\n    ///\n    /// `FBRD` calculation according to the PL011 Technical Reference Manual:\n    /// `INTEGER((0.2552083 * 64) + 0.5) = 16`.\n    ///\n    /// Therefore, the generated baud rate divider is: `3 + 16/64 = 3.25`. Which results in a\n    /// genrated baud rate of `48_000_000 / (16 * 3.25) = 923_077`.\n    ///\n    /// Error = `((923_077 - 921_600) / 921_600) * 100 = 0.16%`.\n    pub fn init(&mut self) {\n        // Execution can arrive here while there are still characters queued in the TX FIFO and\n        // actively being sent out by the UART hardware. If the UART is turned off in this case,\n        // those queued characters would be lost.\n        //\n        // For example, this can happen during runtime on a call to panic!(), because panic!()\n        // initializes its own UART instance and calls init().\n        //\n        // Hence, flush first to ensure all pending characters are transmitted.\n        self.flush();\n\n        // Turn the UART off temporarily.\n        self.registers.CR.set(0);\n\n        // Clear all pending interrupts.\n        self.registers.ICR.write(ICR::ALL::CLEAR);\n\n        // From the PL011 Technical Reference Manual:\n        //\n        // The LCR_H, IBRD, and FBRD registers form the single 30-bit wide LCR Register that is\n        // updated on a single write strobe generated by a LCR_H write. So, to internally update the\n        // contents of IBRD or FBRD, a LCR_H write must always be performed at the end.\n        //\n        // Set the baud rate, 8N1 and FIFO enabled.\n        self.registers.IBRD.write(IBRD::BAUD_DIVINT.val(3));\n        self.registers.FBRD.write(FBRD::BAUD_DIVFRAC.val(16));\n        self.registers\n            .LCR_H\n            .write(LCR_H::WLEN::EightBit + LCR_H::FEN::FifosEnabled);\n\n        // Turn the UART on.\n        self.registers\n            .CR\n            .write(CR::UARTEN::Enabled + CR::TXE::Enabled + CR::RXE::Enabled);\n    }\n\n    /// Send a character.\n    fn write_char(&mut self, c: char) {\n        // Spin while TX FIFO full is set, waiting for an empty slot.\n        while self.registers.FR.matches_all(FR::TXFF::SET) {\n            cpu::nop();\n        }\n\n        // Write the character to the buffer.\n        self.registers.DR.set(c as u32);\n\n        self.chars_written += 1;\n    }\n\n    /// Block execution until the last buffered character has been physically put on the TX wire.\n    fn flush(&self) {\n        // Spin until the busy bit is cleared.\n        while self.registers.FR.matches_all(FR::BUSY::SET) {\n            cpu::nop();\n        }\n    }\n\n    /// Retrieve a character.\n    fn read_char_converting(&mut self, blocking_mode: BlockingMode) -> Option<char> {\n        // If RX FIFO is empty,\n        if self.registers.FR.matches_all(FR::RXFE::SET) {\n            // immediately return in non-blocking mode.\n            if blocking_mode == BlockingMode::NonBlocking {\n                return None;\n            }\n\n            // Otherwise, wait until a char was received.\n            while self.registers.FR.matches_all(FR::RXFE::SET) {\n                cpu::nop();\n            }\n        }\n\n        // Read one character.\n        let mut ret = self.registers.DR.get() as u8 as char;\n\n        // Convert carrige return to newline.\n        if ret == '\\r' {\n            ret = '\\n'\n        }\n\n        // Update statistics.\n        self.chars_read += 1;\n\n        Some(ret)\n    }\n}\n\n/// Implementing `core::fmt::Write` enables usage of the `format_args!` macros, which in turn are\n/// used to implement the `kernel`'s `print!` and `println!` macros. By implementing `write_str()`,\n/// we get `write_fmt()` automatically.\n///\n/// The function takes an `&mut self`, so it must be implemented for the inner struct.\n///\n/// See [`src/print.rs`].\n///\n/// [`src/print.rs`]: ../../print/index.html\nimpl fmt::Write for PL011UartInner {\n    fn write_str(&mut self, s: &str) -> fmt::Result {\n        for c in s.chars() {\n            self.write_char(c);\n        }\n\n        Ok(())\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PL011Uart {\n    pub const COMPATIBLE: &'static str = \"BCM PL011 UART\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            inner: NullLock::new(PL011UartInner::new(mmio_start_addr)),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl driver::interface::DeviceDriver for PL011Uart {\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n\n    unsafe fn init(&self) -> Result<(), &'static str> {\n        self.inner.lock(|inner| inner.init());\n\n        Ok(())\n    }\n}\n\nimpl console::interface::Write for PL011Uart {\n    /// Passthrough of `args` to the `core::fmt::Write` implementation, but guarded by a Mutex to\n    /// serialize access.\n    fn write_char(&self, c: char) {\n        self.inner.lock(|inner| inner.write_char(c));\n    }\n\n    fn write_fmt(&self, args: core::fmt::Arguments) -> fmt::Result {\n        // Fully qualified syntax for the call to `core::fmt::Write::write_fmt()` to increase\n        // readability.\n        self.inner.lock(|inner| fmt::Write::write_fmt(inner, args))\n    }\n\n    fn flush(&self) {\n        // Spin until TX FIFO empty is set.\n        self.inner.lock(|inner| inner.flush());\n    }\n}\n\nimpl console::interface::Read for PL011Uart {\n    fn read_char(&self) -> char {\n        self.inner\n            .lock(|inner| inner.read_char_converting(BlockingMode::Blocking).unwrap())\n    }\n\n    fn clear_rx(&self) {\n        // Read from the RX FIFO until it is indicating empty.\n        while self\n            .inner\n            .lock(|inner| inner.read_char_converting(BlockingMode::NonBlocking))\n            .is_some()\n        {}\n    }\n}\n\nimpl console::interface::Statistics for PL011Uart {\n    fn chars_written(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_written)\n    }\n\n    fn chars_read(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_read)\n    }\n}\n\nimpl console::interface::All for PL011Uart {}\n"
  },
  {
    "path": "07_timestamps/src/bsp/device_driver/bcm.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BCM driver top level.\n\nmod bcm2xxx_gpio;\nmod bcm2xxx_pl011_uart;\n\npub use bcm2xxx_gpio::*;\npub use bcm2xxx_pl011_uart::*;\n"
  },
  {
    "path": "07_timestamps/src/bsp/device_driver/common.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Common device driver code.\n\nuse core::{marker::PhantomData, ops};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct MMIODerefWrapper<T> {\n    start_addr: usize,\n    phantom: PhantomData<fn() -> T>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T> MMIODerefWrapper<T> {\n    /// Create an instance.\n    pub const unsafe fn new(start_addr: usize) -> Self {\n        Self {\n            start_addr,\n            phantom: PhantomData,\n        }\n    }\n}\n\nimpl<T> ops::Deref for MMIODerefWrapper<T> {\n    type Target = T;\n\n    fn deref(&self) -> &Self::Target {\n        unsafe { &*(self.start_addr as *const _) }\n    }\n}\n"
  },
  {
    "path": "07_timestamps/src/bsp/device_driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Device driver.\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod bcm;\nmod common;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use bcm::*;\n"
  },
  {
    "path": "07_timestamps/src/bsp/raspberrypi/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Processor code.\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Used by `arch` code to find the early boot core.\n#[no_mangle]\n#[link_section = \".text._start_arguments\"]\npub static BOOT_CORE_ID: u64 = 0;\n"
  },
  {
    "path": "07_timestamps/src/bsp/raspberrypi/driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP driver support.\n\nuse super::memory::map::mmio;\nuse crate::{bsp::device_driver, console, driver as generic_driver};\nuse core::sync::atomic::{AtomicBool, Ordering};\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic PL011_UART: device_driver::PL011Uart =\n    unsafe { device_driver::PL011Uart::new(mmio::PL011_UART_START) };\nstatic GPIO: device_driver::GPIO = unsafe { device_driver::GPIO::new(mmio::GPIO_START) };\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// This must be called only after successful init of the UART driver.\nfn post_init_uart() -> Result<(), &'static str> {\n    console::register_console(&PL011_UART);\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the GPIO driver.\nfn post_init_gpio() -> Result<(), &'static str> {\n    GPIO.map_pl011_uart();\n    Ok(())\n}\n\nfn driver_uart() -> Result<(), &'static str> {\n    let uart_descriptor =\n        generic_driver::DeviceDriverDescriptor::new(&PL011_UART, Some(post_init_uart));\n    generic_driver::driver_manager().register_driver(uart_descriptor);\n\n    Ok(())\n}\n\nfn driver_gpio() -> Result<(), &'static str> {\n    let gpio_descriptor = generic_driver::DeviceDriverDescriptor::new(&GPIO, Some(post_init_gpio));\n    generic_driver::driver_manager().register_driver(gpio_descriptor);\n\n    Ok(())\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Initialize the driver subsystem.\n///\n/// # Safety\n///\n/// See child function calls.\n///\n/// # Note\n///\n/// Using atomics here relieves us from needing to use `unsafe` for the static variable.\n///\n/// On `AArch64`, which is the only implemented architecture at the time of writing this,\n/// [`AtomicBool::load`] and [`AtomicBool::store`] are lowered to ordinary load and store\n/// instructions. They are therefore safe to use even with MMU + caching deactivated.\n///\n/// [`AtomicBool::load`]: core::sync::atomic::AtomicBool::load\n/// [`AtomicBool::store`]: core::sync::atomic::AtomicBool::store\npub unsafe fn init() -> Result<(), &'static str> {\n    static INIT_DONE: AtomicBool = AtomicBool::new(false);\n    if INIT_DONE.load(Ordering::Relaxed) {\n        return Err(\"Init already done\");\n    }\n\n    driver_uart()?;\n    driver_gpio()?;\n\n    INIT_DONE.store(true, Ordering::Relaxed);\n    Ok(())\n}\n"
  },
  {
    "path": "07_timestamps/src/bsp/raspberrypi/kernel.ld",
    "content": "/* SPDX-License-Identifier: MIT OR Apache-2.0\n *\n * Copyright (c) 2018-2022 Andre Richter <andre.o.richter@gmail.com>\n */\n\n__rpi_phys_dram_start_addr = 0;\n\n/* The physical address at which the the kernel binary will be loaded by the Raspberry's firmware */\n__rpi_phys_binary_load_addr = 0x80000;\n\n\nENTRY(__rpi_phys_binary_load_addr)\n\n/* Flags:\n *     4 == R\n *     5 == RX\n *     6 == RW\n *\n * Segments are marked PT_LOAD below so that the ELF file provides virtual and physical addresses.\n * It doesn't mean all of them need actually be loaded.\n */\nPHDRS\n{\n    segment_boot_core_stack PT_LOAD FLAGS(6);\n    segment_code            PT_LOAD FLAGS(5);\n    segment_data            PT_LOAD FLAGS(6);\n}\n\nSECTIONS\n{\n    . =  __rpi_phys_dram_start_addr;\n\n    /***********************************************************************************************\n    * Boot Core Stack\n    ***********************************************************************************************/\n    .boot_core_stack (NOLOAD) :\n    {\n                                             /*   ^             */\n                                             /*   | stack       */\n        . += __rpi_phys_binary_load_addr;    /*   | growth      */\n                                             /*   | direction   */\n        __boot_core_stack_end_exclusive = .; /*   |             */\n    } :segment_boot_core_stack\n\n    /***********************************************************************************************\n    * Code + RO Data + Global Offset Table\n    ***********************************************************************************************/\n    .text :\n    {\n        KEEP(*(.text._start))\n        *(.text._start_arguments) /* Constants (or statics in Rust speak) read by _start(). */\n        *(.text._start_rust)      /* The Rust entry point */\n        *(.text*)                 /* Everything else */\n    } :segment_code\n\n    .rodata : ALIGN(8) { *(.rodata*) } :segment_code\n\n    /***********************************************************************************************\n    * Data + BSS\n    ***********************************************************************************************/\n    .data : { *(.data*) } :segment_data\n\n    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */\n    .bss (NOLOAD) : ALIGN(16)\n    {\n        __bss_start = .;\n        *(.bss*);\n        . = ALIGN(16);\n        __bss_end_exclusive = .;\n    } :segment_data\n\n    /***********************************************************************************************\n    * Misc\n    ***********************************************************************************************/\n    .got : { *(.got*) }\n    ASSERT(SIZEOF(.got) == 0, \"Relocation support not expected\")\n\n    /DISCARD/ : { *(.comment*) }\n}\n"
  },
  {
    "path": "07_timestamps/src/bsp/raspberrypi/memory.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Memory Management.\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// The board's physical memory map.\n#[rustfmt::skip]\npub(super) mod map {\n\n    pub const GPIO_OFFSET:         usize = 0x0020_0000;\n    pub const UART_OFFSET:         usize = 0x0020_1000;\n\n    /// Physical devices.\n    #[cfg(feature = \"bsp_rpi3\")]\n    pub mod mmio {\n        use super::*;\n\n        pub const START:            usize =         0x3F00_0000;\n        pub const GPIO_START:       usize = START + GPIO_OFFSET;\n        pub const PL011_UART_START: usize = START + UART_OFFSET;\n    }\n\n    /// Physical devices.\n    #[cfg(feature = \"bsp_rpi4\")]\n    pub mod mmio {\n        use super::*;\n\n        pub const START:            usize =         0xFE00_0000;\n        pub const GPIO_START:       usize = START + GPIO_OFFSET;\n        pub const PL011_UART_START: usize = START + UART_OFFSET;\n    }\n}\n"
  },
  {
    "path": "07_timestamps/src/bsp/raspberrypi.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Top-level BSP file for the Raspberry Pi 3 and 4.\n\npub mod cpu;\npub mod driver;\npub mod memory;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Board identification.\npub fn board_name() -> &'static str {\n    #[cfg(feature = \"bsp_rpi3\")]\n    {\n        \"Raspberry Pi 3\"\n    }\n\n    #[cfg(feature = \"bsp_rpi4\")]\n    {\n        \"Raspberry Pi 4\"\n    }\n}\n"
  },
  {
    "path": "07_timestamps/src/bsp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Conditional reexporting of Board Support Packages.\n\nmod device_driver;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod raspberrypi;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use raspberrypi::*;\n"
  },
  {
    "path": "07_timestamps/src/console/null_console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Null console.\n\nuse super::interface;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct NullConsole;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\npub static NULL_CONSOLE: NullConsole = NullConsole {};\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl interface::Write for NullConsole {\n    fn write_char(&self, _c: char) {}\n\n    fn write_fmt(&self, _args: fmt::Arguments) -> fmt::Result {\n        fmt::Result::Ok(())\n    }\n\n    fn flush(&self) {}\n}\n\nimpl interface::Read for NullConsole {\n    fn clear_rx(&self) {}\n}\n\nimpl interface::Statistics for NullConsole {}\nimpl interface::All for NullConsole {}\n"
  },
  {
    "path": "07_timestamps/src/console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! System console.\n\nmod null_console;\n\nuse crate::synchronization::{self, NullLock};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Console interfaces.\npub mod interface {\n    use core::fmt;\n\n    /// Console write functions.\n    pub trait Write {\n        /// Write a single character.\n        fn write_char(&self, c: char);\n\n        /// Write a Rust format string.\n        fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result;\n\n        /// Block until the last buffered character has been physically put on the TX wire.\n        fn flush(&self);\n    }\n\n    /// Console read functions.\n    pub trait Read {\n        /// Read a single character.\n        fn read_char(&self) -> char {\n            ' '\n        }\n\n        /// Clear RX buffers, if any.\n        fn clear_rx(&self);\n    }\n\n    /// Console statistics.\n    pub trait Statistics {\n        /// Return the number of characters written.\n        fn chars_written(&self) -> usize {\n            0\n        }\n\n        /// Return the number of characters read.\n        fn chars_read(&self) -> usize {\n            0\n        }\n    }\n\n    /// Trait alias for a full-fledged console.\n    pub trait All: Write + Read + Statistics {}\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic CUR_CONSOLE: NullLock<&'static (dyn interface::All + Sync)> =\n    NullLock::new(&null_console::NULL_CONSOLE);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\n/// Register a new console.\npub fn register_console(new_console: &'static (dyn interface::All + Sync)) {\n    CUR_CONSOLE.lock(|con| *con = new_console);\n}\n\n/// Return a reference to the currently registered console.\n///\n/// This is the global console used by all printing macros.\npub fn console() -> &'static dyn interface::All {\n    CUR_CONSOLE.lock(|con| *con)\n}\n"
  },
  {
    "path": "07_timestamps/src/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Boot code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/cpu/boot.rs\"]\nmod arch_boot;\n"
  },
  {
    "path": "07_timestamps/src/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Processor code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/cpu.rs\"]\nmod arch_cpu;\n\nmod boot;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_cpu::{nop, wait_forever};\n"
  },
  {
    "path": "07_timestamps/src/driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Driver support.\n\nuse crate::{\n    info,\n    synchronization::{interface::Mutex, NullLock},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nconst NUM_DRIVERS: usize = 5;\n\nstruct DriverManagerInner {\n    next_index: usize,\n    descriptors: [Option<DeviceDriverDescriptor>; NUM_DRIVERS],\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Driver interfaces.\npub mod interface {\n    /// Device Driver functions.\n    pub trait DeviceDriver {\n        /// Return a compatibility string for identifying the driver.\n        fn compatible(&self) -> &'static str;\n\n        /// Called by the kernel to bring up the device.\n        ///\n        /// # Safety\n        ///\n        /// - During init, drivers might do stuff with system-wide impact.\n        unsafe fn init(&self) -> Result<(), &'static str> {\n            Ok(())\n        }\n    }\n}\n\n/// Tpye to be used as an optional callback after a driver's init() has run.\npub type DeviceDriverPostInitCallback = unsafe fn() -> Result<(), &'static str>;\n\n/// A descriptor for device drivers.\n#[derive(Copy, Clone)]\npub struct DeviceDriverDescriptor {\n    device_driver: &'static (dyn interface::DeviceDriver + Sync),\n    post_init_callback: Option<DeviceDriverPostInitCallback>,\n}\n\n/// Provides device driver management functions.\npub struct DriverManager {\n    inner: NullLock<DriverManagerInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic DRIVER_MANAGER: DriverManager = DriverManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl DriverManagerInner {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            next_index: 0,\n            descriptors: [None; NUM_DRIVERS],\n        }\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl DeviceDriverDescriptor {\n    /// Create an instance.\n    pub fn new(\n        device_driver: &'static (dyn interface::DeviceDriver + Sync),\n        post_init_callback: Option<DeviceDriverPostInitCallback>,\n    ) -> Self {\n        Self {\n            device_driver,\n            post_init_callback,\n        }\n    }\n}\n\n/// Return a reference to the global DriverManager.\npub fn driver_manager() -> &'static DriverManager {\n    &DRIVER_MANAGER\n}\n\nimpl DriverManager {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            inner: NullLock::new(DriverManagerInner::new()),\n        }\n    }\n\n    /// Register a device driver with the kernel.\n    pub fn register_driver(&self, descriptor: DeviceDriverDescriptor) {\n        self.inner.lock(|inner| {\n            inner.descriptors[inner.next_index] = Some(descriptor);\n            inner.next_index += 1;\n        })\n    }\n\n    /// Helper for iterating over registered drivers.\n    fn for_each_descriptor<'a>(&'a self, f: impl FnMut(&'a DeviceDriverDescriptor)) {\n        self.inner.lock(|inner| {\n            inner\n                .descriptors\n                .iter()\n                .filter_map(|x| x.as_ref())\n                .for_each(f)\n        })\n    }\n\n    /// Fully initialize all drivers.\n    ///\n    /// # Safety\n    ///\n    /// - During init, drivers might do stuff with system-wide impact.\n    pub unsafe fn init_drivers(&self) {\n        self.for_each_descriptor(|descriptor| {\n            // 1. Initialize driver.\n            if let Err(x) = descriptor.device_driver.init() {\n                panic!(\n                    \"Error initializing driver: {}: {}\",\n                    descriptor.device_driver.compatible(),\n                    x\n                );\n            }\n\n            // 2. Call corresponding post init callback.\n            if let Some(callback) = &descriptor.post_init_callback {\n                if let Err(x) = callback() {\n                    panic!(\n                        \"Error during driver post-init callback: {}: {}\",\n                        descriptor.device_driver.compatible(),\n                        x\n                    );\n                }\n            }\n        });\n    }\n\n    /// Enumerate all registered device drivers.\n    pub fn enumerate(&self) {\n        let mut i: usize = 1;\n        self.for_each_descriptor(|descriptor| {\n            info!(\"      {}. {}\", i, descriptor.device_driver.compatible());\n\n            i += 1;\n        });\n    }\n}\n"
  },
  {
    "path": "07_timestamps/src/main.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n// Rust embedded logo for `make doc`.\n#![doc(\n    html_logo_url = \"https://raw.githubusercontent.com/rust-embedded/wg/master/assets/logo/ewg-logo-blue-white-on-transparent.png\"\n)]\n\n//! The `kernel` binary.\n//!\n//! # Code organization and architecture\n//!\n//! The code is divided into different *modules*, each representing a typical **subsystem** of the\n//! `kernel`. Top-level module files of subsystems reside directly in the `src` folder. For example,\n//! `src/memory.rs` contains code that is concerned with all things memory management.\n//!\n//! ## Visibility of processor architecture code\n//!\n//! Some of the `kernel`'s subsystems depend on low-level code that is specific to the target\n//! processor architecture. For each supported processor architecture, there exists a subfolder in\n//! `src/_arch`, for example, `src/_arch/aarch64`.\n//!\n//! The architecture folders mirror the subsystem modules laid out in `src`. For example,\n//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go\n//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in\n//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic\n//! module's name prefixed with `arch_`.\n//!\n//! For example, this is the top of `src/memory/mmu.rs`:\n//!\n//! ```\n//! #[cfg(target_arch = \"aarch64\")]\n//! #[path = \"../_arch/aarch64/memory/mmu.rs\"]\n//! mod arch_mmu;\n//! ```\n//!\n//! Often times, items from the `arch_ module` will be publicly reexported by the parent module.\n//! This way, each architecture specific module can provide its implementation of an item, while the\n//! caller must not be concerned which architecture has been conditionally compiled.\n//!\n//! ## BSP code\n//!\n//! `BSP` stands for Board Support Package. `BSP` code is organized under `src/bsp.rs` and contains\n//! target board specific definitions and functions. These are things such as the board's memory map\n//! or instances of drivers for devices that are featured on the respective board.\n//!\n//! Just like processor architecture code, the `BSP` code's module structure tries to mirror the\n//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is\n//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`.\n//!\n//! ## Kernel interfaces\n//!\n//! Both `arch` and `bsp` contain code that is conditionally compiled depending on the actual target\n//! and board for which the kernel is compiled. For example, the `interrupt controller` hardware of\n//! the `Raspberry Pi 3` and the `Raspberry Pi 4` is different, but we want the rest of the `kernel`\n//! code to play nicely with any of the two without much hassle.\n//!\n//! In order to provide a clean abstraction between `arch`, `bsp` and `generic kernel code`,\n//! `interface` traits are provided *whenever possible* and *where it makes sense*. They are defined\n//! in the respective subsystem module and help to enforce the idiom of *program to an interface,\n//! not an implementation*. For example, there will be a common IRQ handling interface which the two\n//! different interrupt controller `drivers` of both Raspberrys will implement, and only export the\n//! interface to the rest of the `kernel`.\n//!\n//! ```\n//!         +-------------------+\n//!         | Interface (Trait) |\n//!         |                   |\n//!         +--+-------------+--+\n//!            ^             ^\n//!            |             |\n//!            |             |\n//! +----------+--+       +--+----------+\n//! | kernel code |       |  bsp code   |\n//! |             |       |  arch code  |\n//! +-------------+       +-------------+\n//! ```\n//!\n//! # Summary\n//!\n//! For a logical `kernel` subsystem, corresponding code can be distributed over several physical\n//! locations. Here is an example for the **memory** subsystem:\n//!\n//! - `src/memory.rs` and `src/memory/**/*`\n//!   - Common code that is agnostic of target processor architecture and `BSP` characteristics.\n//!     - Example: A function to zero a chunk of memory.\n//!   - Interfaces for the memory subsystem that are implemented by `arch` or `BSP` code.\n//!     - Example: An `MMU` interface that defines `MMU` function prototypes.\n//! - `src/bsp/__board_name__/memory.rs` and `src/bsp/__board_name__/memory/**/*`\n//!   - `BSP` specific code.\n//!   - Example: The board's memory map (physical addresses of DRAM and MMIO devices).\n//! - `src/_arch/__arch_name__/memory.rs` and `src/_arch/__arch_name__/memory/**/*`\n//!   - Processor architecture specific code.\n//!   - Example: Implementation of the `MMU` interface for the `__arch_name__` processor\n//!     architecture.\n//!\n//! From a namespace perspective, **memory** subsystem code lives in:\n//!\n//! - `crate::memory::*`\n//! - `crate::bsp::memory::*`\n//!\n//! # Boot flow\n//!\n//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.\n//!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.\n//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.\n\n#![allow(clippy::upper_case_acronyms)]\n#![feature(asm_const)]\n#![feature(const_option)]\n#![feature(format_args_nl)]\n#![feature(nonzero_min_max)]\n#![feature(panic_info_message)]\n#![feature(trait_alias)]\n#![feature(unchecked_math)]\n#![no_main]\n#![no_std]\n\nmod bsp;\nmod console;\nmod cpu;\nmod driver;\nmod panic_wait;\nmod print;\nmod synchronization;\nmod time;\n\n/// Early init code.\n///\n/// # Safety\n///\n/// - Only a single core must be active and running this function.\n/// - The init calls in this function must appear in the correct order.\nunsafe fn kernel_init() -> ! {\n    // Initialize the BSP driver subsystem.\n    if let Err(x) = bsp::driver::init() {\n        panic!(\"Error initializing BSP driver subsystem: {}\", x);\n    }\n\n    // Initialize all device drivers.\n    driver::driver_manager().init_drivers();\n    // println! is usable from here on.\n\n    // Transition from unsafe to safe.\n    kernel_main()\n}\n\n/// The main function running after the early init.\nfn kernel_main() -> ! {\n    use core::time::Duration;\n\n    info!(\n        \"{} version {}\",\n        env!(\"CARGO_PKG_NAME\"),\n        env!(\"CARGO_PKG_VERSION\")\n    );\n    info!(\"Booting on: {}\", bsp::board_name());\n\n    info!(\n        \"Architectural timer resolution: {} ns\",\n        time::time_manager().resolution().as_nanos()\n    );\n\n    info!(\"Drivers loaded:\");\n    driver::driver_manager().enumerate();\n\n    // Test a failing timer case.\n    time::time_manager().spin_for(Duration::from_nanos(1));\n\n    loop {\n        info!(\"Spinning for 1 second\");\n        time::time_manager().spin_for(Duration::from_secs(1));\n    }\n}\n"
  },
  {
    "path": "07_timestamps/src/panic_wait.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A panic handler that infinitely waits.\n\nuse crate::{cpu, println};\nuse core::panic::PanicInfo;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Stop immediately if called a second time.\n///\n/// # Note\n///\n/// Using atomics here relieves us from needing to use `unsafe` for the static variable.\n///\n/// On `AArch64`, which is the only implemented architecture at the time of writing this,\n/// [`AtomicBool::load`] and [`AtomicBool::store`] are lowered to ordinary load and store\n/// instructions. They are therefore safe to use even with MMU + caching deactivated.\n///\n/// [`AtomicBool::load`]: core::sync::atomic::AtomicBool::load\n/// [`AtomicBool::store`]: core::sync::atomic::AtomicBool::store\nfn panic_prevent_reenter() {\n    use core::sync::atomic::{AtomicBool, Ordering};\n\n    #[cfg(not(target_arch = \"aarch64\"))]\n    compile_error!(\"Add the target_arch to above's check if the following code is safe to use\");\n\n    static PANIC_IN_PROGRESS: AtomicBool = AtomicBool::new(false);\n\n    if !PANIC_IN_PROGRESS.load(Ordering::Relaxed) {\n        PANIC_IN_PROGRESS.store(true, Ordering::Relaxed);\n\n        return;\n    }\n\n    cpu::wait_forever()\n}\n\n#[panic_handler]\nfn panic(info: &PanicInfo) -> ! {\n    // Protect against panic infinite loops if any of the following code panics itself.\n    panic_prevent_reenter();\n\n    let timestamp = crate::time::time_manager().uptime();\n    let (location, line, column) = match info.location() {\n        Some(loc) => (loc.file(), loc.line(), loc.column()),\n        _ => (\"???\", 0, 0),\n    };\n\n    println!(\n        \"[  {:>3}.{:06}] Kernel panic!\\n\\n\\\n        Panic location:\\n      File '{}', line {}, column {}\\n\\n\\\n        {}\",\n        timestamp.as_secs(),\n        timestamp.subsec_micros(),\n        location,\n        line,\n        column,\n        info.message().unwrap_or(&format_args!(\"\")),\n    );\n\n    cpu::wait_forever()\n}\n"
  },
  {
    "path": "07_timestamps/src/print.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Printing.\n\nuse crate::console;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n#[doc(hidden)]\npub fn _print(args: fmt::Arguments) {\n    console::console().write_fmt(args).unwrap();\n}\n\n/// Prints without a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! print {\n    ($($arg:tt)*) => ($crate::print::_print(format_args!($($arg)*)));\n}\n\n/// Prints with a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! println {\n    () => ($crate::print!(\"\\n\"));\n    ($($arg:tt)*) => ({\n        $crate::print::_print(format_args_nl!($($arg)*));\n    })\n}\n\n/// Prints an info, with a newline.\n#[macro_export]\nmacro_rules! info {\n    ($string:expr) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[  {:>3}.{:06}] \", $string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n        ));\n    });\n    ($format_string:expr, $($arg:tt)*) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[  {:>3}.{:06}] \", $format_string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n            $($arg)*\n        ));\n    })\n}\n\n/// Prints a warning, with a newline.\n#[macro_export]\nmacro_rules! warn {\n    ($string:expr) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[W {:>3}.{:06}] \", $string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n        ));\n    });\n    ($format_string:expr, $($arg:tt)*) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[W {:>3}.{:06}] \", $format_string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n            $($arg)*\n        ));\n    })\n}\n"
  },
  {
    "path": "07_timestamps/src/synchronization.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Synchronization primitives.\n//!\n//! # Resources\n//!\n//!   - <https://doc.rust-lang.org/book/ch16-04-extensible-concurrency-sync-and-send.html>\n//!   - <https://stackoverflow.com/questions/59428096/understanding-the-send-trait>\n//!   - <https://doc.rust-lang.org/std/cell/index.html>\n\nuse core::cell::UnsafeCell;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Synchronization interfaces.\npub mod interface {\n\n    /// Any object implementing this trait guarantees exclusive access to the data wrapped within\n    /// the Mutex for the duration of the provided closure.\n    pub trait Mutex {\n        /// The type of the data that is wrapped by this mutex.\n        type Data;\n\n        /// Locks the mutex and grants the closure temporary mutable access to the wrapped data.\n        fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R;\n    }\n}\n\n/// A pseudo-lock for teaching purposes.\n///\n/// In contrast to a real Mutex implementation, does not protect against concurrent access from\n/// other cores to the contained data. This part is preserved for later lessons.\n///\n/// The lock will only be used as long as it is safe to do so, i.e. as long as the kernel is\n/// executing single-threaded, aka only running on a single core with interrupts disabled.\npub struct NullLock<T>\nwhere\n    T: ?Sized,\n{\n    data: UnsafeCell<T>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nunsafe impl<T> Send for NullLock<T> where T: ?Sized + Send {}\nunsafe impl<T> Sync for NullLock<T> where T: ?Sized + Send {}\n\nimpl<T> NullLock<T> {\n    /// Create an instance.\n    pub const fn new(data: T) -> Self {\n        Self {\n            data: UnsafeCell::new(data),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\n\nimpl<T> interface::Mutex for NullLock<T> {\n    type Data = T;\n\n    fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R {\n        // In a real lock, there would be code encapsulating this line that ensures that this\n        // mutable reference will ever only be given out once at a time.\n        let data = unsafe { &mut *self.data.get() };\n\n        f(data)\n    }\n}\n"
  },
  {
    "path": "07_timestamps/src/time.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Timer primitives.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/time.rs\"]\nmod arch_time;\n\nuse core::time::Duration;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Provides time management functions.\npub struct TimeManager;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic TIME_MANAGER: TimeManager = TimeManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the global TimeManager.\npub fn time_manager() -> &'static TimeManager {\n    &TIME_MANAGER\n}\n\nimpl TimeManager {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self\n    }\n\n    /// The timer's resolution.\n    pub fn resolution(&self) -> Duration {\n        arch_time::resolution()\n    }\n\n    /// The uptime since power-on of the device.\n    ///\n    /// This includes time consumed by firmware and bootloaders.\n    pub fn uptime(&self) -> Duration {\n        arch_time::uptime()\n    }\n\n    /// Spin for a given duration.\n    pub fn spin_for(&self, duration: Duration) {\n        arch_time::spin_for(duration)\n    }\n}\n"
  },
  {
    "path": "07_timestamps/tests/boot_test_string.rb",
    "content": "# frozen_string_literal: true\n\nEXPECTED_PRINT = 'Spinning for 1 second'\n"
  },
  {
    "path": "08_hw_debug_JTAG/.vscode/settings.json",
    "content": "{\n    \"editor.formatOnSave\": true,\n    \"editor.rulers\": [100],\n    \"rust-analyzer.cargo.target\": \"aarch64-unknown-none-softfloat\",\n    \"rust-analyzer.cargo.features\": [\"bsp_rpi3\"],\n    \"rust-analyzer.checkOnSave.allTargets\": false,\n    \"rust-analyzer.checkOnSave.extraArgs\": [\"--bins\"],\n    \"rust-analyzer.lens.debug\": false,\n    \"rust-analyzer.lens.run\": false\n}\n"
  },
  {
    "path": "08_hw_debug_JTAG/Cargo.toml",
    "content": "[package]\nname = \"mingo\"\nversion = \"0.8.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n\n[profile.release]\nlto = true\n\n[features]\ndefault = []\nbsp_rpi3 = [\"tock-registers\"]\nbsp_rpi4 = [\"tock-registers\"]\n\n[[bin]]\nname = \"kernel\"\npath = \"src/main.rs\"\n\n##--------------------------------------------------------------------------------------------------\n## Dependencies\n##--------------------------------------------------------------------------------------------------\n\n[dependencies]\n\n# Optional dependencies\ntock-registers = { version = \"0.8.x\", default-features = false, features = [\"register_types\"], optional = true }\n\n# Platform specific dependencies\n[target.'cfg(target_arch = \"aarch64\")'.dependencies]\naarch64-cpu = { version = \"9.x.x\" }\n"
  },
  {
    "path": "08_hw_debug_JTAG/Makefile",
    "content": "## SPDX-License-Identifier: MIT OR Apache-2.0\n##\n## Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\ninclude ../common/docker.mk\ninclude ../common/format.mk\ninclude ../common/operating_system.mk\n\n##--------------------------------------------------------------------------------------------------\n## Optional, user-provided configuration values\n##--------------------------------------------------------------------------------------------------\n\n# Default to the RPi3.\nBSP ?= rpi3\n\n# Default to a serial device name that is common in Linux.\nDEV_SERIAL ?= /dev/ttyUSB0\n\n\n\n##--------------------------------------------------------------------------------------------------\n## BSP-specific configuration values\n##--------------------------------------------------------------------------------------------------\nQEMU_MISSING_STRING = \"This board is not yet supported for QEMU.\"\n\nifeq ($(BSP),rpi3)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE = raspi3\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    OPENOCD_ARG       = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi3.cfg\n    JTAG_BOOT_IMAGE   = ../X1_JTAG_boot/jtag_boot_rpi3.img\n    LD_SCRIPT_PATH    = $(shell pwd)/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a53\nelse ifeq ($(BSP),rpi4)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE =\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    OPENOCD_ARG       = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi4.cfg\n    JTAG_BOOT_IMAGE   = ../X1_JTAG_boot/jtag_boot_rpi4.img\n    LD_SCRIPT_PATH    = $(shell pwd)/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a72\nendif\n\n# Export for build.rs.\nexport LD_SCRIPT_PATH\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets and Prerequisites\n##--------------------------------------------------------------------------------------------------\nKERNEL_MANIFEST      = Cargo.toml\nKERNEL_LINKER_SCRIPT = kernel.ld\nLAST_BUILD_CONFIG    = target/$(BSP).build_config\n\nKERNEL_ELF      = target/$(TARGET)/release/kernel\n# This parses cargo's dep-info file.\n# https://doc.rust-lang.org/cargo/guide/build-cache.html#dep-info-files\nKERNEL_ELF_DEPS = $(filter-out %: ,$(file < $(KERNEL_ELF).d)) $(KERNEL_MANIFEST) $(LAST_BUILD_CONFIG)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Command building blocks\n##--------------------------------------------------------------------------------------------------\nRUSTFLAGS = $(RUSTC_MISC_ARGS)                   \\\n    -C link-arg=--library-path=$(LD_SCRIPT_PATH) \\\n    -C link-arg=--script=$(KERNEL_LINKER_SCRIPT)\n\nRUSTFLAGS_PEDANTIC = $(RUSTFLAGS) \\\n    -D warnings                   \\\n    -D missing_docs\n\nFEATURES      = --features bsp_$(BSP)\nCOMPILER_ARGS = --target=$(TARGET) \\\n    $(FEATURES)                    \\\n    --release\n\nRUSTC_CMD   = cargo rustc $(COMPILER_ARGS)\nDOC_CMD     = cargo doc $(COMPILER_ARGS)\nCLIPPY_CMD  = cargo clippy $(COMPILER_ARGS)\nOBJCOPY_CMD = rust-objcopy \\\n    --strip-all            \\\n    -O binary\n\nEXEC_QEMU          = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE)\nEXEC_TEST_DISPATCH = ruby ../common/tests/dispatch.rb\nEXEC_MINIPUSH      = ruby ../common/serial/minipush.rb\n\n##------------------------------------------------------------------------------\n## Dockerization\n##------------------------------------------------------------------------------\nDOCKER_CMD            = docker run -t --rm -v $(shell pwd):/work/tutorial -w /work/tutorial\nDOCKER_CMD_INTERACT   = $(DOCKER_CMD) -i\nDOCKER_ARG_DIR_COMMON = -v $(shell pwd)/../common:/work/common\nDOCKER_ARG_DIR_JTAG   = -v $(shell pwd)/../X1_JTAG_boot:/work/X1_JTAG_boot\nDOCKER_ARG_DEV        = --privileged -v /dev:/dev\nDOCKER_ARG_NET        = --network host\n\n# DOCKER_IMAGE defined in include file (see top of this file).\nDOCKER_QEMU  = $(DOCKER_CMD_INTERACT) $(DOCKER_IMAGE)\nDOCKER_TOOLS = $(DOCKER_CMD) $(DOCKER_IMAGE)\nDOCKER_TEST  = $(DOCKER_CMD) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\nDOCKER_GDB   = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_NET) $(DOCKER_IMAGE)\n\n# Dockerize commands, which require USB device passthrough, only on Linux.\nifeq ($(shell uname -s),Linux)\n    DOCKER_CMD_DEV = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_DEV)\n\n    DOCKER_CHAINBOOT = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\n    DOCKER_JTAGBOOT  = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_ARG_DIR_JTAG) $(DOCKER_IMAGE)\n    DOCKER_OPENOCD   = $(DOCKER_CMD_DEV) $(DOCKER_ARG_NET) $(DOCKER_IMAGE)\nelse\n    DOCKER_OPENOCD   = echo \"Not yet supported on non-Linux systems.\"; \\#\nendif\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: all doc qemu chainboot clippy clean readelf objdump nm check\n\nall: $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Save the configuration as a file, so make understands if it changed.\n##------------------------------------------------------------------------------\n$(LAST_BUILD_CONFIG):\n\t@rm -f target/*.build_config\n\t@mkdir -p target\n\t@touch $(LAST_BUILD_CONFIG)\n\n##------------------------------------------------------------------------------\n## Compile the kernel ELF\n##------------------------------------------------------------------------------\n$(KERNEL_ELF): $(KERNEL_ELF_DEPS)\n\t$(call color_header, \"Compiling kernel ELF - $(BSP)\")\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(RUSTC_CMD)\n\n##------------------------------------------------------------------------------\n## Generate the stripped kernel binary\n##------------------------------------------------------------------------------\n$(KERNEL_BIN): $(KERNEL_ELF)\n\t$(call color_header, \"Generating stripped binary\")\n\t@$(OBJCOPY_CMD) $(KERNEL_ELF) $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Name\")\n\t@echo $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Size\")\n\t$(call disk_usage_KiB, $(KERNEL_BIN))\n\n##------------------------------------------------------------------------------\n## Generate the documentation\n##------------------------------------------------------------------------------\ndoc:\n\t$(call color_header, \"Generating docs\")\n\t@$(DOC_CMD) --document-private-items --open\n\n##------------------------------------------------------------------------------\n## Run the kernel in QEMU\n##------------------------------------------------------------------------------\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\nqemu:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\nqemu: $(KERNEL_BIN)\n\t$(call color_header, \"Launching QEMU\")\n\t@$(DOCKER_QEMU) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\nendif\n\n##------------------------------------------------------------------------------\n## Push the kernel to the real HW target\n##------------------------------------------------------------------------------\nchainboot: $(KERNEL_BIN)\n\t@$(DOCKER_CHAINBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run clippy\n##------------------------------------------------------------------------------\nclippy:\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(CLIPPY_CMD)\n\n##------------------------------------------------------------------------------\n## Clean\n##------------------------------------------------------------------------------\nclean:\n\trm -rf target $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run readelf\n##------------------------------------------------------------------------------\nreadelf: $(KERNEL_ELF)\n\t$(call color_header, \"Launching readelf\")\n\t@$(DOCKER_TOOLS) $(READELF_BINARY) --headers $(KERNEL_ELF)\n\n##------------------------------------------------------------------------------\n## Run objdump\n##------------------------------------------------------------------------------\nobjdump: $(KERNEL_ELF)\n\t$(call color_header, \"Launching objdump\")\n\t@$(DOCKER_TOOLS) $(OBJDUMP_BINARY) --disassemble --demangle \\\n                --section .text   \\\n                --section .rodata \\\n                $(KERNEL_ELF) | rustfilt\n\n##------------------------------------------------------------------------------\n## Run nm\n##------------------------------------------------------------------------------\nnm: $(KERNEL_ELF)\n\t$(call color_header, \"Launching nm\")\n\t@$(DOCKER_TOOLS) $(NM_BINARY) --demangle --print-size $(KERNEL_ELF) | sort | rustfilt\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Debugging targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: jtagboot openocd gdb gdb-opt0\n\n##------------------------------------------------------------------------------\n## Push the JTAG boot image to the real HW target\n##------------------------------------------------------------------------------\njtagboot:\n\t@$(DOCKER_JTAGBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(JTAG_BOOT_IMAGE)\n\n##------------------------------------------------------------------------------\n## Start OpenOCD session\n##------------------------------------------------------------------------------\nopenocd:\n\t$(call color_header, \"Launching OpenOCD\")\n\t@$(DOCKER_OPENOCD) openocd $(OPENOCD_ARG)\n\n##------------------------------------------------------------------------------\n## Start GDB session\n##------------------------------------------------------------------------------\ngdb: RUSTC_MISC_ARGS += -C debuginfo=2\ngdb-opt0: RUSTC_MISC_ARGS += -C debuginfo=2 -C opt-level=0\ngdb gdb-opt0: $(KERNEL_ELF)\n\t$(call color_header, \"Launching GDB\")\n\t@$(DOCKER_GDB) gdb-multiarch -q $(KERNEL_ELF)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Testing targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: test test_boot\n\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\ntest_boot test:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\n##------------------------------------------------------------------------------\n## Run boot test\n##------------------------------------------------------------------------------\ntest_boot: $(KERNEL_BIN)\n\t$(call color_header, \"Boot test - $(BSP)\")\n\t@$(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\ntest: test_boot\n\nendif\n"
  },
  {
    "path": "08_hw_debug_JTAG/README.CN.md",
    "content": "# 教程 08 - 使用JTAG进行硬件调试\n\n## tl;dr\n\n按照以下顺序进行操作：\n\n1. 运行`make jtagboot`并保持终端打开。\n2. 连接USB串行设备。\n3. 连接`JTAG`调试器的USB设备。\n4. 在新的终端中，运行`make openocd`并保持终端打开。\n5. 在新的终端中，运行`make gdb`或者运行`make gdb-opt0`。\n\n![Demo](../doc/09_demo.gif)\n\n## 目录\n\n- [简介](#简介)\n- [大纲](#大纲)\n- [软件设置](#软件设置)\n- [硬件设置](#硬件设置)\n  * [线路](#线路)\n- [准备连接](#准备连接)\n- [OpenOCD](#openocd)\n- [GDB](#gdb)\n  * [备注](#备注)\n    + [优化](#优化)\n    + [GDB控制](#GDB控制)\n- [关于USB连接限制的注意事项](#关于USB连接限制的注意事项)\n- [额外资料](#额外资料)\n- [致谢](#致谢)\n- [相比之前的变化（diff）](#相比之前的变化（diff）)\n\n## 简介\n\n在即将到来的教程中，我们将涉及RPi的SoC（系统芯片）的敏感区域，这可能会让我们的调试工作变得非常困难。\n例如，改变处理器的`Privilege Level`或引入`Virtual Memory`。\n\n硬件调试器有时可以成为寻找棘手错误的最后手段。特别是对于调试复杂的、与体系结构相关的硬件问题，它将非常有用，\n因为在这个领域，`QEMU`有时无法提供帮助，因为它对硬件的某些特性进行了抽象，并没有模拟到最后一位。\n\n那么，让我们介绍一下`JTAG`调试。一旦设置好，它将允许我们在真实的硬件上逐步执行我们的内核。这是多么酷啊！\n\n## 大纲\n\n从内核的角度来看，这个教程与之前的教程相同。我们只是在其周围添加了用于JTAG调试的基础设施。\n\n## 软件设置\n\n我们需要在SD卡的`config.txt`文件中添加另一行：\n\n```toml\narm_64bit=1\ninit_uart_clock=48000000\nenable_jtag_gpio=1\n```\n\n## 硬件设置\n\n与我们WG的[Embedded Rust Book]书籍中使用的`STM32F3DISCOVERY`等微控制器板不同，RPi没有在其板上内置调试器。\n因此，您需要购买一个。\n\n在本教程中，我们将使用OLIMEX的[ARM-USB-TINY-H]。它具有标准的[ARM JTAG 20 connector]。\n不幸的是，RPi没有这个连接器，所以我们必须通过跳线连接它。\n\n[Embedded Rust Book]: https://rust-embedded.github.io/book/start/hardware.html\n[ARM-USB-TINY-H]: https://www.olimex.com/Products/ARM/JTAG/ARM-USB-TINY-H\n[ARM JTAG 20 connector]: http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.dui0499dj/BEHEIHCE.html\n\n### 线路\n\n<table>\n    <thead>\n        <tr>\n            <th>GPIO #</th>\n\t\t\t<th>Name</th>\n\t\t\t<th>JTAG #</th>\n\t\t\t<th>Note</th>\n\t\t\t<th width=\"60%\">Diagram</th>\n        </tr>\n    </thead>\n    <tbody>\n        <tr>\n            <td></td>\n            <td>VTREF</td>\n            <td>1</td>\n            <td>to 3.3V</td>\n            <td rowspan=\"8\"><img src=\"../doc/09_wiring_jtag.png\"></td>\n        </tr>\n        <tr>\n            <td></td>\n            <td>GND</td>\n            <td>4</td>\n            <td>to GND</td>\n        </tr>\n        <tr>\n            <td>22</td>\n            <td>TRST</td>\n            <td>3</td>\n            <td></td>\n        </tr>\n        <tr>\n            <td>26</td>\n            <td>TDI</td>\n            <td>5</td>\n            <td></td>\n        </tr>\n        <tr>\n            <td>27</td>\n            <td>TMS</td>\n            <td>7</td>\n            <td></td>\n        </tr>\n        <tr>\n            <td>25</td>\n            <td>TCK</td>\n            <td>9</td>\n            <td></td>\n        </tr>\n        <tr>\n            <td>23</td>\n            <td>RTCK</td>\n            <td>11</td>\n            <td></td>\n        </tr>\n        <tr>\n            <td>24</td>\n            <td>TDO</td>\n            <td>13</td>\n            <td></td>\n        </tr>\n    </tbody>\n</table>\n\n<p align=\"center\"><img src=\"../doc/09_image_jtag_connected.jpg\" width=\"50%\"></p>\n\n## 准备连接\n\n在启动时，由于我们对`config.txt`进行的更改，RPi的固件将配置相应的GPIO引脚以实现`JTAG`功能。\n\n现在剩下的要做的就是暂停RPi的执行，然后通过`JTAG`进行连接。因此，我们添加了一个新的`Makefile` target，\n`make jtagboot`，它使用`chainboot`方法将一个小型辅助二进制文件加载到RPi上，\n该文件只是将执行核心置于等待状态。\n\n文件夹中单独[X1_JTAG_boot]文件夹中单独维护，并且是我们迄今为止在教程中使用的内核的修改版本。\n\n[X1_JTAG_boot]: ../X1_JTAG_boot\n\n```console\n$ make jtagboot\nMinipush 1.0\n\n[MP] ⏳ Waiting for /dev/ttyUSB0\n[MP] ✅ Serial connected\n[MP] 🔌 Please power the target now\n __  __ _      _ _                 _\n|  \\/  (_)_ _ (_) |   ___  __ _ __| |\n| |\\/| | | ' \\| | |__/ _ \\/ _` / _` |\n|_|  |_|_|_||_|_|____\\___/\\__,_\\__,_|\n\n           Raspberry Pi 3\n\n[ML] Requesting binary\n[MP] ⏩ Pushing 7 KiB ==========================================🦀 100% 0 KiB/s Time: 00:00:00\n[ML] Loaded! Executing the payload now\n\n[    0.394532] Parking CPU core. Please connect over JTAG now.\n```\n\n保持USB串口连接和打开运行`jtagboot`的终端非常重要。当我们稍后加载实际的内核时，`UART`输出将显示在这里。\n\n## OpenOCD\n\n接下来，我们需要启动开放式片上调试器 [Open On-Chip Debugger]，也称为`OpenOCD`，以实际连接`JTAG`。\n\n[Open On-Chip Debugger]: http://openocd.org\n\n一如既往，我们的教程力求使开发工具的使用尽可能简单，\n这就是为什么我们将所有内容打包到了[dedicated Docker container]中，该容器已经用于链式引导和`QEMU`。\n\n[dedicated Docker container]: ../docker/rustembedded-osdev-utils\n\n连接Olimex USB JTAG调试器，在同一个文件夹中打开一个新的终端窗口，然后按顺序输入\n`make openocd`命令。你将会看到一些初始输出：\n\n```console\n$ make openocd\n[...]\nOpen On-Chip Debugger 0.10.0\n[...]\nInfo : Listening on port 6666 for tcl connections\nInfo : Listening on port 4444 for telnet connections\nInfo : clock speed 1000 kHz\nInfo : JTAG tap: rpi3.tap tap/device found: 0x4ba00477 (mfg: 0x23b (ARM Ltd.), part: 0xba00, ver: 0x4)\nInfo : rpi3.core0: hardware has 6 breakpoints, 4 watchpoints\nInfo : rpi3.core1: hardware has 6 breakpoints, 4 watchpoints\nInfo : rpi3.core2: hardware has 6 breakpoints, 4 watchpoints\nInfo : rpi3.core3: hardware has 6 breakpoints, 4 watchpoints\nInfo : Listening on port 3333 for gdb connections\nInfo : Listening on port 3334 for gdb connections\nInfo : Listening on port 3335 for gdb connections\nInfo : Listening on port 3336 for gdb connections\n```\n\n`OpenOCD`已检测到RPi的四个核心，并打开了四个网络端口，`gdb`现在可以连接到这些端口来调试各自的核心。\n\n## GDB\n\n最后，我们需要一个支持`AArch64`的`gdb`版本。你猜对了，它已经打包在osdev容器中。\n可以通过`make gdb`命令启动它。\n\n实际上，这个Makefile target做了更多的事情。它构建了一个包含调试信息的特殊版本的内核。\n这使得`gdb`能够显示我们当前正在调试的`Rust`源代码行。\n它还启动了`gdb`，以便它已经加载了这个调试构建（`kernel_for_jtag`）。\n\n现在我们可以使用`gdb`命令行来进行以下操作：\n  1. 在我们的内核中设置断点。\n  2. 通过JTAG将内核加载到内存中（请记住，当前RPi仍在执行最小的JTAG引导二进制文件）。\n  3. 操纵RPi的程序计数器，使其从我们内核的入口点开始执行。\n  4. 逐步执行内核的执行过程。\n\n```console\n$ make gdb\n[...]\n>>> target remote :3333                          # Connect to OpenOCD, core0\n>>> load                                         # Load the kernel into the RPi's DRAM over JTAG.\nLoading section .text, size 0x2454 lma 0x80000\nLoading section .rodata, size 0xa1d lma 0x82460\nLoading section .got, size 0x10 lma 0x82e80\nLoading section .data, size 0x20 lma 0x82e90\nStart address 0x0000000000080000, load size 11937\nTransfer rate: 63 KB/sec, 2984 bytes/write.\n>>> set $pc = 0x80000                            # Set RPI's program counter to the start of the\n                                                 # kernel binary.\n>>> break main.rs:158\nBreakpoint 1 at 0x8025c: file src/main.rs, line 158.\n>>> cont\n>>> step                                         # Single-step through the kernel\n>>> step\n>>> ...\n```\n\n### 备注\n\n#### 优化\n\n在调试操作系统二进制文件时，您需要在可以逐步执行源代码粒度和生成的二进制文件的优化级别之间进行权衡。\n`make`和`make gdb`targets生成一个`--release`二进制文件，其中包含优化级别为3（`-opt-level=3`）。\n然而，在这种情况下，编译器会非常积极地进行内联，并尽可能地将读取和写入操作打包在一起。\n因此，不总是能够在源代码文件的特定行上准确命中断点。\n\n因此，Makefile还提供了`make gdb-opt0` target，它使用了`-opt-level=0`。\n因此，它将允许您拥有更精细的调试粒度。然而，请记住，当调试与硬件密切相关的代码时，\n编译器对易失性寄存器的读取或写入进行压缩的优化可能会对执行产生重大影响。\n请注意，上面的演示GIF是使用`gdb-opt0`录制的。\n\n#### GDB控制\n\n在某些情况下，您可能会遇到延迟循环或等待串行输入的代码。在这种情况下，\n逐步执行可能不可行或无法正常工作。您可以通过在这些区域之外设置其他断点，从而跳过这些障碍。\n并使用`cont`命令到达它们。\n\n在`gdb`中按下`ctrl+c`将再次停止RPi的执行，以防止您在没有进一步断点的情况下继续执行。\n\n## 关于USB连接限制的注意事项\n\n如果您按照教程从头到尾进行操作，关于USB连接的一切应该都没问题。\n\n但是，请注意，根据当前的形式，我们的`Makefile`对连接的USB设备的命名做出了隐含的假设。\n它期望`/dev/ttyUSB0`是`UART`设备。\n\n因此，请确保按照以下顺序将设备连接到您的计算机：\n  1. 首先连接USB串行设备。\n  2. 然后连接Olimex调试器。\n\n这样，主机操作系统会相应地枚举这些设备。这只需要做一次即可。\n可以多次断开和连接串行设备，例如在保持调试器连接的情况下启动不同的`make jtagboot`运行。\n\n## 额外资料\n\n- https://metebalci.com/blog/bare-metal-raspberry-pi-3b-jtag\n- https://www.suse.com/c/debugging-raspberry-pi-3-with-jtag\n\n## 致谢\n\n感谢[@naotaco](https://github.com/naotaco)为本教程奠定了基础。\n\n## 相比之前的变化（diff）\n请检查[英文版本](README.md#diff-to-previous)，这是最新的。\n"
  },
  {
    "path": "08_hw_debug_JTAG/README.md",
    "content": "# Tutorial 08 - Hardware Debugging using JTAG\n\n## tl;dr\n\nIn the exact order as listed:\n\n1. `make jtagboot` and keep terminal open.\n2. Connect USB serial device.\n3. Connect `JTAG` debugger USB device.\n4. In new terminal, `make openocd` and keep terminal open.\n5. In new terminal, `make gdb` or make `make gdb-opt0`.\n\n![Demo](../doc/09_demo.gif)\n\n## Table of Contents\n\n- [Introduction](#introduction)\n- [Outline](#outline)\n- [Software Setup](#software-setup)\n- [Hardware Setup](#hardware-setup)\n  * [Wiring](#wiring)\n- [Getting ready to connect](#getting-ready-to-connect)\n- [OpenOCD](#openocd)\n- [GDB](#gdb)\n  * [Remarks](#remarks)\n    + [Optimization](#optimization)\n    + [GDB control](#gdb-control)\n- [Notes on USB connection constraints](#notes-on-usb-connection-constraints)\n- [Additional resources](#additional-resources)\n- [Acknowledgments](#acknowledgments)\n- [Diff to previous](#diff-to-previous)\n\n## Introduction\n\nIn the upcoming tutorials, we are going to touch sensitive areas of the RPi's SoC that can make our\ndebugging life very hard. For example, changing the processor's `Privilege Level` or introducing\n`Virtual Memory`.\n\nA hardware based debugger can sometimes be the last resort when searching for a tricky bug.\nEspecially for debugging intricate, architecture-specific HW issues, it will be handy, because in\nthis area `QEMU` sometimes can not help, since it abstracts certain features of the HW and doesn't\nsimulate down to the very last bit.\n\nSo lets introduce `JTAG` debugging. Once set up, it will allow us to single-step through our kernel\non the real HW. How cool is that?!\n\n## Outline\n\nFrom kernel perspective, this tutorial is the same as the previous one. We are just wrapping\ninfrastructure for JTAG debugging around it.\n\n## Software Setup\n\nWe need to add another line to the `config.txt` file from the SD Card:\n\n```toml\narm_64bit=1\ninit_uart_clock=48000000\nenable_jtag_gpio=1\n```\n\n## Hardware Setup\n\nUnlike microcontroller boards like the `STM32F3DISCOVERY`, which is used in our WG's [Embedded Rust\nBook], the Raspberry Pi does not have an embedded debugger on its board. Hence, you need to buy one.\n\nFor this tutorial, we will use the [ARM-USB-TINY-H] from OLIMEX. It has a standard [ARM JTAG 20\nconnector]. Unfortunately, the RPi does not, so we have to connect it via jumper wires.\n\n[Embedded Rust Book]: https://rust-embedded.github.io/book/start/hardware.html\n[ARM-USB-TINY-H]: https://www.olimex.com/Products/ARM/JTAG/ARM-USB-TINY-H\n[ARM JTAG 20 connector]: http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.dui0499dj/BEHEIHCE.html\n\n### Wiring\n\n<table>\n    <thead>\n        <tr>\n            <th>GPIO #</th>\n\t\t\t<th>Name</th>\n\t\t\t<th>JTAG #</th>\n\t\t\t<th>Note</th>\n\t\t\t<th width=\"60%\">Diagram</th>\n        </tr>\n    </thead>\n    <tbody>\n        <tr>\n            <td></td>\n            <td>VTREF</td>\n            <td>1</td>\n            <td>to 3.3V</td>\n            <td rowspan=\"8\"><img src=\"../doc/09_wiring_jtag.png\"></td>\n        </tr>\n        <tr>\n            <td></td>\n            <td>GND</td>\n            <td>4</td>\n            <td>to GND</td>\n        </tr>\n        <tr>\n            <td>22</td>\n            <td>TRST</td>\n            <td>3</td>\n            <td></td>\n        </tr>\n        <tr>\n            <td>26</td>\n            <td>TDI</td>\n            <td>5</td>\n            <td></td>\n        </tr>\n        <tr>\n            <td>27</td>\n            <td>TMS</td>\n            <td>7</td>\n            <td></td>\n        </tr>\n        <tr>\n            <td>25</td>\n            <td>TCK</td>\n            <td>9</td>\n            <td></td>\n        </tr>\n        <tr>\n            <td>23</td>\n            <td>RTCK</td>\n            <td>11</td>\n            <td></td>\n        </tr>\n        <tr>\n            <td>24</td>\n            <td>TDO</td>\n            <td>13</td>\n            <td></td>\n        </tr>\n    </tbody>\n</table>\n\n<p align=\"center\"><img src=\"../doc/09_image_jtag_connected.jpg\" width=\"50%\"></p>\n\n## Getting ready to connect\n\nUpon booting, thanks to the changes we made to `config.txt`, the RPi's firmware will configure the\nrespective GPIO pins for `JTAG` functionality.\n\nWhat is left to do now is to pause the execution of the RPi and then connect\nover `JTAG`. Therefore, we add a new `Makefile` target, `make jtagboot`, which\nuses the `chainboot` approach to load a tiny helper binary onto the RPi that\njust parks the executing core into a waiting state.\n\nThe helper binary is maintained separately in this repository's [X1_JTAG_boot] folder, and is a\nmodified version of the kernel we used in our tutorials so far.\n\n[X1_JTAG_boot]: ../X1_JTAG_boot\n\n```console\n$ make jtagboot\nMinipush 1.0\n\n[MP] ⏳ Waiting for /dev/ttyUSB0\n[MP] ✅ Serial connected\n[MP] 🔌 Please power the target now\n __  __ _      _ _                 _\n|  \\/  (_)_ _ (_) |   ___  __ _ __| |\n| |\\/| | | ' \\| | |__/ _ \\/ _` / _` |\n|_|  |_|_|_||_|_|____\\___/\\__,_\\__,_|\n\n           Raspberry Pi 3\n\n[ML] Requesting binary\n[MP] ⏩ Pushing 7 KiB ==========================================🦀 100% 0 KiB/s Time: 00:00:00\n[ML] Loaded! Executing the payload now\n\n[    0.394532] Parking CPU core. Please connect over JTAG now.\n```\n\nIt is important to keep the USB serial connected and the terminal with the `jtagboot` open and\nrunning. When we load the actual kernel later, `UART` output will appear here.\n\n## OpenOCD\n\nNext, we need to launch the [Open On-Chip Debugger], aka `OpenOCD` to actually connect the `JTAG`.\n\n[Open On-Chip Debugger]: http://openocd.org\n\nAs always, our tutorials try to be as painless as possible regarding dev-tools, which is why we have\npackaged everything into the [dedicated Docker container] that is already used for chainbooting and\n`QEMU`.\n\n[dedicated Docker container]: ../docker/rustembedded-osdev-utils\n\nConnect the Olimex USB JTAG debugger, open a new terminal and in the same folder, type `make\nopenocd` (in that order!). You will see some initial output:\n\n```console\n$ make openocd\n[...]\nOpen On-Chip Debugger 0.10.0\n[...]\nInfo : Listening on port 6666 for tcl connections\nInfo : Listening on port 4444 for telnet connections\nInfo : clock speed 1000 kHz\nInfo : JTAG tap: rpi3.tap tap/device found: 0x4ba00477 (mfg: 0x23b (ARM Ltd.), part: 0xba00, ver: 0x4)\nInfo : rpi3.core0: hardware has 6 breakpoints, 4 watchpoints\nInfo : rpi3.core1: hardware has 6 breakpoints, 4 watchpoints\nInfo : rpi3.core2: hardware has 6 breakpoints, 4 watchpoints\nInfo : rpi3.core3: hardware has 6 breakpoints, 4 watchpoints\nInfo : Listening on port 3333 for gdb connections\nInfo : Listening on port 3334 for gdb connections\nInfo : Listening on port 3335 for gdb connections\nInfo : Listening on port 3336 for gdb connections\n```\n\n`OpenOCD` has detected the four cores of the RPi, and opened four network ports to which `gdb` can\nnow connect to debug the respective core.\n\n## GDB\n\nFinally, we need an `AArch64`-capable version of `gdb`. You guessed right, it's already packaged in\nthe osdev container. It can be launched via `make gdb`.\n\nThis Makefile target actually does a little more. It builds a special version of our kernel with\ndebug information included. This enables `gdb` to show the `Rust` source code line we are currently\ndebugging. It also launches `gdb` such that it already loads this debug build (`kernel_for_jtag`).\n\nWe can now use the `gdb` commandline to\n  1. Set breakpoints in our kernel\n  2. Load the kernel via JTAG into memory (remember that currently, the RPi is still executing the\n     minimal JTAG boot binary).\n  3. Manipulate the program counter of the RPi to start execution at our kernel's entry point.\n  4. Single-step through its execution.\n\n```console\n$ make gdb\n[...]\n>>> target remote :3333                          # Connect to OpenOCD, core0\n>>> load                                         # Load the kernel into the RPi's DRAM over JTAG.\nLoading section .text, size 0x2454 lma 0x80000\nLoading section .rodata, size 0xa1d lma 0x82460\nLoading section .got, size 0x10 lma 0x82e80\nLoading section .data, size 0x20 lma 0x82e90\nStart address 0x0000000000080000, load size 11937\nTransfer rate: 63 KB/sec, 2984 bytes/write.\n>>> set $pc = 0x80000                            # Set RPI's program counter to the start of the\n                                                 # kernel binary.\n>>> break main.rs:158\nBreakpoint 1 at 0x8025c: file src/main.rs, line 158.\n>>> cont\n>>> step                                         # Single-step through the kernel\n>>> step\n>>> ...\n```\n\n### Remarks\n\n#### Optimization\n\nWhen debugging an OS binary, you have to make a trade-off between the granularity at which you can\nstep through your Rust source-code and the optimization level of the generated binary. The `make`\nand `make gdb` targets produce a `--release` binary, which includes an optimization level of three\n(`-opt-level=3`). However, in this case, the compiler will inline very aggressively and pack\ntogether reads and writes where possible. As a result, it will not always be possible to hit\nbreakpoints exactly where you want to regarding the line of source code file.\n\nFor this reason, the Makefile also provides the `make gdb-opt0` target, which uses `-opt-level=0`.\nHence, it will allow you to have finer debugging granularity. However, please keep in mind that when\ndebugging code that closely deals with HW, a compiler optimization that squashes reads or writes to\nvolatile registers can make all the difference in execution. FYI, the demo gif above has been\nrecorded with `gdb-opt0`.\n\n#### GDB control\n\nAt some point, you may reach delay loops or code that waits on user input from the serial. Here,\nsingle stepping might not be feasible or work anymore. You can jump over these roadblocks by setting\nother breakpoints beyond these areas, and reach them using the `cont` command.\n\nPressing `ctrl+c` in `gdb` will stop execution of the RPi again in case you continued it without\nfurther breakpoints.\n\n## Notes on USB connection constraints\n\nIf you followed the tutorial from top to bottom, everything should be fine regarding USB\nconnections.\n\nStill, please note that in its current form, our `Makefile` makes implicit assumptions about the\nnaming of the connected USB devices. It expects `/dev/ttyUSB0` to be the `UART` device.\n\nHence, please ensure the following order of connecting the devices to your box:\n  1. Connect the USB serial.\n  2. Afterwards, the Olimex debugger.\n\nThis way, the host OS enumerates the devices accordingly. This has to be done only once. It is fine\nto disconnect and connect the serial multiple times, e.g. for kicking off different `make jtagboot`\nruns, while keeping the debugger connected.\n\n## Additional resources\n\n- https://metebalci.com/blog/bare-metal-raspberry-pi-3b-jtag\n- https://www.suse.com/c/debugging-raspberry-pi-3-with-jtag\n\n## Acknowledgments\n\nThanks to [@naotaco](https://github.com/naotaco) for laying the groundwork for this tutorial.\n\n## Diff to previous\n```diff\n\ndiff -uNr 07_timestamps/Cargo.toml 08_hw_debug_JTAG/Cargo.toml\n--- 07_timestamps/Cargo.toml\n+++ 08_hw_debug_JTAG/Cargo.toml\n@@ -1,6 +1,6 @@\n [package]\n name = \"mingo\"\n-version = \"0.7.0\"\n+version = \"0.8.0\"\n authors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\n edition = \"2021\"\n\n\ndiff -uNr 07_timestamps/Makefile 08_hw_debug_JTAG/Makefile\n--- 07_timestamps/Makefile\n+++ 08_hw_debug_JTAG/Makefile\n@@ -32,6 +32,8 @@\n     OBJDUMP_BINARY    = aarch64-none-elf-objdump\n     NM_BINARY         = aarch64-none-elf-nm\n     READELF_BINARY    = aarch64-none-elf-readelf\n+    OPENOCD_ARG       = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi3.cfg\n+    JTAG_BOOT_IMAGE   = ../X1_JTAG_boot/jtag_boot_rpi3.img\n     LD_SCRIPT_PATH    = $(shell pwd)/src/bsp/raspberrypi\n     RUSTC_MISC_ARGS   = -C target-cpu=cortex-a53\n else ifeq ($(BSP),rpi4)\n@@ -43,6 +45,8 @@\n     OBJDUMP_BINARY    = aarch64-none-elf-objdump\n     NM_BINARY         = aarch64-none-elf-nm\n     READELF_BINARY    = aarch64-none-elf-readelf\n+    OPENOCD_ARG       = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi4.cfg\n+    JTAG_BOOT_IMAGE   = ../X1_JTAG_boot/jtag_boot_rpi4.img\n     LD_SCRIPT_PATH    = $(shell pwd)/src/bsp/raspberrypi\n     RUSTC_MISC_ARGS   = -C target-cpu=cortex-a72\n endif\n@@ -99,18 +103,25 @@\n DOCKER_CMD            = docker run -t --rm -v $(shell pwd):/work/tutorial -w /work/tutorial\n DOCKER_CMD_INTERACT   = $(DOCKER_CMD) -i\n DOCKER_ARG_DIR_COMMON = -v $(shell pwd)/../common:/work/common\n+DOCKER_ARG_DIR_JTAG   = -v $(shell pwd)/../X1_JTAG_boot:/work/X1_JTAG_boot\n DOCKER_ARG_DEV        = --privileged -v /dev:/dev\n+DOCKER_ARG_NET        = --network host\n\n # DOCKER_IMAGE defined in include file (see top of this file).\n DOCKER_QEMU  = $(DOCKER_CMD_INTERACT) $(DOCKER_IMAGE)\n DOCKER_TOOLS = $(DOCKER_CMD) $(DOCKER_IMAGE)\n DOCKER_TEST  = $(DOCKER_CMD) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\n+DOCKER_GDB   = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_NET) $(DOCKER_IMAGE)\n\n # Dockerize commands, which require USB device passthrough, only on Linux.\n ifeq ($(shell uname -s),Linux)\n     DOCKER_CMD_DEV = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_DEV)\n\n     DOCKER_CHAINBOOT = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\n+    DOCKER_JTAGBOOT  = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_ARG_DIR_JTAG) $(DOCKER_IMAGE)\n+    DOCKER_OPENOCD   = $(DOCKER_CMD_DEV) $(DOCKER_ARG_NET) $(DOCKER_IMAGE)\n+else\n+    DOCKER_OPENOCD   = echo \"Not yet supported on non-Linux systems.\"; \\#\n endif\n\n\n@@ -215,6 +226,35 @@\n\n\n\n+##--------------------------------------------------------------------------------------------------\n+## Debugging targets\n+##--------------------------------------------------------------------------------------------------\n+.PHONY: jtagboot openocd gdb gdb-opt0\n+\n+##------------------------------------------------------------------------------\n+## Push the JTAG boot image to the real HW target\n+##------------------------------------------------------------------------------\n+jtagboot:\n+\t@$(DOCKER_JTAGBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(JTAG_BOOT_IMAGE)\n+\n+##------------------------------------------------------------------------------\n+## Start OpenOCD session\n+##------------------------------------------------------------------------------\n+openocd:\n+\t$(call color_header, \"Launching OpenOCD\")\n+\t@$(DOCKER_OPENOCD) openocd $(OPENOCD_ARG)\n+\n+##------------------------------------------------------------------------------\n+## Start GDB session\n+##------------------------------------------------------------------------------\n+gdb: RUSTC_MISC_ARGS += -C debuginfo=2\n+gdb-opt0: RUSTC_MISC_ARGS += -C debuginfo=2 -C opt-level=0\n+gdb gdb-opt0: $(KERNEL_ELF)\n+\t$(call color_header, \"Launching GDB\")\n+\t@$(DOCKER_GDB) gdb-multiarch -q $(KERNEL_ELF)\n+\n+\n+\n ##--------------------------------------------------------------------------------------------------\n ## Testing targets\n ##--------------------------------------------------------------------------------------------------\n\ndiff -uNr 07_timestamps/src/bsp/raspberrypi/driver.rs 08_hw_debug_JTAG/src/bsp/raspberrypi/driver.rs\n--- 07_timestamps/src/bsp/raspberrypi/driver.rs\n+++ 08_hw_debug_JTAG/src/bsp/raspberrypi/driver.rs\n@@ -57,17 +57,6 @@\n /// # Safety\n ///\n /// See child function calls.\n-///\n-/// # Note\n-///\n-/// Using atomics here relieves us from needing to use `unsafe` for the static variable.\n-///\n-/// On `AArch64`, which is the only implemented architecture at the time of writing this,\n-/// [`AtomicBool::load`] and [`AtomicBool::store`] are lowered to ordinary load and store\n-/// instructions. They are therefore safe to use even with MMU + caching deactivated.\n-///\n-/// [`AtomicBool::load`]: core::sync::atomic::AtomicBool::load\n-/// [`AtomicBool::store`]: core::sync::atomic::AtomicBool::store\n pub unsafe fn init() -> Result<(), &'static str> {\n     static INIT_DONE: AtomicBool = AtomicBool::new(false);\n     if INIT_DONE.load(Ordering::Relaxed) {\n\n```\n"
  },
  {
    "path": "08_hw_debug_JTAG/build.rs",
    "content": "use std::{env, fs, process};\n\nfn main() {\n    let ld_script_path = match env::var(\"LD_SCRIPT_PATH\") {\n        Ok(var) => var,\n        _ => process::exit(0),\n    };\n\n    let files = fs::read_dir(ld_script_path).unwrap();\n    files\n        .filter_map(Result::ok)\n        .filter(|d| {\n            if let Some(e) = d.path().extension() {\n                e == \"ld\"\n            } else {\n                false\n            }\n        })\n        .for_each(|f| println!(\"cargo:rerun-if-changed={}\", f.path().display()));\n}\n"
  },
  {
    "path": "08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural boot code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::boot::arch_boot\n\nuse core::arch::global_asm;\n\n// Assembly counterpart to this file.\nglobal_asm!(\n    include_str!(\"boot.s\"),\n    CONST_CORE_ID_MASK = const 0b11\n);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The Rust entry of the `kernel` binary.\n///\n/// The function is called from the assembly `_start` function.\n#[no_mangle]\npub unsafe fn _start_rust() -> ! {\n    crate::kernel_init()\n}\n"
  },
  {
    "path": "08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.s",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2022 Andre Richter <andre.o.richter@gmail.com>\n\n//--------------------------------------------------------------------------------------------------\n// Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Load the address of a symbol into a register, PC-relative.\n//\n// The symbol must lie within +/- 4 GiB of the Program Counter.\n//\n// # Resources\n//\n// - https://sourceware.org/binutils/docs-2.36/as/AArch64_002dRelocations.html\n.macro ADR_REL register, symbol\n\tadrp\t\\register, \\symbol\n\tadd\t\\register, \\register, #:lo12:\\symbol\n.endm\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n.section .text._start\n\n//------------------------------------------------------------------------------\n// fn _start()\n//------------------------------------------------------------------------------\n_start:\n\t// Only proceed on the boot core. Park it otherwise.\n\tmrs\tx0, MPIDR_EL1\n\tand\tx0, x0, {CONST_CORE_ID_MASK}\n\tldr\tx1, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs\n\tcmp\tx0, x1\n\tb.ne\t.L_parking_loop\n\n\t// If execution reaches here, it is the boot core.\n\n\t// Initialize DRAM.\n\tADR_REL\tx0, __bss_start\n\tADR_REL x1, __bss_end_exclusive\n\n.L_bss_init_loop:\n\tcmp\tx0, x1\n\tb.eq\t.L_prepare_rust\n\tstp\txzr, xzr, [x0], #16\n\tb\t.L_bss_init_loop\n\n\t// Prepare the jump to Rust code.\n.L_prepare_rust:\n\t// Set the stack pointer.\n\tADR_REL\tx0, __boot_core_stack_end_exclusive\n\tmov\tsp, x0\n\n\t// Read the CPU's timer counter frequency and store it in ARCH_TIMER_COUNTER_FREQUENCY.\n\t// Abort if the frequency read back as 0.\n\tADR_REL\tx1, ARCH_TIMER_COUNTER_FREQUENCY // provided by aarch64/time.rs\n\tmrs\tx2, CNTFRQ_EL0\n\tcmp\tx2, xzr\n\tb.eq\t.L_parking_loop\n\tstr\tw2, [x1]\n\n\t// Jump to Rust code.\n\tb\t_start_rust\n\n\t// Infinitely wait for events (aka \"park the core\").\n.L_parking_loop:\n\twfe\n\tb\t.L_parking_loop\n\n.size\t_start, . - _start\n.type\t_start, function\n.global\t_start\n"
  },
  {
    "path": "08_hw_debug_JTAG/src/_arch/aarch64/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural processor code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::arch_cpu\n\nuse aarch64_cpu::asm;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\npub use asm::nop;\n\n/// Pause execution on the core.\n#[inline(always)]\npub fn wait_forever() -> ! {\n    loop {\n        asm::wfe()\n    }\n}\n"
  },
  {
    "path": "08_hw_debug_JTAG/src/_arch/aarch64/time.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural timer primitives.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::time::arch_time\n\nuse crate::warn;\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::{\n    num::{NonZeroU128, NonZeroU32, NonZeroU64},\n    ops::{Add, Div},\n    time::Duration,\n};\nuse tock_registers::interfaces::Readable;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nconst NANOSEC_PER_SEC: NonZeroU64 = NonZeroU64::new(1_000_000_000).unwrap();\n\n#[derive(Copy, Clone, PartialOrd, PartialEq)]\nstruct GenericTimerCounterValue(u64);\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n/// Boot assembly code overwrites this value with the value of CNTFRQ_EL0 before any Rust code is\n/// executed. This given value here is just a (safe) dummy.\n#[no_mangle]\nstatic ARCH_TIMER_COUNTER_FREQUENCY: NonZeroU32 = NonZeroU32::MIN;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nfn arch_timer_counter_frequency() -> NonZeroU32 {\n    // Read volatile is needed here to prevent the compiler from optimizing\n    // ARCH_TIMER_COUNTER_FREQUENCY away.\n    //\n    // This is safe, because all the safety requirements as stated in read_volatile()'s\n    // documentation are fulfilled.\n    unsafe { core::ptr::read_volatile(&ARCH_TIMER_COUNTER_FREQUENCY) }\n}\n\nimpl GenericTimerCounterValue {\n    pub const MAX: Self = GenericTimerCounterValue(u64::MAX);\n}\n\nimpl Add for GenericTimerCounterValue {\n    type Output = Self;\n\n    fn add(self, other: Self) -> Self {\n        GenericTimerCounterValue(self.0.wrapping_add(other.0))\n    }\n}\n\nimpl From<GenericTimerCounterValue> for Duration {\n    fn from(counter_value: GenericTimerCounterValue) -> Self {\n        if counter_value.0 == 0 {\n            return Duration::ZERO;\n        }\n\n        let frequency: NonZeroU64 = arch_timer_counter_frequency().into();\n\n        // Div<NonZeroU64> implementation for u64 cannot panic.\n        let secs = counter_value.0.div(frequency);\n\n        // This is safe, because frequency can never be greater than u32::MAX, which means the\n        // largest theoretical value for sub_second_counter_value is (u32::MAX - 1). Therefore,\n        // (sub_second_counter_value * NANOSEC_PER_SEC) cannot overflow an u64.\n        //\n        // The subsequent division ensures the result fits into u32, since the max result is smaller\n        // than NANOSEC_PER_SEC. Therefore, just cast it to u32 using `as`.\n        let sub_second_counter_value = counter_value.0 % frequency;\n        let nanos = unsafe { sub_second_counter_value.unchecked_mul(u64::from(NANOSEC_PER_SEC)) }\n            .div(frequency) as u32;\n\n        Duration::new(secs, nanos)\n    }\n}\n\nfn max_duration() -> Duration {\n    Duration::from(GenericTimerCounterValue::MAX)\n}\n\nimpl TryFrom<Duration> for GenericTimerCounterValue {\n    type Error = &'static str;\n\n    fn try_from(duration: Duration) -> Result<Self, Self::Error> {\n        if duration < resolution() {\n            return Ok(GenericTimerCounterValue(0));\n        }\n\n        if duration > max_duration() {\n            return Err(\"Conversion error. Duration too big\");\n        }\n\n        let frequency: u128 = u32::from(arch_timer_counter_frequency()) as u128;\n        let duration: u128 = duration.as_nanos();\n\n        // This is safe, because frequency can never be greater than u32::MAX, and\n        // (Duration::MAX.as_nanos() * u32::MAX) < u128::MAX.\n        let counter_value =\n            unsafe { duration.unchecked_mul(frequency) }.div(NonZeroU128::from(NANOSEC_PER_SEC));\n\n        // Since we checked above that we are <= max_duration(), just cast to u64.\n        Ok(GenericTimerCounterValue(counter_value as u64))\n    }\n}\n\n#[inline(always)]\nfn read_cntpct() -> GenericTimerCounterValue {\n    // Prevent that the counter is read ahead of time due to out-of-order execution.\n    barrier::isb(barrier::SY);\n    let cnt = CNTPCT_EL0.get();\n\n    GenericTimerCounterValue(cnt)\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The timer's resolution.\npub fn resolution() -> Duration {\n    Duration::from(GenericTimerCounterValue(1))\n}\n\n/// The uptime since power-on of the device.\n///\n/// This includes time consumed by firmware and bootloaders.\npub fn uptime() -> Duration {\n    read_cntpct().into()\n}\n\n/// Spin for a given duration.\npub fn spin_for(duration: Duration) {\n    let curr_counter_value = read_cntpct();\n\n    let counter_value_delta: GenericTimerCounterValue = match duration.try_into() {\n        Err(msg) => {\n            warn!(\"spin_for: {}. Skipping\", msg);\n            return;\n        }\n        Ok(val) => val,\n    };\n    let counter_value_target = curr_counter_value + counter_value_delta;\n\n    // Busy wait.\n    //\n    // Read CNTPCT_EL0 directly to avoid the ISB that is part of [`read_cntpct`].\n    while GenericTimerCounterValue(CNTPCT_EL0.get()) < counter_value_target {}\n}\n"
  },
  {
    "path": "08_hw_debug_JTAG/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GPIO Driver.\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper, driver, synchronization,\n    synchronization::NullLock,\n};\nuse tock_registers::{\n    interfaces::{ReadWriteable, Writeable},\n    register_bitfields, register_structs,\n    registers::ReadWrite,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// GPIO registers.\n//\n// Descriptions taken from\n// - https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf\n// - https://datasheets.raspberrypi.org/bcm2711/bcm2711-peripherals.pdf\nregister_bitfields! {\n    u32,\n\n    /// GPIO Function Select 1\n    GPFSEL1 [\n        /// Pin 15\n        FSEL15 OFFSET(15) NUMBITS(3) [\n            Input = 0b000,\n            Output = 0b001,\n            AltFunc0 = 0b100  // PL011 UART RX\n\n        ],\n\n        /// Pin 14\n        FSEL14 OFFSET(12) NUMBITS(3) [\n            Input = 0b000,\n            Output = 0b001,\n            AltFunc0 = 0b100  // PL011 UART TX\n        ]\n    ],\n\n    /// GPIO Pull-up/down Register\n    ///\n    /// BCM2837 only.\n    GPPUD [\n        /// Controls the actuation of the internal pull-up/down control line to ALL the GPIO pins.\n        PUD OFFSET(0) NUMBITS(2) [\n            Off = 0b00,\n            PullDown = 0b01,\n            PullUp = 0b10\n        ]\n    ],\n\n    /// GPIO Pull-up/down Clock Register 0\n    ///\n    /// BCM2837 only.\n    GPPUDCLK0 [\n        /// Pin 15\n        PUDCLK15 OFFSET(15) NUMBITS(1) [\n            NoEffect = 0,\n            AssertClock = 1\n        ],\n\n        /// Pin 14\n        PUDCLK14 OFFSET(14) NUMBITS(1) [\n            NoEffect = 0,\n            AssertClock = 1\n        ]\n    ],\n\n    /// GPIO Pull-up / Pull-down Register 0\n    ///\n    /// BCM2711 only.\n    GPIO_PUP_PDN_CNTRL_REG0 [\n        /// Pin 15\n        GPIO_PUP_PDN_CNTRL15 OFFSET(30) NUMBITS(2) [\n            NoResistor = 0b00,\n            PullUp = 0b01\n        ],\n\n        /// Pin 14\n        GPIO_PUP_PDN_CNTRL14 OFFSET(28) NUMBITS(2) [\n            NoResistor = 0b00,\n            PullUp = 0b01\n        ]\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    RegisterBlock {\n        (0x00 => _reserved1),\n        (0x04 => GPFSEL1: ReadWrite<u32, GPFSEL1::Register>),\n        (0x08 => _reserved2),\n        (0x94 => GPPUD: ReadWrite<u32, GPPUD::Register>),\n        (0x98 => GPPUDCLK0: ReadWrite<u32, GPPUDCLK0::Register>),\n        (0x9C => _reserved3),\n        (0xE4 => GPIO_PUP_PDN_CNTRL_REG0: ReadWrite<u32, GPIO_PUP_PDN_CNTRL_REG0::Register>),\n        (0xE8 => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\nstruct GPIOInner {\n    registers: Registers,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the GPIO HW.\npub struct GPIO {\n    inner: NullLock<GPIOInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GPIOInner {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n        }\n    }\n\n    /// Disable pull-up/down on pins 14 and 15.\n    #[cfg(feature = \"bsp_rpi3\")]\n    fn disable_pud_14_15_bcm2837(&mut self) {\n        use crate::time;\n        use core::time::Duration;\n\n        // The Linux 2837 GPIO driver waits 1 µs between the steps.\n        const DELAY: Duration = Duration::from_micros(1);\n\n        self.registers.GPPUD.write(GPPUD::PUD::Off);\n        time::time_manager().spin_for(DELAY);\n\n        self.registers\n            .GPPUDCLK0\n            .write(GPPUDCLK0::PUDCLK15::AssertClock + GPPUDCLK0::PUDCLK14::AssertClock);\n        time::time_manager().spin_for(DELAY);\n\n        self.registers.GPPUD.write(GPPUD::PUD::Off);\n        self.registers.GPPUDCLK0.set(0);\n    }\n\n    /// Disable pull-up/down on pins 14 and 15.\n    #[cfg(feature = \"bsp_rpi4\")]\n    fn disable_pud_14_15_bcm2711(&mut self) {\n        self.registers.GPIO_PUP_PDN_CNTRL_REG0.write(\n            GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL15::PullUp\n                + GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL14::PullUp,\n        );\n    }\n\n    /// Map PL011 UART as standard output.\n    ///\n    /// TX to pin 14\n    /// RX to pin 15\n    pub fn map_pl011_uart(&mut self) {\n        // Select the UART on pins 14 and 15.\n        self.registers\n            .GPFSEL1\n            .modify(GPFSEL1::FSEL15::AltFunc0 + GPFSEL1::FSEL14::AltFunc0);\n\n        // Disable pull-up/down on pins 14 and 15.\n        #[cfg(feature = \"bsp_rpi3\")]\n        self.disable_pud_14_15_bcm2837();\n\n        #[cfg(feature = \"bsp_rpi4\")]\n        self.disable_pud_14_15_bcm2711();\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GPIO {\n    pub const COMPATIBLE: &'static str = \"BCM GPIO\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            inner: NullLock::new(GPIOInner::new(mmio_start_addr)),\n        }\n    }\n\n    /// Concurrency safe version of `GPIOInner.map_pl011_uart()`\n    pub fn map_pl011_uart(&self) {\n        self.inner.lock(|inner| inner.map_pl011_uart())\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl driver::interface::DeviceDriver for GPIO {\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n}\n"
  },
  {
    "path": "08_hw_debug_JTAG/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! PL011 UART driver.\n//!\n//! # Resources\n//!\n//! - <https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf>\n//! - <https://developer.arm.com/documentation/ddi0183/latest>\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper, console, cpu, driver, synchronization,\n    synchronization::NullLock,\n};\nuse core::fmt;\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields, register_structs,\n    registers::{ReadOnly, ReadWrite, WriteOnly},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// PL011 UART registers.\n//\n// Descriptions taken from \"PrimeCell UART (PL011) Technical Reference Manual\" r1p5.\nregister_bitfields! {\n    u32,\n\n    /// Flag Register.\n    FR [\n        /// Transmit FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n        /// Line Control Register, LCR_H.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the transmit holding register is empty.\n        /// - If the FIFO is enabled, the TXFE bit is set when the transmit FIFO is empty.\n        /// - This bit does not indicate if there is data in the transmit shift register.\n        TXFE OFFSET(7) NUMBITS(1) [],\n\n        /// Transmit FIFO full. The meaning of this bit depends on the state of the FEN bit in the\n        /// LCR_H Register.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the transmit holding register is full.\n        /// - If the FIFO is enabled, the TXFF bit is set when the transmit FIFO is full.\n        TXFF OFFSET(5) NUMBITS(1) [],\n\n        /// Receive FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n        /// LCR_H Register.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the receive holding register is empty.\n        /// - If the FIFO is enabled, the RXFE bit is set when the receive FIFO is empty.\n        RXFE OFFSET(4) NUMBITS(1) [],\n\n        /// UART busy. If this bit is set to 1, the UART is busy transmitting data. This bit remains\n        /// set until the complete byte, including all the stop bits, has been sent from the shift\n        /// register.\n        ///\n        /// This bit is set as soon as the transmit FIFO becomes non-empty, regardless of whether\n        /// the UART is enabled or not.\n        BUSY OFFSET(3) NUMBITS(1) []\n    ],\n\n    /// Integer Baud Rate Divisor.\n    IBRD [\n        /// The integer baud rate divisor.\n        BAUD_DIVINT OFFSET(0) NUMBITS(16) []\n    ],\n\n    /// Fractional Baud Rate Divisor.\n    FBRD [\n        ///  The fractional baud rate divisor.\n        BAUD_DIVFRAC OFFSET(0) NUMBITS(6) []\n    ],\n\n    /// Line Control Register.\n    LCR_H [\n        /// Word length. These bits indicate the number of data bits transmitted or received in a\n        /// frame.\n        #[allow(clippy::enum_variant_names)]\n        WLEN OFFSET(5) NUMBITS(2) [\n            FiveBit = 0b00,\n            SixBit = 0b01,\n            SevenBit = 0b10,\n            EightBit = 0b11\n        ],\n\n        /// Enable FIFOs:\n        ///\n        /// 0 = FIFOs are disabled (character mode) that is, the FIFOs become 1-byte-deep holding\n        /// registers.\n        ///\n        /// 1 = Transmit and receive FIFO buffers are enabled (FIFO mode).\n        FEN  OFFSET(4) NUMBITS(1) [\n            FifosDisabled = 0,\n            FifosEnabled = 1\n        ]\n    ],\n\n    /// Control Register.\n    CR [\n        /// Receive enable. If this bit is set to 1, the receive section of the UART is enabled.\n        /// Data reception occurs for either UART signals or SIR signals depending on the setting of\n        /// the SIREN bit. When the UART is disabled in the middle of reception, it completes the\n        /// current character before stopping.\n        RXE OFFSET(9) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// Transmit enable. If this bit is set to 1, the transmit section of the UART is enabled.\n        /// Data transmission occurs for either UART signals, or SIR signals depending on the\n        /// setting of the SIREN bit. When the UART is disabled in the middle of transmission, it\n        /// completes the current character before stopping.\n        TXE OFFSET(8) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// UART enable:\n        ///\n        /// 0 = UART is disabled. If the UART is disabled in the middle of transmission or\n        /// reception, it completes the current character before stopping.\n        ///\n        /// 1 = The UART is enabled. Data transmission and reception occurs for either UART signals\n        /// or SIR signals depending on the setting of the SIREN bit\n        UARTEN OFFSET(0) NUMBITS(1) [\n            /// If the UART is disabled in the middle of transmission or reception, it completes the\n            /// current character before stopping.\n            Disabled = 0,\n            Enabled = 1\n        ]\n    ],\n\n    /// Interrupt Clear Register.\n    ICR [\n        /// Meta field for all pending interrupts.\n        ALL OFFSET(0) NUMBITS(11) []\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    pub RegisterBlock {\n        (0x00 => DR: ReadWrite<u32>),\n        (0x04 => _reserved1),\n        (0x18 => FR: ReadOnly<u32, FR::Register>),\n        (0x1c => _reserved2),\n        (0x24 => IBRD: WriteOnly<u32, IBRD::Register>),\n        (0x28 => FBRD: WriteOnly<u32, FBRD::Register>),\n        (0x2c => LCR_H: WriteOnly<u32, LCR_H::Register>),\n        (0x30 => CR: WriteOnly<u32, CR::Register>),\n        (0x34 => _reserved3),\n        (0x44 => ICR: WriteOnly<u32, ICR::Register>),\n        (0x48 => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\n#[derive(PartialEq)]\nenum BlockingMode {\n    Blocking,\n    NonBlocking,\n}\n\nstruct PL011UartInner {\n    registers: Registers,\n    chars_written: usize,\n    chars_read: usize,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the UART.\npub struct PL011Uart {\n    inner: NullLock<PL011UartInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PL011UartInner {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n            chars_written: 0,\n            chars_read: 0,\n        }\n    }\n\n    /// Set up baud rate and characteristics.\n    ///\n    /// This results in 8N1 and 921_600 baud.\n    ///\n    /// The calculation for the BRD is (we set the clock to 48 MHz in config.txt):\n    /// `(48_000_000 / 16) / 921_600 = 3.2552083`.\n    ///\n    /// This means the integer part is `3` and goes into the `IBRD`.\n    /// The fractional part is `0.2552083`.\n    ///\n    /// `FBRD` calculation according to the PL011 Technical Reference Manual:\n    /// `INTEGER((0.2552083 * 64) + 0.5) = 16`.\n    ///\n    /// Therefore, the generated baud rate divider is: `3 + 16/64 = 3.25`. Which results in a\n    /// genrated baud rate of `48_000_000 / (16 * 3.25) = 923_077`.\n    ///\n    /// Error = `((923_077 - 921_600) / 921_600) * 100 = 0.16%`.\n    pub fn init(&mut self) {\n        // Execution can arrive here while there are still characters queued in the TX FIFO and\n        // actively being sent out by the UART hardware. If the UART is turned off in this case,\n        // those queued characters would be lost.\n        //\n        // For example, this can happen during runtime on a call to panic!(), because panic!()\n        // initializes its own UART instance and calls init().\n        //\n        // Hence, flush first to ensure all pending characters are transmitted.\n        self.flush();\n\n        // Turn the UART off temporarily.\n        self.registers.CR.set(0);\n\n        // Clear all pending interrupts.\n        self.registers.ICR.write(ICR::ALL::CLEAR);\n\n        // From the PL011 Technical Reference Manual:\n        //\n        // The LCR_H, IBRD, and FBRD registers form the single 30-bit wide LCR Register that is\n        // updated on a single write strobe generated by a LCR_H write. So, to internally update the\n        // contents of IBRD or FBRD, a LCR_H write must always be performed at the end.\n        //\n        // Set the baud rate, 8N1 and FIFO enabled.\n        self.registers.IBRD.write(IBRD::BAUD_DIVINT.val(3));\n        self.registers.FBRD.write(FBRD::BAUD_DIVFRAC.val(16));\n        self.registers\n            .LCR_H\n            .write(LCR_H::WLEN::EightBit + LCR_H::FEN::FifosEnabled);\n\n        // Turn the UART on.\n        self.registers\n            .CR\n            .write(CR::UARTEN::Enabled + CR::TXE::Enabled + CR::RXE::Enabled);\n    }\n\n    /// Send a character.\n    fn write_char(&mut self, c: char) {\n        // Spin while TX FIFO full is set, waiting for an empty slot.\n        while self.registers.FR.matches_all(FR::TXFF::SET) {\n            cpu::nop();\n        }\n\n        // Write the character to the buffer.\n        self.registers.DR.set(c as u32);\n\n        self.chars_written += 1;\n    }\n\n    /// Block execution until the last buffered character has been physically put on the TX wire.\n    fn flush(&self) {\n        // Spin until the busy bit is cleared.\n        while self.registers.FR.matches_all(FR::BUSY::SET) {\n            cpu::nop();\n        }\n    }\n\n    /// Retrieve a character.\n    fn read_char_converting(&mut self, blocking_mode: BlockingMode) -> Option<char> {\n        // If RX FIFO is empty,\n        if self.registers.FR.matches_all(FR::RXFE::SET) {\n            // immediately return in non-blocking mode.\n            if blocking_mode == BlockingMode::NonBlocking {\n                return None;\n            }\n\n            // Otherwise, wait until a char was received.\n            while self.registers.FR.matches_all(FR::RXFE::SET) {\n                cpu::nop();\n            }\n        }\n\n        // Read one character.\n        let mut ret = self.registers.DR.get() as u8 as char;\n\n        // Convert carrige return to newline.\n        if ret == '\\r' {\n            ret = '\\n'\n        }\n\n        // Update statistics.\n        self.chars_read += 1;\n\n        Some(ret)\n    }\n}\n\n/// Implementing `core::fmt::Write` enables usage of the `format_args!` macros, which in turn are\n/// used to implement the `kernel`'s `print!` and `println!` macros. By implementing `write_str()`,\n/// we get `write_fmt()` automatically.\n///\n/// The function takes an `&mut self`, so it must be implemented for the inner struct.\n///\n/// See [`src/print.rs`].\n///\n/// [`src/print.rs`]: ../../print/index.html\nimpl fmt::Write for PL011UartInner {\n    fn write_str(&mut self, s: &str) -> fmt::Result {\n        for c in s.chars() {\n            self.write_char(c);\n        }\n\n        Ok(())\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PL011Uart {\n    pub const COMPATIBLE: &'static str = \"BCM PL011 UART\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            inner: NullLock::new(PL011UartInner::new(mmio_start_addr)),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl driver::interface::DeviceDriver for PL011Uart {\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n\n    unsafe fn init(&self) -> Result<(), &'static str> {\n        self.inner.lock(|inner| inner.init());\n\n        Ok(())\n    }\n}\n\nimpl console::interface::Write for PL011Uart {\n    /// Passthrough of `args` to the `core::fmt::Write` implementation, but guarded by a Mutex to\n    /// serialize access.\n    fn write_char(&self, c: char) {\n        self.inner.lock(|inner| inner.write_char(c));\n    }\n\n    fn write_fmt(&self, args: core::fmt::Arguments) -> fmt::Result {\n        // Fully qualified syntax for the call to `core::fmt::Write::write_fmt()` to increase\n        // readability.\n        self.inner.lock(|inner| fmt::Write::write_fmt(inner, args))\n    }\n\n    fn flush(&self) {\n        // Spin until TX FIFO empty is set.\n        self.inner.lock(|inner| inner.flush());\n    }\n}\n\nimpl console::interface::Read for PL011Uart {\n    fn read_char(&self) -> char {\n        self.inner\n            .lock(|inner| inner.read_char_converting(BlockingMode::Blocking).unwrap())\n    }\n\n    fn clear_rx(&self) {\n        // Read from the RX FIFO until it is indicating empty.\n        while self\n            .inner\n            .lock(|inner| inner.read_char_converting(BlockingMode::NonBlocking))\n            .is_some()\n        {}\n    }\n}\n\nimpl console::interface::Statistics for PL011Uart {\n    fn chars_written(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_written)\n    }\n\n    fn chars_read(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_read)\n    }\n}\n\nimpl console::interface::All for PL011Uart {}\n"
  },
  {
    "path": "08_hw_debug_JTAG/src/bsp/device_driver/bcm.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BCM driver top level.\n\nmod bcm2xxx_gpio;\nmod bcm2xxx_pl011_uart;\n\npub use bcm2xxx_gpio::*;\npub use bcm2xxx_pl011_uart::*;\n"
  },
  {
    "path": "08_hw_debug_JTAG/src/bsp/device_driver/common.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Common device driver code.\n\nuse core::{marker::PhantomData, ops};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct MMIODerefWrapper<T> {\n    start_addr: usize,\n    phantom: PhantomData<fn() -> T>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T> MMIODerefWrapper<T> {\n    /// Create an instance.\n    pub const unsafe fn new(start_addr: usize) -> Self {\n        Self {\n            start_addr,\n            phantom: PhantomData,\n        }\n    }\n}\n\nimpl<T> ops::Deref for MMIODerefWrapper<T> {\n    type Target = T;\n\n    fn deref(&self) -> &Self::Target {\n        unsafe { &*(self.start_addr as *const _) }\n    }\n}\n"
  },
  {
    "path": "08_hw_debug_JTAG/src/bsp/device_driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Device driver.\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod bcm;\nmod common;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use bcm::*;\n"
  },
  {
    "path": "08_hw_debug_JTAG/src/bsp/raspberrypi/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Processor code.\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Used by `arch` code to find the early boot core.\n#[no_mangle]\n#[link_section = \".text._start_arguments\"]\npub static BOOT_CORE_ID: u64 = 0;\n"
  },
  {
    "path": "08_hw_debug_JTAG/src/bsp/raspberrypi/driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP driver support.\n\nuse super::memory::map::mmio;\nuse crate::{bsp::device_driver, console, driver as generic_driver};\nuse core::sync::atomic::{AtomicBool, Ordering};\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic PL011_UART: device_driver::PL011Uart =\n    unsafe { device_driver::PL011Uart::new(mmio::PL011_UART_START) };\nstatic GPIO: device_driver::GPIO = unsafe { device_driver::GPIO::new(mmio::GPIO_START) };\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// This must be called only after successful init of the UART driver.\nfn post_init_uart() -> Result<(), &'static str> {\n    console::register_console(&PL011_UART);\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the GPIO driver.\nfn post_init_gpio() -> Result<(), &'static str> {\n    GPIO.map_pl011_uart();\n    Ok(())\n}\n\nfn driver_uart() -> Result<(), &'static str> {\n    let uart_descriptor =\n        generic_driver::DeviceDriverDescriptor::new(&PL011_UART, Some(post_init_uart));\n    generic_driver::driver_manager().register_driver(uart_descriptor);\n\n    Ok(())\n}\n\nfn driver_gpio() -> Result<(), &'static str> {\n    let gpio_descriptor = generic_driver::DeviceDriverDescriptor::new(&GPIO, Some(post_init_gpio));\n    generic_driver::driver_manager().register_driver(gpio_descriptor);\n\n    Ok(())\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Initialize the driver subsystem.\n///\n/// # Safety\n///\n/// See child function calls.\npub unsafe fn init() -> Result<(), &'static str> {\n    static INIT_DONE: AtomicBool = AtomicBool::new(false);\n    if INIT_DONE.load(Ordering::Relaxed) {\n        return Err(\"Init already done\");\n    }\n\n    driver_uart()?;\n    driver_gpio()?;\n\n    INIT_DONE.store(true, Ordering::Relaxed);\n    Ok(())\n}\n"
  },
  {
    "path": "08_hw_debug_JTAG/src/bsp/raspberrypi/kernel.ld",
    "content": "/* SPDX-License-Identifier: MIT OR Apache-2.0\n *\n * Copyright (c) 2018-2022 Andre Richter <andre.o.richter@gmail.com>\n */\n\n__rpi_phys_dram_start_addr = 0;\n\n/* The physical address at which the the kernel binary will be loaded by the Raspberry's firmware */\n__rpi_phys_binary_load_addr = 0x80000;\n\n\nENTRY(__rpi_phys_binary_load_addr)\n\n/* Flags:\n *     4 == R\n *     5 == RX\n *     6 == RW\n *\n * Segments are marked PT_LOAD below so that the ELF file provides virtual and physical addresses.\n * It doesn't mean all of them need actually be loaded.\n */\nPHDRS\n{\n    segment_boot_core_stack PT_LOAD FLAGS(6);\n    segment_code            PT_LOAD FLAGS(5);\n    segment_data            PT_LOAD FLAGS(6);\n}\n\nSECTIONS\n{\n    . =  __rpi_phys_dram_start_addr;\n\n    /***********************************************************************************************\n    * Boot Core Stack\n    ***********************************************************************************************/\n    .boot_core_stack (NOLOAD) :\n    {\n                                             /*   ^             */\n                                             /*   | stack       */\n        . += __rpi_phys_binary_load_addr;    /*   | growth      */\n                                             /*   | direction   */\n        __boot_core_stack_end_exclusive = .; /*   |             */\n    } :segment_boot_core_stack\n\n    /***********************************************************************************************\n    * Code + RO Data + Global Offset Table\n    ***********************************************************************************************/\n    .text :\n    {\n        KEEP(*(.text._start))\n        *(.text._start_arguments) /* Constants (or statics in Rust speak) read by _start(). */\n        *(.text._start_rust)      /* The Rust entry point */\n        *(.text*)                 /* Everything else */\n    } :segment_code\n\n    .rodata : ALIGN(8) { *(.rodata*) } :segment_code\n\n    /***********************************************************************************************\n    * Data + BSS\n    ***********************************************************************************************/\n    .data : { *(.data*) } :segment_data\n\n    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */\n    .bss (NOLOAD) : ALIGN(16)\n    {\n        __bss_start = .;\n        *(.bss*);\n        . = ALIGN(16);\n        __bss_end_exclusive = .;\n    } :segment_data\n\n    /***********************************************************************************************\n    * Misc\n    ***********************************************************************************************/\n    .got : { *(.got*) }\n    ASSERT(SIZEOF(.got) == 0, \"Relocation support not expected\")\n\n    /DISCARD/ : { *(.comment*) }\n}\n"
  },
  {
    "path": "08_hw_debug_JTAG/src/bsp/raspberrypi/memory.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Memory Management.\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// The board's physical memory map.\n#[rustfmt::skip]\npub(super) mod map {\n\n    pub const GPIO_OFFSET:         usize = 0x0020_0000;\n    pub const UART_OFFSET:         usize = 0x0020_1000;\n\n    /// Physical devices.\n    #[cfg(feature = \"bsp_rpi3\")]\n    pub mod mmio {\n        use super::*;\n\n        pub const START:            usize =         0x3F00_0000;\n        pub const GPIO_START:       usize = START + GPIO_OFFSET;\n        pub const PL011_UART_START: usize = START + UART_OFFSET;\n    }\n\n    /// Physical devices.\n    #[cfg(feature = \"bsp_rpi4\")]\n    pub mod mmio {\n        use super::*;\n\n        pub const START:            usize =         0xFE00_0000;\n        pub const GPIO_START:       usize = START + GPIO_OFFSET;\n        pub const PL011_UART_START: usize = START + UART_OFFSET;\n    }\n}\n"
  },
  {
    "path": "08_hw_debug_JTAG/src/bsp/raspberrypi.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Top-level BSP file for the Raspberry Pi 3 and 4.\n\npub mod cpu;\npub mod driver;\npub mod memory;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Board identification.\npub fn board_name() -> &'static str {\n    #[cfg(feature = \"bsp_rpi3\")]\n    {\n        \"Raspberry Pi 3\"\n    }\n\n    #[cfg(feature = \"bsp_rpi4\")]\n    {\n        \"Raspberry Pi 4\"\n    }\n}\n"
  },
  {
    "path": "08_hw_debug_JTAG/src/bsp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Conditional reexporting of Board Support Packages.\n\nmod device_driver;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod raspberrypi;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use raspberrypi::*;\n"
  },
  {
    "path": "08_hw_debug_JTAG/src/console/null_console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Null console.\n\nuse super::interface;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct NullConsole;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\npub static NULL_CONSOLE: NullConsole = NullConsole {};\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl interface::Write for NullConsole {\n    fn write_char(&self, _c: char) {}\n\n    fn write_fmt(&self, _args: fmt::Arguments) -> fmt::Result {\n        fmt::Result::Ok(())\n    }\n\n    fn flush(&self) {}\n}\n\nimpl interface::Read for NullConsole {\n    fn clear_rx(&self) {}\n}\n\nimpl interface::Statistics for NullConsole {}\nimpl interface::All for NullConsole {}\n"
  },
  {
    "path": "08_hw_debug_JTAG/src/console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! System console.\n\nmod null_console;\n\nuse crate::synchronization::{self, NullLock};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Console interfaces.\npub mod interface {\n    use core::fmt;\n\n    /// Console write functions.\n    pub trait Write {\n        /// Write a single character.\n        fn write_char(&self, c: char);\n\n        /// Write a Rust format string.\n        fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result;\n\n        /// Block until the last buffered character has been physically put on the TX wire.\n        fn flush(&self);\n    }\n\n    /// Console read functions.\n    pub trait Read {\n        /// Read a single character.\n        fn read_char(&self) -> char {\n            ' '\n        }\n\n        /// Clear RX buffers, if any.\n        fn clear_rx(&self);\n    }\n\n    /// Console statistics.\n    pub trait Statistics {\n        /// Return the number of characters written.\n        fn chars_written(&self) -> usize {\n            0\n        }\n\n        /// Return the number of characters read.\n        fn chars_read(&self) -> usize {\n            0\n        }\n    }\n\n    /// Trait alias for a full-fledged console.\n    pub trait All: Write + Read + Statistics {}\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic CUR_CONSOLE: NullLock<&'static (dyn interface::All + Sync)> =\n    NullLock::new(&null_console::NULL_CONSOLE);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\n/// Register a new console.\npub fn register_console(new_console: &'static (dyn interface::All + Sync)) {\n    CUR_CONSOLE.lock(|con| *con = new_console);\n}\n\n/// Return a reference to the currently registered console.\n///\n/// This is the global console used by all printing macros.\npub fn console() -> &'static dyn interface::All {\n    CUR_CONSOLE.lock(|con| *con)\n}\n"
  },
  {
    "path": "08_hw_debug_JTAG/src/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Boot code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/cpu/boot.rs\"]\nmod arch_boot;\n"
  },
  {
    "path": "08_hw_debug_JTAG/src/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Processor code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/cpu.rs\"]\nmod arch_cpu;\n\nmod boot;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_cpu::{nop, wait_forever};\n"
  },
  {
    "path": "08_hw_debug_JTAG/src/driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Driver support.\n\nuse crate::{\n    info,\n    synchronization::{interface::Mutex, NullLock},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nconst NUM_DRIVERS: usize = 5;\n\nstruct DriverManagerInner {\n    next_index: usize,\n    descriptors: [Option<DeviceDriverDescriptor>; NUM_DRIVERS],\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Driver interfaces.\npub mod interface {\n    /// Device Driver functions.\n    pub trait DeviceDriver {\n        /// Return a compatibility string for identifying the driver.\n        fn compatible(&self) -> &'static str;\n\n        /// Called by the kernel to bring up the device.\n        ///\n        /// # Safety\n        ///\n        /// - During init, drivers might do stuff with system-wide impact.\n        unsafe fn init(&self) -> Result<(), &'static str> {\n            Ok(())\n        }\n    }\n}\n\n/// Tpye to be used as an optional callback after a driver's init() has run.\npub type DeviceDriverPostInitCallback = unsafe fn() -> Result<(), &'static str>;\n\n/// A descriptor for device drivers.\n#[derive(Copy, Clone)]\npub struct DeviceDriverDescriptor {\n    device_driver: &'static (dyn interface::DeviceDriver + Sync),\n    post_init_callback: Option<DeviceDriverPostInitCallback>,\n}\n\n/// Provides device driver management functions.\npub struct DriverManager {\n    inner: NullLock<DriverManagerInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic DRIVER_MANAGER: DriverManager = DriverManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl DriverManagerInner {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            next_index: 0,\n            descriptors: [None; NUM_DRIVERS],\n        }\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl DeviceDriverDescriptor {\n    /// Create an instance.\n    pub fn new(\n        device_driver: &'static (dyn interface::DeviceDriver + Sync),\n        post_init_callback: Option<DeviceDriverPostInitCallback>,\n    ) -> Self {\n        Self {\n            device_driver,\n            post_init_callback,\n        }\n    }\n}\n\n/// Return a reference to the global DriverManager.\npub fn driver_manager() -> &'static DriverManager {\n    &DRIVER_MANAGER\n}\n\nimpl DriverManager {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            inner: NullLock::new(DriverManagerInner::new()),\n        }\n    }\n\n    /// Register a device driver with the kernel.\n    pub fn register_driver(&self, descriptor: DeviceDriverDescriptor) {\n        self.inner.lock(|inner| {\n            inner.descriptors[inner.next_index] = Some(descriptor);\n            inner.next_index += 1;\n        })\n    }\n\n    /// Helper for iterating over registered drivers.\n    fn for_each_descriptor<'a>(&'a self, f: impl FnMut(&'a DeviceDriverDescriptor)) {\n        self.inner.lock(|inner| {\n            inner\n                .descriptors\n                .iter()\n                .filter_map(|x| x.as_ref())\n                .for_each(f)\n        })\n    }\n\n    /// Fully initialize all drivers.\n    ///\n    /// # Safety\n    ///\n    /// - During init, drivers might do stuff with system-wide impact.\n    pub unsafe fn init_drivers(&self) {\n        self.for_each_descriptor(|descriptor| {\n            // 1. Initialize driver.\n            if let Err(x) = descriptor.device_driver.init() {\n                panic!(\n                    \"Error initializing driver: {}: {}\",\n                    descriptor.device_driver.compatible(),\n                    x\n                );\n            }\n\n            // 2. Call corresponding post init callback.\n            if let Some(callback) = &descriptor.post_init_callback {\n                if let Err(x) = callback() {\n                    panic!(\n                        \"Error during driver post-init callback: {}: {}\",\n                        descriptor.device_driver.compatible(),\n                        x\n                    );\n                }\n            }\n        });\n    }\n\n    /// Enumerate all registered device drivers.\n    pub fn enumerate(&self) {\n        let mut i: usize = 1;\n        self.for_each_descriptor(|descriptor| {\n            info!(\"      {}. {}\", i, descriptor.device_driver.compatible());\n\n            i += 1;\n        });\n    }\n}\n"
  },
  {
    "path": "08_hw_debug_JTAG/src/main.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n// Rust embedded logo for `make doc`.\n#![doc(\n    html_logo_url = \"https://raw.githubusercontent.com/rust-embedded/wg/master/assets/logo/ewg-logo-blue-white-on-transparent.png\"\n)]\n\n//! The `kernel` binary.\n//!\n//! # Code organization and architecture\n//!\n//! The code is divided into different *modules*, each representing a typical **subsystem** of the\n//! `kernel`. Top-level module files of subsystems reside directly in the `src` folder. For example,\n//! `src/memory.rs` contains code that is concerned with all things memory management.\n//!\n//! ## Visibility of processor architecture code\n//!\n//! Some of the `kernel`'s subsystems depend on low-level code that is specific to the target\n//! processor architecture. For each supported processor architecture, there exists a subfolder in\n//! `src/_arch`, for example, `src/_arch/aarch64`.\n//!\n//! The architecture folders mirror the subsystem modules laid out in `src`. For example,\n//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go\n//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in\n//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic\n//! module's name prefixed with `arch_`.\n//!\n//! For example, this is the top of `src/memory/mmu.rs`:\n//!\n//! ```\n//! #[cfg(target_arch = \"aarch64\")]\n//! #[path = \"../_arch/aarch64/memory/mmu.rs\"]\n//! mod arch_mmu;\n//! ```\n//!\n//! Often times, items from the `arch_ module` will be publicly reexported by the parent module.\n//! This way, each architecture specific module can provide its implementation of an item, while the\n//! caller must not be concerned which architecture has been conditionally compiled.\n//!\n//! ## BSP code\n//!\n//! `BSP` stands for Board Support Package. `BSP` code is organized under `src/bsp.rs` and contains\n//! target board specific definitions and functions. These are things such as the board's memory map\n//! or instances of drivers for devices that are featured on the respective board.\n//!\n//! Just like processor architecture code, the `BSP` code's module structure tries to mirror the\n//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is\n//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`.\n//!\n//! ## Kernel interfaces\n//!\n//! Both `arch` and `bsp` contain code that is conditionally compiled depending on the actual target\n//! and board for which the kernel is compiled. For example, the `interrupt controller` hardware of\n//! the `Raspberry Pi 3` and the `Raspberry Pi 4` is different, but we want the rest of the `kernel`\n//! code to play nicely with any of the two without much hassle.\n//!\n//! In order to provide a clean abstraction between `arch`, `bsp` and `generic kernel code`,\n//! `interface` traits are provided *whenever possible* and *where it makes sense*. They are defined\n//! in the respective subsystem module and help to enforce the idiom of *program to an interface,\n//! not an implementation*. For example, there will be a common IRQ handling interface which the two\n//! different interrupt controller `drivers` of both Raspberrys will implement, and only export the\n//! interface to the rest of the `kernel`.\n//!\n//! ```\n//!         +-------------------+\n//!         | Interface (Trait) |\n//!         |                   |\n//!         +--+-------------+--+\n//!            ^             ^\n//!            |             |\n//!            |             |\n//! +----------+--+       +--+----------+\n//! | kernel code |       |  bsp code   |\n//! |             |       |  arch code  |\n//! +-------------+       +-------------+\n//! ```\n//!\n//! # Summary\n//!\n//! For a logical `kernel` subsystem, corresponding code can be distributed over several physical\n//! locations. Here is an example for the **memory** subsystem:\n//!\n//! - `src/memory.rs` and `src/memory/**/*`\n//!   - Common code that is agnostic of target processor architecture and `BSP` characteristics.\n//!     - Example: A function to zero a chunk of memory.\n//!   - Interfaces for the memory subsystem that are implemented by `arch` or `BSP` code.\n//!     - Example: An `MMU` interface that defines `MMU` function prototypes.\n//! - `src/bsp/__board_name__/memory.rs` and `src/bsp/__board_name__/memory/**/*`\n//!   - `BSP` specific code.\n//!   - Example: The board's memory map (physical addresses of DRAM and MMIO devices).\n//! - `src/_arch/__arch_name__/memory.rs` and `src/_arch/__arch_name__/memory/**/*`\n//!   - Processor architecture specific code.\n//!   - Example: Implementation of the `MMU` interface for the `__arch_name__` processor\n//!     architecture.\n//!\n//! From a namespace perspective, **memory** subsystem code lives in:\n//!\n//! - `crate::memory::*`\n//! - `crate::bsp::memory::*`\n//!\n//! # Boot flow\n//!\n//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.\n//!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.\n//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.\n\n#![allow(clippy::upper_case_acronyms)]\n#![feature(asm_const)]\n#![feature(const_option)]\n#![feature(format_args_nl)]\n#![feature(nonzero_min_max)]\n#![feature(panic_info_message)]\n#![feature(trait_alias)]\n#![feature(unchecked_math)]\n#![no_main]\n#![no_std]\n\nmod bsp;\nmod console;\nmod cpu;\nmod driver;\nmod panic_wait;\nmod print;\nmod synchronization;\nmod time;\n\n/// Early init code.\n///\n/// # Safety\n///\n/// - Only a single core must be active and running this function.\n/// - The init calls in this function must appear in the correct order.\nunsafe fn kernel_init() -> ! {\n    // Initialize the BSP driver subsystem.\n    if let Err(x) = bsp::driver::init() {\n        panic!(\"Error initializing BSP driver subsystem: {}\", x);\n    }\n\n    // Initialize all device drivers.\n    driver::driver_manager().init_drivers();\n    // println! is usable from here on.\n\n    // Transition from unsafe to safe.\n    kernel_main()\n}\n\n/// The main function running after the early init.\nfn kernel_main() -> ! {\n    use core::time::Duration;\n\n    info!(\n        \"{} version {}\",\n        env!(\"CARGO_PKG_NAME\"),\n        env!(\"CARGO_PKG_VERSION\")\n    );\n    info!(\"Booting on: {}\", bsp::board_name());\n\n    info!(\n        \"Architectural timer resolution: {} ns\",\n        time::time_manager().resolution().as_nanos()\n    );\n\n    info!(\"Drivers loaded:\");\n    driver::driver_manager().enumerate();\n\n    // Test a failing timer case.\n    time::time_manager().spin_for(Duration::from_nanos(1));\n\n    loop {\n        info!(\"Spinning for 1 second\");\n        time::time_manager().spin_for(Duration::from_secs(1));\n    }\n}\n"
  },
  {
    "path": "08_hw_debug_JTAG/src/panic_wait.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A panic handler that infinitely waits.\n\nuse crate::{cpu, println};\nuse core::panic::PanicInfo;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Stop immediately if called a second time.\n///\n/// # Note\n///\n/// Using atomics here relieves us from needing to use `unsafe` for the static variable.\n///\n/// On `AArch64`, which is the only implemented architecture at the time of writing this,\n/// [`AtomicBool::load`] and [`AtomicBool::store`] are lowered to ordinary load and store\n/// instructions. They are therefore safe to use even with MMU + caching deactivated.\n///\n/// [`AtomicBool::load`]: core::sync::atomic::AtomicBool::load\n/// [`AtomicBool::store`]: core::sync::atomic::AtomicBool::store\nfn panic_prevent_reenter() {\n    use core::sync::atomic::{AtomicBool, Ordering};\n\n    #[cfg(not(target_arch = \"aarch64\"))]\n    compile_error!(\"Add the target_arch to above's check if the following code is safe to use\");\n\n    static PANIC_IN_PROGRESS: AtomicBool = AtomicBool::new(false);\n\n    if !PANIC_IN_PROGRESS.load(Ordering::Relaxed) {\n        PANIC_IN_PROGRESS.store(true, Ordering::Relaxed);\n\n        return;\n    }\n\n    cpu::wait_forever()\n}\n\n#[panic_handler]\nfn panic(info: &PanicInfo) -> ! {\n    // Protect against panic infinite loops if any of the following code panics itself.\n    panic_prevent_reenter();\n\n    let timestamp = crate::time::time_manager().uptime();\n    let (location, line, column) = match info.location() {\n        Some(loc) => (loc.file(), loc.line(), loc.column()),\n        _ => (\"???\", 0, 0),\n    };\n\n    println!(\n        \"[  {:>3}.{:06}] Kernel panic!\\n\\n\\\n        Panic location:\\n      File '{}', line {}, column {}\\n\\n\\\n        {}\",\n        timestamp.as_secs(),\n        timestamp.subsec_micros(),\n        location,\n        line,\n        column,\n        info.message().unwrap_or(&format_args!(\"\")),\n    );\n\n    cpu::wait_forever()\n}\n"
  },
  {
    "path": "08_hw_debug_JTAG/src/print.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Printing.\n\nuse crate::console;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n#[doc(hidden)]\npub fn _print(args: fmt::Arguments) {\n    console::console().write_fmt(args).unwrap();\n}\n\n/// Prints without a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! print {\n    ($($arg:tt)*) => ($crate::print::_print(format_args!($($arg)*)));\n}\n\n/// Prints with a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! println {\n    () => ($crate::print!(\"\\n\"));\n    ($($arg:tt)*) => ({\n        $crate::print::_print(format_args_nl!($($arg)*));\n    })\n}\n\n/// Prints an info, with a newline.\n#[macro_export]\nmacro_rules! info {\n    ($string:expr) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[  {:>3}.{:06}] \", $string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n        ));\n    });\n    ($format_string:expr, $($arg:tt)*) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[  {:>3}.{:06}] \", $format_string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n            $($arg)*\n        ));\n    })\n}\n\n/// Prints a warning, with a newline.\n#[macro_export]\nmacro_rules! warn {\n    ($string:expr) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[W {:>3}.{:06}] \", $string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n        ));\n    });\n    ($format_string:expr, $($arg:tt)*) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[W {:>3}.{:06}] \", $format_string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n            $($arg)*\n        ));\n    })\n}\n"
  },
  {
    "path": "08_hw_debug_JTAG/src/synchronization.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Synchronization primitives.\n//!\n//! # Resources\n//!\n//!   - <https://doc.rust-lang.org/book/ch16-04-extensible-concurrency-sync-and-send.html>\n//!   - <https://stackoverflow.com/questions/59428096/understanding-the-send-trait>\n//!   - <https://doc.rust-lang.org/std/cell/index.html>\n\nuse core::cell::UnsafeCell;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Synchronization interfaces.\npub mod interface {\n\n    /// Any object implementing this trait guarantees exclusive access to the data wrapped within\n    /// the Mutex for the duration of the provided closure.\n    pub trait Mutex {\n        /// The type of the data that is wrapped by this mutex.\n        type Data;\n\n        /// Locks the mutex and grants the closure temporary mutable access to the wrapped data.\n        fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R;\n    }\n}\n\n/// A pseudo-lock for teaching purposes.\n///\n/// In contrast to a real Mutex implementation, does not protect against concurrent access from\n/// other cores to the contained data. This part is preserved for later lessons.\n///\n/// The lock will only be used as long as it is safe to do so, i.e. as long as the kernel is\n/// executing single-threaded, aka only running on a single core with interrupts disabled.\npub struct NullLock<T>\nwhere\n    T: ?Sized,\n{\n    data: UnsafeCell<T>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nunsafe impl<T> Send for NullLock<T> where T: ?Sized + Send {}\nunsafe impl<T> Sync for NullLock<T> where T: ?Sized + Send {}\n\nimpl<T> NullLock<T> {\n    /// Create an instance.\n    pub const fn new(data: T) -> Self {\n        Self {\n            data: UnsafeCell::new(data),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\n\nimpl<T> interface::Mutex for NullLock<T> {\n    type Data = T;\n\n    fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R {\n        // In a real lock, there would be code encapsulating this line that ensures that this\n        // mutable reference will ever only be given out once at a time.\n        let data = unsafe { &mut *self.data.get() };\n\n        f(data)\n    }\n}\n"
  },
  {
    "path": "08_hw_debug_JTAG/src/time.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Timer primitives.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/time.rs\"]\nmod arch_time;\n\nuse core::time::Duration;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Provides time management functions.\npub struct TimeManager;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic TIME_MANAGER: TimeManager = TimeManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the global TimeManager.\npub fn time_manager() -> &'static TimeManager {\n    &TIME_MANAGER\n}\n\nimpl TimeManager {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self\n    }\n\n    /// The timer's resolution.\n    pub fn resolution(&self) -> Duration {\n        arch_time::resolution()\n    }\n\n    /// The uptime since power-on of the device.\n    ///\n    /// This includes time consumed by firmware and bootloaders.\n    pub fn uptime(&self) -> Duration {\n        arch_time::uptime()\n    }\n\n    /// Spin for a given duration.\n    pub fn spin_for(&self, duration: Duration) {\n        arch_time::spin_for(duration)\n    }\n}\n"
  },
  {
    "path": "08_hw_debug_JTAG/tests/boot_test_string.rb",
    "content": "# frozen_string_literal: true\n\nEXPECTED_PRINT = 'Spinning for 1 second'\n"
  },
  {
    "path": "09_privilege_level/.vscode/settings.json",
    "content": "{\n    \"editor.formatOnSave\": true,\n    \"editor.rulers\": [100],\n    \"rust-analyzer.cargo.target\": \"aarch64-unknown-none-softfloat\",\n    \"rust-analyzer.cargo.features\": [\"bsp_rpi3\"],\n    \"rust-analyzer.checkOnSave.allTargets\": false,\n    \"rust-analyzer.checkOnSave.extraArgs\": [\"--bins\"],\n    \"rust-analyzer.lens.debug\": false,\n    \"rust-analyzer.lens.run\": false\n}\n"
  },
  {
    "path": "09_privilege_level/Cargo.toml",
    "content": "[package]\nname = \"mingo\"\nversion = \"0.9.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n\n[profile.release]\nlto = true\n\n[features]\ndefault = []\nbsp_rpi3 = [\"tock-registers\"]\nbsp_rpi4 = [\"tock-registers\"]\n\n[[bin]]\nname = \"kernel\"\npath = \"src/main.rs\"\n\n##--------------------------------------------------------------------------------------------------\n## Dependencies\n##--------------------------------------------------------------------------------------------------\n\n[dependencies]\n\n# Optional dependencies\ntock-registers = { version = \"0.8.x\", default-features = false, features = [\"register_types\"], optional = true }\n\n# Platform specific dependencies\n[target.'cfg(target_arch = \"aarch64\")'.dependencies]\naarch64-cpu = { version = \"9.x.x\" }\n"
  },
  {
    "path": "09_privilege_level/Makefile",
    "content": "## SPDX-License-Identifier: MIT OR Apache-2.0\n##\n## Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\ninclude ../common/docker.mk\ninclude ../common/format.mk\ninclude ../common/operating_system.mk\n\n##--------------------------------------------------------------------------------------------------\n## Optional, user-provided configuration values\n##--------------------------------------------------------------------------------------------------\n\n# Default to the RPi3.\nBSP ?= rpi3\n\n# Default to a serial device name that is common in Linux.\nDEV_SERIAL ?= /dev/ttyUSB0\n\n\n\n##--------------------------------------------------------------------------------------------------\n## BSP-specific configuration values\n##--------------------------------------------------------------------------------------------------\nQEMU_MISSING_STRING = \"This board is not yet supported for QEMU.\"\n\nifeq ($(BSP),rpi3)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE = raspi3\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    OPENOCD_ARG       = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi3.cfg\n    JTAG_BOOT_IMAGE   = ../X1_JTAG_boot/jtag_boot_rpi3.img\n    LD_SCRIPT_PATH    = $(shell pwd)/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a53\nelse ifeq ($(BSP),rpi4)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE =\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    OPENOCD_ARG       = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi4.cfg\n    JTAG_BOOT_IMAGE   = ../X1_JTAG_boot/jtag_boot_rpi4.img\n    LD_SCRIPT_PATH    = $(shell pwd)/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a72\nendif\n\n# Export for build.rs.\nexport LD_SCRIPT_PATH\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets and Prerequisites\n##--------------------------------------------------------------------------------------------------\nKERNEL_MANIFEST      = Cargo.toml\nKERNEL_LINKER_SCRIPT = kernel.ld\nLAST_BUILD_CONFIG    = target/$(BSP).build_config\n\nKERNEL_ELF      = target/$(TARGET)/release/kernel\n# This parses cargo's dep-info file.\n# https://doc.rust-lang.org/cargo/guide/build-cache.html#dep-info-files\nKERNEL_ELF_DEPS = $(filter-out %: ,$(file < $(KERNEL_ELF).d)) $(KERNEL_MANIFEST) $(LAST_BUILD_CONFIG)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Command building blocks\n##--------------------------------------------------------------------------------------------------\nRUSTFLAGS = $(RUSTC_MISC_ARGS)                   \\\n    -C link-arg=--library-path=$(LD_SCRIPT_PATH) \\\n    -C link-arg=--script=$(KERNEL_LINKER_SCRIPT)\n\nRUSTFLAGS_PEDANTIC = $(RUSTFLAGS) \\\n    -D warnings                   \\\n    -D missing_docs\n\nFEATURES      = --features bsp_$(BSP)\nCOMPILER_ARGS = --target=$(TARGET) \\\n    $(FEATURES)                    \\\n    --release\n\nRUSTC_CMD   = cargo rustc $(COMPILER_ARGS)\nDOC_CMD     = cargo doc $(COMPILER_ARGS)\nCLIPPY_CMD  = cargo clippy $(COMPILER_ARGS)\nOBJCOPY_CMD = rust-objcopy \\\n    --strip-all            \\\n    -O binary\n\nEXEC_QEMU          = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE)\nEXEC_TEST_DISPATCH = ruby ../common/tests/dispatch.rb\nEXEC_MINIPUSH      = ruby ../common/serial/minipush.rb\n\n##------------------------------------------------------------------------------\n## Dockerization\n##------------------------------------------------------------------------------\nDOCKER_CMD            = docker run -t --rm -v $(shell pwd):/work/tutorial -w /work/tutorial\nDOCKER_CMD_INTERACT   = $(DOCKER_CMD) -i\nDOCKER_ARG_DIR_COMMON = -v $(shell pwd)/../common:/work/common\nDOCKER_ARG_DIR_JTAG   = -v $(shell pwd)/../X1_JTAG_boot:/work/X1_JTAG_boot\nDOCKER_ARG_DEV        = --privileged -v /dev:/dev\nDOCKER_ARG_NET        = --network host\n\n# DOCKER_IMAGE defined in include file (see top of this file).\nDOCKER_QEMU  = $(DOCKER_CMD_INTERACT) $(DOCKER_IMAGE)\nDOCKER_TOOLS = $(DOCKER_CMD) $(DOCKER_IMAGE)\nDOCKER_TEST  = $(DOCKER_CMD) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\nDOCKER_GDB   = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_NET) $(DOCKER_IMAGE)\n\n# Dockerize commands, which require USB device passthrough, only on Linux.\nifeq ($(shell uname -s),Linux)\n    DOCKER_CMD_DEV = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_DEV)\n\n    DOCKER_CHAINBOOT = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\n    DOCKER_JTAGBOOT  = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_ARG_DIR_JTAG) $(DOCKER_IMAGE)\n    DOCKER_OPENOCD   = $(DOCKER_CMD_DEV) $(DOCKER_ARG_NET) $(DOCKER_IMAGE)\nelse\n    DOCKER_OPENOCD   = echo \"Not yet supported on non-Linux systems.\"; \\#\nendif\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: all doc qemu chainboot clippy clean readelf objdump nm check\n\nall: $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Save the configuration as a file, so make understands if it changed.\n##------------------------------------------------------------------------------\n$(LAST_BUILD_CONFIG):\n\t@rm -f target/*.build_config\n\t@mkdir -p target\n\t@touch $(LAST_BUILD_CONFIG)\n\n##------------------------------------------------------------------------------\n## Compile the kernel ELF\n##------------------------------------------------------------------------------\n$(KERNEL_ELF): $(KERNEL_ELF_DEPS)\n\t$(call color_header, \"Compiling kernel ELF - $(BSP)\")\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(RUSTC_CMD)\n\n##------------------------------------------------------------------------------\n## Generate the stripped kernel binary\n##------------------------------------------------------------------------------\n$(KERNEL_BIN): $(KERNEL_ELF)\n\t$(call color_header, \"Generating stripped binary\")\n\t@$(OBJCOPY_CMD) $(KERNEL_ELF) $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Name\")\n\t@echo $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Size\")\n\t$(call disk_usage_KiB, $(KERNEL_BIN))\n\n##------------------------------------------------------------------------------\n## Generate the documentation\n##------------------------------------------------------------------------------\ndoc:\n\t$(call color_header, \"Generating docs\")\n\t@$(DOC_CMD) --document-private-items --open\n\n##------------------------------------------------------------------------------\n## Run the kernel in QEMU\n##------------------------------------------------------------------------------\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\nqemu:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\nqemu: $(KERNEL_BIN)\n\t$(call color_header, \"Launching QEMU\")\n\t@$(DOCKER_QEMU) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\nendif\n\n##------------------------------------------------------------------------------\n## Push the kernel to the real HW target\n##------------------------------------------------------------------------------\nchainboot: $(KERNEL_BIN)\n\t@$(DOCKER_CHAINBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run clippy\n##------------------------------------------------------------------------------\nclippy:\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(CLIPPY_CMD)\n\n##------------------------------------------------------------------------------\n## Clean\n##------------------------------------------------------------------------------\nclean:\n\trm -rf target $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run readelf\n##------------------------------------------------------------------------------\nreadelf: $(KERNEL_ELF)\n\t$(call color_header, \"Launching readelf\")\n\t@$(DOCKER_TOOLS) $(READELF_BINARY) --headers $(KERNEL_ELF)\n\n##------------------------------------------------------------------------------\n## Run objdump\n##------------------------------------------------------------------------------\nobjdump: $(KERNEL_ELF)\n\t$(call color_header, \"Launching objdump\")\n\t@$(DOCKER_TOOLS) $(OBJDUMP_BINARY) --disassemble --demangle \\\n                --section .text   \\\n                --section .rodata \\\n                $(KERNEL_ELF) | rustfilt\n\n##------------------------------------------------------------------------------\n## Run nm\n##------------------------------------------------------------------------------\nnm: $(KERNEL_ELF)\n\t$(call color_header, \"Launching nm\")\n\t@$(DOCKER_TOOLS) $(NM_BINARY) --demangle --print-size $(KERNEL_ELF) | sort | rustfilt\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Debugging targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: jtagboot openocd gdb gdb-opt0\n\n##------------------------------------------------------------------------------\n## Push the JTAG boot image to the real HW target\n##------------------------------------------------------------------------------\njtagboot:\n\t@$(DOCKER_JTAGBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(JTAG_BOOT_IMAGE)\n\n##------------------------------------------------------------------------------\n## Start OpenOCD session\n##------------------------------------------------------------------------------\nopenocd:\n\t$(call color_header, \"Launching OpenOCD\")\n\t@$(DOCKER_OPENOCD) openocd $(OPENOCD_ARG)\n\n##------------------------------------------------------------------------------\n## Start GDB session\n##------------------------------------------------------------------------------\ngdb: RUSTC_MISC_ARGS += -C debuginfo=2\ngdb-opt0: RUSTC_MISC_ARGS += -C debuginfo=2 -C opt-level=0\ngdb gdb-opt0: $(KERNEL_ELF)\n\t$(call color_header, \"Launching GDB\")\n\t@$(DOCKER_GDB) gdb-multiarch -q $(KERNEL_ELF)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Testing targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: test test_boot\n\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\ntest_boot test:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\n##------------------------------------------------------------------------------\n## Run boot test\n##------------------------------------------------------------------------------\ntest_boot: $(KERNEL_BIN)\n\t$(call color_header, \"Boot test - $(BSP)\")\n\t@$(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\ntest: test_boot\n\nendif\n"
  },
  {
    "path": "09_privilege_level/README.CN.md",
    "content": "# 教程 09 - 特权级别\n\n## tl;dr\n\n- 在早期引导代码中，我们从`Hypervisor`特权级别（AArch64中的`EL2`）过渡到`Kernel` （`EL1`）特权级别。\n\n## 目录\n\n- [介绍](#介绍)\n- [本教程的范围](#本教程的范围)\n- [在入口点检查EL2](#在入口点检查EL2)\n- [过渡准备](#过渡准备)\n- [从未发生的异常中返回](#从未发生的异常中返回)\n- [测试](#测试)\n- [相比之前的变化（diff）](#相比之前的变化（diff）)\n\n## 介绍\n\n应用级别的CPU具有所谓的`privilege levels`，它们具有不同的目的：\n\n| Typically used for | AArch64 | RISC-V | x86 |\n| ------------- | ------------- | ------------- | ------------- |\n| Userspace applications | EL0 | U/VU | Ring 3 |\n| OS Kernel | EL1 | S/VS | Ring 0 |\n| Hypervisor | EL2 | HS | Ring -1 |\n| Low-Level Firmware | EL3 | M | |\n\n在AArch64中，`EL`代表`Exception Level`（异常级别）。如果您想获取有关其他体系结构的更多信息，请查看以下链接：\n- [x86 privilege rings](https://en.wikipedia.org/wiki/Protection_ring).\n- [RISC-V privilege modes](https://content.riscv.org/wp-content/uploads/2017/12/Tue0942-riscv-hypervisor-waterman.pdf).\n\n在继续之前，我强烈建议您先浏览一下[Programmer’s Guide for ARMv8-A]`的第3章`。它提供了关于该主题的简明概述。\n\n[Programmer’s Guide for ARMv8-A]: http://infocenter.arm.com/help/topic/com.arm.doc.den0024a/DEN0024A_v8_architecture_PG.pdf\n\n## 本教程的范围\n\n默认情况下，树莓派将始终在`EL2`中开始执行。由于我们正在编写一个传统的`Kernel`，我们需要过渡到更合适的`EL1`。\n\n## 在入口点检查EL2\n\n首先，我们需要确保我们实际上是在`EL2`中执行，然后才能调用相应的代码过渡到`EL1`。\n因此，我们在`boot.s`的顶部添加了一个新的检查，如果CPU核心不在`EL2`中，则将其停止。\n\n```\n// Only proceed if the core executes in EL2. Park it otherwise.\nmrs\tx0, CurrentEL\ncmp\tx0, {CONST_CURRENTEL_EL2}\nb.ne\t.L_parking_loop\n```\n\n接下来，在`boot.rs`中继续准备从`EL2`到`EL1`的过渡，通过调用`prepare_el2_to_el1_transition()`函数。\n\n```rust\n#[no_mangle]\npub unsafe extern \"C\" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! {\n    prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr);\n\n    // Use `eret` to \"return\" to EL1. This results in execution of kernel_init() in EL1.\n    asm::eret()\n}\n```\n\n## 过渡准备\n\n由于`EL2`比`EL1`更具特权，它可以控制各种处理器功能，并允许或禁止`EL1`代码使用它们。\n其中一个例子是访问计时器和计数器寄存器。我们已经在[tutorial 07](../07_timestamps/)中使用了它们，所以当然我们希望保留它们。\n因此，我们在[Counter-timer Hypervisor Control register]中设置相应的标志，并将虚拟偏移量设置为零，以获取真实的物理值。\n\n[Counter-timer Hypervisor Control register]:  https://docs.rs/aarch64-cpu/9.0.0/src/aarch64_cpu/registers/cnthctl_el2.rs.html\n\n```rust\n// Enable timer counter registers for EL1.\nCNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET);\n\n// No offset for reading the counters.\nCNTVOFF_EL2.set(0);\n```\n\n接下来，我们配置[Hypervisor Configuration Register]，使`EL1`在`AArch64`模式下运行，而不是在`AArch32`模式下运行，这也是可能的。\n\n[Hypervisor Configuration Register]: https://docs.rs/aarch64-cpu/9.0.0/src/aarch64_cpu/registers/hcr_el2.rs.html\n\n```rust\n// Set EL1 execution state to AArch64.\nHCR_EL2.write(HCR_EL2::RW::EL1IsAarch64);\n```\n\n## 从未发生的异常中返回\n\n实际上，从较高的EL过渡到较低的EL只有一种方式，即通过执行[ERET]指令。\n\n[ERET]: https://docs.rs/aarch64-cpu/9.0.0/src/aarch64_cpu/asm.rs.html#92-101\n\n在这个指令中，它将会将[Saved Program Status Register - EL2]的内容复制到`Current Program Status Register - EL1`，并跳转到存储在[Exception Link Register - EL2]。\n\n这基本上是在发生异常时所发生的相反过程。您将在即将发布的教程中了解更多相关内容。\n\n[Saved Program Status Register - EL2]: https://docs.rs/aarch64-cpu/9.0.0/src/aarch64_cpu/registers/spsr_el2.rs.html\n[Exception Link Register - EL2]: https://docs.rs/aarch64-cpu/9.0.0/src/aarch64_cpu/registers/elr_el2.rs.html\n\n```rust\n// Set up a simulated exception return.\n//\n// First, fake a saved program status where all interrupts were masked and SP_EL1 was used as a\n// stack pointer.\nSPSR_EL2.write(\n    SPSR_EL2::D::Masked\n        + SPSR_EL2::A::Masked\n        + SPSR_EL2::I::Masked\n        + SPSR_EL2::F::Masked\n        + SPSR_EL2::M::EL1h,\n);\n\n// Second, let the link register point to kernel_init().\nELR_EL2.set(crate::kernel_init as *const () as u64);\n\n// Set up SP_EL1 (stack pointer), which will be used by EL1 once we \"return\" to it. Since there\n// are no plans to ever return to EL2, just re-use the same stack.\nSP_EL1.set(phys_boot_core_stack_end_exclusive_addr);\n```\n\n正如您所看到的，我们将`ELR_EL2`的值设置为之前直接从入口点调用的`kernel_init()`函数的地址。最后，我们设置了`SP_EL1`的堆栈指针。\n\n您可能已经注意到，堆栈的地址作为函数参数进行了传递。正如您可能记得的，在`boot.s`的`_start()`函数中，\n我们已经为`EL2`设置了堆栈。由于没有计划返回到`EL2`，我们可以直接重用相同的堆栈作为`EL1`的堆栈，\n因此使用函数参数将其地址传递。\n\n最后，在`_start_rust()`函数中调用了`ERET`指令。\n\n```rust\n#[no_mangle]\npub unsafe extern \"C\" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! {\n    prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr);\n\n    // Use `eret` to \"return\" to EL1. This results in execution of kernel_init() in EL1.\n    asm::eret()\n}\n```\n\n## 测试\n\n在`main.rs`中，我们打印`current privilege level`，并额外检查`SPSR_EL2`中的掩码位是否传递到了`EL1`：\n\n```console\n$ make chainboot\n[...]\nMinipush 1.0\n\n[MP] ⏳ Waiting for /dev/ttyUSB0\n[MP] ✅ Serial connected\n[MP] 🔌 Please power the target now\n\n __  __ _      _ _                 _\n|  \\/  (_)_ _ (_) |   ___  __ _ __| |\n| |\\/| | | ' \\| | |__/ _ \\/ _` / _` |\n|_|  |_|_|_||_|_|____\\___/\\__,_\\__,_|\n\n           Raspberry Pi 3\n\n[ML] Requesting binary\n[MP] ⏩ Pushing 14 KiB =========================================🦀 100% 0 KiB/s Time: 00:00:00\n[ML] Loaded! Executing the payload now\n\n[    0.162546] mingo version 0.9.0\n[    0.162745] Booting on: Raspberry Pi 3\n[    0.163201] Current privilege level: EL1\n[    0.163677] Exception handling state:\n[    0.164122]       Debug:  Masked\n[    0.164511]       SError: Masked\n[    0.164901]       IRQ:    Masked\n[    0.165291]       FIQ:    Masked\n[    0.165681] Architectural timer resolution: 52 ns\n[    0.166255] Drivers loaded:\n[    0.166592]       1. BCM PL011 UART\n[    0.167014]       2. BCM GPIO\n[    0.167371] Timer test, spinning for 1 second\n[    1.167904] Echoing input now\n```\n\n## 相比之前的变化（diff）\n请检查[英文版本](README.md#diff-to-previous)，这是最新的。\n"
  },
  {
    "path": "09_privilege_level/README.md",
    "content": "# Tutorial 09 - Privilege Level\n\n## tl;dr\n\n- In early boot code, we transition from the `Hypervisor` privilege level (`EL2` in AArch64) to the\n  `Kernel` (`EL1`) privilege level.\n\n## Table of Contents\n\n- [Introduction](#introduction)\n- [Scope of this tutorial](#scope-of-this-tutorial)\n- [Checking for EL2 in the entrypoint](#checking-for-el2-in-the-entrypoint)\n- [Transition preparation](#transition-preparation)\n- [Returning from an exception that never happened](#returning-from-an-exception-that-never-happened)\n- [Test it](#test-it)\n- [Diff to previous](#diff-to-previous)\n\n## Introduction\n\nApplication-grade CPUs have so-called `privilege levels`, which have different purposes:\n\n| Typically used for | AArch64 | RISC-V | x86 |\n| ------------- | ------------- | ------------- | ------------- |\n| Userspace applications | EL0 | U/VU | Ring 3 |\n| OS Kernel | EL1 | S/VS | Ring 0 |\n| Hypervisor | EL2 | HS | Ring -1 |\n| Low-Level Firmware | EL3 | M | |\n\n`EL` in AArch64 stands for `Exception Level`. If you want more information regarding the other\narchitectures, please have a look at the following links:\n- [x86 privilege rings](https://en.wikipedia.org/wiki/Protection_ring).\n- [RISC-V privilege modes](https://content.riscv.org/wp-content/uploads/2017/12/Tue0942-riscv-hypervisor-waterman.pdf).\n\nAt this point, I strongly recommend that you glimpse over `Chapter 3` of the [Programmer’s Guide for\nARMv8-A] before you continue. It gives a concise overview about the topic.\n\n[Programmer’s Guide for ARMv8-A]: http://infocenter.arm.com/help/topic/com.arm.doc.den0024a/DEN0024A_v8_architecture_PG.pdf\n\n## Scope of this tutorial\n\nBy default, the Raspberry will always start executing in `EL2`. Since we are writing a traditional\n`Kernel`, we have to transition into the more appropriate `EL1`.\n\n## Checking for EL2 in the entrypoint\n\nFirst of all, we need to ensure that we actually execute in `EL2` before we can call respective code\nto transition to `EL1`. Therefore, we add a new checkt to the top of `boot.s`, which parks the CPU\ncore should it not be in `EL2`.\n\n```\n// Only proceed if the core executes in EL2. Park it otherwise.\nmrs\tx0, CurrentEL\ncmp\tx0, {CONST_CURRENTEL_EL2}\nb.ne\t.L_parking_loop\n```\n\nAfterwards, we continue with preparing the `EL2` -> `EL1` transition by calling\n`prepare_el2_to_el1_transition()` in `boot.rs`:\n\n```rust\n#[no_mangle]\npub unsafe extern \"C\" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! {\n    prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr);\n\n    // Use `eret` to \"return\" to EL1. This results in execution of kernel_init() in EL1.\n    asm::eret()\n}\n```\n\n## Transition preparation\n\nSince `EL2` is more privileged than `EL1`, it has control over various processor features and can\nallow or disallow `EL1` code to use them. One such example is access to timer and counter registers.\nWe are already using them since [tutorial 07](../07_timestamps/), so of course we want to keep them.\nTherefore we set the respective flags in the [Counter-timer Hypervisor Control register] and\nadditionally set the virtual offset to zero so that we get the real physical value everytime:\n\n[Counter-timer Hypervisor Control register]:  https://docs.rs/aarch64-cpu/9.0.0/src/aarch64_cpu/registers/cnthctl_el2.rs.html\n\n```rust\n// Enable timer counter registers for EL1.\nCNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET);\n\n// No offset for reading the counters.\nCNTVOFF_EL2.set(0);\n```\n\nNext, we configure the [Hypervisor Configuration Register] such that `EL1` runs in `AArch64` mode,\nand not in `AArch32`, which would also be possible.\n\n[Hypervisor Configuration Register]: https://docs.rs/aarch64-cpu/9.0.0/src/aarch64_cpu/registers/hcr_el2.rs.html\n\n```rust\n// Set EL1 execution state to AArch64.\nHCR_EL2.write(HCR_EL2::RW::EL1IsAarch64);\n```\n\n## Returning from an exception that never happened\n\nThere is actually only one way to transition from a higher EL to a lower EL, which is by way of\nexecuting the [ERET] instruction.\n\n[ERET]: https://docs.rs/aarch64-cpu/9.0.0/src/aarch64_cpu/asm.rs.html#92-101\n\nThis instruction will copy the contents of the [Saved Program Status Register - EL2] to `Current\nProgram Status Register - EL1` and jump to the instruction address that is stored in the [Exception\nLink Register - EL2].\n\nThis is basically the reverse of what is happening when an exception is taken. You'll learn about\nthat in an upcoming tutorial.\n\n[Saved Program Status Register - EL2]: https://docs.rs/aarch64-cpu/9.0.0/src/aarch64_cpu/registers/spsr_el2.rs.html\n[Exception Link Register - EL2]: https://docs.rs/aarch64-cpu/9.0.0/src/aarch64_cpu/registers/elr_el2.rs.html\n\n```rust\n// Set up a simulated exception return.\n//\n// First, fake a saved program status where all interrupts were masked and SP_EL1 was used as a\n// stack pointer.\nSPSR_EL2.write(\n    SPSR_EL2::D::Masked\n        + SPSR_EL2::A::Masked\n        + SPSR_EL2::I::Masked\n        + SPSR_EL2::F::Masked\n        + SPSR_EL2::M::EL1h,\n);\n\n// Second, let the link register point to kernel_init().\nELR_EL2.set(crate::kernel_init as *const () as u64);\n\n// Set up SP_EL1 (stack pointer), which will be used by EL1 once we \"return\" to it. Since there\n// are no plans to ever return to EL2, just re-use the same stack.\nSP_EL1.set(phys_boot_core_stack_end_exclusive_addr);\n```\n\nAs you can see, we are populating `ELR_EL2` with the address of the `kernel_init()` function that we\nearlier used to call directly from the entrypoint. Finally, we set the stack pointer for `SP_EL1`.\n\nYou might have noticed that the stack's address was supplied as a function argument. As you might\nremember, in  `_start()` in `boot.s`, we are already setting up the stack for `EL2`. Since there\nare no plans to ever return to `EL2`, we can just re-use the same stack for `EL1`, so its address is\nforwarded using function arguments.\n\nLastly, back in `_start_rust()` a call to `ERET` is made:\n\n```rust\n#[no_mangle]\npub unsafe extern \"C\" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! {\n    prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr);\n\n    // Use `eret` to \"return\" to EL1. This results in execution of kernel_init() in EL1.\n    asm::eret()\n}\n```\n\n## Test it\n\nIn `main.rs`, we print the `current privilege level` and additionally inspect if the mask bits in\n`SPSR_EL2` made it to `EL1` as well:\n\n```console\n$ make chainboot\n[...]\nMinipush 1.0\n\n[MP] ⏳ Waiting for /dev/ttyUSB0\n[MP] ✅ Serial connected\n[MP] 🔌 Please power the target now\n\n __  __ _      _ _                 _\n|  \\/  (_)_ _ (_) |   ___  __ _ __| |\n| |\\/| | | ' \\| | |__/ _ \\/ _` / _` |\n|_|  |_|_|_||_|_|____\\___/\\__,_\\__,_|\n\n           Raspberry Pi 3\n\n[ML] Requesting binary\n[MP] ⏩ Pushing 14 KiB =========================================🦀 100% 0 KiB/s Time: 00:00:00\n[ML] Loaded! Executing the payload now\n\n[    0.162546] mingo version 0.9.0\n[    0.162745] Booting on: Raspberry Pi 3\n[    0.163201] Current privilege level: EL1\n[    0.163677] Exception handling state:\n[    0.164122]       Debug:  Masked\n[    0.164511]       SError: Masked\n[    0.164901]       IRQ:    Masked\n[    0.165291]       FIQ:    Masked\n[    0.165681] Architectural timer resolution: 52 ns\n[    0.166255] Drivers loaded:\n[    0.166592]       1. BCM PL011 UART\n[    0.167014]       2. BCM GPIO\n[    0.167371] Timer test, spinning for 1 second\n[    1.167904] Echoing input now\n```\n\n## Diff to previous\n```diff\n\ndiff -uNr 08_hw_debug_JTAG/Cargo.toml 09_privilege_level/Cargo.toml\n--- 08_hw_debug_JTAG/Cargo.toml\n+++ 09_privilege_level/Cargo.toml\n@@ -1,6 +1,6 @@\n [package]\n name = \"mingo\"\n-version = \"0.8.0\"\n+version = \"0.9.0\"\n authors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\n edition = \"2021\"\n\n\ndiff -uNr 08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs 09_privilege_level/src/_arch/aarch64/cpu/boot.rs\n--- 08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs\n+++ 09_privilege_level/src/_arch/aarch64/cpu/boot.rs\n@@ -11,22 +11,73 @@\n //!\n //! crate::cpu::boot::arch_boot\n\n+use aarch64_cpu::{asm, registers::*};\n use core::arch::global_asm;\n+use tock_registers::interfaces::Writeable;\n\n // Assembly counterpart to this file.\n global_asm!(\n     include_str!(\"boot.s\"),\n+    CONST_CURRENTEL_EL2 = const 0x8,\n     CONST_CORE_ID_MASK = const 0b11\n );\n\n //--------------------------------------------------------------------------------------------------\n+// Private Code\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Prepares the transition from EL2 to EL1.\n+///\n+/// # Safety\n+///\n+/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.\n+/// - The HW state of EL1 must be prepared in a sound way.\n+#[inline(always)]\n+unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: u64) {\n+    // Enable timer counter registers for EL1.\n+    CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET);\n+\n+    // No offset for reading the counters.\n+    CNTVOFF_EL2.set(0);\n+\n+    // Set EL1 execution state to AArch64.\n+    HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64);\n+\n+    // Set up a simulated exception return.\n+    //\n+    // First, fake a saved program status where all interrupts were masked and SP_EL1 was used as a\n+    // stack pointer.\n+    SPSR_EL2.write(\n+        SPSR_EL2::D::Masked\n+            + SPSR_EL2::A::Masked\n+            + SPSR_EL2::I::Masked\n+            + SPSR_EL2::F::Masked\n+            + SPSR_EL2::M::EL1h,\n+    );\n+\n+    // Second, let the link register point to kernel_init().\n+    ELR_EL2.set(crate::kernel_init as *const () as u64);\n+\n+    // Set up SP_EL1 (stack pointer), which will be used by EL1 once we \"return\" to it. Since there\n+    // are no plans to ever return to EL2, just re-use the same stack.\n+    SP_EL1.set(phys_boot_core_stack_end_exclusive_addr);\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n // Public Code\n //--------------------------------------------------------------------------------------------------\n\n /// The Rust entry of the `kernel` binary.\n ///\n /// The function is called from the assembly `_start` function.\n+///\n+/// # Safety\n+///\n+/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`.\n #[no_mangle]\n-pub unsafe fn _start_rust() -> ! {\n-    crate::kernel_init()\n+pub unsafe extern \"C\" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! {\n+    prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr);\n+\n+    // Use `eret` to \"return\" to EL1. This results in execution of kernel_init() in EL1.\n+    asm::eret()\n }\n\ndiff -uNr 08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.s 09_privilege_level/src/_arch/aarch64/cpu/boot.s\n--- 08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.s\n+++ 09_privilege_level/src/_arch/aarch64/cpu/boot.s\n@@ -27,11 +27,16 @@\n // fn _start()\n //------------------------------------------------------------------------------\n _start:\n+\t// Only proceed if the core executes in EL2. Park it otherwise.\n+\tmrs\tx0, CurrentEL\n+\tcmp\tx0, {CONST_CURRENTEL_EL2}\n+\tb.ne\t.L_parking_loop\n+\n \t// Only proceed on the boot core. Park it otherwise.\n-\tmrs\tx0, MPIDR_EL1\n-\tand\tx0, x0, {CONST_CORE_ID_MASK}\n-\tldr\tx1, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs\n-\tcmp\tx0, x1\n+\tmrs\tx1, MPIDR_EL1\n+\tand\tx1, x1, {CONST_CORE_ID_MASK}\n+\tldr\tx2, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs\n+\tcmp\tx1, x2\n \tb.ne\t.L_parking_loop\n\n \t// If execution reaches here, it is the boot core.\n@@ -48,7 +53,7 @@\n\n \t// Prepare the jump to Rust code.\n .L_prepare_rust:\n-\t// Set the stack pointer.\n+\t// Set the stack pointer. This ensures that any code in EL2 that needs the stack will work.\n \tADR_REL\tx0, __boot_core_stack_end_exclusive\n \tmov\tsp, x0\n\n@@ -60,7 +65,7 @@\n \tb.eq\t.L_parking_loop\n \tstr\tw2, [x1]\n\n-\t// Jump to Rust code.\n+\t// Jump to Rust code. x0 holds the function argument provided to _start_rust().\n \tb\t_start_rust\n\n \t// Infinitely wait for events (aka \"park the core\").\n\ndiff -uNr 08_hw_debug_JTAG/src/_arch/aarch64/exception/asynchronous.rs 09_privilege_level/src/_arch/aarch64/exception/asynchronous.rs\n--- 08_hw_debug_JTAG/src/_arch/aarch64/exception/asynchronous.rs\n+++ 09_privilege_level/src/_arch/aarch64/exception/asynchronous.rs\n@@ -0,0 +1,82 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Architectural asynchronous exception handling.\n+//!\n+//! # Orientation\n+//!\n+//! Since arch modules are imported into generic modules using the path attribute, the path of this\n+//! file is:\n+//!\n+//! crate::exception::asynchronous::arch_asynchronous\n+\n+use aarch64_cpu::registers::*;\n+use tock_registers::interfaces::Readable;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+trait DaifField {\n+    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register>;\n+}\n+\n+struct Debug;\n+struct SError;\n+struct IRQ;\n+struct FIQ;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Code\n+//--------------------------------------------------------------------------------------------------\n+\n+impl DaifField for Debug {\n+    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n+        DAIF::D\n+    }\n+}\n+\n+impl DaifField for SError {\n+    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n+        DAIF::A\n+    }\n+}\n+\n+impl DaifField for IRQ {\n+    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n+        DAIF::I\n+    }\n+}\n+\n+impl DaifField for FIQ {\n+    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n+        DAIF::F\n+    }\n+}\n+\n+fn is_masked<T>() -> bool\n+where\n+    T: DaifField,\n+{\n+    DAIF.is_set(T::daif_field())\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Print the AArch64 exceptions status.\n+#[rustfmt::skip]\n+pub fn print_state() {\n+    use crate::info;\n+\n+    let to_mask_str = |x| -> _ {\n+        if x { \"Masked\" } else { \"Unmasked\" }\n+    };\n+\n+    info!(\"      Debug:  {}\", to_mask_str(is_masked::<Debug>()));\n+    info!(\"      SError: {}\", to_mask_str(is_masked::<SError>()));\n+    info!(\"      IRQ:    {}\", to_mask_str(is_masked::<IRQ>()));\n+    info!(\"      FIQ:    {}\", to_mask_str(is_masked::<FIQ>()));\n+}\n\ndiff -uNr 08_hw_debug_JTAG/src/_arch/aarch64/exception.rs 09_privilege_level/src/_arch/aarch64/exception.rs\n--- 08_hw_debug_JTAG/src/_arch/aarch64/exception.rs\n+++ 09_privilege_level/src/_arch/aarch64/exception.rs\n@@ -0,0 +1,31 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Architectural synchronous and asynchronous exception handling.\n+//!\n+//! # Orientation\n+//!\n+//! Since arch modules are imported into generic modules using the path attribute, the path of this\n+//! file is:\n+//!\n+//! crate::exception::arch_exception\n+\n+use aarch64_cpu::registers::*;\n+use tock_registers::interfaces::Readable;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+use crate::exception::PrivilegeLevel;\n+\n+/// The processing element's current privilege level.\n+pub fn current_privilege_level() -> (PrivilegeLevel, &'static str) {\n+    let el = CurrentEL.read_as_enum(CurrentEL::EL);\n+    match el {\n+        Some(CurrentEL::EL::Value::EL2) => (PrivilegeLevel::Hypervisor, \"EL2\"),\n+        Some(CurrentEL::EL::Value::EL1) => (PrivilegeLevel::Kernel, \"EL1\"),\n+        Some(CurrentEL::EL::Value::EL0) => (PrivilegeLevel::User, \"EL0\"),\n+        _ => (PrivilegeLevel::Unknown, \"Unknown\"),\n+    }\n+}\n\ndiff -uNr 08_hw_debug_JTAG/src/exception/asynchronous.rs 09_privilege_level/src/exception/asynchronous.rs\n--- 08_hw_debug_JTAG/src/exception/asynchronous.rs\n+++ 09_privilege_level/src/exception/asynchronous.rs\n@@ -0,0 +1,14 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Asynchronous exception handling.\n+\n+#[cfg(target_arch = \"aarch64\")]\n+#[path = \"../_arch/aarch64/exception/asynchronous.rs\"]\n+mod arch_asynchronous;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Architectural Public Reexports\n+//--------------------------------------------------------------------------------------------------\n+pub use arch_asynchronous::print_state;\n\ndiff -uNr 08_hw_debug_JTAG/src/exception.rs 09_privilege_level/src/exception.rs\n--- 08_hw_debug_JTAG/src/exception.rs\n+++ 09_privilege_level/src/exception.rs\n@@ -0,0 +1,30 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Synchronous and asynchronous exception handling.\n+\n+#[cfg(target_arch = \"aarch64\")]\n+#[path = \"_arch/aarch64/exception.rs\"]\n+mod arch_exception;\n+\n+pub mod asynchronous;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Architectural Public Reexports\n+//--------------------------------------------------------------------------------------------------\n+pub use arch_exception::current_privilege_level;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Kernel privilege levels.\n+#[allow(missing_docs)]\n+#[derive(Eq, PartialEq)]\n+pub enum PrivilegeLevel {\n+    User,\n+    Kernel,\n+    Hypervisor,\n+    Unknown,\n+}\n\ndiff -uNr 08_hw_debug_JTAG/src/main.rs 09_privilege_level/src/main.rs\n--- 08_hw_debug_JTAG/src/main.rs\n+++ 09_privilege_level/src/main.rs\n@@ -121,6 +121,7 @@\n mod console;\n mod cpu;\n mod driver;\n+mod exception;\n mod panic_wait;\n mod print;\n mod synchronization;\n@@ -148,6 +149,7 @@\n\n /// The main function running after the early init.\n fn kernel_main() -> ! {\n+    use console::console;\n     use core::time::Duration;\n\n     info!(\n@@ -157,6 +159,12 @@\n     );\n     info!(\"Booting on: {}\", bsp::board_name());\n\n+    let (_, privilege_level) = exception::current_privilege_level();\n+    info!(\"Current privilege level: {}\", privilege_level);\n+\n+    info!(\"Exception handling state:\");\n+    exception::asynchronous::print_state();\n+\n     info!(\n         \"Architectural timer resolution: {} ns\",\n         time::time_manager().resolution().as_nanos()\n@@ -165,11 +173,15 @@\n     info!(\"Drivers loaded:\");\n     driver::driver_manager().enumerate();\n\n-    // Test a failing timer case.\n-    time::time_manager().spin_for(Duration::from_nanos(1));\n+    info!(\"Timer test, spinning for 1 second\");\n+    time::time_manager().spin_for(Duration::from_secs(1));\n+\n+    info!(\"Echoing input now\");\n\n+    // Discard any spurious received characters before going into echo mode.\n+    console().clear_rx();\n     loop {\n-        info!(\"Spinning for 1 second\");\n-        time::time_manager().spin_for(Duration::from_secs(1));\n+        let c = console().read_char();\n+        console().write_char(c);\n     }\n }\n\ndiff -uNr 08_hw_debug_JTAG/tests/boot_test_string.rb 09_privilege_level/tests/boot_test_string.rb\n--- 08_hw_debug_JTAG/tests/boot_test_string.rb\n+++ 09_privilege_level/tests/boot_test_string.rb\n@@ -1,3 +1,3 @@\n # frozen_string_literal: true\n\n-EXPECTED_PRINT = 'Spinning for 1 second'\n+EXPECTED_PRINT = 'Echoing input now'\n\n```\n"
  },
  {
    "path": "09_privilege_level/build.rs",
    "content": "use std::{env, fs, process};\n\nfn main() {\n    let ld_script_path = match env::var(\"LD_SCRIPT_PATH\") {\n        Ok(var) => var,\n        _ => process::exit(0),\n    };\n\n    let files = fs::read_dir(ld_script_path).unwrap();\n    files\n        .filter_map(Result::ok)\n        .filter(|d| {\n            if let Some(e) = d.path().extension() {\n                e == \"ld\"\n            } else {\n                false\n            }\n        })\n        .for_each(|f| println!(\"cargo:rerun-if-changed={}\", f.path().display()));\n}\n"
  },
  {
    "path": "09_privilege_level/src/_arch/aarch64/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural boot code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::boot::arch_boot\n\nuse aarch64_cpu::{asm, registers::*};\nuse core::arch::global_asm;\nuse tock_registers::interfaces::Writeable;\n\n// Assembly counterpart to this file.\nglobal_asm!(\n    include_str!(\"boot.s\"),\n    CONST_CURRENTEL_EL2 = const 0x8,\n    CONST_CORE_ID_MASK = const 0b11\n);\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Prepares the transition from EL2 to EL1.\n///\n/// # Safety\n///\n/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.\n/// - The HW state of EL1 must be prepared in a sound way.\n#[inline(always)]\nunsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: u64) {\n    // Enable timer counter registers for EL1.\n    CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET);\n\n    // No offset for reading the counters.\n    CNTVOFF_EL2.set(0);\n\n    // Set EL1 execution state to AArch64.\n    HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64);\n\n    // Set up a simulated exception return.\n    //\n    // First, fake a saved program status where all interrupts were masked and SP_EL1 was used as a\n    // stack pointer.\n    SPSR_EL2.write(\n        SPSR_EL2::D::Masked\n            + SPSR_EL2::A::Masked\n            + SPSR_EL2::I::Masked\n            + SPSR_EL2::F::Masked\n            + SPSR_EL2::M::EL1h,\n    );\n\n    // Second, let the link register point to kernel_init().\n    ELR_EL2.set(crate::kernel_init as *const () as u64);\n\n    // Set up SP_EL1 (stack pointer), which will be used by EL1 once we \"return\" to it. Since there\n    // are no plans to ever return to EL2, just re-use the same stack.\n    SP_EL1.set(phys_boot_core_stack_end_exclusive_addr);\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The Rust entry of the `kernel` binary.\n///\n/// The function is called from the assembly `_start` function.\n///\n/// # Safety\n///\n/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`.\n#[no_mangle]\npub unsafe extern \"C\" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! {\n    prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr);\n\n    // Use `eret` to \"return\" to EL1. This results in execution of kernel_init() in EL1.\n    asm::eret()\n}\n"
  },
  {
    "path": "09_privilege_level/src/_arch/aarch64/cpu/boot.s",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2022 Andre Richter <andre.o.richter@gmail.com>\n\n//--------------------------------------------------------------------------------------------------\n// Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Load the address of a symbol into a register, PC-relative.\n//\n// The symbol must lie within +/- 4 GiB of the Program Counter.\n//\n// # Resources\n//\n// - https://sourceware.org/binutils/docs-2.36/as/AArch64_002dRelocations.html\n.macro ADR_REL register, symbol\n\tadrp\t\\register, \\symbol\n\tadd\t\\register, \\register, #:lo12:\\symbol\n.endm\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n.section .text._start\n\n//------------------------------------------------------------------------------\n// fn _start()\n//------------------------------------------------------------------------------\n_start:\n\t// Only proceed if the core executes in EL2. Park it otherwise.\n\tmrs\tx0, CurrentEL\n\tcmp\tx0, {CONST_CURRENTEL_EL2}\n\tb.ne\t.L_parking_loop\n\n\t// Only proceed on the boot core. Park it otherwise.\n\tmrs\tx1, MPIDR_EL1\n\tand\tx1, x1, {CONST_CORE_ID_MASK}\n\tldr\tx2, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs\n\tcmp\tx1, x2\n\tb.ne\t.L_parking_loop\n\n\t// If execution reaches here, it is the boot core.\n\n\t// Initialize DRAM.\n\tADR_REL\tx0, __bss_start\n\tADR_REL x1, __bss_end_exclusive\n\n.L_bss_init_loop:\n\tcmp\tx0, x1\n\tb.eq\t.L_prepare_rust\n\tstp\txzr, xzr, [x0], #16\n\tb\t.L_bss_init_loop\n\n\t// Prepare the jump to Rust code.\n.L_prepare_rust:\n\t// Set the stack pointer. This ensures that any code in EL2 that needs the stack will work.\n\tADR_REL\tx0, __boot_core_stack_end_exclusive\n\tmov\tsp, x0\n\n\t// Read the CPU's timer counter frequency and store it in ARCH_TIMER_COUNTER_FREQUENCY.\n\t// Abort if the frequency read back as 0.\n\tADR_REL\tx1, ARCH_TIMER_COUNTER_FREQUENCY // provided by aarch64/time.rs\n\tmrs\tx2, CNTFRQ_EL0\n\tcmp\tx2, xzr\n\tb.eq\t.L_parking_loop\n\tstr\tw2, [x1]\n\n\t// Jump to Rust code. x0 holds the function argument provided to _start_rust().\n\tb\t_start_rust\n\n\t// Infinitely wait for events (aka \"park the core\").\n.L_parking_loop:\n\twfe\n\tb\t.L_parking_loop\n\n.size\t_start, . - _start\n.type\t_start, function\n.global\t_start\n"
  },
  {
    "path": "09_privilege_level/src/_arch/aarch64/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural processor code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::arch_cpu\n\nuse aarch64_cpu::asm;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\npub use asm::nop;\n\n/// Pause execution on the core.\n#[inline(always)]\npub fn wait_forever() -> ! {\n    loop {\n        asm::wfe()\n    }\n}\n"
  },
  {
    "path": "09_privilege_level/src/_arch/aarch64/exception/asynchronous.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural asynchronous exception handling.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::exception::asynchronous::arch_asynchronous\n\nuse aarch64_cpu::registers::*;\nuse tock_registers::interfaces::Readable;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\ntrait DaifField {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register>;\n}\n\nstruct Debug;\nstruct SError;\nstruct IRQ;\nstruct FIQ;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl DaifField for Debug {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::D\n    }\n}\n\nimpl DaifField for SError {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::A\n    }\n}\n\nimpl DaifField for IRQ {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::I\n    }\n}\n\nimpl DaifField for FIQ {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::F\n    }\n}\n\nfn is_masked<T>() -> bool\nwhere\n    T: DaifField,\n{\n    DAIF.is_set(T::daif_field())\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Print the AArch64 exceptions status.\n#[rustfmt::skip]\npub fn print_state() {\n    use crate::info;\n\n    let to_mask_str = |x| -> _ {\n        if x { \"Masked\" } else { \"Unmasked\" }\n    };\n\n    info!(\"      Debug:  {}\", to_mask_str(is_masked::<Debug>()));\n    info!(\"      SError: {}\", to_mask_str(is_masked::<SError>()));\n    info!(\"      IRQ:    {}\", to_mask_str(is_masked::<IRQ>()));\n    info!(\"      FIQ:    {}\", to_mask_str(is_masked::<FIQ>()));\n}\n"
  },
  {
    "path": "09_privilege_level/src/_arch/aarch64/exception.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural synchronous and asynchronous exception handling.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::exception::arch_exception\n\nuse aarch64_cpu::registers::*;\nuse tock_registers::interfaces::Readable;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse crate::exception::PrivilegeLevel;\n\n/// The processing element's current privilege level.\npub fn current_privilege_level() -> (PrivilegeLevel, &'static str) {\n    let el = CurrentEL.read_as_enum(CurrentEL::EL);\n    match el {\n        Some(CurrentEL::EL::Value::EL2) => (PrivilegeLevel::Hypervisor, \"EL2\"),\n        Some(CurrentEL::EL::Value::EL1) => (PrivilegeLevel::Kernel, \"EL1\"),\n        Some(CurrentEL::EL::Value::EL0) => (PrivilegeLevel::User, \"EL0\"),\n        _ => (PrivilegeLevel::Unknown, \"Unknown\"),\n    }\n}\n"
  },
  {
    "path": "09_privilege_level/src/_arch/aarch64/time.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural timer primitives.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::time::arch_time\n\nuse crate::warn;\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::{\n    num::{NonZeroU128, NonZeroU32, NonZeroU64},\n    ops::{Add, Div},\n    time::Duration,\n};\nuse tock_registers::interfaces::Readable;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nconst NANOSEC_PER_SEC: NonZeroU64 = NonZeroU64::new(1_000_000_000).unwrap();\n\n#[derive(Copy, Clone, PartialOrd, PartialEq)]\nstruct GenericTimerCounterValue(u64);\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n/// Boot assembly code overwrites this value with the value of CNTFRQ_EL0 before any Rust code is\n/// executed. This given value here is just a (safe) dummy.\n#[no_mangle]\nstatic ARCH_TIMER_COUNTER_FREQUENCY: NonZeroU32 = NonZeroU32::MIN;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nfn arch_timer_counter_frequency() -> NonZeroU32 {\n    // Read volatile is needed here to prevent the compiler from optimizing\n    // ARCH_TIMER_COUNTER_FREQUENCY away.\n    //\n    // This is safe, because all the safety requirements as stated in read_volatile()'s\n    // documentation are fulfilled.\n    unsafe { core::ptr::read_volatile(&ARCH_TIMER_COUNTER_FREQUENCY) }\n}\n\nimpl GenericTimerCounterValue {\n    pub const MAX: Self = GenericTimerCounterValue(u64::MAX);\n}\n\nimpl Add for GenericTimerCounterValue {\n    type Output = Self;\n\n    fn add(self, other: Self) -> Self {\n        GenericTimerCounterValue(self.0.wrapping_add(other.0))\n    }\n}\n\nimpl From<GenericTimerCounterValue> for Duration {\n    fn from(counter_value: GenericTimerCounterValue) -> Self {\n        if counter_value.0 == 0 {\n            return Duration::ZERO;\n        }\n\n        let frequency: NonZeroU64 = arch_timer_counter_frequency().into();\n\n        // Div<NonZeroU64> implementation for u64 cannot panic.\n        let secs = counter_value.0.div(frequency);\n\n        // This is safe, because frequency can never be greater than u32::MAX, which means the\n        // largest theoretical value for sub_second_counter_value is (u32::MAX - 1). Therefore,\n        // (sub_second_counter_value * NANOSEC_PER_SEC) cannot overflow an u64.\n        //\n        // The subsequent division ensures the result fits into u32, since the max result is smaller\n        // than NANOSEC_PER_SEC. Therefore, just cast it to u32 using `as`.\n        let sub_second_counter_value = counter_value.0 % frequency;\n        let nanos = unsafe { sub_second_counter_value.unchecked_mul(u64::from(NANOSEC_PER_SEC)) }\n            .div(frequency) as u32;\n\n        Duration::new(secs, nanos)\n    }\n}\n\nfn max_duration() -> Duration {\n    Duration::from(GenericTimerCounterValue::MAX)\n}\n\nimpl TryFrom<Duration> for GenericTimerCounterValue {\n    type Error = &'static str;\n\n    fn try_from(duration: Duration) -> Result<Self, Self::Error> {\n        if duration < resolution() {\n            return Ok(GenericTimerCounterValue(0));\n        }\n\n        if duration > max_duration() {\n            return Err(\"Conversion error. Duration too big\");\n        }\n\n        let frequency: u128 = u32::from(arch_timer_counter_frequency()) as u128;\n        let duration: u128 = duration.as_nanos();\n\n        // This is safe, because frequency can never be greater than u32::MAX, and\n        // (Duration::MAX.as_nanos() * u32::MAX) < u128::MAX.\n        let counter_value =\n            unsafe { duration.unchecked_mul(frequency) }.div(NonZeroU128::from(NANOSEC_PER_SEC));\n\n        // Since we checked above that we are <= max_duration(), just cast to u64.\n        Ok(GenericTimerCounterValue(counter_value as u64))\n    }\n}\n\n#[inline(always)]\nfn read_cntpct() -> GenericTimerCounterValue {\n    // Prevent that the counter is read ahead of time due to out-of-order execution.\n    barrier::isb(barrier::SY);\n    let cnt = CNTPCT_EL0.get();\n\n    GenericTimerCounterValue(cnt)\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The timer's resolution.\npub fn resolution() -> Duration {\n    Duration::from(GenericTimerCounterValue(1))\n}\n\n/// The uptime since power-on of the device.\n///\n/// This includes time consumed by firmware and bootloaders.\npub fn uptime() -> Duration {\n    read_cntpct().into()\n}\n\n/// Spin for a given duration.\npub fn spin_for(duration: Duration) {\n    let curr_counter_value = read_cntpct();\n\n    let counter_value_delta: GenericTimerCounterValue = match duration.try_into() {\n        Err(msg) => {\n            warn!(\"spin_for: {}. Skipping\", msg);\n            return;\n        }\n        Ok(val) => val,\n    };\n    let counter_value_target = curr_counter_value + counter_value_delta;\n\n    // Busy wait.\n    //\n    // Read CNTPCT_EL0 directly to avoid the ISB that is part of [`read_cntpct`].\n    while GenericTimerCounterValue(CNTPCT_EL0.get()) < counter_value_target {}\n}\n"
  },
  {
    "path": "09_privilege_level/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GPIO Driver.\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper, driver, synchronization,\n    synchronization::NullLock,\n};\nuse tock_registers::{\n    interfaces::{ReadWriteable, Writeable},\n    register_bitfields, register_structs,\n    registers::ReadWrite,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// GPIO registers.\n//\n// Descriptions taken from\n// - https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf\n// - https://datasheets.raspberrypi.org/bcm2711/bcm2711-peripherals.pdf\nregister_bitfields! {\n    u32,\n\n    /// GPIO Function Select 1\n    GPFSEL1 [\n        /// Pin 15\n        FSEL15 OFFSET(15) NUMBITS(3) [\n            Input = 0b000,\n            Output = 0b001,\n            AltFunc0 = 0b100  // PL011 UART RX\n\n        ],\n\n        /// Pin 14\n        FSEL14 OFFSET(12) NUMBITS(3) [\n            Input = 0b000,\n            Output = 0b001,\n            AltFunc0 = 0b100  // PL011 UART TX\n        ]\n    ],\n\n    /// GPIO Pull-up/down Register\n    ///\n    /// BCM2837 only.\n    GPPUD [\n        /// Controls the actuation of the internal pull-up/down control line to ALL the GPIO pins.\n        PUD OFFSET(0) NUMBITS(2) [\n            Off = 0b00,\n            PullDown = 0b01,\n            PullUp = 0b10\n        ]\n    ],\n\n    /// GPIO Pull-up/down Clock Register 0\n    ///\n    /// BCM2837 only.\n    GPPUDCLK0 [\n        /// Pin 15\n        PUDCLK15 OFFSET(15) NUMBITS(1) [\n            NoEffect = 0,\n            AssertClock = 1\n        ],\n\n        /// Pin 14\n        PUDCLK14 OFFSET(14) NUMBITS(1) [\n            NoEffect = 0,\n            AssertClock = 1\n        ]\n    ],\n\n    /// GPIO Pull-up / Pull-down Register 0\n    ///\n    /// BCM2711 only.\n    GPIO_PUP_PDN_CNTRL_REG0 [\n        /// Pin 15\n        GPIO_PUP_PDN_CNTRL15 OFFSET(30) NUMBITS(2) [\n            NoResistor = 0b00,\n            PullUp = 0b01\n        ],\n\n        /// Pin 14\n        GPIO_PUP_PDN_CNTRL14 OFFSET(28) NUMBITS(2) [\n            NoResistor = 0b00,\n            PullUp = 0b01\n        ]\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    RegisterBlock {\n        (0x00 => _reserved1),\n        (0x04 => GPFSEL1: ReadWrite<u32, GPFSEL1::Register>),\n        (0x08 => _reserved2),\n        (0x94 => GPPUD: ReadWrite<u32, GPPUD::Register>),\n        (0x98 => GPPUDCLK0: ReadWrite<u32, GPPUDCLK0::Register>),\n        (0x9C => _reserved3),\n        (0xE4 => GPIO_PUP_PDN_CNTRL_REG0: ReadWrite<u32, GPIO_PUP_PDN_CNTRL_REG0::Register>),\n        (0xE8 => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\nstruct GPIOInner {\n    registers: Registers,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the GPIO HW.\npub struct GPIO {\n    inner: NullLock<GPIOInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GPIOInner {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n        }\n    }\n\n    /// Disable pull-up/down on pins 14 and 15.\n    #[cfg(feature = \"bsp_rpi3\")]\n    fn disable_pud_14_15_bcm2837(&mut self) {\n        use crate::time;\n        use core::time::Duration;\n\n        // The Linux 2837 GPIO driver waits 1 µs between the steps.\n        const DELAY: Duration = Duration::from_micros(1);\n\n        self.registers.GPPUD.write(GPPUD::PUD::Off);\n        time::time_manager().spin_for(DELAY);\n\n        self.registers\n            .GPPUDCLK0\n            .write(GPPUDCLK0::PUDCLK15::AssertClock + GPPUDCLK0::PUDCLK14::AssertClock);\n        time::time_manager().spin_for(DELAY);\n\n        self.registers.GPPUD.write(GPPUD::PUD::Off);\n        self.registers.GPPUDCLK0.set(0);\n    }\n\n    /// Disable pull-up/down on pins 14 and 15.\n    #[cfg(feature = \"bsp_rpi4\")]\n    fn disable_pud_14_15_bcm2711(&mut self) {\n        self.registers.GPIO_PUP_PDN_CNTRL_REG0.write(\n            GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL15::PullUp\n                + GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL14::PullUp,\n        );\n    }\n\n    /// Map PL011 UART as standard output.\n    ///\n    /// TX to pin 14\n    /// RX to pin 15\n    pub fn map_pl011_uart(&mut self) {\n        // Select the UART on pins 14 and 15.\n        self.registers\n            .GPFSEL1\n            .modify(GPFSEL1::FSEL15::AltFunc0 + GPFSEL1::FSEL14::AltFunc0);\n\n        // Disable pull-up/down on pins 14 and 15.\n        #[cfg(feature = \"bsp_rpi3\")]\n        self.disable_pud_14_15_bcm2837();\n\n        #[cfg(feature = \"bsp_rpi4\")]\n        self.disable_pud_14_15_bcm2711();\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GPIO {\n    pub const COMPATIBLE: &'static str = \"BCM GPIO\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            inner: NullLock::new(GPIOInner::new(mmio_start_addr)),\n        }\n    }\n\n    /// Concurrency safe version of `GPIOInner.map_pl011_uart()`\n    pub fn map_pl011_uart(&self) {\n        self.inner.lock(|inner| inner.map_pl011_uart())\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl driver::interface::DeviceDriver for GPIO {\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n}\n"
  },
  {
    "path": "09_privilege_level/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! PL011 UART driver.\n//!\n//! # Resources\n//!\n//! - <https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf>\n//! - <https://developer.arm.com/documentation/ddi0183/latest>\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper, console, cpu, driver, synchronization,\n    synchronization::NullLock,\n};\nuse core::fmt;\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields, register_structs,\n    registers::{ReadOnly, ReadWrite, WriteOnly},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// PL011 UART registers.\n//\n// Descriptions taken from \"PrimeCell UART (PL011) Technical Reference Manual\" r1p5.\nregister_bitfields! {\n    u32,\n\n    /// Flag Register.\n    FR [\n        /// Transmit FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n        /// Line Control Register, LCR_H.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the transmit holding register is empty.\n        /// - If the FIFO is enabled, the TXFE bit is set when the transmit FIFO is empty.\n        /// - This bit does not indicate if there is data in the transmit shift register.\n        TXFE OFFSET(7) NUMBITS(1) [],\n\n        /// Transmit FIFO full. The meaning of this bit depends on the state of the FEN bit in the\n        /// LCR_H Register.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the transmit holding register is full.\n        /// - If the FIFO is enabled, the TXFF bit is set when the transmit FIFO is full.\n        TXFF OFFSET(5) NUMBITS(1) [],\n\n        /// Receive FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n        /// LCR_H Register.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the receive holding register is empty.\n        /// - If the FIFO is enabled, the RXFE bit is set when the receive FIFO is empty.\n        RXFE OFFSET(4) NUMBITS(1) [],\n\n        /// UART busy. If this bit is set to 1, the UART is busy transmitting data. This bit remains\n        /// set until the complete byte, including all the stop bits, has been sent from the shift\n        /// register.\n        ///\n        /// This bit is set as soon as the transmit FIFO becomes non-empty, regardless of whether\n        /// the UART is enabled or not.\n        BUSY OFFSET(3) NUMBITS(1) []\n    ],\n\n    /// Integer Baud Rate Divisor.\n    IBRD [\n        /// The integer baud rate divisor.\n        BAUD_DIVINT OFFSET(0) NUMBITS(16) []\n    ],\n\n    /// Fractional Baud Rate Divisor.\n    FBRD [\n        ///  The fractional baud rate divisor.\n        BAUD_DIVFRAC OFFSET(0) NUMBITS(6) []\n    ],\n\n    /// Line Control Register.\n    LCR_H [\n        /// Word length. These bits indicate the number of data bits transmitted or received in a\n        /// frame.\n        #[allow(clippy::enum_variant_names)]\n        WLEN OFFSET(5) NUMBITS(2) [\n            FiveBit = 0b00,\n            SixBit = 0b01,\n            SevenBit = 0b10,\n            EightBit = 0b11\n        ],\n\n        /// Enable FIFOs:\n        ///\n        /// 0 = FIFOs are disabled (character mode) that is, the FIFOs become 1-byte-deep holding\n        /// registers.\n        ///\n        /// 1 = Transmit and receive FIFO buffers are enabled (FIFO mode).\n        FEN  OFFSET(4) NUMBITS(1) [\n            FifosDisabled = 0,\n            FifosEnabled = 1\n        ]\n    ],\n\n    /// Control Register.\n    CR [\n        /// Receive enable. If this bit is set to 1, the receive section of the UART is enabled.\n        /// Data reception occurs for either UART signals or SIR signals depending on the setting of\n        /// the SIREN bit. When the UART is disabled in the middle of reception, it completes the\n        /// current character before stopping.\n        RXE OFFSET(9) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// Transmit enable. If this bit is set to 1, the transmit section of the UART is enabled.\n        /// Data transmission occurs for either UART signals, or SIR signals depending on the\n        /// setting of the SIREN bit. When the UART is disabled in the middle of transmission, it\n        /// completes the current character before stopping.\n        TXE OFFSET(8) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// UART enable:\n        ///\n        /// 0 = UART is disabled. If the UART is disabled in the middle of transmission or\n        /// reception, it completes the current character before stopping.\n        ///\n        /// 1 = The UART is enabled. Data transmission and reception occurs for either UART signals\n        /// or SIR signals depending on the setting of the SIREN bit\n        UARTEN OFFSET(0) NUMBITS(1) [\n            /// If the UART is disabled in the middle of transmission or reception, it completes the\n            /// current character before stopping.\n            Disabled = 0,\n            Enabled = 1\n        ]\n    ],\n\n    /// Interrupt Clear Register.\n    ICR [\n        /// Meta field for all pending interrupts.\n        ALL OFFSET(0) NUMBITS(11) []\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    pub RegisterBlock {\n        (0x00 => DR: ReadWrite<u32>),\n        (0x04 => _reserved1),\n        (0x18 => FR: ReadOnly<u32, FR::Register>),\n        (0x1c => _reserved2),\n        (0x24 => IBRD: WriteOnly<u32, IBRD::Register>),\n        (0x28 => FBRD: WriteOnly<u32, FBRD::Register>),\n        (0x2c => LCR_H: WriteOnly<u32, LCR_H::Register>),\n        (0x30 => CR: WriteOnly<u32, CR::Register>),\n        (0x34 => _reserved3),\n        (0x44 => ICR: WriteOnly<u32, ICR::Register>),\n        (0x48 => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\n#[derive(PartialEq)]\nenum BlockingMode {\n    Blocking,\n    NonBlocking,\n}\n\nstruct PL011UartInner {\n    registers: Registers,\n    chars_written: usize,\n    chars_read: usize,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the UART.\npub struct PL011Uart {\n    inner: NullLock<PL011UartInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PL011UartInner {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n            chars_written: 0,\n            chars_read: 0,\n        }\n    }\n\n    /// Set up baud rate and characteristics.\n    ///\n    /// This results in 8N1 and 921_600 baud.\n    ///\n    /// The calculation for the BRD is (we set the clock to 48 MHz in config.txt):\n    /// `(48_000_000 / 16) / 921_600 = 3.2552083`.\n    ///\n    /// This means the integer part is `3` and goes into the `IBRD`.\n    /// The fractional part is `0.2552083`.\n    ///\n    /// `FBRD` calculation according to the PL011 Technical Reference Manual:\n    /// `INTEGER((0.2552083 * 64) + 0.5) = 16`.\n    ///\n    /// Therefore, the generated baud rate divider is: `3 + 16/64 = 3.25`. Which results in a\n    /// genrated baud rate of `48_000_000 / (16 * 3.25) = 923_077`.\n    ///\n    /// Error = `((923_077 - 921_600) / 921_600) * 100 = 0.16%`.\n    pub fn init(&mut self) {\n        // Execution can arrive here while there are still characters queued in the TX FIFO and\n        // actively being sent out by the UART hardware. If the UART is turned off in this case,\n        // those queued characters would be lost.\n        //\n        // For example, this can happen during runtime on a call to panic!(), because panic!()\n        // initializes its own UART instance and calls init().\n        //\n        // Hence, flush first to ensure all pending characters are transmitted.\n        self.flush();\n\n        // Turn the UART off temporarily.\n        self.registers.CR.set(0);\n\n        // Clear all pending interrupts.\n        self.registers.ICR.write(ICR::ALL::CLEAR);\n\n        // From the PL011 Technical Reference Manual:\n        //\n        // The LCR_H, IBRD, and FBRD registers form the single 30-bit wide LCR Register that is\n        // updated on a single write strobe generated by a LCR_H write. So, to internally update the\n        // contents of IBRD or FBRD, a LCR_H write must always be performed at the end.\n        //\n        // Set the baud rate, 8N1 and FIFO enabled.\n        self.registers.IBRD.write(IBRD::BAUD_DIVINT.val(3));\n        self.registers.FBRD.write(FBRD::BAUD_DIVFRAC.val(16));\n        self.registers\n            .LCR_H\n            .write(LCR_H::WLEN::EightBit + LCR_H::FEN::FifosEnabled);\n\n        // Turn the UART on.\n        self.registers\n            .CR\n            .write(CR::UARTEN::Enabled + CR::TXE::Enabled + CR::RXE::Enabled);\n    }\n\n    /// Send a character.\n    fn write_char(&mut self, c: char) {\n        // Spin while TX FIFO full is set, waiting for an empty slot.\n        while self.registers.FR.matches_all(FR::TXFF::SET) {\n            cpu::nop();\n        }\n\n        // Write the character to the buffer.\n        self.registers.DR.set(c as u32);\n\n        self.chars_written += 1;\n    }\n\n    /// Block execution until the last buffered character has been physically put on the TX wire.\n    fn flush(&self) {\n        // Spin until the busy bit is cleared.\n        while self.registers.FR.matches_all(FR::BUSY::SET) {\n            cpu::nop();\n        }\n    }\n\n    /// Retrieve a character.\n    fn read_char_converting(&mut self, blocking_mode: BlockingMode) -> Option<char> {\n        // If RX FIFO is empty,\n        if self.registers.FR.matches_all(FR::RXFE::SET) {\n            // immediately return in non-blocking mode.\n            if blocking_mode == BlockingMode::NonBlocking {\n                return None;\n            }\n\n            // Otherwise, wait until a char was received.\n            while self.registers.FR.matches_all(FR::RXFE::SET) {\n                cpu::nop();\n            }\n        }\n\n        // Read one character.\n        let mut ret = self.registers.DR.get() as u8 as char;\n\n        // Convert carrige return to newline.\n        if ret == '\\r' {\n            ret = '\\n'\n        }\n\n        // Update statistics.\n        self.chars_read += 1;\n\n        Some(ret)\n    }\n}\n\n/// Implementing `core::fmt::Write` enables usage of the `format_args!` macros, which in turn are\n/// used to implement the `kernel`'s `print!` and `println!` macros. By implementing `write_str()`,\n/// we get `write_fmt()` automatically.\n///\n/// The function takes an `&mut self`, so it must be implemented for the inner struct.\n///\n/// See [`src/print.rs`].\n///\n/// [`src/print.rs`]: ../../print/index.html\nimpl fmt::Write for PL011UartInner {\n    fn write_str(&mut self, s: &str) -> fmt::Result {\n        for c in s.chars() {\n            self.write_char(c);\n        }\n\n        Ok(())\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PL011Uart {\n    pub const COMPATIBLE: &'static str = \"BCM PL011 UART\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            inner: NullLock::new(PL011UartInner::new(mmio_start_addr)),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl driver::interface::DeviceDriver for PL011Uart {\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n\n    unsafe fn init(&self) -> Result<(), &'static str> {\n        self.inner.lock(|inner| inner.init());\n\n        Ok(())\n    }\n}\n\nimpl console::interface::Write for PL011Uart {\n    /// Passthrough of `args` to the `core::fmt::Write` implementation, but guarded by a Mutex to\n    /// serialize access.\n    fn write_char(&self, c: char) {\n        self.inner.lock(|inner| inner.write_char(c));\n    }\n\n    fn write_fmt(&self, args: core::fmt::Arguments) -> fmt::Result {\n        // Fully qualified syntax for the call to `core::fmt::Write::write_fmt()` to increase\n        // readability.\n        self.inner.lock(|inner| fmt::Write::write_fmt(inner, args))\n    }\n\n    fn flush(&self) {\n        // Spin until TX FIFO empty is set.\n        self.inner.lock(|inner| inner.flush());\n    }\n}\n\nimpl console::interface::Read for PL011Uart {\n    fn read_char(&self) -> char {\n        self.inner\n            .lock(|inner| inner.read_char_converting(BlockingMode::Blocking).unwrap())\n    }\n\n    fn clear_rx(&self) {\n        // Read from the RX FIFO until it is indicating empty.\n        while self\n            .inner\n            .lock(|inner| inner.read_char_converting(BlockingMode::NonBlocking))\n            .is_some()\n        {}\n    }\n}\n\nimpl console::interface::Statistics for PL011Uart {\n    fn chars_written(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_written)\n    }\n\n    fn chars_read(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_read)\n    }\n}\n\nimpl console::interface::All for PL011Uart {}\n"
  },
  {
    "path": "09_privilege_level/src/bsp/device_driver/bcm.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BCM driver top level.\n\nmod bcm2xxx_gpio;\nmod bcm2xxx_pl011_uart;\n\npub use bcm2xxx_gpio::*;\npub use bcm2xxx_pl011_uart::*;\n"
  },
  {
    "path": "09_privilege_level/src/bsp/device_driver/common.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Common device driver code.\n\nuse core::{marker::PhantomData, ops};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct MMIODerefWrapper<T> {\n    start_addr: usize,\n    phantom: PhantomData<fn() -> T>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T> MMIODerefWrapper<T> {\n    /// Create an instance.\n    pub const unsafe fn new(start_addr: usize) -> Self {\n        Self {\n            start_addr,\n            phantom: PhantomData,\n        }\n    }\n}\n\nimpl<T> ops::Deref for MMIODerefWrapper<T> {\n    type Target = T;\n\n    fn deref(&self) -> &Self::Target {\n        unsafe { &*(self.start_addr as *const _) }\n    }\n}\n"
  },
  {
    "path": "09_privilege_level/src/bsp/device_driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Device driver.\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod bcm;\nmod common;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use bcm::*;\n"
  },
  {
    "path": "09_privilege_level/src/bsp/raspberrypi/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Processor code.\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Used by `arch` code to find the early boot core.\n#[no_mangle]\n#[link_section = \".text._start_arguments\"]\npub static BOOT_CORE_ID: u64 = 0;\n"
  },
  {
    "path": "09_privilege_level/src/bsp/raspberrypi/driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP driver support.\n\nuse super::memory::map::mmio;\nuse crate::{bsp::device_driver, console, driver as generic_driver};\nuse core::sync::atomic::{AtomicBool, Ordering};\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic PL011_UART: device_driver::PL011Uart =\n    unsafe { device_driver::PL011Uart::new(mmio::PL011_UART_START) };\nstatic GPIO: device_driver::GPIO = unsafe { device_driver::GPIO::new(mmio::GPIO_START) };\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// This must be called only after successful init of the UART driver.\nfn post_init_uart() -> Result<(), &'static str> {\n    console::register_console(&PL011_UART);\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the GPIO driver.\nfn post_init_gpio() -> Result<(), &'static str> {\n    GPIO.map_pl011_uart();\n    Ok(())\n}\n\nfn driver_uart() -> Result<(), &'static str> {\n    let uart_descriptor =\n        generic_driver::DeviceDriverDescriptor::new(&PL011_UART, Some(post_init_uart));\n    generic_driver::driver_manager().register_driver(uart_descriptor);\n\n    Ok(())\n}\n\nfn driver_gpio() -> Result<(), &'static str> {\n    let gpio_descriptor = generic_driver::DeviceDriverDescriptor::new(&GPIO, Some(post_init_gpio));\n    generic_driver::driver_manager().register_driver(gpio_descriptor);\n\n    Ok(())\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Initialize the driver subsystem.\n///\n/// # Safety\n///\n/// See child function calls.\npub unsafe fn init() -> Result<(), &'static str> {\n    static INIT_DONE: AtomicBool = AtomicBool::new(false);\n    if INIT_DONE.load(Ordering::Relaxed) {\n        return Err(\"Init already done\");\n    }\n\n    driver_uart()?;\n    driver_gpio()?;\n\n    INIT_DONE.store(true, Ordering::Relaxed);\n    Ok(())\n}\n"
  },
  {
    "path": "09_privilege_level/src/bsp/raspberrypi/kernel.ld",
    "content": "/* SPDX-License-Identifier: MIT OR Apache-2.0\n *\n * Copyright (c) 2018-2022 Andre Richter <andre.o.richter@gmail.com>\n */\n\n__rpi_phys_dram_start_addr = 0;\n\n/* The physical address at which the the kernel binary will be loaded by the Raspberry's firmware */\n__rpi_phys_binary_load_addr = 0x80000;\n\n\nENTRY(__rpi_phys_binary_load_addr)\n\n/* Flags:\n *     4 == R\n *     5 == RX\n *     6 == RW\n *\n * Segments are marked PT_LOAD below so that the ELF file provides virtual and physical addresses.\n * It doesn't mean all of them need actually be loaded.\n */\nPHDRS\n{\n    segment_boot_core_stack PT_LOAD FLAGS(6);\n    segment_code            PT_LOAD FLAGS(5);\n    segment_data            PT_LOAD FLAGS(6);\n}\n\nSECTIONS\n{\n    . =  __rpi_phys_dram_start_addr;\n\n    /***********************************************************************************************\n    * Boot Core Stack\n    ***********************************************************************************************/\n    .boot_core_stack (NOLOAD) :\n    {\n                                             /*   ^             */\n                                             /*   | stack       */\n        . += __rpi_phys_binary_load_addr;    /*   | growth      */\n                                             /*   | direction   */\n        __boot_core_stack_end_exclusive = .; /*   |             */\n    } :segment_boot_core_stack\n\n    /***********************************************************************************************\n    * Code + RO Data + Global Offset Table\n    ***********************************************************************************************/\n    .text :\n    {\n        KEEP(*(.text._start))\n        *(.text._start_arguments) /* Constants (or statics in Rust speak) read by _start(). */\n        *(.text._start_rust)      /* The Rust entry point */\n        *(.text*)                 /* Everything else */\n    } :segment_code\n\n    .rodata : ALIGN(8) { *(.rodata*) } :segment_code\n\n    /***********************************************************************************************\n    * Data + BSS\n    ***********************************************************************************************/\n    .data : { *(.data*) } :segment_data\n\n    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */\n    .bss (NOLOAD) : ALIGN(16)\n    {\n        __bss_start = .;\n        *(.bss*);\n        . = ALIGN(16);\n        __bss_end_exclusive = .;\n    } :segment_data\n\n    /***********************************************************************************************\n    * Misc\n    ***********************************************************************************************/\n    .got : { *(.got*) }\n    ASSERT(SIZEOF(.got) == 0, \"Relocation support not expected\")\n\n    /DISCARD/ : { *(.comment*) }\n}\n"
  },
  {
    "path": "09_privilege_level/src/bsp/raspberrypi/memory.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Memory Management.\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// The board's physical memory map.\n#[rustfmt::skip]\npub(super) mod map {\n\n    pub const GPIO_OFFSET:         usize = 0x0020_0000;\n    pub const UART_OFFSET:         usize = 0x0020_1000;\n\n    /// Physical devices.\n    #[cfg(feature = \"bsp_rpi3\")]\n    pub mod mmio {\n        use super::*;\n\n        pub const START:            usize =         0x3F00_0000;\n        pub const GPIO_START:       usize = START + GPIO_OFFSET;\n        pub const PL011_UART_START: usize = START + UART_OFFSET;\n    }\n\n    /// Physical devices.\n    #[cfg(feature = \"bsp_rpi4\")]\n    pub mod mmio {\n        use super::*;\n\n        pub const START:            usize =         0xFE00_0000;\n        pub const GPIO_START:       usize = START + GPIO_OFFSET;\n        pub const PL011_UART_START: usize = START + UART_OFFSET;\n    }\n}\n"
  },
  {
    "path": "09_privilege_level/src/bsp/raspberrypi.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Top-level BSP file for the Raspberry Pi 3 and 4.\n\npub mod cpu;\npub mod driver;\npub mod memory;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Board identification.\npub fn board_name() -> &'static str {\n    #[cfg(feature = \"bsp_rpi3\")]\n    {\n        \"Raspberry Pi 3\"\n    }\n\n    #[cfg(feature = \"bsp_rpi4\")]\n    {\n        \"Raspberry Pi 4\"\n    }\n}\n"
  },
  {
    "path": "09_privilege_level/src/bsp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Conditional reexporting of Board Support Packages.\n\nmod device_driver;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod raspberrypi;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use raspberrypi::*;\n"
  },
  {
    "path": "09_privilege_level/src/console/null_console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Null console.\n\nuse super::interface;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct NullConsole;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\npub static NULL_CONSOLE: NullConsole = NullConsole {};\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl interface::Write for NullConsole {\n    fn write_char(&self, _c: char) {}\n\n    fn write_fmt(&self, _args: fmt::Arguments) -> fmt::Result {\n        fmt::Result::Ok(())\n    }\n\n    fn flush(&self) {}\n}\n\nimpl interface::Read for NullConsole {\n    fn clear_rx(&self) {}\n}\n\nimpl interface::Statistics for NullConsole {}\nimpl interface::All for NullConsole {}\n"
  },
  {
    "path": "09_privilege_level/src/console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! System console.\n\nmod null_console;\n\nuse crate::synchronization::{self, NullLock};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Console interfaces.\npub mod interface {\n    use core::fmt;\n\n    /// Console write functions.\n    pub trait Write {\n        /// Write a single character.\n        fn write_char(&self, c: char);\n\n        /// Write a Rust format string.\n        fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result;\n\n        /// Block until the last buffered character has been physically put on the TX wire.\n        fn flush(&self);\n    }\n\n    /// Console read functions.\n    pub trait Read {\n        /// Read a single character.\n        fn read_char(&self) -> char {\n            ' '\n        }\n\n        /// Clear RX buffers, if any.\n        fn clear_rx(&self);\n    }\n\n    /// Console statistics.\n    pub trait Statistics {\n        /// Return the number of characters written.\n        fn chars_written(&self) -> usize {\n            0\n        }\n\n        /// Return the number of characters read.\n        fn chars_read(&self) -> usize {\n            0\n        }\n    }\n\n    /// Trait alias for a full-fledged console.\n    pub trait All: Write + Read + Statistics {}\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic CUR_CONSOLE: NullLock<&'static (dyn interface::All + Sync)> =\n    NullLock::new(&null_console::NULL_CONSOLE);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\n/// Register a new console.\npub fn register_console(new_console: &'static (dyn interface::All + Sync)) {\n    CUR_CONSOLE.lock(|con| *con = new_console);\n}\n\n/// Return a reference to the currently registered console.\n///\n/// This is the global console used by all printing macros.\npub fn console() -> &'static dyn interface::All {\n    CUR_CONSOLE.lock(|con| *con)\n}\n"
  },
  {
    "path": "09_privilege_level/src/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Boot code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/cpu/boot.rs\"]\nmod arch_boot;\n"
  },
  {
    "path": "09_privilege_level/src/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Processor code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/cpu.rs\"]\nmod arch_cpu;\n\nmod boot;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_cpu::{nop, wait_forever};\n"
  },
  {
    "path": "09_privilege_level/src/driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Driver support.\n\nuse crate::{\n    info,\n    synchronization::{interface::Mutex, NullLock},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nconst NUM_DRIVERS: usize = 5;\n\nstruct DriverManagerInner {\n    next_index: usize,\n    descriptors: [Option<DeviceDriverDescriptor>; NUM_DRIVERS],\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Driver interfaces.\npub mod interface {\n    /// Device Driver functions.\n    pub trait DeviceDriver {\n        /// Return a compatibility string for identifying the driver.\n        fn compatible(&self) -> &'static str;\n\n        /// Called by the kernel to bring up the device.\n        ///\n        /// # Safety\n        ///\n        /// - During init, drivers might do stuff with system-wide impact.\n        unsafe fn init(&self) -> Result<(), &'static str> {\n            Ok(())\n        }\n    }\n}\n\n/// Tpye to be used as an optional callback after a driver's init() has run.\npub type DeviceDriverPostInitCallback = unsafe fn() -> Result<(), &'static str>;\n\n/// A descriptor for device drivers.\n#[derive(Copy, Clone)]\npub struct DeviceDriverDescriptor {\n    device_driver: &'static (dyn interface::DeviceDriver + Sync),\n    post_init_callback: Option<DeviceDriverPostInitCallback>,\n}\n\n/// Provides device driver management functions.\npub struct DriverManager {\n    inner: NullLock<DriverManagerInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic DRIVER_MANAGER: DriverManager = DriverManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl DriverManagerInner {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            next_index: 0,\n            descriptors: [None; NUM_DRIVERS],\n        }\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl DeviceDriverDescriptor {\n    /// Create an instance.\n    pub fn new(\n        device_driver: &'static (dyn interface::DeviceDriver + Sync),\n        post_init_callback: Option<DeviceDriverPostInitCallback>,\n    ) -> Self {\n        Self {\n            device_driver,\n            post_init_callback,\n        }\n    }\n}\n\n/// Return a reference to the global DriverManager.\npub fn driver_manager() -> &'static DriverManager {\n    &DRIVER_MANAGER\n}\n\nimpl DriverManager {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            inner: NullLock::new(DriverManagerInner::new()),\n        }\n    }\n\n    /// Register a device driver with the kernel.\n    pub fn register_driver(&self, descriptor: DeviceDriverDescriptor) {\n        self.inner.lock(|inner| {\n            inner.descriptors[inner.next_index] = Some(descriptor);\n            inner.next_index += 1;\n        })\n    }\n\n    /// Helper for iterating over registered drivers.\n    fn for_each_descriptor<'a>(&'a self, f: impl FnMut(&'a DeviceDriverDescriptor)) {\n        self.inner.lock(|inner| {\n            inner\n                .descriptors\n                .iter()\n                .filter_map(|x| x.as_ref())\n                .for_each(f)\n        })\n    }\n\n    /// Fully initialize all drivers.\n    ///\n    /// # Safety\n    ///\n    /// - During init, drivers might do stuff with system-wide impact.\n    pub unsafe fn init_drivers(&self) {\n        self.for_each_descriptor(|descriptor| {\n            // 1. Initialize driver.\n            if let Err(x) = descriptor.device_driver.init() {\n                panic!(\n                    \"Error initializing driver: {}: {}\",\n                    descriptor.device_driver.compatible(),\n                    x\n                );\n            }\n\n            // 2. Call corresponding post init callback.\n            if let Some(callback) = &descriptor.post_init_callback {\n                if let Err(x) = callback() {\n                    panic!(\n                        \"Error during driver post-init callback: {}: {}\",\n                        descriptor.device_driver.compatible(),\n                        x\n                    );\n                }\n            }\n        });\n    }\n\n    /// Enumerate all registered device drivers.\n    pub fn enumerate(&self) {\n        let mut i: usize = 1;\n        self.for_each_descriptor(|descriptor| {\n            info!(\"      {}. {}\", i, descriptor.device_driver.compatible());\n\n            i += 1;\n        });\n    }\n}\n"
  },
  {
    "path": "09_privilege_level/src/exception/asynchronous.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Asynchronous exception handling.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/exception/asynchronous.rs\"]\nmod arch_asynchronous;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_asynchronous::print_state;\n"
  },
  {
    "path": "09_privilege_level/src/exception.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Synchronous and asynchronous exception handling.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/exception.rs\"]\nmod arch_exception;\n\npub mod asynchronous;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_exception::current_privilege_level;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Kernel privilege levels.\n#[allow(missing_docs)]\n#[derive(Eq, PartialEq)]\npub enum PrivilegeLevel {\n    User,\n    Kernel,\n    Hypervisor,\n    Unknown,\n}\n"
  },
  {
    "path": "09_privilege_level/src/main.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n// Rust embedded logo for `make doc`.\n#![doc(\n    html_logo_url = \"https://raw.githubusercontent.com/rust-embedded/wg/master/assets/logo/ewg-logo-blue-white-on-transparent.png\"\n)]\n\n//! The `kernel` binary.\n//!\n//! # Code organization and architecture\n//!\n//! The code is divided into different *modules*, each representing a typical **subsystem** of the\n//! `kernel`. Top-level module files of subsystems reside directly in the `src` folder. For example,\n//! `src/memory.rs` contains code that is concerned with all things memory management.\n//!\n//! ## Visibility of processor architecture code\n//!\n//! Some of the `kernel`'s subsystems depend on low-level code that is specific to the target\n//! processor architecture. For each supported processor architecture, there exists a subfolder in\n//! `src/_arch`, for example, `src/_arch/aarch64`.\n//!\n//! The architecture folders mirror the subsystem modules laid out in `src`. For example,\n//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go\n//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in\n//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic\n//! module's name prefixed with `arch_`.\n//!\n//! For example, this is the top of `src/memory/mmu.rs`:\n//!\n//! ```\n//! #[cfg(target_arch = \"aarch64\")]\n//! #[path = \"../_arch/aarch64/memory/mmu.rs\"]\n//! mod arch_mmu;\n//! ```\n//!\n//! Often times, items from the `arch_ module` will be publicly reexported by the parent module.\n//! This way, each architecture specific module can provide its implementation of an item, while the\n//! caller must not be concerned which architecture has been conditionally compiled.\n//!\n//! ## BSP code\n//!\n//! `BSP` stands for Board Support Package. `BSP` code is organized under `src/bsp.rs` and contains\n//! target board specific definitions and functions. These are things such as the board's memory map\n//! or instances of drivers for devices that are featured on the respective board.\n//!\n//! Just like processor architecture code, the `BSP` code's module structure tries to mirror the\n//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is\n//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`.\n//!\n//! ## Kernel interfaces\n//!\n//! Both `arch` and `bsp` contain code that is conditionally compiled depending on the actual target\n//! and board for which the kernel is compiled. For example, the `interrupt controller` hardware of\n//! the `Raspberry Pi 3` and the `Raspberry Pi 4` is different, but we want the rest of the `kernel`\n//! code to play nicely with any of the two without much hassle.\n//!\n//! In order to provide a clean abstraction between `arch`, `bsp` and `generic kernel code`,\n//! `interface` traits are provided *whenever possible* and *where it makes sense*. They are defined\n//! in the respective subsystem module and help to enforce the idiom of *program to an interface,\n//! not an implementation*. For example, there will be a common IRQ handling interface which the two\n//! different interrupt controller `drivers` of both Raspberrys will implement, and only export the\n//! interface to the rest of the `kernel`.\n//!\n//! ```\n//!         +-------------------+\n//!         | Interface (Trait) |\n//!         |                   |\n//!         +--+-------------+--+\n//!            ^             ^\n//!            |             |\n//!            |             |\n//! +----------+--+       +--+----------+\n//! | kernel code |       |  bsp code   |\n//! |             |       |  arch code  |\n//! +-------------+       +-------------+\n//! ```\n//!\n//! # Summary\n//!\n//! For a logical `kernel` subsystem, corresponding code can be distributed over several physical\n//! locations. Here is an example for the **memory** subsystem:\n//!\n//! - `src/memory.rs` and `src/memory/**/*`\n//!   - Common code that is agnostic of target processor architecture and `BSP` characteristics.\n//!     - Example: A function to zero a chunk of memory.\n//!   - Interfaces for the memory subsystem that are implemented by `arch` or `BSP` code.\n//!     - Example: An `MMU` interface that defines `MMU` function prototypes.\n//! - `src/bsp/__board_name__/memory.rs` and `src/bsp/__board_name__/memory/**/*`\n//!   - `BSP` specific code.\n//!   - Example: The board's memory map (physical addresses of DRAM and MMIO devices).\n//! - `src/_arch/__arch_name__/memory.rs` and `src/_arch/__arch_name__/memory/**/*`\n//!   - Processor architecture specific code.\n//!   - Example: Implementation of the `MMU` interface for the `__arch_name__` processor\n//!     architecture.\n//!\n//! From a namespace perspective, **memory** subsystem code lives in:\n//!\n//! - `crate::memory::*`\n//! - `crate::bsp::memory::*`\n//!\n//! # Boot flow\n//!\n//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.\n//!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.\n//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.\n\n#![allow(clippy::upper_case_acronyms)]\n#![feature(asm_const)]\n#![feature(const_option)]\n#![feature(format_args_nl)]\n#![feature(nonzero_min_max)]\n#![feature(panic_info_message)]\n#![feature(trait_alias)]\n#![feature(unchecked_math)]\n#![no_main]\n#![no_std]\n\nmod bsp;\nmod console;\nmod cpu;\nmod driver;\nmod exception;\nmod panic_wait;\nmod print;\nmod synchronization;\nmod time;\n\n/// Early init code.\n///\n/// # Safety\n///\n/// - Only a single core must be active and running this function.\n/// - The init calls in this function must appear in the correct order.\nunsafe fn kernel_init() -> ! {\n    // Initialize the BSP driver subsystem.\n    if let Err(x) = bsp::driver::init() {\n        panic!(\"Error initializing BSP driver subsystem: {}\", x);\n    }\n\n    // Initialize all device drivers.\n    driver::driver_manager().init_drivers();\n    // println! is usable from here on.\n\n    // Transition from unsafe to safe.\n    kernel_main()\n}\n\n/// The main function running after the early init.\nfn kernel_main() -> ! {\n    use console::console;\n    use core::time::Duration;\n\n    info!(\n        \"{} version {}\",\n        env!(\"CARGO_PKG_NAME\"),\n        env!(\"CARGO_PKG_VERSION\")\n    );\n    info!(\"Booting on: {}\", bsp::board_name());\n\n    let (_, privilege_level) = exception::current_privilege_level();\n    info!(\"Current privilege level: {}\", privilege_level);\n\n    info!(\"Exception handling state:\");\n    exception::asynchronous::print_state();\n\n    info!(\n        \"Architectural timer resolution: {} ns\",\n        time::time_manager().resolution().as_nanos()\n    );\n\n    info!(\"Drivers loaded:\");\n    driver::driver_manager().enumerate();\n\n    info!(\"Timer test, spinning for 1 second\");\n    time::time_manager().spin_for(Duration::from_secs(1));\n\n    info!(\"Echoing input now\");\n\n    // Discard any spurious received characters before going into echo mode.\n    console().clear_rx();\n    loop {\n        let c = console().read_char();\n        console().write_char(c);\n    }\n}\n"
  },
  {
    "path": "09_privilege_level/src/panic_wait.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A panic handler that infinitely waits.\n\nuse crate::{cpu, println};\nuse core::panic::PanicInfo;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Stop immediately if called a second time.\n///\n/// # Note\n///\n/// Using atomics here relieves us from needing to use `unsafe` for the static variable.\n///\n/// On `AArch64`, which is the only implemented architecture at the time of writing this,\n/// [`AtomicBool::load`] and [`AtomicBool::store`] are lowered to ordinary load and store\n/// instructions. They are therefore safe to use even with MMU + caching deactivated.\n///\n/// [`AtomicBool::load`]: core::sync::atomic::AtomicBool::load\n/// [`AtomicBool::store`]: core::sync::atomic::AtomicBool::store\nfn panic_prevent_reenter() {\n    use core::sync::atomic::{AtomicBool, Ordering};\n\n    #[cfg(not(target_arch = \"aarch64\"))]\n    compile_error!(\"Add the target_arch to above's check if the following code is safe to use\");\n\n    static PANIC_IN_PROGRESS: AtomicBool = AtomicBool::new(false);\n\n    if !PANIC_IN_PROGRESS.load(Ordering::Relaxed) {\n        PANIC_IN_PROGRESS.store(true, Ordering::Relaxed);\n\n        return;\n    }\n\n    cpu::wait_forever()\n}\n\n#[panic_handler]\nfn panic(info: &PanicInfo) -> ! {\n    // Protect against panic infinite loops if any of the following code panics itself.\n    panic_prevent_reenter();\n\n    let timestamp = crate::time::time_manager().uptime();\n    let (location, line, column) = match info.location() {\n        Some(loc) => (loc.file(), loc.line(), loc.column()),\n        _ => (\"???\", 0, 0),\n    };\n\n    println!(\n        \"[  {:>3}.{:06}] Kernel panic!\\n\\n\\\n        Panic location:\\n      File '{}', line {}, column {}\\n\\n\\\n        {}\",\n        timestamp.as_secs(),\n        timestamp.subsec_micros(),\n        location,\n        line,\n        column,\n        info.message().unwrap_or(&format_args!(\"\")),\n    );\n\n    cpu::wait_forever()\n}\n"
  },
  {
    "path": "09_privilege_level/src/print.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Printing.\n\nuse crate::console;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n#[doc(hidden)]\npub fn _print(args: fmt::Arguments) {\n    console::console().write_fmt(args).unwrap();\n}\n\n/// Prints without a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! print {\n    ($($arg:tt)*) => ($crate::print::_print(format_args!($($arg)*)));\n}\n\n/// Prints with a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! println {\n    () => ($crate::print!(\"\\n\"));\n    ($($arg:tt)*) => ({\n        $crate::print::_print(format_args_nl!($($arg)*));\n    })\n}\n\n/// Prints an info, with a newline.\n#[macro_export]\nmacro_rules! info {\n    ($string:expr) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[  {:>3}.{:06}] \", $string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n        ));\n    });\n    ($format_string:expr, $($arg:tt)*) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[  {:>3}.{:06}] \", $format_string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n            $($arg)*\n        ));\n    })\n}\n\n/// Prints a warning, with a newline.\n#[macro_export]\nmacro_rules! warn {\n    ($string:expr) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[W {:>3}.{:06}] \", $string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n        ));\n    });\n    ($format_string:expr, $($arg:tt)*) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[W {:>3}.{:06}] \", $format_string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n            $($arg)*\n        ));\n    })\n}\n"
  },
  {
    "path": "09_privilege_level/src/synchronization.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Synchronization primitives.\n//!\n//! # Resources\n//!\n//!   - <https://doc.rust-lang.org/book/ch16-04-extensible-concurrency-sync-and-send.html>\n//!   - <https://stackoverflow.com/questions/59428096/understanding-the-send-trait>\n//!   - <https://doc.rust-lang.org/std/cell/index.html>\n\nuse core::cell::UnsafeCell;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Synchronization interfaces.\npub mod interface {\n\n    /// Any object implementing this trait guarantees exclusive access to the data wrapped within\n    /// the Mutex for the duration of the provided closure.\n    pub trait Mutex {\n        /// The type of the data that is wrapped by this mutex.\n        type Data;\n\n        /// Locks the mutex and grants the closure temporary mutable access to the wrapped data.\n        fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R;\n    }\n}\n\n/// A pseudo-lock for teaching purposes.\n///\n/// In contrast to a real Mutex implementation, does not protect against concurrent access from\n/// other cores to the contained data. This part is preserved for later lessons.\n///\n/// The lock will only be used as long as it is safe to do so, i.e. as long as the kernel is\n/// executing single-threaded, aka only running on a single core with interrupts disabled.\npub struct NullLock<T>\nwhere\n    T: ?Sized,\n{\n    data: UnsafeCell<T>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nunsafe impl<T> Send for NullLock<T> where T: ?Sized + Send {}\nunsafe impl<T> Sync for NullLock<T> where T: ?Sized + Send {}\n\nimpl<T> NullLock<T> {\n    /// Create an instance.\n    pub const fn new(data: T) -> Self {\n        Self {\n            data: UnsafeCell::new(data),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\n\nimpl<T> interface::Mutex for NullLock<T> {\n    type Data = T;\n\n    fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R {\n        // In a real lock, there would be code encapsulating this line that ensures that this\n        // mutable reference will ever only be given out once at a time.\n        let data = unsafe { &mut *self.data.get() };\n\n        f(data)\n    }\n}\n"
  },
  {
    "path": "09_privilege_level/src/time.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Timer primitives.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/time.rs\"]\nmod arch_time;\n\nuse core::time::Duration;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Provides time management functions.\npub struct TimeManager;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic TIME_MANAGER: TimeManager = TimeManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the global TimeManager.\npub fn time_manager() -> &'static TimeManager {\n    &TIME_MANAGER\n}\n\nimpl TimeManager {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self\n    }\n\n    /// The timer's resolution.\n    pub fn resolution(&self) -> Duration {\n        arch_time::resolution()\n    }\n\n    /// The uptime since power-on of the device.\n    ///\n    /// This includes time consumed by firmware and bootloaders.\n    pub fn uptime(&self) -> Duration {\n        arch_time::uptime()\n    }\n\n    /// Spin for a given duration.\n    pub fn spin_for(&self, duration: Duration) {\n        arch_time::spin_for(duration)\n    }\n}\n"
  },
  {
    "path": "09_privilege_level/tests/boot_test_string.rb",
    "content": "# frozen_string_literal: true\n\nEXPECTED_PRINT = 'Echoing input now'\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/.vscode/settings.json",
    "content": "{\n    \"editor.formatOnSave\": true,\n    \"editor.rulers\": [100],\n    \"rust-analyzer.cargo.target\": \"aarch64-unknown-none-softfloat\",\n    \"rust-analyzer.cargo.features\": [\"bsp_rpi3\"],\n    \"rust-analyzer.checkOnSave.allTargets\": false,\n    \"rust-analyzer.checkOnSave.extraArgs\": [\"--bins\"],\n    \"rust-analyzer.lens.debug\": false,\n    \"rust-analyzer.lens.run\": false\n}\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/Cargo.toml",
    "content": "[package]\nname = \"mingo\"\nversion = \"0.10.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n\n[profile.release]\nlto = true\n\n[features]\ndefault = []\nbsp_rpi3 = [\"tock-registers\"]\nbsp_rpi4 = [\"tock-registers\"]\n\n[[bin]]\nname = \"kernel\"\npath = \"src/main.rs\"\n\n##--------------------------------------------------------------------------------------------------\n## Dependencies\n##--------------------------------------------------------------------------------------------------\n\n[dependencies]\n\n# Optional dependencies\ntock-registers = { version = \"0.8.x\", default-features = false, features = [\"register_types\"], optional = true }\n\n# Platform specific dependencies\n[target.'cfg(target_arch = \"aarch64\")'.dependencies]\naarch64-cpu = { version = \"9.x.x\" }\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/Makefile",
    "content": "## SPDX-License-Identifier: MIT OR Apache-2.0\n##\n## Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\ninclude ../common/docker.mk\ninclude ../common/format.mk\ninclude ../common/operating_system.mk\n\n##--------------------------------------------------------------------------------------------------\n## Optional, user-provided configuration values\n##--------------------------------------------------------------------------------------------------\n\n# Default to the RPi3.\nBSP ?= rpi3\n\n# Default to a serial device name that is common in Linux.\nDEV_SERIAL ?= /dev/ttyUSB0\n\n\n\n##--------------------------------------------------------------------------------------------------\n## BSP-specific configuration values\n##--------------------------------------------------------------------------------------------------\nQEMU_MISSING_STRING = \"This board is not yet supported for QEMU.\"\n\nifeq ($(BSP),rpi3)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE = raspi3\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    OPENOCD_ARG       = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi3.cfg\n    JTAG_BOOT_IMAGE   = ../X1_JTAG_boot/jtag_boot_rpi3.img\n    LD_SCRIPT_PATH    = $(shell pwd)/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a53\nelse ifeq ($(BSP),rpi4)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE =\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    OPENOCD_ARG       = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi4.cfg\n    JTAG_BOOT_IMAGE   = ../X1_JTAG_boot/jtag_boot_rpi4.img\n    LD_SCRIPT_PATH    = $(shell pwd)/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a72\nendif\n\n# Export for build.rs.\nexport LD_SCRIPT_PATH\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets and Prerequisites\n##--------------------------------------------------------------------------------------------------\nKERNEL_MANIFEST      = Cargo.toml\nKERNEL_LINKER_SCRIPT = kernel.ld\nLAST_BUILD_CONFIG    = target/$(BSP).build_config\n\nKERNEL_ELF      = target/$(TARGET)/release/kernel\n# This parses cargo's dep-info file.\n# https://doc.rust-lang.org/cargo/guide/build-cache.html#dep-info-files\nKERNEL_ELF_DEPS = $(filter-out %: ,$(file < $(KERNEL_ELF).d)) $(KERNEL_MANIFEST) $(LAST_BUILD_CONFIG)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Command building blocks\n##--------------------------------------------------------------------------------------------------\nRUSTFLAGS = $(RUSTC_MISC_ARGS)                   \\\n    -C link-arg=--library-path=$(LD_SCRIPT_PATH) \\\n    -C link-arg=--script=$(KERNEL_LINKER_SCRIPT)\n\nRUSTFLAGS_PEDANTIC = $(RUSTFLAGS) \\\n    -D warnings                   \\\n    -D missing_docs\n\nFEATURES      = --features bsp_$(BSP)\nCOMPILER_ARGS = --target=$(TARGET) \\\n    $(FEATURES)                    \\\n    --release\n\nRUSTC_CMD   = cargo rustc $(COMPILER_ARGS)\nDOC_CMD     = cargo doc $(COMPILER_ARGS)\nCLIPPY_CMD  = cargo clippy $(COMPILER_ARGS)\nOBJCOPY_CMD = rust-objcopy \\\n    --strip-all            \\\n    -O binary\n\nEXEC_QEMU          = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE)\nEXEC_TEST_DISPATCH = ruby ../common/tests/dispatch.rb\nEXEC_MINIPUSH      = ruby ../common/serial/minipush.rb\n\n##------------------------------------------------------------------------------\n## Dockerization\n##------------------------------------------------------------------------------\nDOCKER_CMD            = docker run -t --rm -v $(shell pwd):/work/tutorial -w /work/tutorial\nDOCKER_CMD_INTERACT   = $(DOCKER_CMD) -i\nDOCKER_ARG_DIR_COMMON = -v $(shell pwd)/../common:/work/common\nDOCKER_ARG_DIR_JTAG   = -v $(shell pwd)/../X1_JTAG_boot:/work/X1_JTAG_boot\nDOCKER_ARG_DEV        = --privileged -v /dev:/dev\nDOCKER_ARG_NET        = --network host\n\n# DOCKER_IMAGE defined in include file (see top of this file).\nDOCKER_QEMU  = $(DOCKER_CMD_INTERACT) $(DOCKER_IMAGE)\nDOCKER_TOOLS = $(DOCKER_CMD) $(DOCKER_IMAGE)\nDOCKER_TEST  = $(DOCKER_CMD) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\nDOCKER_GDB   = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_NET) $(DOCKER_IMAGE)\n\n# Dockerize commands, which require USB device passthrough, only on Linux.\nifeq ($(shell uname -s),Linux)\n    DOCKER_CMD_DEV = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_DEV)\n\n    DOCKER_CHAINBOOT = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\n    DOCKER_JTAGBOOT  = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_ARG_DIR_JTAG) $(DOCKER_IMAGE)\n    DOCKER_OPENOCD   = $(DOCKER_CMD_DEV) $(DOCKER_ARG_NET) $(DOCKER_IMAGE)\nelse\n    DOCKER_OPENOCD   = echo \"Not yet supported on non-Linux systems.\"; \\#\nendif\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: all doc qemu chainboot clippy clean readelf objdump nm check\n\nall: $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Save the configuration as a file, so make understands if it changed.\n##------------------------------------------------------------------------------\n$(LAST_BUILD_CONFIG):\n\t@rm -f target/*.build_config\n\t@mkdir -p target\n\t@touch $(LAST_BUILD_CONFIG)\n\n##------------------------------------------------------------------------------\n## Compile the kernel ELF\n##------------------------------------------------------------------------------\n$(KERNEL_ELF): $(KERNEL_ELF_DEPS)\n\t$(call color_header, \"Compiling kernel ELF - $(BSP)\")\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(RUSTC_CMD)\n\n##------------------------------------------------------------------------------\n## Generate the stripped kernel binary\n##------------------------------------------------------------------------------\n$(KERNEL_BIN): $(KERNEL_ELF)\n\t$(call color_header, \"Generating stripped binary\")\n\t@$(OBJCOPY_CMD) $(KERNEL_ELF) $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Name\")\n\t@echo $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Size\")\n\t$(call disk_usage_KiB, $(KERNEL_BIN))\n\n##------------------------------------------------------------------------------\n## Generate the documentation\n##------------------------------------------------------------------------------\ndoc:\n\t$(call color_header, \"Generating docs\")\n\t@$(DOC_CMD) --document-private-items --open\n\n##------------------------------------------------------------------------------\n## Run the kernel in QEMU\n##------------------------------------------------------------------------------\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\nqemu:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\nqemu: $(KERNEL_BIN)\n\t$(call color_header, \"Launching QEMU\")\n\t@$(DOCKER_QEMU) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\nendif\n\n##------------------------------------------------------------------------------\n## Push the kernel to the real HW target\n##------------------------------------------------------------------------------\nchainboot: $(KERNEL_BIN)\n\t@$(DOCKER_CHAINBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run clippy\n##------------------------------------------------------------------------------\nclippy:\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(CLIPPY_CMD)\n\n##------------------------------------------------------------------------------\n## Clean\n##------------------------------------------------------------------------------\nclean:\n\trm -rf target $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run readelf\n##------------------------------------------------------------------------------\nreadelf: $(KERNEL_ELF)\n\t$(call color_header, \"Launching readelf\")\n\t@$(DOCKER_TOOLS) $(READELF_BINARY) --headers $(KERNEL_ELF)\n\n##------------------------------------------------------------------------------\n## Run objdump\n##------------------------------------------------------------------------------\nobjdump: $(KERNEL_ELF)\n\t$(call color_header, \"Launching objdump\")\n\t@$(DOCKER_TOOLS) $(OBJDUMP_BINARY) --disassemble --demangle \\\n                --section .text   \\\n                --section .rodata \\\n                $(KERNEL_ELF) | rustfilt\n\n##------------------------------------------------------------------------------\n## Run nm\n##------------------------------------------------------------------------------\nnm: $(KERNEL_ELF)\n\t$(call color_header, \"Launching nm\")\n\t@$(DOCKER_TOOLS) $(NM_BINARY) --demangle --print-size $(KERNEL_ELF) | sort | rustfilt\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Debugging targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: jtagboot openocd gdb gdb-opt0\n\n##------------------------------------------------------------------------------\n## Push the JTAG boot image to the real HW target\n##------------------------------------------------------------------------------\njtagboot:\n\t@$(DOCKER_JTAGBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(JTAG_BOOT_IMAGE)\n\n##------------------------------------------------------------------------------\n## Start OpenOCD session\n##------------------------------------------------------------------------------\nopenocd:\n\t$(call color_header, \"Launching OpenOCD\")\n\t@$(DOCKER_OPENOCD) openocd $(OPENOCD_ARG)\n\n##------------------------------------------------------------------------------\n## Start GDB session\n##------------------------------------------------------------------------------\ngdb: RUSTC_MISC_ARGS += -C debuginfo=2\ngdb-opt0: RUSTC_MISC_ARGS += -C debuginfo=2 -C opt-level=0\ngdb gdb-opt0: $(KERNEL_ELF)\n\t$(call color_header, \"Launching GDB\")\n\t@$(DOCKER_GDB) gdb-multiarch -q $(KERNEL_ELF)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Testing targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: test test_boot\n\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\ntest_boot test:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\n##------------------------------------------------------------------------------\n## Run boot test\n##------------------------------------------------------------------------------\ntest_boot: $(KERNEL_BIN)\n\t$(call color_header, \"Boot test - $(BSP)\")\n\t@$(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\ntest: test_boot\n\nendif\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/README.CN.md",
    "content": "# 教程10 - 虚拟内存第一部分：将所有内容进行身份映射！\n\n## tl;dr\n\n- 打开`MMU`。\n- 使用简单的方案：静态的`64 KiB`转换表。\n- 为了教学目的，我们将数据写入重新映射的`UART`，并对其他所有内容进行`identity map`。\n\n## 目录\n\n- [介绍](#introduction)\n- [MMU和分页理论](#MMU和分页理论)\n- [方法](#方法)\n  * [通用内核代码：`memory/mmu.rs`](#通用内核代码：`memory/mmu.rs`)\n  * [BSP：`bsp/raspberrypi/memory/mmu.rs`](#bsp-bspraspberrypimemorymmurs)\n  * [AArch64：`_arch/aarch64/memory/*`](#aarch64-_archaarch64memory)\n  * [`kernel.ld`](#kernelld)\n- [地址转换示例](#地址转换示例)\n  * [使用64 KiB页描述符进行地址转换](#使用64KiB页描述符进行地址转换)\n- [零成本抽象](#零成本抽象)\n- [测试](#测试)\n- [相比之前的变化（diff）](#相比之前的变化（diff）)\n\n## 介绍\n\n虚拟内存是一个非常复杂但重要且强大的主题。在本教程中，我们从简单易懂的方式开始，\n通过打开`MMU`，使用静态转换表和一次性进行`identity-map`\n（除了为教育目的而重新映射的`UART`之外；在下一个教程中，这将被取消）。\n\n## MMU和分页理论\n\n在这一点上，我们不会重新发明轮子并详细描述现代应用级处理器中分页的工作原理。\n互联网上有很多关于这个主题的优秀资源，我们鼓励您阅读其中一些以获得对该主题的高层理解。\n\n继续阅读本`AArch64`特定的教程，我强烈建议您在此处停下来，首先阅读[ARM Cortex-A Series Programmer's Guide for ARMv8-A]的`第12章`，\n以便在继续之前获得所有所需的`AArch64`特定知识。\n\n已经阅读完`第12章`了吗？做得好 :+1:!\n\n[ARM Cortex-A Series Programmer's Guide for ARMv8-A]: http://infocenter.arm.com/help/topic/com.arm.doc.den0024a/DEN0024A_v8_architecture_PG.pdf\n\n## 方法\n\n1. 通用的`kernel`部分：`src/memory/mmu.rs`及其子模块提供了与体系结构无关的描述符类型，\n   用于组合一个高级数据结构，描述内核的虚拟内存布局：`memory::mmu::KernelVirtualLayout`。\n2. `BSP`部分：`src/bsp/raspberrypi/memory/mmu.rs`包含一个`KernelVirtualLayout`的静态实例，并通过函数\n   `bsp::memory::mmu::virt_mem_layout()`使其可访问。\n3. `aarch64`部分：`src/_arch/aarch64/memory/mmu.rs`及其子模块包含实际的`MMU`驱动程序。它使用`64 KiB`粒度获取\n   `BSP`的高级`KernelVirtualLayout`并进行映射。\n\n### 通用内核代码：`memory/mmu.rs`\n\n在这个文件中提供的描述符类型是构建块，用于描述不同内存区域的属性。\n例如，`R/W`（读/写）、`no-execute`（不执行）、`cached/uncached`（缓存/非缓存）等等。\n\n这些描述符与硬件`MMU`的实际描述符无关。不同的`BSP`可以使用这些类型来生成内核虚拟内存布局的高级描述。\n真实硬件的实际`MMU`驱动程序将使用这些类型作为输入。\n\n通过这种方式，我们在`BSP`和`_arch`代码之间实现了清晰的抽象，这样可以在不需要调整另一个的情况下进行交换。\n\n### BSP: `bsp/raspberrypi/memory/mmu.rs`\n\n这个文件包含了一个`KernelVirtualLayout`的实例，用于存储先前提到的描述符。\n将其放在`BSP`中是正确的位置，因为它具有目标板的内存映射知识。\n\n策略是只描述**不是**普通的、可缓存的DRAM的区域。然而，如果您希望，也可以定义这些区域。\n这里是一个设备MMIO区域的示例：\n\n```rust\nTranslationDescriptor {\n    name: \"Device MMIO\",\n    virtual_range: mmio_range_inclusive,\n    physical_range_translation: Translation::Identity,\n    attribute_fields: AttributeFields {\n        mem_attributes: MemAttributes::Device,\n        acc_perms: AccessPermissions::ReadWrite,\n        execute_never: true,\n    },\n},\n```\n\n`KernelVirtualLayout`本身实现了以下方法：\n\n```rust\npub fn virt_addr_properties(\n    &self,\n    virt_addr: usize,\n) -> Result<(usize, AttributeFields), &'static str>\n```\n\n它将被`_arch/aarch64`的`MMU`代码使用，用于请求虚拟地址和转换的属性，该转换提供物理输出地址\n（返回元组中的`usize`）。该函数扫描包含查询地址的描述符，并返回第一个匹配的条目的相应结果。\n如果找不到条目，则返回普通可缓存DRAM的默认属性和输入地址，从而告诉`MMU`代码请求的地址应该是`identity mapped`。\n\n由于这种默认行为，不需要定义普通可缓存DRAM区域。\n\n### AArch64: `_arch/aarch64/memory/*`\n\n这些模块包含了`AArch64`的`MMU`驱动程序。粒度在这里被硬编码为（`64 KiB`页描述符）。\n\n在`translation_table.rs`中，有一个实际的转换表结构的定义，它对`LVL2`表的数量进行了泛化。\n后者取决于目标板的内存大小。自然地，`BSP`了解目标板的这些细节，并通过常量\n`bsp::memory::mmu::KernelAddrSpace::SIZE`提供大小信息。\n\n`translation_table.rs`使用这些信息来计算所需的`LVL2`表的数量。由于在`64 KiB`配置中，\n一个`LVL2`表可以覆盖`512 MiB`，所以只需要将`KernelAddrSpace::SIZE`除以`512 MiB`\n（有几个编译时检查确保`KernelAddrSpace::SIZE`是`512 MiB`的倍数）。\n\n最终的表类型被导出为`KernelTranslationTable`。以下是来自`translation_table.rs`的相关代码：\n\n```rust\n/// A table descriptor for 64 KiB aperture.\n///\n/// The output points to the next table.\n#[derive(Copy, Clone)]\n#[repr(C)]\nstruct TableDescriptor {\n    value: u64,\n}\n\n/// A page descriptor with 64 KiB aperture.\n///\n/// The output points to physical memory.\n#[derive(Copy, Clone)]\n#[repr(C)]\nstruct PageDescriptor {\n    value: u64,\n}\n\nconst NUM_LVL2_TABLES: usize = bsp::memory::mmu::KernelAddrSpace::SIZE >> Granule512MiB::SHIFT;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB\n/// aligned, hence the \"reverse\" order of appearance.\n#[repr(C)]\n#[repr(align(65536))]\npub struct FixedSizeTranslationTable<const NUM_TABLES: usize> {\n    /// Page descriptors, covering 64 KiB windows per entry.\n    lvl3: [[PageDescriptor; 8192]; NUM_TABLES],\n\n    /// Table descriptors, covering 512 MiB windows.\n    lvl2: [TableDescriptor; NUM_TABLES],\n}\n\n/// A translation table type for the kernel space.\npub type KernelTranslationTable = FixedSizeTranslationTable<NUM_LVL2_TABLES>;\n```\n\n在`mmu.rs`中，`KernelTranslationTable`用于创建内核表的最终实例：\n\n```rust\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n/// The kernel translation tables.\nstatic mut KERNEL_TABLES: KernelTranslationTable = KernelTranslationTable::new();\n```\n\n它们在`MMU::init()`期间通过调用`KERNEL_TABLES.populate_tt_entries()`进行填充，\n该函数利用`bsp::memory::mmu::virt_mem_layout().virt_addr_properties()`和一系列实用函数，将内核通用描述符转换为\n`AArch64 MMU`硬件所需的实际`64 bit`整数条目，用于填充转换表数组。\n\n一个值得注意的事情是，每个页描述符都有一个索引（`AttrIndex`），它索引到[MAIR_EL1]寄存器，\n该寄存器保存了有关相应页面的缓存属性的信息。我们目前定义了普通可缓存内存和设备内存（不被缓存）。\n\n[MAIR_EL1]: http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.ddi0500d/CIHDHJBB.html\n\n```rust\nimpl MemoryManagementUnit {\n    /// Setup function for the MAIR_EL1 register.\n    fn set_up_mair(&self) {\n        // Define the memory types being mapped.\n        MAIR_EL1.write(\n            // Attribute 1 - Cacheable normal DRAM.\n            MAIR_EL1::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc +\n        MAIR_EL1::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc +\n\n        // Attribute 0 - Device.\n        MAIR_EL1::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck,\n        );\n    }\n```\n\n然后，[Translation Table Base Register 0 - EL1]使用`lvl2`表的基地址进行设置，同时配置[Translation Control Register - EL1]：\n\n```rust\n// Set the \"Translation Table Base Register\".\nTTBR0_EL1.set_baddr(KERNEL_TABLES.phys_base_address());\n\nself.configure_translation_control();\n```\n\n最后，通过[System Control Register - EL1]打开`MMU`。最后一步还启用了数据和指令的缓存。\n\n[Translation Table Base Register 0 - EL1]: https://docs.rs/aarch64-cpu/9.0.0/src/aarch64_cpu/registers/ttbr0_el1.rs.html\n[Translation Control Register - EL1]: https://docs.rs/aarch64-cpu/9.0.0/src/aarch64_cpu/registers/tcr_el1.rs.html\n[System Control Register - EL1]: https://docs.rs/aarch64-cpu/9.0.0/src/aarch64_cpu/registers/sctlr_el1.rs.html\n\n### `kernel.ld`\n\n我们需要将`code`段对齐到`64 KiB`，这样它就不会与下一个需要读/写属性而不是读/执行属性的部分重叠。\n\n```ld.s\n. = ALIGN(PAGE_SIZE);\n__code_end_exclusive = .;\n```\n\n这会增加二进制文件的大小，但考虑到与传统的`4 KiB`粒度相比，它显著减少了静态分页条目的数量，这是一个小小的代价。\n\n## 地址转换示例\n\n出于教育目的，定义了一个布局，允许通过两个不同的虚拟地址访问`UART`\n- 由于我们对整个`Device MMIO`区域进行了身份映射，所以在`MMU`打开后，可以通过断言其物理基地址\n  （`0x3F20_1000`或`0xFA20_1000`，取决于使用的是哪个RPi版本）来访问它。\n- 此外，它还映射到第一个`512 MiB`中的最后一个`64 KiB`槽位，使其可以通过基地址`0x1FFF_1000`访问。\n\n以下块图可视化了第二个映射的底层转换。\n\n### 使用64KiB页描述符进行地址转换\n\n<img src=\"../doc/11_page_tables_64KiB.png\" alt=\"Page Tables 64KiB\" width=\"90%\">\n\n## 零成本抽象\n\n初始化代码再次是展示Rust零成本抽象在嵌入式编程中巨大潜力的一个很好的例子[[1]][[2]]。\n\n让我们再次看一下使用[aarch64-cpu]crate设置`MAIR_EL1`寄存器的代码片段：\n\n[1]: https://blog.rust-lang.org/2015/05/11/traits.html\n[2]: https://ruudvanasseldonk.com/2016/11/30/zero-cost-abstractions\n[aarch64-cpu]: https://crates.io/crates/aarch64-cpu\n\n```rust\n/// Setup function for the MAIR_EL1 register.\nfn set_up_mair(&self) {\n    // Define the memory types being mapped.\n    MAIR_EL1.write(\n        // Attribute 1 - Cacheable normal DRAM.\n        MAIR_EL1::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc +\n    MAIR_EL1::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc +\n\n    // Attribute 0 - Device.\n    MAIR_EL1::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck,\n    );\n}\n```\n\n这段代码具有超强的表达能力，它利用`traits`，不同的`types`和`constants`来提供类型安全的寄存器操作。\n\n最后，此代码根据数据表将寄存器的前四个字节设置为特定值。查看生成的代码，\n我们可以看到，尽管有所有的类型安全和抽象，但它可以归结为两条汇编指令：\n\n```text\n   800a8:       529fe089        mov     w9, #0xff04                     // #65284\n   800ac:       d518a209        msr     mair_el1, x9\n```\n\n## 测试\n\n打开虚拟内存现在是我们在内核初始化过程中要做的第一件事：\n\n```rust\nunsafe fn kernel_init() -> ! {\n    use memory::mmu::interface::MMU;\n\n    if let Err(string) = memory::mmu::mmu().enable_mmu_and_caching() {\n        panic!(\"MMU: {}\", string);\n    }\n```\n\n稍后在引导过程中，可以观察到有关映射的打印：\n\n```console\n$ make chainboot\n[...]\nMinipush 1.0\n\n[MP] ⏳ Waiting for /dev/ttyUSB0\n[MP] ✅ Serial connected\n[MP] 🔌 Please power the target now\n\n __  __ _      _ _                 _\n|  \\/  (_)_ _ (_) |   ___  __ _ __| |\n| |\\/| | | ' \\| | |__/ _ \\/ _` / _` |\n|_|  |_|_|_||_|_|____\\___/\\__,_\\__,_|\n\n           Raspberry Pi 3\n\n[ML] Requesting binary\n[MP] ⏩ Pushing 64 KiB =========================================🦀 100% 0 KiB/s Time: 00:00:00\n[ML] Loaded! Executing the payload now\n\n[    0.811167] mingo version 0.10.0\n[    0.811374] Booting on: Raspberry Pi 3\n[    0.811829] MMU online. Special regions:\n[    0.812306]       0x00080000 - 0x0008ffff |  64 KiB | C   RO PX  | Kernel code and RO data\n[    0.813324]       0x1fff0000 - 0x1fffffff |  64 KiB | Dev RW PXN | Remapped Device MMIO\n[    0.814310]       0x3f000000 - 0x4000ffff |  17 MiB | Dev RW PXN | Device MMIO\n[    0.815198] Current privilege level: EL1\n[    0.815675] Exception handling state:\n[    0.816119]       Debug:  Masked\n[    0.816509]       SError: Masked\n[    0.816899]       IRQ:    Masked\n[    0.817289]       FIQ:    Masked\n[    0.817679] Architectural timer resolution: 52 ns\n[    0.818253] Drivers loaded:\n[    0.818589]       1. BCM PL011 UART\n[    0.819011]       2. BCM GPIO\n[    0.819369] Timer test, spinning for 1 second\n[     !!!    ] Writing through the remapped UART at 0x1FFF_1000\n[    1.820409] Echoing input now\n```\n\n## 相比之前的变化（diff）\n请检查[英文版本](README.md#diff-to-previous)，这是最新的。\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/README.md",
    "content": "# Tutorial 10 - Virtual Memory Part 1: Identity Map All The Things!\n\n## tl;dr\n\n- The `MMU` is turned on.\n- A simple scheme is used: Static `64 KiB` translation tables.\n- For educational purposes, we write to a remapped `UART`, and `identity map` everything else.\n\n## Table of Contents\n\n- [Introduction](#introduction)\n- [MMU and paging theory](#mmu-and-paging-theory)\n- [Approach](#approach)\n  * [Generic Kernel code: `memory/mmu.rs`](#generic-kernel-code-memorymmurs)\n  * [BSP: `bsp/raspberrypi/memory/mmu.rs`](#bsp-bspraspberrypimemorymmurs)\n  * [AArch64: `_arch/aarch64/memory/*`](#aarch64-_archaarch64memory)\n  * [`kernel.ld`](#kernelld)\n- [Address translation examples](#address-translation-examples)\n  * [Address translation using a 64 KiB page descriptor](#address-translation-using-a-64-kib-page-descriptor)\n- [Zero-cost abstraction](#zero-cost-abstraction)\n- [Test it](#test-it)\n- [Diff to previous](#diff-to-previous)\n\n## Introduction\n\nVirtual memory is an immensely complex, but important and powerful topic. In this tutorial, we start\nslow and easy by switching on the `MMU`, using static translation tables and `identity-map`\neverything at once (except for the `UART`, which we also remap a second time for educational\npurposes; This will be gone again in the next tutorial).\n\n## MMU and paging theory\n\nAt this point, we will not re-invent the wheel and go into detailed descriptions of how paging in\nmodern application-grade processors works. The internet is full of great resources regarding this\ntopic, and we encourage you to read some of it to get a high-level understanding of the topic.\n\nTo follow the rest of this `AArch64` specific tutorial, I strongly recommend that you stop right\nhere and first read `Chapter 12` of the [ARM Cortex-A Series Programmer's Guide for ARMv8-A] before\nyou continue. This will set you up with all the `AArch64`-specific knowledge needed to follow along.\n\nBack from reading `Chapter 12` already? Good job :+1:!\n\n[ARM Cortex-A Series Programmer's Guide for ARMv8-A]: http://infocenter.arm.com/help/topic/com.arm.doc.den0024a/DEN0024A_v8_architecture_PG.pdf\n\n## Approach\n\n1. The generic `kernel` part: `src/memory/mmu.rs` and its submodules provide architecture-agnostic\n   descriptor types for composing a high-level data structure that describes the kernel's virtual\n   memory layout: `memory::mmu::KernelVirtualLayout`.\n2. The `BSP` part: `src/bsp/raspberrypi/memory/mmu.rs` contains a static instance of\n   `KernelVirtualLayout` and makes it accessible through the function\n   `bsp::memory::mmu::virt_mem_layout()`.\n3. The `aarch64` part: `src/_arch/aarch64/memory/mmu.rs` and its submodules contain the actual `MMU`\n   driver. It picks up the `BSP`'s high-level `KernelVirtualLayout` and maps it using a `64 KiB`\n   granule.\n\n### Generic Kernel code: `memory/mmu.rs`\n\nThe descriptor types provided in this file are building blocks which help to describe attributes of\ndifferent memory regions. For example, `R/W`, `no-execute`, `cached/uncached`, and so on.\n\nThe descriptors are agnostic of the hardware `MMU`'s actual descriptors. Different `BSP`s can use\nthese types to produce a high-level description of the kernel's virtual memory layout. The actual\n`MMU` driver for the real HW will consume these types as an input.\n\nThis way, we achieve a clean abstraction between `BSP` and `_arch` code, which allows exchanging one\nwithout needing to adapt the other.\n\n### BSP: `bsp/raspberrypi/memory/mmu.rs`\n\nThis file contains an instance of `KernelVirtualLayout`, which stores the descriptors mentioned\npreviously. The `BSP` is the correct place to do this, because it has knowledge of the target\nboard's memory map.\n\nThe policy is to only describe regions that are **not** ordinary, normal cacheable DRAM. However,\nnothing prevents you from defining those too if you wish to. Here is an example for the device MMIO\nregion:\n\n```rust\nTranslationDescriptor {\n    name: \"Device MMIO\",\n    virtual_range: mmio_range_inclusive,\n    physical_range_translation: Translation::Identity,\n    attribute_fields: AttributeFields {\n        mem_attributes: MemAttributes::Device,\n        acc_perms: AccessPermissions::ReadWrite,\n        execute_never: true,\n    },\n},\n```\n\n`KernelVirtualLayout` itself implements the following method:\n\n```rust\npub fn virt_addr_properties(\n    &self,\n    virt_addr: usize,\n) -> Result<(usize, AttributeFields), &'static str>\n```\n\nIt will be used by `_arch/aarch64`'s `MMU` code to request attributes for a virtual address and the\ntranslation, which delivers the physical output address (the `usize` in the return-tuple). The\nfunction scans for a descriptor that contains the queried address, and returns the respective\nfindings for the first entry that is a hit. If no entry is found, it returns default attributes for\nnormal cacheable DRAM and the input address, hence telling the `MMU` code that the requested\naddress should be `identity mapped`.\n\nDue to this default behavior, it is not needed to define normal cacheable DRAM regions.\n\n### AArch64: `_arch/aarch64/memory/*`\n\nThese modules contain the `AArch64` `MMU` driver. The granule is hardcoded here (`64 KiB` page\ndescriptors).\n\nIn `translation_table.rs`, there is a definition of the actual translation table struct which is\ngeneric over the number of `LVL2` tables. The latter depends on the size of the target board's\nmemory. Naturally, the `BSP` knows these details about the target board, and provides the size\nthrough the constant `bsp::memory::mmu::KernelAddrSpace::SIZE`.\n\nThis information is used by `translation_table.rs` to calculate the number of needed `LVL2` tables.\nSince one `LVL2` table in a `64 KiB` configuration covers `512 MiB`, all that needs to be done is to\ndivide `KernelAddrSpace::SIZE` by `512 MiB` (there are several compile-time checks in place that\nensure that `KernelAddrSpace::SIZE` is a multiple of `512 MiB`).\n\nThe final table type is exported as `KernelTranslationTable`. Below is the respective excerpt from\n`translation_table.rs`:\n\n```rust\n/// A table descriptor for 64 KiB aperture.\n///\n/// The output points to the next table.\n#[derive(Copy, Clone)]\n#[repr(C)]\nstruct TableDescriptor {\n    value: u64,\n}\n\n/// A page descriptor with 64 KiB aperture.\n///\n/// The output points to physical memory.\n#[derive(Copy, Clone)]\n#[repr(C)]\nstruct PageDescriptor {\n    value: u64,\n}\n\nconst NUM_LVL2_TABLES: usize = bsp::memory::mmu::KernelAddrSpace::SIZE >> Granule512MiB::SHIFT;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB\n/// aligned, hence the \"reverse\" order of appearance.\n#[repr(C)]\n#[repr(align(65536))]\npub struct FixedSizeTranslationTable<const NUM_TABLES: usize> {\n    /// Page descriptors, covering 64 KiB windows per entry.\n    lvl3: [[PageDescriptor; 8192]; NUM_TABLES],\n\n    /// Table descriptors, covering 512 MiB windows.\n    lvl2: [TableDescriptor; NUM_TABLES],\n}\n\n/// A translation table type for the kernel space.\npub type KernelTranslationTable = FixedSizeTranslationTable<NUM_LVL2_TABLES>;\n```\n\nIn `mmu.rs`, `KernelTranslationTable` is then used to create the final instance of the kernel's\ntables:\n\n```rust\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n/// The kernel translation tables.\nstatic mut KERNEL_TABLES: KernelTranslationTable = KernelTranslationTable::new();\n```\n\nThey are populated during `MMU::init()` by calling `KERNEL_TABLES.populate_tt_entries()`, which\nutilizes `bsp::memory::mmu::virt_mem_layout().virt_addr_properties()` and a bunch of utility\nfunctions that convert the kernel generic descriptors to the actual `64 bit` integer entries needed\nby the `AArch64 MMU` hardware for the translation table arrays.\n\nOne notable thing is that each page descriptor has an entry (`AttrIndex`) that indexes into the\n[MAIR_EL1] register, which holds information about the cacheability of the respective page. We\ncurrently define normal cacheable memory and device memory (which is not cached).\n\n[MAIR_EL1]: http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.ddi0500d/CIHDHJBB.html\n\n```rust\nimpl MemoryManagementUnit {\n    /// Setup function for the MAIR_EL1 register.\n    fn set_up_mair(&self) {\n        // Define the memory types being mapped.\n        MAIR_EL1.write(\n            // Attribute 1 - Cacheable normal DRAM.\n            MAIR_EL1::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc +\n        MAIR_EL1::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc +\n\n        // Attribute 0 - Device.\n        MAIR_EL1::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck,\n        );\n    }\n```\n\nAfterwards, the [Translation Table Base Register 0 - EL1] is set up with the base address of the\n`lvl2` tables and the [Translation Control Register - EL1] is configured:\n\n```rust\n// Set the \"Translation Table Base Register\".\nTTBR0_EL1.set_baddr(KERNEL_TABLES.phys_base_address());\n\nself.configure_translation_control();\n```\n\nFinally, the `MMU` is turned on through the [System Control Register - EL1]. The last step also\nenables caching for data and instructions.\n\n[Translation Table Base Register 0 - EL1]: https://docs.rs/aarch64-cpu/9.0.0/src/aarch64_cpu/registers/ttbr0_el1.rs.html\n[Translation Control Register - EL1]: https://docs.rs/aarch64-cpu/9.0.0/src/aarch64_cpu/registers/tcr_el1.rs.html\n[System Control Register - EL1]: https://docs.rs/aarch64-cpu/9.0.0/src/aarch64_cpu/registers/sctlr_el1.rs.html\n\n### `kernel.ld`\n\nWe need to align the `code` segment to `64 KiB` so that it doesn't overlap with the next section\nthat needs read/write attributes instead of read/execute attributes:\n\n```ld.s\n. = ALIGN(PAGE_SIZE);\n__code_end_exclusive = .;\n```\n\nThis blows up the binary in size, but is a small price to pay considering that it reduces the amount\nof static paging entries significantly, when compared to the classical `4 KiB` granule.\n\n## Address translation examples\n\nFor educational purposes, a layout is defined which allows to access the `UART` via two different\nvirtual addresses:\n- Since we identity map the whole `Device MMIO` region, it is accessible by asserting its physical\n  base address (`0x3F20_1000` or `0xFA20_1000` depending on which RPi you use) after the `MMU` is\n  turned on.\n- Additionally, it is also mapped into the last `64 KiB` slot in the first `512 MiB`, making it\n  accessible through base address `0x1FFF_1000`.\n\nThe following block diagram visualizes the underlying translation for the second mapping.\n\n### Address translation using a 64 KiB page descriptor\n\n<img src=\"../doc/11_page_tables_64KiB.png\" alt=\"Page Tables 64KiB\" width=\"90%\">\n\n## Zero-cost abstraction\n\nThe MMU init code is again a good example to see the great potential of Rust's zero-cost\nabstractions[[1]][[2]] for embedded programming.\n\nLet's take a look again at the piece of code for setting up the `MAIR_EL1` register using the\n[aarch64-cpu] crate:\n\n[1]: https://blog.rust-lang.org/2015/05/11/traits.html\n[2]: https://ruudvanasseldonk.com/2016/11/30/zero-cost-abstractions\n[aarch64-cpu]: https://crates.io/crates/aarch64-cpu\n\n```rust\n/// Setup function for the MAIR_EL1 register.\nfn set_up_mair(&self) {\n    // Define the memory types being mapped.\n    MAIR_EL1.write(\n        // Attribute 1 - Cacheable normal DRAM.\n        MAIR_EL1::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc +\n    MAIR_EL1::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc +\n\n    // Attribute 0 - Device.\n    MAIR_EL1::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck,\n    );\n}\n```\n\nThis piece of code is super expressive, and it makes use of `traits`, different `types` and\n`constants` to provide type-safe register manipulation.\n\nIn the end, this code sets the first four bytes of the register to certain values according to the\ndata sheet. Looking at the generated code, we can see that despite all the type-safety and\nabstractions, it boils down to two assembly instructions:\n\n```text\n   800a8:       529fe089        mov     w9, #0xff04                     // #65284\n   800ac:       d518a209        msr     mair_el1, x9\n```\n\n## Test it\n\nTurning on virtual memory is now the first thing we do during kernel init:\n\n```rust\nunsafe fn kernel_init() -> ! {\n    use memory::mmu::interface::MMU;\n\n    if let Err(string) = memory::mmu::mmu().enable_mmu_and_caching() {\n        panic!(\"MMU: {}\", string);\n    }\n```\n\nLater in the boot process, prints about the mappings can be observed:\n\n```console\n$ make chainboot\n[...]\nMinipush 1.0\n\n[MP] ⏳ Waiting for /dev/ttyUSB0\n[MP] ✅ Serial connected\n[MP] 🔌 Please power the target now\n\n __  __ _      _ _                 _\n|  \\/  (_)_ _ (_) |   ___  __ _ __| |\n| |\\/| | | ' \\| | |__/ _ \\/ _` / _` |\n|_|  |_|_|_||_|_|____\\___/\\__,_\\__,_|\n\n           Raspberry Pi 3\n\n[ML] Requesting binary\n[MP] ⏩ Pushing 64 KiB =========================================🦀 100% 0 KiB/s Time: 00:00:00\n[ML] Loaded! Executing the payload now\n\n[    0.811167] mingo version 0.10.0\n[    0.811374] Booting on: Raspberry Pi 3\n[    0.811829] MMU online. Special regions:\n[    0.812306]       0x00080000 - 0x0008ffff |  64 KiB | C   RO PX  | Kernel code and RO data\n[    0.813324]       0x1fff0000 - 0x1fffffff |  64 KiB | Dev RW PXN | Remapped Device MMIO\n[    0.814310]       0x3f000000 - 0x4000ffff |  17 MiB | Dev RW PXN | Device MMIO\n[    0.815198] Current privilege level: EL1\n[    0.815675] Exception handling state:\n[    0.816119]       Debug:  Masked\n[    0.816509]       SError: Masked\n[    0.816899]       IRQ:    Masked\n[    0.817289]       FIQ:    Masked\n[    0.817679] Architectural timer resolution: 52 ns\n[    0.818253] Drivers loaded:\n[    0.818589]       1. BCM PL011 UART\n[    0.819011]       2. BCM GPIO\n[    0.819369] Timer test, spinning for 1 second\n[     !!!    ] Writing through the remapped UART at 0x1FFF_1000\n[    1.820409] Echoing input now\n```\n\n## Diff to previous\n```diff\n\ndiff -uNr 09_privilege_level/Cargo.toml 10_virtual_mem_part1_identity_mapping/Cargo.toml\n--- 09_privilege_level/Cargo.toml\n+++ 10_virtual_mem_part1_identity_mapping/Cargo.toml\n@@ -1,6 +1,6 @@\n [package]\n name = \"mingo\"\n-version = \"0.9.0\"\n+version = \"0.10.0\"\n authors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\n edition = \"2021\"\n\n\ndiff -uNr 09_privilege_level/src/_arch/aarch64/memory/mmu/translation_table.rs 10_virtual_mem_part1_identity_mapping/src/_arch/aarch64/memory/mmu/translation_table.rs\n--- 09_privilege_level/src/_arch/aarch64/memory/mmu/translation_table.rs\n+++ 10_virtual_mem_part1_identity_mapping/src/_arch/aarch64/memory/mmu/translation_table.rs\n@@ -0,0 +1,292 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Architectural translation table.\n+//!\n+//! Only 64 KiB granule is supported.\n+//!\n+//! # Orientation\n+//!\n+//! Since arch modules are imported into generic modules using the path attribute, the path of this\n+//! file is:\n+//!\n+//! crate::memory::mmu::translation_table::arch_translation_table\n+\n+use crate::{\n+    bsp, memory,\n+    memory::mmu::{\n+        arch_mmu::{Granule512MiB, Granule64KiB},\n+        AccessPermissions, AttributeFields, MemAttributes,\n+    },\n+};\n+use core::convert;\n+use tock_registers::{\n+    interfaces::{Readable, Writeable},\n+    register_bitfields,\n+    registers::InMemoryRegister,\n+};\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+// A table descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-15.\n+register_bitfields! {u64,\n+    STAGE1_TABLE_DESCRIPTOR [\n+        /// Physical address of the next descriptor.\n+        NEXT_LEVEL_TABLE_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16]\n+\n+        TYPE  OFFSET(1) NUMBITS(1) [\n+            Block = 0,\n+            Table = 1\n+        ],\n+\n+        VALID OFFSET(0) NUMBITS(1) [\n+            False = 0,\n+            True = 1\n+        ]\n+    ]\n+}\n+\n+// A level 3 page descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-17.\n+register_bitfields! {u64,\n+    STAGE1_PAGE_DESCRIPTOR [\n+        /// Unprivileged execute-never.\n+        UXN      OFFSET(54) NUMBITS(1) [\n+            False = 0,\n+            True = 1\n+        ],\n+\n+        /// Privileged execute-never.\n+        PXN      OFFSET(53) NUMBITS(1) [\n+            False = 0,\n+            True = 1\n+        ],\n+\n+        /// Physical address of the next table descriptor (lvl2) or the page descriptor (lvl3).\n+        OUTPUT_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16]\n+\n+        /// Access flag.\n+        AF       OFFSET(10) NUMBITS(1) [\n+            False = 0,\n+            True = 1\n+        ],\n+\n+        /// Shareability field.\n+        SH       OFFSET(8) NUMBITS(2) [\n+            OuterShareable = 0b10,\n+            InnerShareable = 0b11\n+        ],\n+\n+        /// Access Permissions.\n+        AP       OFFSET(6) NUMBITS(2) [\n+            RW_EL1 = 0b00,\n+            RW_EL1_EL0 = 0b01,\n+            RO_EL1 = 0b10,\n+            RO_EL1_EL0 = 0b11\n+        ],\n+\n+        /// Memory attributes index into the MAIR_EL1 register.\n+        AttrIndx OFFSET(2) NUMBITS(3) [],\n+\n+        TYPE     OFFSET(1) NUMBITS(1) [\n+            Reserved_Invalid = 0,\n+            Page = 1\n+        ],\n+\n+        VALID    OFFSET(0) NUMBITS(1) [\n+            False = 0,\n+            True = 1\n+        ]\n+    ]\n+}\n+\n+/// A table descriptor for 64 KiB aperture.\n+///\n+/// The output points to the next table.\n+#[derive(Copy, Clone)]\n+#[repr(C)]\n+struct TableDescriptor {\n+    value: u64,\n+}\n+\n+/// A page descriptor with 64 KiB aperture.\n+///\n+/// The output points to physical memory.\n+#[derive(Copy, Clone)]\n+#[repr(C)]\n+struct PageDescriptor {\n+    value: u64,\n+}\n+\n+trait StartAddr {\n+    fn phys_start_addr_u64(&self) -> u64;\n+    fn phys_start_addr_usize(&self) -> usize;\n+}\n+\n+const NUM_LVL2_TABLES: usize = bsp::memory::mmu::KernelAddrSpace::SIZE >> Granule512MiB::SHIFT;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB\n+/// aligned, so the lvl3 is put first.\n+#[repr(C)]\n+#[repr(align(65536))]\n+pub struct FixedSizeTranslationTable<const NUM_TABLES: usize> {\n+    /// Page descriptors, covering 64 KiB windows per entry.\n+    lvl3: [[PageDescriptor; 8192]; NUM_TABLES],\n+\n+    /// Table descriptors, covering 512 MiB windows.\n+    lvl2: [TableDescriptor; NUM_TABLES],\n+}\n+\n+/// A translation table type for the kernel space.\n+pub type KernelTranslationTable = FixedSizeTranslationTable<NUM_LVL2_TABLES>;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Code\n+//--------------------------------------------------------------------------------------------------\n+\n+// The binary is still identity mapped, so we don't need to convert here.\n+impl<T, const N: usize> StartAddr for [T; N] {\n+    fn phys_start_addr_u64(&self) -> u64 {\n+        self as *const T as u64\n+    }\n+\n+    fn phys_start_addr_usize(&self) -> usize {\n+        self as *const _ as usize\n+    }\n+}\n+\n+impl TableDescriptor {\n+    /// Create an instance.\n+    ///\n+    /// Descriptor is invalid by default.\n+    pub const fn new_zeroed() -> Self {\n+        Self { value: 0 }\n+    }\n+\n+    /// Create an instance pointing to the supplied address.\n+    pub fn from_next_lvl_table_addr(phys_next_lvl_table_addr: usize) -> Self {\n+        let val = InMemoryRegister::<u64, STAGE1_TABLE_DESCRIPTOR::Register>::new(0);\n+\n+        let shifted = phys_next_lvl_table_addr >> Granule64KiB::SHIFT;\n+        val.write(\n+            STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64)\n+                + STAGE1_TABLE_DESCRIPTOR::TYPE::Table\n+                + STAGE1_TABLE_DESCRIPTOR::VALID::True,\n+        );\n+\n+        TableDescriptor { value: val.get() }\n+    }\n+}\n+\n+/// Convert the kernel's generic memory attributes to HW-specific attributes of the MMU.\n+impl convert::From<AttributeFields>\n+    for tock_registers::fields::FieldValue<u64, STAGE1_PAGE_DESCRIPTOR::Register>\n+{\n+    fn from(attribute_fields: AttributeFields) -> Self {\n+        // Memory attributes.\n+        let mut desc = match attribute_fields.mem_attributes {\n+            MemAttributes::CacheableDRAM => {\n+                STAGE1_PAGE_DESCRIPTOR::SH::InnerShareable\n+                    + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::NORMAL)\n+            }\n+            MemAttributes::Device => {\n+                STAGE1_PAGE_DESCRIPTOR::SH::OuterShareable\n+                    + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::DEVICE)\n+            }\n+        };\n+\n+        // Access Permissions.\n+        desc += match attribute_fields.acc_perms {\n+            AccessPermissions::ReadOnly => STAGE1_PAGE_DESCRIPTOR::AP::RO_EL1,\n+            AccessPermissions::ReadWrite => STAGE1_PAGE_DESCRIPTOR::AP::RW_EL1,\n+        };\n+\n+        // The execute-never attribute is mapped to PXN in AArch64.\n+        desc += if attribute_fields.execute_never {\n+            STAGE1_PAGE_DESCRIPTOR::PXN::True\n+        } else {\n+            STAGE1_PAGE_DESCRIPTOR::PXN::False\n+        };\n+\n+        // Always set unprivileged exectue-never as long as userspace is not implemented yet.\n+        desc += STAGE1_PAGE_DESCRIPTOR::UXN::True;\n+\n+        desc\n+    }\n+}\n+\n+impl PageDescriptor {\n+    /// Create an instance.\n+    ///\n+    /// Descriptor is invalid by default.\n+    pub const fn new_zeroed() -> Self {\n+        Self { value: 0 }\n+    }\n+\n+    /// Create an instance.\n+    pub fn from_output_addr(phys_output_addr: usize, attribute_fields: &AttributeFields) -> Self {\n+        let val = InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(0);\n+\n+        let shifted = phys_output_addr as u64 >> Granule64KiB::SHIFT;\n+        val.write(\n+            STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted)\n+                + STAGE1_PAGE_DESCRIPTOR::AF::True\n+                + STAGE1_PAGE_DESCRIPTOR::TYPE::Page\n+                + STAGE1_PAGE_DESCRIPTOR::VALID::True\n+                + (*attribute_fields).into(),\n+        );\n+\n+        Self { value: val.get() }\n+    }\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+impl<const NUM_TABLES: usize> FixedSizeTranslationTable<NUM_TABLES> {\n+    /// Create an instance.\n+    pub const fn new() -> Self {\n+        // Can't have a zero-sized address space.\n+        assert!(NUM_TABLES > 0);\n+\n+        Self {\n+            lvl3: [[PageDescriptor::new_zeroed(); 8192]; NUM_TABLES],\n+            lvl2: [TableDescriptor::new_zeroed(); NUM_TABLES],\n+        }\n+    }\n+\n+    /// Iterates over all static translation table entries and fills them at once.\n+    ///\n+    /// # Safety\n+    ///\n+    /// - Modifies a `static mut`. Ensure it only happens from here.\n+    pub unsafe fn populate_tt_entries(&mut self) -> Result<(), &'static str> {\n+        for (l2_nr, l2_entry) in self.lvl2.iter_mut().enumerate() {\n+            *l2_entry =\n+                TableDescriptor::from_next_lvl_table_addr(self.lvl3[l2_nr].phys_start_addr_usize());\n+\n+            for (l3_nr, l3_entry) in self.lvl3[l2_nr].iter_mut().enumerate() {\n+                let virt_addr = (l2_nr << Granule512MiB::SHIFT) + (l3_nr << Granule64KiB::SHIFT);\n+\n+                let (phys_output_addr, attribute_fields) =\n+                    bsp::memory::mmu::virt_mem_layout().virt_addr_properties(virt_addr)?;\n+\n+                *l3_entry = PageDescriptor::from_output_addr(phys_output_addr, &attribute_fields);\n+            }\n+        }\n+\n+        Ok(())\n+    }\n+\n+    /// The translation table's base address to be used for programming the MMU.\n+    pub fn phys_base_address(&self) -> u64 {\n+        self.lvl2.phys_start_addr_u64()\n+    }\n+}\n\ndiff -uNr 09_privilege_level/src/_arch/aarch64/memory/mmu.rs 10_virtual_mem_part1_identity_mapping/src/_arch/aarch64/memory/mmu.rs\n--- 09_privilege_level/src/_arch/aarch64/memory/mmu.rs\n+++ 10_virtual_mem_part1_identity_mapping/src/_arch/aarch64/memory/mmu.rs\n@@ -0,0 +1,165 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Memory Management Unit Driver.\n+//!\n+//! Only 64 KiB granule is supported.\n+//!\n+//! # Orientation\n+//!\n+//! Since arch modules are imported into generic modules using the path attribute, the path of this\n+//! file is:\n+//!\n+//! crate::memory::mmu::arch_mmu\n+\n+use crate::{\n+    bsp, memory,\n+    memory::mmu::{translation_table::KernelTranslationTable, TranslationGranule},\n+};\n+use aarch64_cpu::{asm::barrier, registers::*};\n+use core::intrinsics::unlikely;\n+use tock_registers::interfaces::{ReadWriteable, Readable, Writeable};\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Memory Management Unit type.\n+struct MemoryManagementUnit;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+pub type Granule512MiB = TranslationGranule<{ 512 * 1024 * 1024 }>;\n+pub type Granule64KiB = TranslationGranule<{ 64 * 1024 }>;\n+\n+/// Constants for indexing the MAIR_EL1.\n+#[allow(dead_code)]\n+pub mod mair {\n+    pub const DEVICE: u64 = 0;\n+    pub const NORMAL: u64 = 1;\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Global instances\n+//--------------------------------------------------------------------------------------------------\n+\n+/// The kernel translation tables.\n+///\n+/// # Safety\n+///\n+/// - Supposed to land in `.bss`. Therefore, ensure that all initial member values boil down to \"0\".\n+static mut KERNEL_TABLES: KernelTranslationTable = KernelTranslationTable::new();\n+\n+static MMU: MemoryManagementUnit = MemoryManagementUnit;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Code\n+//--------------------------------------------------------------------------------------------------\n+\n+impl<const AS_SIZE: usize> memory::mmu::AddressSpace<AS_SIZE> {\n+    /// Checks for architectural restrictions.\n+    pub const fn arch_address_space_size_sanity_checks() {\n+        // Size must be at least one full 512 MiB table.\n+        assert!((AS_SIZE modulo Granule512MiB::SIZE) == 0);\n+\n+        // Check for 48 bit virtual address size as maximum, which is supported by any ARMv8\n+        // version.\n+        assert!(AS_SIZE <= (1 << 48));\n+    }\n+}\n+\n+impl MemoryManagementUnit {\n+    /// Setup function for the MAIR_EL1 register.\n+    fn set_up_mair(&self) {\n+        // Define the memory types being mapped.\n+        MAIR_EL1.write(\n+            // Attribute 1 - Cacheable normal DRAM.\n+            MAIR_EL1::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc +\n+        MAIR_EL1::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc +\n+\n+        // Attribute 0 - Device.\n+        MAIR_EL1::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck,\n+        );\n+    }\n+\n+    /// Configure various settings of stage 1 of the EL1 translation regime.\n+    fn configure_translation_control(&self) {\n+        let t0sz = (64 - bsp::memory::mmu::KernelAddrSpace::SIZE_SHIFT) as u64;\n+\n+        TCR_EL1.write(\n+            TCR_EL1::TBI0::Used\n+                + TCR_EL1::IPS::Bits_40\n+                + TCR_EL1::TG0::KiB_64\n+                + TCR_EL1::SH0::Inner\n+                + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable\n+                + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable\n+                + TCR_EL1::EPD0::EnableTTBR0Walks\n+                + TCR_EL1::A1::TTBR0\n+                + TCR_EL1::T0SZ.val(t0sz)\n+                + TCR_EL1::EPD1::DisableTTBR1Walks,\n+        );\n+    }\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Return a reference to the MMU instance.\n+pub fn mmu() -> &'static impl memory::mmu::interface::MMU {\n+    &MMU\n+}\n+\n+//------------------------------------------------------------------------------\n+// OS Interface Code\n+//------------------------------------------------------------------------------\n+use memory::mmu::MMUEnableError;\n+\n+impl memory::mmu::interface::MMU for MemoryManagementUnit {\n+    unsafe fn enable_mmu_and_caching(&self) -> Result<(), MMUEnableError> {\n+        if unlikely(self.is_enabled()) {\n+            return Err(MMUEnableError::AlreadyEnabled);\n+        }\n+\n+        // Fail early if translation granule is not supported.\n+        if unlikely(!ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran64::Supported)) {\n+            return Err(MMUEnableError::Other(\n+                \"Translation granule not supported in HW\",\n+            ));\n+        }\n+\n+        // Prepare the memory attribute indirection register.\n+        self.set_up_mair();\n+\n+        // Populate translation tables.\n+        KERNEL_TABLES\n+            .populate_tt_entries()\n+            .map_err(MMUEnableError::Other)?;\n+\n+        // Set the \"Translation Table Base Register\".\n+        TTBR0_EL1.set_baddr(KERNEL_TABLES.phys_base_address());\n+\n+        self.configure_translation_control();\n+\n+        // Switch the MMU on.\n+        //\n+        // First, force all previous changes to be seen before the MMU is enabled.\n+        barrier::isb(barrier::SY);\n+\n+        // Enable the MMU and turn on data and instruction caching.\n+        SCTLR_EL1.modify(SCTLR_EL1::M::Enable + SCTLR_EL1::C::Cacheable + SCTLR_EL1::I::Cacheable);\n+\n+        // Force MMU init to complete before next instruction.\n+        barrier::isb(barrier::SY);\n+\n+        Ok(())\n+    }\n+\n+    #[inline(always)]\n+    fn is_enabled(&self) -> bool {\n+        SCTLR_EL1.matches_all(SCTLR_EL1::M::Enable)\n+    }\n+}\n\ndiff -uNr 09_privilege_level/src/bsp/raspberrypi/kernel.ld 10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/kernel.ld\n--- 09_privilege_level/src/bsp/raspberrypi/kernel.ld\n+++ 10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/kernel.ld\n@@ -3,6 +3,9 @@\n  * Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n  */\n\n+PAGE_SIZE = 64K;\n+PAGE_MASK = PAGE_SIZE - 1;\n+\n __rpi_phys_dram_start_addr = 0;\n\n /* The physical address at which the the kernel binary will be loaded by the Raspberry's firmware */\n@@ -42,9 +45,12 @@\n         __boot_core_stack_end_exclusive = .; /*   |             */\n     } :segment_boot_core_stack\n\n+    ASSERT((. & PAGE_MASK) == 0, \"End of boot core stack is not page aligned\")\n+\n     /***********************************************************************************************\n     * Code + RO Data + Global Offset Table\n     ***********************************************************************************************/\n+    __code_start = .;\n     .text :\n     {\n         KEEP(*(.text._start))\n@@ -55,6 +61,9 @@\n\n     .rodata : ALIGN(8) { *(.rodata*) } :segment_code\n\n+    . = ALIGN(PAGE_SIZE);\n+    __code_end_exclusive = .;\n+\n     /***********************************************************************************************\n     * Data + BSS\n     ***********************************************************************************************/\n\ndiff -uNr 09_privilege_level/src/bsp/raspberrypi/memory/mmu.rs 10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory/mmu.rs\n--- 09_privilege_level/src/bsp/raspberrypi/memory/mmu.rs\n+++ 10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory/mmu.rs\n@@ -0,0 +1,86 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! BSP Memory Management Unit.\n+\n+use super::map as memory_map;\n+use crate::memory::mmu::*;\n+use core::ops::RangeInclusive;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// The kernel's address space defined by this BSP.\n+pub type KernelAddrSpace = AddressSpace<{ memory_map::END_INCLUSIVE + 1 }>;\n+\n+const NUM_MEM_RANGES: usize = 3;\n+\n+/// The virtual memory layout.\n+///\n+/// The layout must contain only special ranges, aka anything that is _not_ normal cacheable DRAM.\n+/// It is agnostic of the paging granularity that the architecture's MMU will use.\n+pub static LAYOUT: KernelVirtualLayout<NUM_MEM_RANGES> = KernelVirtualLayout::new(\n+    memory_map::END_INCLUSIVE,\n+    [\n+        TranslationDescriptor {\n+            name: \"Kernel code and RO data\",\n+            virtual_range: code_range_inclusive,\n+            physical_range_translation: Translation::Identity,\n+            attribute_fields: AttributeFields {\n+                mem_attributes: MemAttributes::CacheableDRAM,\n+                acc_perms: AccessPermissions::ReadOnly,\n+                execute_never: false,\n+            },\n+        },\n+        TranslationDescriptor {\n+            name: \"Remapped Device MMIO\",\n+            virtual_range: remapped_mmio_range_inclusive,\n+            physical_range_translation: Translation::Offset(memory_map::mmio::START + 0x20_0000),\n+            attribute_fields: AttributeFields {\n+                mem_attributes: MemAttributes::Device,\n+                acc_perms: AccessPermissions::ReadWrite,\n+                execute_never: true,\n+            },\n+        },\n+        TranslationDescriptor {\n+            name: \"Device MMIO\",\n+            virtual_range: mmio_range_inclusive,\n+            physical_range_translation: Translation::Identity,\n+            attribute_fields: AttributeFields {\n+                mem_attributes: MemAttributes::Device,\n+                acc_perms: AccessPermissions::ReadWrite,\n+                execute_never: true,\n+            },\n+        },\n+    ],\n+);\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Code\n+//--------------------------------------------------------------------------------------------------\n+\n+fn code_range_inclusive() -> RangeInclusive<usize> {\n+    // Notice the subtraction to turn the exclusive end into an inclusive end.\n+    #[allow(clippy::range_minus_one)]\n+    RangeInclusive::new(super::code_start(), super::code_end_exclusive() - 1)\n+}\n+\n+fn remapped_mmio_range_inclusive() -> RangeInclusive<usize> {\n+    // The last 64 KiB slot in the first 512 MiB\n+    RangeInclusive::new(0x1FFF_0000, 0x1FFF_FFFF)\n+}\n+\n+fn mmio_range_inclusive() -> RangeInclusive<usize> {\n+    RangeInclusive::new(memory_map::mmio::START, memory_map::mmio::END_INCLUSIVE)\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Return a reference to the virtual memory layout.\n+pub fn virt_mem_layout() -> &'static KernelVirtualLayout<NUM_MEM_RANGES> {\n+    &LAYOUT\n+}\n\ndiff -uNr 09_privilege_level/src/bsp/raspberrypi/memory.rs 10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory.rs\n--- 09_privilege_level/src/bsp/raspberrypi/memory.rs\n+++ 10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory.rs\n@@ -3,6 +3,45 @@\n // Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n //! BSP Memory Management.\n+//!\n+//! The physical memory layout.\n+//!\n+//! The Raspberry's firmware copies the kernel binary to 0x8_0000. The preceding region will be used\n+//! as the boot core's stack.\n+//!\n+//! +---------------------------------------+\n+//! |                                       | 0x0\n+//! |                                       |                                ^\n+//! | Boot-core Stack                       |                                | stack\n+//! |                                       |                                | growth\n+//! |                                       |                                | direction\n+//! +---------------------------------------+\n+//! |                                       | code_start @ 0x8_0000\n+//! | .text                                 |\n+//! | .rodata                               |\n+//! | .got                                  |\n+//! |                                       |\n+//! +---------------------------------------+\n+//! |                                       | code_end_exclusive\n+//! | .data                                 |\n+//! | .bss                                  |\n+//! |                                       |\n+//! +---------------------------------------+\n+//! |                                       |\n+//! |                                       |\n+pub mod mmu;\n+\n+use core::cell::UnsafeCell;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+// Symbols from the linker script.\n+extern \"Rust\" {\n+    static __code_start: UnsafeCell<()>;\n+    static __code_end_exclusive: UnsafeCell<()>;\n+}\n\n //--------------------------------------------------------------------------------------------------\n // Public Definitions\n@@ -11,6 +50,20 @@\n /// The board's physical memory map.\n #[rustfmt::skip]\n pub(super) mod map {\n+    /// The inclusive end address of the memory map.\n+    ///\n+    /// End address + 1 must be power of two.\n+    ///\n+    /// # Note\n+    ///\n+    /// RPi3 and RPi4 boards can have different amounts of RAM. To make our code lean for\n+    /// educational purposes, we set the max size of the address space to 4 GiB regardless of board.\n+    /// This way, we can map the entire range that we need (end of MMIO for RPi4) in one take.\n+    ///\n+    /// However, making this trade-off has the downside of making it possible for the CPU to assert a\n+    /// physical address that is not backed by any DRAM (e.g. accessing an address close to 4 GiB on\n+    /// an RPi3 that comes with 1 GiB of RAM). This would result in a crash or other kind of error.\n+    pub const END_INCLUSIVE:       usize = 0xFFFF_FFFF;\n\n     pub const GPIO_OFFSET:         usize = 0x0020_0000;\n     pub const UART_OFFSET:         usize = 0x0020_1000;\n@@ -23,6 +76,7 @@\n         pub const START:            usize =         0x3F00_0000;\n         pub const GPIO_START:       usize = START + GPIO_OFFSET;\n         pub const PL011_UART_START: usize = START + UART_OFFSET;\n+        pub const END_INCLUSIVE:    usize =         0x4000_FFFF;\n     }\n\n     /// Physical devices.\n@@ -33,5 +87,29 @@\n         pub const START:            usize =         0xFE00_0000;\n         pub const GPIO_START:       usize = START + GPIO_OFFSET;\n         pub const PL011_UART_START: usize = START + UART_OFFSET;\n+        pub const END_INCLUSIVE:    usize =         0xFF84_FFFF;\n     }\n }\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Code\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Start page address of the code segment.\n+///\n+/// # Safety\n+///\n+/// - Value is provided by the linker script and must be trusted as-is.\n+#[inline(always)]\n+fn code_start() -> usize {\n+    unsafe { __code_start.get() as usize }\n+}\n+\n+/// Exclusive end page address of the code segment.\n+/// # Safety\n+///\n+/// - Value is provided by the linker script and must be trusted as-is.\n+#[inline(always)]\n+fn code_end_exclusive() -> usize {\n+    unsafe { __code_end_exclusive.get() as usize }\n+}\n\ndiff -uNr 09_privilege_level/src/bsp.rs 10_virtual_mem_part1_identity_mapping/src/bsp.rs\n--- 09_privilege_level/src/bsp.rs\n+++ 10_virtual_mem_part1_identity_mapping/src/bsp.rs\n@@ -4,7 +4,7 @@\n\n //! Conditional reexporting of Board Support Packages.\n\n-mod device_driver;\n+pub mod device_driver;\n\n #[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\n mod raspberrypi;\n\ndiff -uNr 09_privilege_level/src/common.rs 10_virtual_mem_part1_identity_mapping/src/common.rs\n--- 09_privilege_level/src/common.rs\n+++ 10_virtual_mem_part1_identity_mapping/src/common.rs\n@@ -0,0 +1,22 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! General purpose code.\n+\n+/// Convert a size into human readable format.\n+pub const fn size_human_readable_ceil(size: usize) -> (usize, &'static str) {\n+    const KIB: usize = 1024;\n+    const MIB: usize = 1024 * 1024;\n+    const GIB: usize = 1024 * 1024 * 1024;\n+\n+    if (size / GIB) > 0 {\n+        (size.div_ceil(GIB), \"GiB\")\n+    } else if (size / MIB) > 0 {\n+        (size.div_ceil(MIB), \"MiB\")\n+    } else if (size / KIB) > 0 {\n+        (size.div_ceil(KIB), \"KiB\")\n+    } else {\n+        (size, \"Byte\")\n+    }\n+}\n\ndiff -uNr 09_privilege_level/src/main.rs 10_virtual_mem_part1_identity_mapping/src/main.rs\n--- 09_privilege_level/src/main.rs\n+++ 10_virtual_mem_part1_identity_mapping/src/main.rs\n@@ -107,9 +107,12 @@\n //! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.\n\n #![allow(clippy::upper_case_acronyms)]\n+#![allow(incomplete_features)]\n #![feature(asm_const)]\n #![feature(const_option)]\n+#![feature(core_intrinsics)]\n #![feature(format_args_nl)]\n+#![feature(int_roundings)]\n #![feature(nonzero_min_max)]\n #![feature(panic_info_message)]\n #![feature(trait_alias)]\n@@ -118,10 +121,12 @@\n #![no_std]\n\n mod bsp;\n+mod common;\n mod console;\n mod cpu;\n mod driver;\n mod exception;\n+mod memory;\n mod panic_wait;\n mod print;\n mod synchronization;\n@@ -132,8 +137,17 @@\n /// # Safety\n ///\n /// - Only a single core must be active and running this function.\n-/// - The init calls in this function must appear in the correct order.\n+/// - The init calls in this function must appear in the correct order:\n+///     - MMU + Data caching must be activated at the earliest. Without it, any atomic operations,\n+///       e.g. the yet-to-be-introduced spinlocks in the device drivers (which currently employ\n+///       NullLocks instead of spinlocks), will fail to work (properly) on the RPi SoCs.\n unsafe fn kernel_init() -> ! {\n+    use memory::mmu::interface::MMU;\n+\n+    if let Err(string) = memory::mmu::mmu().enable_mmu_and_caching() {\n+        panic!(\"MMU: {}\", string);\n+    }\n+\n     // Initialize the BSP driver subsystem.\n     if let Err(x) = bsp::driver::init() {\n         panic!(\"Error initializing BSP driver subsystem: {}\", x);\n@@ -149,7 +163,7 @@\n\n /// The main function running after the early init.\n fn kernel_main() -> ! {\n-    use console::console;\n+    use console::{console, interface::Write};\n     use core::time::Duration;\n\n     info!(\n@@ -159,6 +173,9 @@\n     );\n     info!(\"Booting on: {}\", bsp::board_name());\n\n+    info!(\"MMU online. Special regions:\");\n+    bsp::memory::mmu::virt_mem_layout().print_layout();\n+\n     let (_, privilege_level) = exception::current_privilege_level();\n     info!(\"Current privilege level: {}\", privilege_level);\n\n@@ -176,6 +193,13 @@\n     info!(\"Timer test, spinning for 1 second\");\n     time::time_manager().spin_for(Duration::from_secs(1));\n\n+    let remapped_uart = unsafe { bsp::device_driver::PL011Uart::new(0x1FFF_1000) };\n+    writeln!(\n+        remapped_uart,\n+        \"[     !!!    ] Writing through the remapped UART at 0x1FFF_1000\"\n+    )\n+    .unwrap();\n+\n     info!(\"Echoing input now\");\n\n     // Discard any spurious received characters before going into echo mode.\n\ndiff -uNr 09_privilege_level/src/memory/mmu/translation_table.rs 10_virtual_mem_part1_identity_mapping/src/memory/mmu/translation_table.rs\n--- 09_privilege_level/src/memory/mmu/translation_table.rs\n+++ 10_virtual_mem_part1_identity_mapping/src/memory/mmu/translation_table.rs\n@@ -0,0 +1,14 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Translation table.\n+\n+#[cfg(target_arch = \"aarch64\")]\n+#[path = \"../../_arch/aarch64/memory/mmu/translation_table.rs\"]\n+mod arch_translation_table;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Architectural Public Reexports\n+//--------------------------------------------------------------------------------------------------\n+pub use arch_translation_table::KernelTranslationTable;\n\ndiff -uNr 09_privilege_level/src/memory/mmu.rs 10_virtual_mem_part1_identity_mapping/src/memory/mmu.rs\n--- 09_privilege_level/src/memory/mmu.rs\n+++ 10_virtual_mem_part1_identity_mapping/src/memory/mmu.rs\n@@ -0,0 +1,253 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Memory Management Unit.\n+//!\n+//! In order to decouple `BSP` and `arch` parts of the MMU code (to keep them pluggable), this file\n+//! provides types for composing an architecture-agnostic description of the kernel's virtual memory\n+//! layout.\n+//!\n+//! The `BSP` provides such a description through the `bsp::memory::mmu::virt_mem_layout()`\n+//! function.\n+//!\n+//! The `MMU` driver of the `arch` code uses `bsp::memory::mmu::virt_mem_layout()` to compile and\n+//! install respective translation tables.\n+\n+#[cfg(target_arch = \"aarch64\")]\n+#[path = \"../_arch/aarch64/memory/mmu.rs\"]\n+mod arch_mmu;\n+\n+mod translation_table;\n+\n+use crate::common;\n+use core::{fmt, ops::RangeInclusive};\n+\n+//--------------------------------------------------------------------------------------------------\n+// Architectural Public Reexports\n+//--------------------------------------------------------------------------------------------------\n+pub use arch_mmu::mmu;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// MMU enable errors variants.\n+#[allow(missing_docs)]\n+#[derive(Debug)]\n+pub enum MMUEnableError {\n+    AlreadyEnabled,\n+    Other(&'static str),\n+}\n+\n+/// Memory Management interfaces.\n+pub mod interface {\n+    use super::*;\n+\n+    /// MMU functions.\n+    pub trait MMU {\n+        /// Called by the kernel during early init. Supposed to take the translation tables from the\n+        /// `BSP`-supplied `virt_mem_layout()` and install/activate them for the respective MMU.\n+        ///\n+        /// # Safety\n+        ///\n+        /// - Changes the HW's global state.\n+        unsafe fn enable_mmu_and_caching(&self) -> Result<(), MMUEnableError>;\n+\n+        /// Returns true if the MMU is enabled, false otherwise.\n+        fn is_enabled(&self) -> bool;\n+    }\n+}\n+\n+/// Describes the characteristics of a translation granule.\n+pub struct TranslationGranule<const GRANULE_SIZE: usize>;\n+\n+/// Describes properties of an address space.\n+pub struct AddressSpace<const AS_SIZE: usize>;\n+\n+/// Architecture agnostic translation types.\n+#[allow(missing_docs)]\n+#[allow(dead_code)]\n+#[derive(Copy, Clone)]\n+pub enum Translation {\n+    Identity,\n+    Offset(usize),\n+}\n+\n+/// Architecture agnostic memory attributes.\n+#[allow(missing_docs)]\n+#[derive(Copy, Clone)]\n+pub enum MemAttributes {\n+    CacheableDRAM,\n+    Device,\n+}\n+\n+/// Architecture agnostic access permissions.\n+#[allow(missing_docs)]\n+#[derive(Copy, Clone)]\n+pub enum AccessPermissions {\n+    ReadOnly,\n+    ReadWrite,\n+}\n+\n+/// Collection of memory attributes.\n+#[allow(missing_docs)]\n+#[derive(Copy, Clone)]\n+pub struct AttributeFields {\n+    pub mem_attributes: MemAttributes,\n+    pub acc_perms: AccessPermissions,\n+    pub execute_never: bool,\n+}\n+\n+/// Architecture agnostic descriptor for a memory range.\n+#[allow(missing_docs)]\n+pub struct TranslationDescriptor {\n+    pub name: &'static str,\n+    pub virtual_range: fn() -> RangeInclusive<usize>,\n+    pub physical_range_translation: Translation,\n+    pub attribute_fields: AttributeFields,\n+}\n+\n+/// Type for expressing the kernel's virtual memory layout.\n+pub struct KernelVirtualLayout<const NUM_SPECIAL_RANGES: usize> {\n+    /// The last (inclusive) address of the address space.\n+    max_virt_addr_inclusive: usize,\n+\n+    /// Array of descriptors for non-standard (normal cacheable DRAM) memory regions.\n+    inner: [TranslationDescriptor; NUM_SPECIAL_RANGES],\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+impl fmt::Display for MMUEnableError {\n+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n+        match self {\n+            MMUEnableError::AlreadyEnabled => write!(f, \"MMU is already enabled\"),\n+            MMUEnableError::Other(x) => write!(f, \"{}\", x),\n+        }\n+    }\n+}\n+\n+impl<const GRANULE_SIZE: usize> TranslationGranule<GRANULE_SIZE> {\n+    /// The granule's size.\n+    pub const SIZE: usize = Self::size_checked();\n+\n+    /// The granule's shift, aka log2(size).\n+    pub const SHIFT: usize = Self::SIZE.trailing_zeros() as usize;\n+\n+    const fn size_checked() -> usize {\n+        assert!(GRANULE_SIZE.is_power_of_two());\n+\n+        GRANULE_SIZE\n+    }\n+}\n+\n+impl<const AS_SIZE: usize> AddressSpace<AS_SIZE> {\n+    /// The address space size.\n+    pub const SIZE: usize = Self::size_checked();\n+\n+    /// The address space shift, aka log2(size).\n+    pub const SIZE_SHIFT: usize = Self::SIZE.trailing_zeros() as usize;\n+\n+    const fn size_checked() -> usize {\n+        assert!(AS_SIZE.is_power_of_two());\n+\n+        // Check for architectural restrictions as well.\n+        Self::arch_address_space_size_sanity_checks();\n+\n+        AS_SIZE\n+    }\n+}\n+\n+impl Default for AttributeFields {\n+    fn default() -> AttributeFields {\n+        AttributeFields {\n+            mem_attributes: MemAttributes::CacheableDRAM,\n+            acc_perms: AccessPermissions::ReadWrite,\n+            execute_never: true,\n+        }\n+    }\n+}\n+\n+/// Human-readable output of a TranslationDescriptor.\n+impl fmt::Display for TranslationDescriptor {\n+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n+        // Call the function to which self.range points, and dereference the result, which causes\n+        // Rust to copy the value.\n+        let start = *(self.virtual_range)().start();\n+        let end = *(self.virtual_range)().end();\n+        let size = end - start + 1;\n+\n+        let (size, unit) = common::size_human_readable_ceil(size);\n+\n+        let attr = match self.attribute_fields.mem_attributes {\n+            MemAttributes::CacheableDRAM => \"C\",\n+            MemAttributes::Device => \"Dev\",\n+        };\n+\n+        let acc_p = match self.attribute_fields.acc_perms {\n+            AccessPermissions::ReadOnly => \"RO\",\n+            AccessPermissions::ReadWrite => \"RW\",\n+        };\n+\n+        let xn = if self.attribute_fields.execute_never {\n+            \"PXN\"\n+        } else {\n+            \"PX\"\n+        };\n+\n+        write!(\n+            f,\n+            \"      {:#010x} - {:#010x} | {: >3} {} | {: <3} {} {: <3} | {}\",\n+            start, end, size, unit, attr, acc_p, xn, self.name\n+        )\n+    }\n+}\n+\n+impl<const NUM_SPECIAL_RANGES: usize> KernelVirtualLayout<{ NUM_SPECIAL_RANGES }> {\n+    /// Create a new instance.\n+    pub const fn new(max: usize, layout: [TranslationDescriptor; NUM_SPECIAL_RANGES]) -> Self {\n+        Self {\n+            max_virt_addr_inclusive: max,\n+            inner: layout,\n+        }\n+    }\n+\n+    /// For a virtual address, find and return the physical output address and corresponding\n+    /// attributes.\n+    ///\n+    /// If the address is not found in `inner`, return an identity mapped default with normal\n+    /// cacheable DRAM attributes.\n+    pub fn virt_addr_properties(\n+        &self,\n+        virt_addr: usize,\n+    ) -> Result<(usize, AttributeFields), &'static str> {\n+        if virt_addr > self.max_virt_addr_inclusive {\n+            return Err(\"Address out of range\");\n+        }\n+\n+        for i in self.inner.iter() {\n+            if (i.virtual_range)().contains(&virt_addr) {\n+                let output_addr = match i.physical_range_translation {\n+                    Translation::Identity => virt_addr,\n+                    Translation::Offset(a) => a + (virt_addr - (i.virtual_range)().start()),\n+                };\n+\n+                return Ok((output_addr, i.attribute_fields));\n+            }\n+        }\n+\n+        Ok((virt_addr, AttributeFields::default()))\n+    }\n+\n+    /// Print the memory layout.\n+    pub fn print_layout(&self) {\n+        use crate::info;\n+\n+        for i in self.inner.iter() {\n+            info!(\"{}\", i);\n+        }\n+    }\n+}\n\ndiff -uNr 09_privilege_level/src/memory.rs 10_virtual_mem_part1_identity_mapping/src/memory.rs\n--- 09_privilege_level/src/memory.rs\n+++ 10_virtual_mem_part1_identity_mapping/src/memory.rs\n@@ -0,0 +1,7 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Memory Management.\n+\n+pub mod mmu;\n\n```\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/build.rs",
    "content": "use std::{env, fs, process};\n\nfn main() {\n    let ld_script_path = match env::var(\"LD_SCRIPT_PATH\") {\n        Ok(var) => var,\n        _ => process::exit(0),\n    };\n\n    let files = fs::read_dir(ld_script_path).unwrap();\n    files\n        .filter_map(Result::ok)\n        .filter(|d| {\n            if let Some(e) = d.path().extension() {\n                e == \"ld\"\n            } else {\n                false\n            }\n        })\n        .for_each(|f| println!(\"cargo:rerun-if-changed={}\", f.path().display()));\n}\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural boot code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::boot::arch_boot\n\nuse aarch64_cpu::{asm, registers::*};\nuse core::arch::global_asm;\nuse tock_registers::interfaces::Writeable;\n\n// Assembly counterpart to this file.\nglobal_asm!(\n    include_str!(\"boot.s\"),\n    CONST_CURRENTEL_EL2 = const 0x8,\n    CONST_CORE_ID_MASK = const 0b11\n);\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Prepares the transition from EL2 to EL1.\n///\n/// # Safety\n///\n/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.\n/// - The HW state of EL1 must be prepared in a sound way.\n#[inline(always)]\nunsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: u64) {\n    // Enable timer counter registers for EL1.\n    CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET);\n\n    // No offset for reading the counters.\n    CNTVOFF_EL2.set(0);\n\n    // Set EL1 execution state to AArch64.\n    HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64);\n\n    // Set up a simulated exception return.\n    //\n    // First, fake a saved program status where all interrupts were masked and SP_EL1 was used as a\n    // stack pointer.\n    SPSR_EL2.write(\n        SPSR_EL2::D::Masked\n            + SPSR_EL2::A::Masked\n            + SPSR_EL2::I::Masked\n            + SPSR_EL2::F::Masked\n            + SPSR_EL2::M::EL1h,\n    );\n\n    // Second, let the link register point to kernel_init().\n    ELR_EL2.set(crate::kernel_init as *const () as u64);\n\n    // Set up SP_EL1 (stack pointer), which will be used by EL1 once we \"return\" to it. Since there\n    // are no plans to ever return to EL2, just re-use the same stack.\n    SP_EL1.set(phys_boot_core_stack_end_exclusive_addr);\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The Rust entry of the `kernel` binary.\n///\n/// The function is called from the assembly `_start` function.\n///\n/// # Safety\n///\n/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`.\n#[no_mangle]\npub unsafe extern \"C\" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! {\n    prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr);\n\n    // Use `eret` to \"return\" to EL1. This results in execution of kernel_init() in EL1.\n    asm::eret()\n}\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/boot.s",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2022 Andre Richter <andre.o.richter@gmail.com>\n\n//--------------------------------------------------------------------------------------------------\n// Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Load the address of a symbol into a register, PC-relative.\n//\n// The symbol must lie within +/- 4 GiB of the Program Counter.\n//\n// # Resources\n//\n// - https://sourceware.org/binutils/docs-2.36/as/AArch64_002dRelocations.html\n.macro ADR_REL register, symbol\n\tadrp\t\\register, \\symbol\n\tadd\t\\register, \\register, #:lo12:\\symbol\n.endm\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n.section .text._start\n\n//------------------------------------------------------------------------------\n// fn _start()\n//------------------------------------------------------------------------------\n_start:\n\t// Only proceed if the core executes in EL2. Park it otherwise.\n\tmrs\tx0, CurrentEL\n\tcmp\tx0, {CONST_CURRENTEL_EL2}\n\tb.ne\t.L_parking_loop\n\n\t// Only proceed on the boot core. Park it otherwise.\n\tmrs\tx1, MPIDR_EL1\n\tand\tx1, x1, {CONST_CORE_ID_MASK}\n\tldr\tx2, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs\n\tcmp\tx1, x2\n\tb.ne\t.L_parking_loop\n\n\t// If execution reaches here, it is the boot core.\n\n\t// Initialize DRAM.\n\tADR_REL\tx0, __bss_start\n\tADR_REL x1, __bss_end_exclusive\n\n.L_bss_init_loop:\n\tcmp\tx0, x1\n\tb.eq\t.L_prepare_rust\n\tstp\txzr, xzr, [x0], #16\n\tb\t.L_bss_init_loop\n\n\t// Prepare the jump to Rust code.\n.L_prepare_rust:\n\t// Set the stack pointer. This ensures that any code in EL2 that needs the stack will work.\n\tADR_REL\tx0, __boot_core_stack_end_exclusive\n\tmov\tsp, x0\n\n\t// Read the CPU's timer counter frequency and store it in ARCH_TIMER_COUNTER_FREQUENCY.\n\t// Abort if the frequency read back as 0.\n\tADR_REL\tx1, ARCH_TIMER_COUNTER_FREQUENCY // provided by aarch64/time.rs\n\tmrs\tx2, CNTFRQ_EL0\n\tcmp\tx2, xzr\n\tb.eq\t.L_parking_loop\n\tstr\tw2, [x1]\n\n\t// Jump to Rust code. x0 holds the function argument provided to _start_rust().\n\tb\t_start_rust\n\n\t// Infinitely wait for events (aka \"park the core\").\n.L_parking_loop:\n\twfe\n\tb\t.L_parking_loop\n\n.size\t_start, . - _start\n.type\t_start, function\n.global\t_start\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural processor code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::arch_cpu\n\nuse aarch64_cpu::asm;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\npub use asm::nop;\n\n/// Pause execution on the core.\n#[inline(always)]\npub fn wait_forever() -> ! {\n    loop {\n        asm::wfe()\n    }\n}\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/_arch/aarch64/exception/asynchronous.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural asynchronous exception handling.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::exception::asynchronous::arch_asynchronous\n\nuse aarch64_cpu::registers::*;\nuse tock_registers::interfaces::Readable;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\ntrait DaifField {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register>;\n}\n\nstruct Debug;\nstruct SError;\nstruct IRQ;\nstruct FIQ;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl DaifField for Debug {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::D\n    }\n}\n\nimpl DaifField for SError {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::A\n    }\n}\n\nimpl DaifField for IRQ {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::I\n    }\n}\n\nimpl DaifField for FIQ {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::F\n    }\n}\n\nfn is_masked<T>() -> bool\nwhere\n    T: DaifField,\n{\n    DAIF.is_set(T::daif_field())\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Print the AArch64 exceptions status.\n#[rustfmt::skip]\npub fn print_state() {\n    use crate::info;\n\n    let to_mask_str = |x| -> _ {\n        if x { \"Masked\" } else { \"Unmasked\" }\n    };\n\n    info!(\"      Debug:  {}\", to_mask_str(is_masked::<Debug>()));\n    info!(\"      SError: {}\", to_mask_str(is_masked::<SError>()));\n    info!(\"      IRQ:    {}\", to_mask_str(is_masked::<IRQ>()));\n    info!(\"      FIQ:    {}\", to_mask_str(is_masked::<FIQ>()));\n}\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/_arch/aarch64/exception.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural synchronous and asynchronous exception handling.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::exception::arch_exception\n\nuse aarch64_cpu::registers::*;\nuse tock_registers::interfaces::Readable;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse crate::exception::PrivilegeLevel;\n\n/// The processing element's current privilege level.\npub fn current_privilege_level() -> (PrivilegeLevel, &'static str) {\n    let el = CurrentEL.read_as_enum(CurrentEL::EL);\n    match el {\n        Some(CurrentEL::EL::Value::EL2) => (PrivilegeLevel::Hypervisor, \"EL2\"),\n        Some(CurrentEL::EL::Value::EL1) => (PrivilegeLevel::Kernel, \"EL1\"),\n        Some(CurrentEL::EL::Value::EL0) => (PrivilegeLevel::User, \"EL0\"),\n        _ => (PrivilegeLevel::Unknown, \"Unknown\"),\n    }\n}\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/_arch/aarch64/memory/mmu/translation_table.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural translation table.\n//!\n//! Only 64 KiB granule is supported.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::memory::mmu::translation_table::arch_translation_table\n\nuse crate::{\n    bsp, memory,\n    memory::mmu::{\n        arch_mmu::{Granule512MiB, Granule64KiB},\n        AccessPermissions, AttributeFields, MemAttributes,\n    },\n};\nuse core::convert;\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields,\n    registers::InMemoryRegister,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// A table descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-15.\nregister_bitfields! {u64,\n    STAGE1_TABLE_DESCRIPTOR [\n        /// Physical address of the next descriptor.\n        NEXT_LEVEL_TABLE_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16]\n\n        TYPE  OFFSET(1) NUMBITS(1) [\n            Block = 0,\n            Table = 1\n        ],\n\n        VALID OFFSET(0) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ]\n    ]\n}\n\n// A level 3 page descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-17.\nregister_bitfields! {u64,\n    STAGE1_PAGE_DESCRIPTOR [\n        /// Unprivileged execute-never.\n        UXN      OFFSET(54) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ],\n\n        /// Privileged execute-never.\n        PXN      OFFSET(53) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ],\n\n        /// Physical address of the next table descriptor (lvl2) or the page descriptor (lvl3).\n        OUTPUT_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16]\n\n        /// Access flag.\n        AF       OFFSET(10) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ],\n\n        /// Shareability field.\n        SH       OFFSET(8) NUMBITS(2) [\n            OuterShareable = 0b10,\n            InnerShareable = 0b11\n        ],\n\n        /// Access Permissions.\n        AP       OFFSET(6) NUMBITS(2) [\n            RW_EL1 = 0b00,\n            RW_EL1_EL0 = 0b01,\n            RO_EL1 = 0b10,\n            RO_EL1_EL0 = 0b11\n        ],\n\n        /// Memory attributes index into the MAIR_EL1 register.\n        AttrIndx OFFSET(2) NUMBITS(3) [],\n\n        TYPE     OFFSET(1) NUMBITS(1) [\n            Reserved_Invalid = 0,\n            Page = 1\n        ],\n\n        VALID    OFFSET(0) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ]\n    ]\n}\n\n/// A table descriptor for 64 KiB aperture.\n///\n/// The output points to the next table.\n#[derive(Copy, Clone)]\n#[repr(C)]\nstruct TableDescriptor {\n    value: u64,\n}\n\n/// A page descriptor with 64 KiB aperture.\n///\n/// The output points to physical memory.\n#[derive(Copy, Clone)]\n#[repr(C)]\nstruct PageDescriptor {\n    value: u64,\n}\n\ntrait StartAddr {\n    fn phys_start_addr_u64(&self) -> u64;\n    fn phys_start_addr_usize(&self) -> usize;\n}\n\nconst NUM_LVL2_TABLES: usize = bsp::memory::mmu::KernelAddrSpace::SIZE >> Granule512MiB::SHIFT;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB\n/// aligned, so the lvl3 is put first.\n#[repr(C)]\n#[repr(align(65536))]\npub struct FixedSizeTranslationTable<const NUM_TABLES: usize> {\n    /// Page descriptors, covering 64 KiB windows per entry.\n    lvl3: [[PageDescriptor; 8192]; NUM_TABLES],\n\n    /// Table descriptors, covering 512 MiB windows.\n    lvl2: [TableDescriptor; NUM_TABLES],\n}\n\n/// A translation table type for the kernel space.\npub type KernelTranslationTable = FixedSizeTranslationTable<NUM_LVL2_TABLES>;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n// The binary is still identity mapped, so we don't need to convert here.\nimpl<T, const N: usize> StartAddr for [T; N] {\n    fn phys_start_addr_u64(&self) -> u64 {\n        self as *const T as u64\n    }\n\n    fn phys_start_addr_usize(&self) -> usize {\n        self as *const _ as usize\n    }\n}\n\nimpl TableDescriptor {\n    /// Create an instance.\n    ///\n    /// Descriptor is invalid by default.\n    pub const fn new_zeroed() -> Self {\n        Self { value: 0 }\n    }\n\n    /// Create an instance pointing to the supplied address.\n    pub fn from_next_lvl_table_addr(phys_next_lvl_table_addr: usize) -> Self {\n        let val = InMemoryRegister::<u64, STAGE1_TABLE_DESCRIPTOR::Register>::new(0);\n\n        let shifted = phys_next_lvl_table_addr >> Granule64KiB::SHIFT;\n        val.write(\n            STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64)\n                + STAGE1_TABLE_DESCRIPTOR::TYPE::Table\n                + STAGE1_TABLE_DESCRIPTOR::VALID::True,\n        );\n\n        TableDescriptor { value: val.get() }\n    }\n}\n\n/// Convert the kernel's generic memory attributes to HW-specific attributes of the MMU.\nimpl convert::From<AttributeFields>\n    for tock_registers::fields::FieldValue<u64, STAGE1_PAGE_DESCRIPTOR::Register>\n{\n    fn from(attribute_fields: AttributeFields) -> Self {\n        // Memory attributes.\n        let mut desc = match attribute_fields.mem_attributes {\n            MemAttributes::CacheableDRAM => {\n                STAGE1_PAGE_DESCRIPTOR::SH::InnerShareable\n                    + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::NORMAL)\n            }\n            MemAttributes::Device => {\n                STAGE1_PAGE_DESCRIPTOR::SH::OuterShareable\n                    + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::DEVICE)\n            }\n        };\n\n        // Access Permissions.\n        desc += match attribute_fields.acc_perms {\n            AccessPermissions::ReadOnly => STAGE1_PAGE_DESCRIPTOR::AP::RO_EL1,\n            AccessPermissions::ReadWrite => STAGE1_PAGE_DESCRIPTOR::AP::RW_EL1,\n        };\n\n        // The execute-never attribute is mapped to PXN in AArch64.\n        desc += if attribute_fields.execute_never {\n            STAGE1_PAGE_DESCRIPTOR::PXN::True\n        } else {\n            STAGE1_PAGE_DESCRIPTOR::PXN::False\n        };\n\n        // Always set unprivileged exectue-never as long as userspace is not implemented yet.\n        desc += STAGE1_PAGE_DESCRIPTOR::UXN::True;\n\n        desc\n    }\n}\n\nimpl PageDescriptor {\n    /// Create an instance.\n    ///\n    /// Descriptor is invalid by default.\n    pub const fn new_zeroed() -> Self {\n        Self { value: 0 }\n    }\n\n    /// Create an instance.\n    pub fn from_output_addr(phys_output_addr: usize, attribute_fields: &AttributeFields) -> Self {\n        let val = InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(0);\n\n        let shifted = phys_output_addr as u64 >> Granule64KiB::SHIFT;\n        val.write(\n            STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted)\n                + STAGE1_PAGE_DESCRIPTOR::AF::True\n                + STAGE1_PAGE_DESCRIPTOR::TYPE::Page\n                + STAGE1_PAGE_DESCRIPTOR::VALID::True\n                + (*attribute_fields).into(),\n        );\n\n        Self { value: val.get() }\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<const NUM_TABLES: usize> FixedSizeTranslationTable<NUM_TABLES> {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        // Can't have a zero-sized address space.\n        assert!(NUM_TABLES > 0);\n\n        Self {\n            lvl3: [[PageDescriptor::new_zeroed(); 8192]; NUM_TABLES],\n            lvl2: [TableDescriptor::new_zeroed(); NUM_TABLES],\n        }\n    }\n\n    /// Iterates over all static translation table entries and fills them at once.\n    ///\n    /// # Safety\n    ///\n    /// - Modifies a `static mut`. Ensure it only happens from here.\n    pub unsafe fn populate_tt_entries(&mut self) -> Result<(), &'static str> {\n        for (l2_nr, l2_entry) in self.lvl2.iter_mut().enumerate() {\n            *l2_entry =\n                TableDescriptor::from_next_lvl_table_addr(self.lvl3[l2_nr].phys_start_addr_usize());\n\n            for (l3_nr, l3_entry) in self.lvl3[l2_nr].iter_mut().enumerate() {\n                let virt_addr = (l2_nr << Granule512MiB::SHIFT) + (l3_nr << Granule64KiB::SHIFT);\n\n                let (phys_output_addr, attribute_fields) =\n                    bsp::memory::mmu::virt_mem_layout().virt_addr_properties(virt_addr)?;\n\n                *l3_entry = PageDescriptor::from_output_addr(phys_output_addr, &attribute_fields);\n            }\n        }\n\n        Ok(())\n    }\n\n    /// The translation table's base address to be used for programming the MMU.\n    pub fn phys_base_address(&self) -> u64 {\n        self.lvl2.phys_start_addr_u64()\n    }\n}\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/_arch/aarch64/memory/mmu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management Unit Driver.\n//!\n//! Only 64 KiB granule is supported.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::memory::mmu::arch_mmu\n\nuse crate::{\n    bsp, memory,\n    memory::mmu::{translation_table::KernelTranslationTable, TranslationGranule},\n};\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::intrinsics::unlikely;\nuse tock_registers::interfaces::{ReadWriteable, Readable, Writeable};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Memory Management Unit type.\nstruct MemoryManagementUnit;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub type Granule512MiB = TranslationGranule<{ 512 * 1024 * 1024 }>;\npub type Granule64KiB = TranslationGranule<{ 64 * 1024 }>;\n\n/// Constants for indexing the MAIR_EL1.\n#[allow(dead_code)]\npub mod mair {\n    pub const DEVICE: u64 = 0;\n    pub const NORMAL: u64 = 1;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n/// The kernel translation tables.\n///\n/// # Safety\n///\n/// - Supposed to land in `.bss`. Therefore, ensure that all initial member values boil down to \"0\".\nstatic mut KERNEL_TABLES: KernelTranslationTable = KernelTranslationTable::new();\n\nstatic MMU: MemoryManagementUnit = MemoryManagementUnit;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<const AS_SIZE: usize> memory::mmu::AddressSpace<AS_SIZE> {\n    /// Checks for architectural restrictions.\n    pub const fn arch_address_space_size_sanity_checks() {\n        // Size must be at least one full 512 MiB table.\n        assert!((AS_SIZE % Granule512MiB::SIZE) == 0);\n\n        // Check for 48 bit virtual address size as maximum, which is supported by any ARMv8\n        // version.\n        assert!(AS_SIZE <= (1 << 48));\n    }\n}\n\nimpl MemoryManagementUnit {\n    /// Setup function for the MAIR_EL1 register.\n    fn set_up_mair(&self) {\n        // Define the memory types being mapped.\n        MAIR_EL1.write(\n            // Attribute 1 - Cacheable normal DRAM.\n            MAIR_EL1::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc +\n        MAIR_EL1::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc +\n\n        // Attribute 0 - Device.\n        MAIR_EL1::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck,\n        );\n    }\n\n    /// Configure various settings of stage 1 of the EL1 translation regime.\n    fn configure_translation_control(&self) {\n        let t0sz = (64 - bsp::memory::mmu::KernelAddrSpace::SIZE_SHIFT) as u64;\n\n        TCR_EL1.write(\n            TCR_EL1::TBI0::Used\n                + TCR_EL1::IPS::Bits_40\n                + TCR_EL1::TG0::KiB_64\n                + TCR_EL1::SH0::Inner\n                + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable\n                + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable\n                + TCR_EL1::EPD0::EnableTTBR0Walks\n                + TCR_EL1::A1::TTBR0\n                + TCR_EL1::T0SZ.val(t0sz)\n                + TCR_EL1::EPD1::DisableTTBR1Walks,\n        );\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the MMU instance.\npub fn mmu() -> &'static impl memory::mmu::interface::MMU {\n    &MMU\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse memory::mmu::MMUEnableError;\n\nimpl memory::mmu::interface::MMU for MemoryManagementUnit {\n    unsafe fn enable_mmu_and_caching(&self) -> Result<(), MMUEnableError> {\n        if unlikely(self.is_enabled()) {\n            return Err(MMUEnableError::AlreadyEnabled);\n        }\n\n        // Fail early if translation granule is not supported.\n        if unlikely(!ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran64::Supported)) {\n            return Err(MMUEnableError::Other(\n                \"Translation granule not supported in HW\",\n            ));\n        }\n\n        // Prepare the memory attribute indirection register.\n        self.set_up_mair();\n\n        // Populate translation tables.\n        KERNEL_TABLES\n            .populate_tt_entries()\n            .map_err(MMUEnableError::Other)?;\n\n        // Set the \"Translation Table Base Register\".\n        TTBR0_EL1.set_baddr(KERNEL_TABLES.phys_base_address());\n\n        self.configure_translation_control();\n\n        // Switch the MMU on.\n        //\n        // First, force all previous changes to be seen before the MMU is enabled.\n        barrier::isb(barrier::SY);\n\n        // Enable the MMU and turn on data and instruction caching.\n        SCTLR_EL1.modify(SCTLR_EL1::M::Enable + SCTLR_EL1::C::Cacheable + SCTLR_EL1::I::Cacheable);\n\n        // Force MMU init to complete before next instruction.\n        barrier::isb(barrier::SY);\n\n        Ok(())\n    }\n\n    #[inline(always)]\n    fn is_enabled(&self) -> bool {\n        SCTLR_EL1.matches_all(SCTLR_EL1::M::Enable)\n    }\n}\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/_arch/aarch64/time.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural timer primitives.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::time::arch_time\n\nuse crate::warn;\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::{\n    num::{NonZeroU128, NonZeroU32, NonZeroU64},\n    ops::{Add, Div},\n    time::Duration,\n};\nuse tock_registers::interfaces::Readable;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nconst NANOSEC_PER_SEC: NonZeroU64 = NonZeroU64::new(1_000_000_000).unwrap();\n\n#[derive(Copy, Clone, PartialOrd, PartialEq)]\nstruct GenericTimerCounterValue(u64);\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n/// Boot assembly code overwrites this value with the value of CNTFRQ_EL0 before any Rust code is\n/// executed. This given value here is just a (safe) dummy.\n#[no_mangle]\nstatic ARCH_TIMER_COUNTER_FREQUENCY: NonZeroU32 = NonZeroU32::MIN;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nfn arch_timer_counter_frequency() -> NonZeroU32 {\n    // Read volatile is needed here to prevent the compiler from optimizing\n    // ARCH_TIMER_COUNTER_FREQUENCY away.\n    //\n    // This is safe, because all the safety requirements as stated in read_volatile()'s\n    // documentation are fulfilled.\n    unsafe { core::ptr::read_volatile(&ARCH_TIMER_COUNTER_FREQUENCY) }\n}\n\nimpl GenericTimerCounterValue {\n    pub const MAX: Self = GenericTimerCounterValue(u64::MAX);\n}\n\nimpl Add for GenericTimerCounterValue {\n    type Output = Self;\n\n    fn add(self, other: Self) -> Self {\n        GenericTimerCounterValue(self.0.wrapping_add(other.0))\n    }\n}\n\nimpl From<GenericTimerCounterValue> for Duration {\n    fn from(counter_value: GenericTimerCounterValue) -> Self {\n        if counter_value.0 == 0 {\n            return Duration::ZERO;\n        }\n\n        let frequency: NonZeroU64 = arch_timer_counter_frequency().into();\n\n        // Div<NonZeroU64> implementation for u64 cannot panic.\n        let secs = counter_value.0.div(frequency);\n\n        // This is safe, because frequency can never be greater than u32::MAX, which means the\n        // largest theoretical value for sub_second_counter_value is (u32::MAX - 1). Therefore,\n        // (sub_second_counter_value * NANOSEC_PER_SEC) cannot overflow an u64.\n        //\n        // The subsequent division ensures the result fits into u32, since the max result is smaller\n        // than NANOSEC_PER_SEC. Therefore, just cast it to u32 using `as`.\n        let sub_second_counter_value = counter_value.0 % frequency;\n        let nanos = unsafe { sub_second_counter_value.unchecked_mul(u64::from(NANOSEC_PER_SEC)) }\n            .div(frequency) as u32;\n\n        Duration::new(secs, nanos)\n    }\n}\n\nfn max_duration() -> Duration {\n    Duration::from(GenericTimerCounterValue::MAX)\n}\n\nimpl TryFrom<Duration> for GenericTimerCounterValue {\n    type Error = &'static str;\n\n    fn try_from(duration: Duration) -> Result<Self, Self::Error> {\n        if duration < resolution() {\n            return Ok(GenericTimerCounterValue(0));\n        }\n\n        if duration > max_duration() {\n            return Err(\"Conversion error. Duration too big\");\n        }\n\n        let frequency: u128 = u32::from(arch_timer_counter_frequency()) as u128;\n        let duration: u128 = duration.as_nanos();\n\n        // This is safe, because frequency can never be greater than u32::MAX, and\n        // (Duration::MAX.as_nanos() * u32::MAX) < u128::MAX.\n        let counter_value =\n            unsafe { duration.unchecked_mul(frequency) }.div(NonZeroU128::from(NANOSEC_PER_SEC));\n\n        // Since we checked above that we are <= max_duration(), just cast to u64.\n        Ok(GenericTimerCounterValue(counter_value as u64))\n    }\n}\n\n#[inline(always)]\nfn read_cntpct() -> GenericTimerCounterValue {\n    // Prevent that the counter is read ahead of time due to out-of-order execution.\n    barrier::isb(barrier::SY);\n    let cnt = CNTPCT_EL0.get();\n\n    GenericTimerCounterValue(cnt)\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The timer's resolution.\npub fn resolution() -> Duration {\n    Duration::from(GenericTimerCounterValue(1))\n}\n\n/// The uptime since power-on of the device.\n///\n/// This includes time consumed by firmware and bootloaders.\npub fn uptime() -> Duration {\n    read_cntpct().into()\n}\n\n/// Spin for a given duration.\npub fn spin_for(duration: Duration) {\n    let curr_counter_value = read_cntpct();\n\n    let counter_value_delta: GenericTimerCounterValue = match duration.try_into() {\n        Err(msg) => {\n            warn!(\"spin_for: {}. Skipping\", msg);\n            return;\n        }\n        Ok(val) => val,\n    };\n    let counter_value_target = curr_counter_value + counter_value_delta;\n\n    // Busy wait.\n    //\n    // Read CNTPCT_EL0 directly to avoid the ISB that is part of [`read_cntpct`].\n    while GenericTimerCounterValue(CNTPCT_EL0.get()) < counter_value_target {}\n}\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GPIO Driver.\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper, driver, synchronization,\n    synchronization::NullLock,\n};\nuse tock_registers::{\n    interfaces::{ReadWriteable, Writeable},\n    register_bitfields, register_structs,\n    registers::ReadWrite,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// GPIO registers.\n//\n// Descriptions taken from\n// - https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf\n// - https://datasheets.raspberrypi.org/bcm2711/bcm2711-peripherals.pdf\nregister_bitfields! {\n    u32,\n\n    /// GPIO Function Select 1\n    GPFSEL1 [\n        /// Pin 15\n        FSEL15 OFFSET(15) NUMBITS(3) [\n            Input = 0b000,\n            Output = 0b001,\n            AltFunc0 = 0b100  // PL011 UART RX\n\n        ],\n\n        /// Pin 14\n        FSEL14 OFFSET(12) NUMBITS(3) [\n            Input = 0b000,\n            Output = 0b001,\n            AltFunc0 = 0b100  // PL011 UART TX\n        ]\n    ],\n\n    /// GPIO Pull-up/down Register\n    ///\n    /// BCM2837 only.\n    GPPUD [\n        /// Controls the actuation of the internal pull-up/down control line to ALL the GPIO pins.\n        PUD OFFSET(0) NUMBITS(2) [\n            Off = 0b00,\n            PullDown = 0b01,\n            PullUp = 0b10\n        ]\n    ],\n\n    /// GPIO Pull-up/down Clock Register 0\n    ///\n    /// BCM2837 only.\n    GPPUDCLK0 [\n        /// Pin 15\n        PUDCLK15 OFFSET(15) NUMBITS(1) [\n            NoEffect = 0,\n            AssertClock = 1\n        ],\n\n        /// Pin 14\n        PUDCLK14 OFFSET(14) NUMBITS(1) [\n            NoEffect = 0,\n            AssertClock = 1\n        ]\n    ],\n\n    /// GPIO Pull-up / Pull-down Register 0\n    ///\n    /// BCM2711 only.\n    GPIO_PUP_PDN_CNTRL_REG0 [\n        /// Pin 15\n        GPIO_PUP_PDN_CNTRL15 OFFSET(30) NUMBITS(2) [\n            NoResistor = 0b00,\n            PullUp = 0b01\n        ],\n\n        /// Pin 14\n        GPIO_PUP_PDN_CNTRL14 OFFSET(28) NUMBITS(2) [\n            NoResistor = 0b00,\n            PullUp = 0b01\n        ]\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    RegisterBlock {\n        (0x00 => _reserved1),\n        (0x04 => GPFSEL1: ReadWrite<u32, GPFSEL1::Register>),\n        (0x08 => _reserved2),\n        (0x94 => GPPUD: ReadWrite<u32, GPPUD::Register>),\n        (0x98 => GPPUDCLK0: ReadWrite<u32, GPPUDCLK0::Register>),\n        (0x9C => _reserved3),\n        (0xE4 => GPIO_PUP_PDN_CNTRL_REG0: ReadWrite<u32, GPIO_PUP_PDN_CNTRL_REG0::Register>),\n        (0xE8 => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\nstruct GPIOInner {\n    registers: Registers,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the GPIO HW.\npub struct GPIO {\n    inner: NullLock<GPIOInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GPIOInner {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n        }\n    }\n\n    /// Disable pull-up/down on pins 14 and 15.\n    #[cfg(feature = \"bsp_rpi3\")]\n    fn disable_pud_14_15_bcm2837(&mut self) {\n        use crate::time;\n        use core::time::Duration;\n\n        // The Linux 2837 GPIO driver waits 1 µs between the steps.\n        const DELAY: Duration = Duration::from_micros(1);\n\n        self.registers.GPPUD.write(GPPUD::PUD::Off);\n        time::time_manager().spin_for(DELAY);\n\n        self.registers\n            .GPPUDCLK0\n            .write(GPPUDCLK0::PUDCLK15::AssertClock + GPPUDCLK0::PUDCLK14::AssertClock);\n        time::time_manager().spin_for(DELAY);\n\n        self.registers.GPPUD.write(GPPUD::PUD::Off);\n        self.registers.GPPUDCLK0.set(0);\n    }\n\n    /// Disable pull-up/down on pins 14 and 15.\n    #[cfg(feature = \"bsp_rpi4\")]\n    fn disable_pud_14_15_bcm2711(&mut self) {\n        self.registers.GPIO_PUP_PDN_CNTRL_REG0.write(\n            GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL15::PullUp\n                + GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL14::PullUp,\n        );\n    }\n\n    /// Map PL011 UART as standard output.\n    ///\n    /// TX to pin 14\n    /// RX to pin 15\n    pub fn map_pl011_uart(&mut self) {\n        // Select the UART on pins 14 and 15.\n        self.registers\n            .GPFSEL1\n            .modify(GPFSEL1::FSEL15::AltFunc0 + GPFSEL1::FSEL14::AltFunc0);\n\n        // Disable pull-up/down on pins 14 and 15.\n        #[cfg(feature = \"bsp_rpi3\")]\n        self.disable_pud_14_15_bcm2837();\n\n        #[cfg(feature = \"bsp_rpi4\")]\n        self.disable_pud_14_15_bcm2711();\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GPIO {\n    pub const COMPATIBLE: &'static str = \"BCM GPIO\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            inner: NullLock::new(GPIOInner::new(mmio_start_addr)),\n        }\n    }\n\n    /// Concurrency safe version of `GPIOInner.map_pl011_uart()`\n    pub fn map_pl011_uart(&self) {\n        self.inner.lock(|inner| inner.map_pl011_uart())\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl driver::interface::DeviceDriver for GPIO {\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n}\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! PL011 UART driver.\n//!\n//! # Resources\n//!\n//! - <https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf>\n//! - <https://developer.arm.com/documentation/ddi0183/latest>\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper, console, cpu, driver, synchronization,\n    synchronization::NullLock,\n};\nuse core::fmt;\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields, register_structs,\n    registers::{ReadOnly, ReadWrite, WriteOnly},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// PL011 UART registers.\n//\n// Descriptions taken from \"PrimeCell UART (PL011) Technical Reference Manual\" r1p5.\nregister_bitfields! {\n    u32,\n\n    /// Flag Register.\n    FR [\n        /// Transmit FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n        /// Line Control Register, LCR_H.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the transmit holding register is empty.\n        /// - If the FIFO is enabled, the TXFE bit is set when the transmit FIFO is empty.\n        /// - This bit does not indicate if there is data in the transmit shift register.\n        TXFE OFFSET(7) NUMBITS(1) [],\n\n        /// Transmit FIFO full. The meaning of this bit depends on the state of the FEN bit in the\n        /// LCR_H Register.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the transmit holding register is full.\n        /// - If the FIFO is enabled, the TXFF bit is set when the transmit FIFO is full.\n        TXFF OFFSET(5) NUMBITS(1) [],\n\n        /// Receive FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n        /// LCR_H Register.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the receive holding register is empty.\n        /// - If the FIFO is enabled, the RXFE bit is set when the receive FIFO is empty.\n        RXFE OFFSET(4) NUMBITS(1) [],\n\n        /// UART busy. If this bit is set to 1, the UART is busy transmitting data. This bit remains\n        /// set until the complete byte, including all the stop bits, has been sent from the shift\n        /// register.\n        ///\n        /// This bit is set as soon as the transmit FIFO becomes non-empty, regardless of whether\n        /// the UART is enabled or not.\n        BUSY OFFSET(3) NUMBITS(1) []\n    ],\n\n    /// Integer Baud Rate Divisor.\n    IBRD [\n        /// The integer baud rate divisor.\n        BAUD_DIVINT OFFSET(0) NUMBITS(16) []\n    ],\n\n    /// Fractional Baud Rate Divisor.\n    FBRD [\n        ///  The fractional baud rate divisor.\n        BAUD_DIVFRAC OFFSET(0) NUMBITS(6) []\n    ],\n\n    /// Line Control Register.\n    LCR_H [\n        /// Word length. These bits indicate the number of data bits transmitted or received in a\n        /// frame.\n        #[allow(clippy::enum_variant_names)]\n        WLEN OFFSET(5) NUMBITS(2) [\n            FiveBit = 0b00,\n            SixBit = 0b01,\n            SevenBit = 0b10,\n            EightBit = 0b11\n        ],\n\n        /// Enable FIFOs:\n        ///\n        /// 0 = FIFOs are disabled (character mode) that is, the FIFOs become 1-byte-deep holding\n        /// registers.\n        ///\n        /// 1 = Transmit and receive FIFO buffers are enabled (FIFO mode).\n        FEN  OFFSET(4) NUMBITS(1) [\n            FifosDisabled = 0,\n            FifosEnabled = 1\n        ]\n    ],\n\n    /// Control Register.\n    CR [\n        /// Receive enable. If this bit is set to 1, the receive section of the UART is enabled.\n        /// Data reception occurs for either UART signals or SIR signals depending on the setting of\n        /// the SIREN bit. When the UART is disabled in the middle of reception, it completes the\n        /// current character before stopping.\n        RXE OFFSET(9) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// Transmit enable. If this bit is set to 1, the transmit section of the UART is enabled.\n        /// Data transmission occurs for either UART signals, or SIR signals depending on the\n        /// setting of the SIREN bit. When the UART is disabled in the middle of transmission, it\n        /// completes the current character before stopping.\n        TXE OFFSET(8) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// UART enable:\n        ///\n        /// 0 = UART is disabled. If the UART is disabled in the middle of transmission or\n        /// reception, it completes the current character before stopping.\n        ///\n        /// 1 = The UART is enabled. Data transmission and reception occurs for either UART signals\n        /// or SIR signals depending on the setting of the SIREN bit\n        UARTEN OFFSET(0) NUMBITS(1) [\n            /// If the UART is disabled in the middle of transmission or reception, it completes the\n            /// current character before stopping.\n            Disabled = 0,\n            Enabled = 1\n        ]\n    ],\n\n    /// Interrupt Clear Register.\n    ICR [\n        /// Meta field for all pending interrupts.\n        ALL OFFSET(0) NUMBITS(11) []\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    pub RegisterBlock {\n        (0x00 => DR: ReadWrite<u32>),\n        (0x04 => _reserved1),\n        (0x18 => FR: ReadOnly<u32, FR::Register>),\n        (0x1c => _reserved2),\n        (0x24 => IBRD: WriteOnly<u32, IBRD::Register>),\n        (0x28 => FBRD: WriteOnly<u32, FBRD::Register>),\n        (0x2c => LCR_H: WriteOnly<u32, LCR_H::Register>),\n        (0x30 => CR: WriteOnly<u32, CR::Register>),\n        (0x34 => _reserved3),\n        (0x44 => ICR: WriteOnly<u32, ICR::Register>),\n        (0x48 => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\n#[derive(PartialEq)]\nenum BlockingMode {\n    Blocking,\n    NonBlocking,\n}\n\nstruct PL011UartInner {\n    registers: Registers,\n    chars_written: usize,\n    chars_read: usize,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the UART.\npub struct PL011Uart {\n    inner: NullLock<PL011UartInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PL011UartInner {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n            chars_written: 0,\n            chars_read: 0,\n        }\n    }\n\n    /// Set up baud rate and characteristics.\n    ///\n    /// This results in 8N1 and 921_600 baud.\n    ///\n    /// The calculation for the BRD is (we set the clock to 48 MHz in config.txt):\n    /// `(48_000_000 / 16) / 921_600 = 3.2552083`.\n    ///\n    /// This means the integer part is `3` and goes into the `IBRD`.\n    /// The fractional part is `0.2552083`.\n    ///\n    /// `FBRD` calculation according to the PL011 Technical Reference Manual:\n    /// `INTEGER((0.2552083 * 64) + 0.5) = 16`.\n    ///\n    /// Therefore, the generated baud rate divider is: `3 + 16/64 = 3.25`. Which results in a\n    /// genrated baud rate of `48_000_000 / (16 * 3.25) = 923_077`.\n    ///\n    /// Error = `((923_077 - 921_600) / 921_600) * 100 = 0.16%`.\n    pub fn init(&mut self) {\n        // Execution can arrive here while there are still characters queued in the TX FIFO and\n        // actively being sent out by the UART hardware. If the UART is turned off in this case,\n        // those queued characters would be lost.\n        //\n        // For example, this can happen during runtime on a call to panic!(), because panic!()\n        // initializes its own UART instance and calls init().\n        //\n        // Hence, flush first to ensure all pending characters are transmitted.\n        self.flush();\n\n        // Turn the UART off temporarily.\n        self.registers.CR.set(0);\n\n        // Clear all pending interrupts.\n        self.registers.ICR.write(ICR::ALL::CLEAR);\n\n        // From the PL011 Technical Reference Manual:\n        //\n        // The LCR_H, IBRD, and FBRD registers form the single 30-bit wide LCR Register that is\n        // updated on a single write strobe generated by a LCR_H write. So, to internally update the\n        // contents of IBRD or FBRD, a LCR_H write must always be performed at the end.\n        //\n        // Set the baud rate, 8N1 and FIFO enabled.\n        self.registers.IBRD.write(IBRD::BAUD_DIVINT.val(3));\n        self.registers.FBRD.write(FBRD::BAUD_DIVFRAC.val(16));\n        self.registers\n            .LCR_H\n            .write(LCR_H::WLEN::EightBit + LCR_H::FEN::FifosEnabled);\n\n        // Turn the UART on.\n        self.registers\n            .CR\n            .write(CR::UARTEN::Enabled + CR::TXE::Enabled + CR::RXE::Enabled);\n    }\n\n    /// Send a character.\n    fn write_char(&mut self, c: char) {\n        // Spin while TX FIFO full is set, waiting for an empty slot.\n        while self.registers.FR.matches_all(FR::TXFF::SET) {\n            cpu::nop();\n        }\n\n        // Write the character to the buffer.\n        self.registers.DR.set(c as u32);\n\n        self.chars_written += 1;\n    }\n\n    /// Block execution until the last buffered character has been physically put on the TX wire.\n    fn flush(&self) {\n        // Spin until the busy bit is cleared.\n        while self.registers.FR.matches_all(FR::BUSY::SET) {\n            cpu::nop();\n        }\n    }\n\n    /// Retrieve a character.\n    fn read_char_converting(&mut self, blocking_mode: BlockingMode) -> Option<char> {\n        // If RX FIFO is empty,\n        if self.registers.FR.matches_all(FR::RXFE::SET) {\n            // immediately return in non-blocking mode.\n            if blocking_mode == BlockingMode::NonBlocking {\n                return None;\n            }\n\n            // Otherwise, wait until a char was received.\n            while self.registers.FR.matches_all(FR::RXFE::SET) {\n                cpu::nop();\n            }\n        }\n\n        // Read one character.\n        let mut ret = self.registers.DR.get() as u8 as char;\n\n        // Convert carrige return to newline.\n        if ret == '\\r' {\n            ret = '\\n'\n        }\n\n        // Update statistics.\n        self.chars_read += 1;\n\n        Some(ret)\n    }\n}\n\n/// Implementing `core::fmt::Write` enables usage of the `format_args!` macros, which in turn are\n/// used to implement the `kernel`'s `print!` and `println!` macros. By implementing `write_str()`,\n/// we get `write_fmt()` automatically.\n///\n/// The function takes an `&mut self`, so it must be implemented for the inner struct.\n///\n/// See [`src/print.rs`].\n///\n/// [`src/print.rs`]: ../../print/index.html\nimpl fmt::Write for PL011UartInner {\n    fn write_str(&mut self, s: &str) -> fmt::Result {\n        for c in s.chars() {\n            self.write_char(c);\n        }\n\n        Ok(())\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PL011Uart {\n    pub const COMPATIBLE: &'static str = \"BCM PL011 UART\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            inner: NullLock::new(PL011UartInner::new(mmio_start_addr)),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl driver::interface::DeviceDriver for PL011Uart {\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n\n    unsafe fn init(&self) -> Result<(), &'static str> {\n        self.inner.lock(|inner| inner.init());\n\n        Ok(())\n    }\n}\n\nimpl console::interface::Write for PL011Uart {\n    /// Passthrough of `args` to the `core::fmt::Write` implementation, but guarded by a Mutex to\n    /// serialize access.\n    fn write_char(&self, c: char) {\n        self.inner.lock(|inner| inner.write_char(c));\n    }\n\n    fn write_fmt(&self, args: core::fmt::Arguments) -> fmt::Result {\n        // Fully qualified syntax for the call to `core::fmt::Write::write_fmt()` to increase\n        // readability.\n        self.inner.lock(|inner| fmt::Write::write_fmt(inner, args))\n    }\n\n    fn flush(&self) {\n        // Spin until TX FIFO empty is set.\n        self.inner.lock(|inner| inner.flush());\n    }\n}\n\nimpl console::interface::Read for PL011Uart {\n    fn read_char(&self) -> char {\n        self.inner\n            .lock(|inner| inner.read_char_converting(BlockingMode::Blocking).unwrap())\n    }\n\n    fn clear_rx(&self) {\n        // Read from the RX FIFO until it is indicating empty.\n        while self\n            .inner\n            .lock(|inner| inner.read_char_converting(BlockingMode::NonBlocking))\n            .is_some()\n        {}\n    }\n}\n\nimpl console::interface::Statistics for PL011Uart {\n    fn chars_written(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_written)\n    }\n\n    fn chars_read(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_read)\n    }\n}\n\nimpl console::interface::All for PL011Uart {}\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/bsp/device_driver/bcm.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BCM driver top level.\n\nmod bcm2xxx_gpio;\nmod bcm2xxx_pl011_uart;\n\npub use bcm2xxx_gpio::*;\npub use bcm2xxx_pl011_uart::*;\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/bsp/device_driver/common.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Common device driver code.\n\nuse core::{marker::PhantomData, ops};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct MMIODerefWrapper<T> {\n    start_addr: usize,\n    phantom: PhantomData<fn() -> T>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T> MMIODerefWrapper<T> {\n    /// Create an instance.\n    pub const unsafe fn new(start_addr: usize) -> Self {\n        Self {\n            start_addr,\n            phantom: PhantomData,\n        }\n    }\n}\n\nimpl<T> ops::Deref for MMIODerefWrapper<T> {\n    type Target = T;\n\n    fn deref(&self) -> &Self::Target {\n        unsafe { &*(self.start_addr as *const _) }\n    }\n}\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/bsp/device_driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Device driver.\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod bcm;\nmod common;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use bcm::*;\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Processor code.\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Used by `arch` code to find the early boot core.\n#[no_mangle]\n#[link_section = \".text._start_arguments\"]\npub static BOOT_CORE_ID: u64 = 0;\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP driver support.\n\nuse super::memory::map::mmio;\nuse crate::{bsp::device_driver, console, driver as generic_driver};\nuse core::sync::atomic::{AtomicBool, Ordering};\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic PL011_UART: device_driver::PL011Uart =\n    unsafe { device_driver::PL011Uart::new(mmio::PL011_UART_START) };\nstatic GPIO: device_driver::GPIO = unsafe { device_driver::GPIO::new(mmio::GPIO_START) };\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// This must be called only after successful init of the UART driver.\nfn post_init_uart() -> Result<(), &'static str> {\n    console::register_console(&PL011_UART);\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the GPIO driver.\nfn post_init_gpio() -> Result<(), &'static str> {\n    GPIO.map_pl011_uart();\n    Ok(())\n}\n\nfn driver_uart() -> Result<(), &'static str> {\n    let uart_descriptor =\n        generic_driver::DeviceDriverDescriptor::new(&PL011_UART, Some(post_init_uart));\n    generic_driver::driver_manager().register_driver(uart_descriptor);\n\n    Ok(())\n}\n\nfn driver_gpio() -> Result<(), &'static str> {\n    let gpio_descriptor = generic_driver::DeviceDriverDescriptor::new(&GPIO, Some(post_init_gpio));\n    generic_driver::driver_manager().register_driver(gpio_descriptor);\n\n    Ok(())\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Initialize the driver subsystem.\n///\n/// # Safety\n///\n/// See child function calls.\npub unsafe fn init() -> Result<(), &'static str> {\n    static INIT_DONE: AtomicBool = AtomicBool::new(false);\n    if INIT_DONE.load(Ordering::Relaxed) {\n        return Err(\"Init already done\");\n    }\n\n    driver_uart()?;\n    driver_gpio()?;\n\n    INIT_DONE.store(true, Ordering::Relaxed);\n    Ok(())\n}\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/kernel.ld",
    "content": "/* SPDX-License-Identifier: MIT OR Apache-2.0\n *\n * Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n */\n\nPAGE_SIZE = 64K;\nPAGE_MASK = PAGE_SIZE - 1;\n\n__rpi_phys_dram_start_addr = 0;\n\n/* The physical address at which the the kernel binary will be loaded by the Raspberry's firmware */\n__rpi_phys_binary_load_addr = 0x80000;\n\n\nENTRY(__rpi_phys_binary_load_addr)\n\n/* Flags:\n *     4 == R\n *     5 == RX\n *     6 == RW\n *\n * Segments are marked PT_LOAD below so that the ELF file provides virtual and physical addresses.\n * It doesn't mean all of them need actually be loaded.\n */\nPHDRS\n{\n    segment_boot_core_stack PT_LOAD FLAGS(6);\n    segment_code            PT_LOAD FLAGS(5);\n    segment_data            PT_LOAD FLAGS(6);\n}\n\nSECTIONS\n{\n    . =  __rpi_phys_dram_start_addr;\n\n    /***********************************************************************************************\n    * Boot Core Stack\n    ***********************************************************************************************/\n    .boot_core_stack (NOLOAD) :\n    {\n                                             /*   ^             */\n                                             /*   | stack       */\n        . += __rpi_phys_binary_load_addr;    /*   | growth      */\n                                             /*   | direction   */\n        __boot_core_stack_end_exclusive = .; /*   |             */\n    } :segment_boot_core_stack\n\n    ASSERT((. & PAGE_MASK) == 0, \"End of boot core stack is not page aligned\")\n\n    /***********************************************************************************************\n    * Code + RO Data + Global Offset Table\n    ***********************************************************************************************/\n    __code_start = .;\n    .text :\n    {\n        KEEP(*(.text._start))\n        *(.text._start_arguments) /* Constants (or statics in Rust speak) read by _start(). */\n        *(.text._start_rust)      /* The Rust entry point */\n        *(.text*)                 /* Everything else */\n    } :segment_code\n\n    .rodata : ALIGN(8) { *(.rodata*) } :segment_code\n\n    . = ALIGN(PAGE_SIZE);\n    __code_end_exclusive = .;\n\n    /***********************************************************************************************\n    * Data + BSS\n    ***********************************************************************************************/\n    .data : { *(.data*) } :segment_data\n\n    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */\n    .bss (NOLOAD) : ALIGN(16)\n    {\n        __bss_start = .;\n        *(.bss*);\n        . = ALIGN(16);\n        __bss_end_exclusive = .;\n    } :segment_data\n\n    /***********************************************************************************************\n    * Misc\n    ***********************************************************************************************/\n    .got : { *(.got*) }\n    ASSERT(SIZEOF(.got) == 0, \"Relocation support not expected\")\n\n    /DISCARD/ : { *(.comment*) }\n}\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory/mmu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Memory Management Unit.\n\nuse super::map as memory_map;\nuse crate::memory::mmu::*;\nuse core::ops::RangeInclusive;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// The kernel's address space defined by this BSP.\npub type KernelAddrSpace = AddressSpace<{ memory_map::END_INCLUSIVE + 1 }>;\n\nconst NUM_MEM_RANGES: usize = 3;\n\n/// The virtual memory layout.\n///\n/// The layout must contain only special ranges, aka anything that is _not_ normal cacheable DRAM.\n/// It is agnostic of the paging granularity that the architecture's MMU will use.\npub static LAYOUT: KernelVirtualLayout<NUM_MEM_RANGES> = KernelVirtualLayout::new(\n    memory_map::END_INCLUSIVE,\n    [\n        TranslationDescriptor {\n            name: \"Kernel code and RO data\",\n            virtual_range: code_range_inclusive,\n            physical_range_translation: Translation::Identity,\n            attribute_fields: AttributeFields {\n                mem_attributes: MemAttributes::CacheableDRAM,\n                acc_perms: AccessPermissions::ReadOnly,\n                execute_never: false,\n            },\n        },\n        TranslationDescriptor {\n            name: \"Remapped Device MMIO\",\n            virtual_range: remapped_mmio_range_inclusive,\n            physical_range_translation: Translation::Offset(memory_map::mmio::START + 0x20_0000),\n            attribute_fields: AttributeFields {\n                mem_attributes: MemAttributes::Device,\n                acc_perms: AccessPermissions::ReadWrite,\n                execute_never: true,\n            },\n        },\n        TranslationDescriptor {\n            name: \"Device MMIO\",\n            virtual_range: mmio_range_inclusive,\n            physical_range_translation: Translation::Identity,\n            attribute_fields: AttributeFields {\n                mem_attributes: MemAttributes::Device,\n                acc_perms: AccessPermissions::ReadWrite,\n                execute_never: true,\n            },\n        },\n    ],\n);\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nfn code_range_inclusive() -> RangeInclusive<usize> {\n    // Notice the subtraction to turn the exclusive end into an inclusive end.\n    #[allow(clippy::range_minus_one)]\n    RangeInclusive::new(super::code_start(), super::code_end_exclusive() - 1)\n}\n\nfn remapped_mmio_range_inclusive() -> RangeInclusive<usize> {\n    // The last 64 KiB slot in the first 512 MiB\n    RangeInclusive::new(0x1FFF_0000, 0x1FFF_FFFF)\n}\n\nfn mmio_range_inclusive() -> RangeInclusive<usize> {\n    RangeInclusive::new(memory_map::mmio::START, memory_map::mmio::END_INCLUSIVE)\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the virtual memory layout.\npub fn virt_mem_layout() -> &'static KernelVirtualLayout<NUM_MEM_RANGES> {\n    &LAYOUT\n}\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Memory Management.\n//!\n//! The physical memory layout.\n//!\n//! The Raspberry's firmware copies the kernel binary to 0x8_0000. The preceding region will be used\n//! as the boot core's stack.\n//!\n//! +---------------------------------------+\n//! |                                       | 0x0\n//! |                                       |                                ^\n//! | Boot-core Stack                       |                                | stack\n//! |                                       |                                | growth\n//! |                                       |                                | direction\n//! +---------------------------------------+\n//! |                                       | code_start @ 0x8_0000\n//! | .text                                 |\n//! | .rodata                               |\n//! | .got                                  |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       | code_end_exclusive\n//! | .data                                 |\n//! | .bss                                  |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       |\n//! |                                       |\npub mod mmu;\n\nuse core::cell::UnsafeCell;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Symbols from the linker script.\nextern \"Rust\" {\n    static __code_start: UnsafeCell<()>;\n    static __code_end_exclusive: UnsafeCell<()>;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// The board's physical memory map.\n#[rustfmt::skip]\npub(super) mod map {\n    /// The inclusive end address of the memory map.\n    ///\n    /// End address + 1 must be power of two.\n    ///\n    /// # Note\n    ///\n    /// RPi3 and RPi4 boards can have different amounts of RAM. To make our code lean for\n    /// educational purposes, we set the max size of the address space to 4 GiB regardless of board.\n    /// This way, we can map the entire range that we need (end of MMIO for RPi4) in one take.\n    ///\n    /// However, making this trade-off has the downside of making it possible for the CPU to assert a\n    /// physical address that is not backed by any DRAM (e.g. accessing an address close to 4 GiB on\n    /// an RPi3 that comes with 1 GiB of RAM). This would result in a crash or other kind of error.\n    pub const END_INCLUSIVE:       usize = 0xFFFF_FFFF;\n\n    pub const GPIO_OFFSET:         usize = 0x0020_0000;\n    pub const UART_OFFSET:         usize = 0x0020_1000;\n\n    /// Physical devices.\n    #[cfg(feature = \"bsp_rpi3\")]\n    pub mod mmio {\n        use super::*;\n\n        pub const START:            usize =         0x3F00_0000;\n        pub const GPIO_START:       usize = START + GPIO_OFFSET;\n        pub const PL011_UART_START: usize = START + UART_OFFSET;\n        pub const END_INCLUSIVE:    usize =         0x4000_FFFF;\n    }\n\n    /// Physical devices.\n    #[cfg(feature = \"bsp_rpi4\")]\n    pub mod mmio {\n        use super::*;\n\n        pub const START:            usize =         0xFE00_0000;\n        pub const GPIO_START:       usize = START + GPIO_OFFSET;\n        pub const PL011_UART_START: usize = START + UART_OFFSET;\n        pub const END_INCLUSIVE:    usize =         0xFF84_FFFF;\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Start page address of the code segment.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn code_start() -> usize {\n    unsafe { __code_start.get() as usize }\n}\n\n/// Exclusive end page address of the code segment.\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn code_end_exclusive() -> usize {\n    unsafe { __code_end_exclusive.get() as usize }\n}\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Top-level BSP file for the Raspberry Pi 3 and 4.\n\npub mod cpu;\npub mod driver;\npub mod memory;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Board identification.\npub fn board_name() -> &'static str {\n    #[cfg(feature = \"bsp_rpi3\")]\n    {\n        \"Raspberry Pi 3\"\n    }\n\n    #[cfg(feature = \"bsp_rpi4\")]\n    {\n        \"Raspberry Pi 4\"\n    }\n}\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/bsp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Conditional reexporting of Board Support Packages.\n\npub mod device_driver;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod raspberrypi;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use raspberrypi::*;\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/common.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! General purpose code.\n\n/// Convert a size into human readable format.\npub const fn size_human_readable_ceil(size: usize) -> (usize, &'static str) {\n    const KIB: usize = 1024;\n    const MIB: usize = 1024 * 1024;\n    const GIB: usize = 1024 * 1024 * 1024;\n\n    if (size / GIB) > 0 {\n        (size.div_ceil(GIB), \"GiB\")\n    } else if (size / MIB) > 0 {\n        (size.div_ceil(MIB), \"MiB\")\n    } else if (size / KIB) > 0 {\n        (size.div_ceil(KIB), \"KiB\")\n    } else {\n        (size, \"Byte\")\n    }\n}\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/console/null_console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Null console.\n\nuse super::interface;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct NullConsole;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\npub static NULL_CONSOLE: NullConsole = NullConsole {};\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl interface::Write for NullConsole {\n    fn write_char(&self, _c: char) {}\n\n    fn write_fmt(&self, _args: fmt::Arguments) -> fmt::Result {\n        fmt::Result::Ok(())\n    }\n\n    fn flush(&self) {}\n}\n\nimpl interface::Read for NullConsole {\n    fn clear_rx(&self) {}\n}\n\nimpl interface::Statistics for NullConsole {}\nimpl interface::All for NullConsole {}\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! System console.\n\nmod null_console;\n\nuse crate::synchronization::{self, NullLock};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Console interfaces.\npub mod interface {\n    use core::fmt;\n\n    /// Console write functions.\n    pub trait Write {\n        /// Write a single character.\n        fn write_char(&self, c: char);\n\n        /// Write a Rust format string.\n        fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result;\n\n        /// Block until the last buffered character has been physically put on the TX wire.\n        fn flush(&self);\n    }\n\n    /// Console read functions.\n    pub trait Read {\n        /// Read a single character.\n        fn read_char(&self) -> char {\n            ' '\n        }\n\n        /// Clear RX buffers, if any.\n        fn clear_rx(&self);\n    }\n\n    /// Console statistics.\n    pub trait Statistics {\n        /// Return the number of characters written.\n        fn chars_written(&self) -> usize {\n            0\n        }\n\n        /// Return the number of characters read.\n        fn chars_read(&self) -> usize {\n            0\n        }\n    }\n\n    /// Trait alias for a full-fledged console.\n    pub trait All: Write + Read + Statistics {}\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic CUR_CONSOLE: NullLock<&'static (dyn interface::All + Sync)> =\n    NullLock::new(&null_console::NULL_CONSOLE);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\n/// Register a new console.\npub fn register_console(new_console: &'static (dyn interface::All + Sync)) {\n    CUR_CONSOLE.lock(|con| *con = new_console);\n}\n\n/// Return a reference to the currently registered console.\n///\n/// This is the global console used by all printing macros.\npub fn console() -> &'static dyn interface::All {\n    CUR_CONSOLE.lock(|con| *con)\n}\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Boot code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/cpu/boot.rs\"]\nmod arch_boot;\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Processor code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/cpu.rs\"]\nmod arch_cpu;\n\nmod boot;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_cpu::{nop, wait_forever};\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Driver support.\n\nuse crate::{\n    info,\n    synchronization::{interface::Mutex, NullLock},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nconst NUM_DRIVERS: usize = 5;\n\nstruct DriverManagerInner {\n    next_index: usize,\n    descriptors: [Option<DeviceDriverDescriptor>; NUM_DRIVERS],\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Driver interfaces.\npub mod interface {\n    /// Device Driver functions.\n    pub trait DeviceDriver {\n        /// Return a compatibility string for identifying the driver.\n        fn compatible(&self) -> &'static str;\n\n        /// Called by the kernel to bring up the device.\n        ///\n        /// # Safety\n        ///\n        /// - During init, drivers might do stuff with system-wide impact.\n        unsafe fn init(&self) -> Result<(), &'static str> {\n            Ok(())\n        }\n    }\n}\n\n/// Tpye to be used as an optional callback after a driver's init() has run.\npub type DeviceDriverPostInitCallback = unsafe fn() -> Result<(), &'static str>;\n\n/// A descriptor for device drivers.\n#[derive(Copy, Clone)]\npub struct DeviceDriverDescriptor {\n    device_driver: &'static (dyn interface::DeviceDriver + Sync),\n    post_init_callback: Option<DeviceDriverPostInitCallback>,\n}\n\n/// Provides device driver management functions.\npub struct DriverManager {\n    inner: NullLock<DriverManagerInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic DRIVER_MANAGER: DriverManager = DriverManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl DriverManagerInner {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            next_index: 0,\n            descriptors: [None; NUM_DRIVERS],\n        }\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl DeviceDriverDescriptor {\n    /// Create an instance.\n    pub fn new(\n        device_driver: &'static (dyn interface::DeviceDriver + Sync),\n        post_init_callback: Option<DeviceDriverPostInitCallback>,\n    ) -> Self {\n        Self {\n            device_driver,\n            post_init_callback,\n        }\n    }\n}\n\n/// Return a reference to the global DriverManager.\npub fn driver_manager() -> &'static DriverManager {\n    &DRIVER_MANAGER\n}\n\nimpl DriverManager {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            inner: NullLock::new(DriverManagerInner::new()),\n        }\n    }\n\n    /// Register a device driver with the kernel.\n    pub fn register_driver(&self, descriptor: DeviceDriverDescriptor) {\n        self.inner.lock(|inner| {\n            inner.descriptors[inner.next_index] = Some(descriptor);\n            inner.next_index += 1;\n        })\n    }\n\n    /// Helper for iterating over registered drivers.\n    fn for_each_descriptor<'a>(&'a self, f: impl FnMut(&'a DeviceDriverDescriptor)) {\n        self.inner.lock(|inner| {\n            inner\n                .descriptors\n                .iter()\n                .filter_map(|x| x.as_ref())\n                .for_each(f)\n        })\n    }\n\n    /// Fully initialize all drivers.\n    ///\n    /// # Safety\n    ///\n    /// - During init, drivers might do stuff with system-wide impact.\n    pub unsafe fn init_drivers(&self) {\n        self.for_each_descriptor(|descriptor| {\n            // 1. Initialize driver.\n            if let Err(x) = descriptor.device_driver.init() {\n                panic!(\n                    \"Error initializing driver: {}: {}\",\n                    descriptor.device_driver.compatible(),\n                    x\n                );\n            }\n\n            // 2. Call corresponding post init callback.\n            if let Some(callback) = &descriptor.post_init_callback {\n                if let Err(x) = callback() {\n                    panic!(\n                        \"Error during driver post-init callback: {}: {}\",\n                        descriptor.device_driver.compatible(),\n                        x\n                    );\n                }\n            }\n        });\n    }\n\n    /// Enumerate all registered device drivers.\n    pub fn enumerate(&self) {\n        let mut i: usize = 1;\n        self.for_each_descriptor(|descriptor| {\n            info!(\"      {}. {}\", i, descriptor.device_driver.compatible());\n\n            i += 1;\n        });\n    }\n}\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/exception/asynchronous.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Asynchronous exception handling.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/exception/asynchronous.rs\"]\nmod arch_asynchronous;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_asynchronous::print_state;\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/exception.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Synchronous and asynchronous exception handling.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/exception.rs\"]\nmod arch_exception;\n\npub mod asynchronous;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_exception::current_privilege_level;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Kernel privilege levels.\n#[allow(missing_docs)]\n#[derive(Eq, PartialEq)]\npub enum PrivilegeLevel {\n    User,\n    Kernel,\n    Hypervisor,\n    Unknown,\n}\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/main.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n// Rust embedded logo for `make doc`.\n#![doc(\n    html_logo_url = \"https://raw.githubusercontent.com/rust-embedded/wg/master/assets/logo/ewg-logo-blue-white-on-transparent.png\"\n)]\n\n//! The `kernel` binary.\n//!\n//! # Code organization and architecture\n//!\n//! The code is divided into different *modules*, each representing a typical **subsystem** of the\n//! `kernel`. Top-level module files of subsystems reside directly in the `src` folder. For example,\n//! `src/memory.rs` contains code that is concerned with all things memory management.\n//!\n//! ## Visibility of processor architecture code\n//!\n//! Some of the `kernel`'s subsystems depend on low-level code that is specific to the target\n//! processor architecture. For each supported processor architecture, there exists a subfolder in\n//! `src/_arch`, for example, `src/_arch/aarch64`.\n//!\n//! The architecture folders mirror the subsystem modules laid out in `src`. For example,\n//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go\n//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in\n//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic\n//! module's name prefixed with `arch_`.\n//!\n//! For example, this is the top of `src/memory/mmu.rs`:\n//!\n//! ```\n//! #[cfg(target_arch = \"aarch64\")]\n//! #[path = \"../_arch/aarch64/memory/mmu.rs\"]\n//! mod arch_mmu;\n//! ```\n//!\n//! Often times, items from the `arch_ module` will be publicly reexported by the parent module.\n//! This way, each architecture specific module can provide its implementation of an item, while the\n//! caller must not be concerned which architecture has been conditionally compiled.\n//!\n//! ## BSP code\n//!\n//! `BSP` stands for Board Support Package. `BSP` code is organized under `src/bsp.rs` and contains\n//! target board specific definitions and functions. These are things such as the board's memory map\n//! or instances of drivers for devices that are featured on the respective board.\n//!\n//! Just like processor architecture code, the `BSP` code's module structure tries to mirror the\n//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is\n//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`.\n//!\n//! ## Kernel interfaces\n//!\n//! Both `arch` and `bsp` contain code that is conditionally compiled depending on the actual target\n//! and board for which the kernel is compiled. For example, the `interrupt controller` hardware of\n//! the `Raspberry Pi 3` and the `Raspberry Pi 4` is different, but we want the rest of the `kernel`\n//! code to play nicely with any of the two without much hassle.\n//!\n//! In order to provide a clean abstraction between `arch`, `bsp` and `generic kernel code`,\n//! `interface` traits are provided *whenever possible* and *where it makes sense*. They are defined\n//! in the respective subsystem module and help to enforce the idiom of *program to an interface,\n//! not an implementation*. For example, there will be a common IRQ handling interface which the two\n//! different interrupt controller `drivers` of both Raspberrys will implement, and only export the\n//! interface to the rest of the `kernel`.\n//!\n//! ```\n//!         +-------------------+\n//!         | Interface (Trait) |\n//!         |                   |\n//!         +--+-------------+--+\n//!            ^             ^\n//!            |             |\n//!            |             |\n//! +----------+--+       +--+----------+\n//! | kernel code |       |  bsp code   |\n//! |             |       |  arch code  |\n//! +-------------+       +-------------+\n//! ```\n//!\n//! # Summary\n//!\n//! For a logical `kernel` subsystem, corresponding code can be distributed over several physical\n//! locations. Here is an example for the **memory** subsystem:\n//!\n//! - `src/memory.rs` and `src/memory/**/*`\n//!   - Common code that is agnostic of target processor architecture and `BSP` characteristics.\n//!     - Example: A function to zero a chunk of memory.\n//!   - Interfaces for the memory subsystem that are implemented by `arch` or `BSP` code.\n//!     - Example: An `MMU` interface that defines `MMU` function prototypes.\n//! - `src/bsp/__board_name__/memory.rs` and `src/bsp/__board_name__/memory/**/*`\n//!   - `BSP` specific code.\n//!   - Example: The board's memory map (physical addresses of DRAM and MMIO devices).\n//! - `src/_arch/__arch_name__/memory.rs` and `src/_arch/__arch_name__/memory/**/*`\n//!   - Processor architecture specific code.\n//!   - Example: Implementation of the `MMU` interface for the `__arch_name__` processor\n//!     architecture.\n//!\n//! From a namespace perspective, **memory** subsystem code lives in:\n//!\n//! - `crate::memory::*`\n//! - `crate::bsp::memory::*`\n//!\n//! # Boot flow\n//!\n//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.\n//!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.\n//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.\n\n#![allow(clippy::upper_case_acronyms)]\n#![allow(incomplete_features)]\n#![feature(asm_const)]\n#![feature(const_option)]\n#![feature(core_intrinsics)]\n#![feature(format_args_nl)]\n#![feature(int_roundings)]\n#![feature(nonzero_min_max)]\n#![feature(panic_info_message)]\n#![feature(trait_alias)]\n#![feature(unchecked_math)]\n#![no_main]\n#![no_std]\n\nmod bsp;\nmod common;\nmod console;\nmod cpu;\nmod driver;\nmod exception;\nmod memory;\nmod panic_wait;\nmod print;\nmod synchronization;\nmod time;\n\n/// Early init code.\n///\n/// # Safety\n///\n/// - Only a single core must be active and running this function.\n/// - The init calls in this function must appear in the correct order:\n///     - MMU + Data caching must be activated at the earliest. Without it, any atomic operations,\n///       e.g. the yet-to-be-introduced spinlocks in the device drivers (which currently employ\n///       NullLocks instead of spinlocks), will fail to work (properly) on the RPi SoCs.\nunsafe fn kernel_init() -> ! {\n    use memory::mmu::interface::MMU;\n\n    if let Err(string) = memory::mmu::mmu().enable_mmu_and_caching() {\n        panic!(\"MMU: {}\", string);\n    }\n\n    // Initialize the BSP driver subsystem.\n    if let Err(x) = bsp::driver::init() {\n        panic!(\"Error initializing BSP driver subsystem: {}\", x);\n    }\n\n    // Initialize all device drivers.\n    driver::driver_manager().init_drivers();\n    // println! is usable from here on.\n\n    // Transition from unsafe to safe.\n    kernel_main()\n}\n\n/// The main function running after the early init.\nfn kernel_main() -> ! {\n    use console::{console, interface::Write};\n    use core::time::Duration;\n\n    info!(\n        \"{} version {}\",\n        env!(\"CARGO_PKG_NAME\"),\n        env!(\"CARGO_PKG_VERSION\")\n    );\n    info!(\"Booting on: {}\", bsp::board_name());\n\n    info!(\"MMU online. Special regions:\");\n    bsp::memory::mmu::virt_mem_layout().print_layout();\n\n    let (_, privilege_level) = exception::current_privilege_level();\n    info!(\"Current privilege level: {}\", privilege_level);\n\n    info!(\"Exception handling state:\");\n    exception::asynchronous::print_state();\n\n    info!(\n        \"Architectural timer resolution: {} ns\",\n        time::time_manager().resolution().as_nanos()\n    );\n\n    info!(\"Drivers loaded:\");\n    driver::driver_manager().enumerate();\n\n    info!(\"Timer test, spinning for 1 second\");\n    time::time_manager().spin_for(Duration::from_secs(1));\n\n    let remapped_uart = unsafe { bsp::device_driver::PL011Uart::new(0x1FFF_1000) };\n    writeln!(\n        remapped_uart,\n        \"[     !!!    ] Writing through the remapped UART at 0x1FFF_1000\"\n    )\n    .unwrap();\n\n    info!(\"Echoing input now\");\n\n    // Discard any spurious received characters before going into echo mode.\n    console().clear_rx();\n    loop {\n        let c = console().read_char();\n        console().write_char(c);\n    }\n}\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/memory/mmu/translation_table.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Translation table.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../../_arch/aarch64/memory/mmu/translation_table.rs\"]\nmod arch_translation_table;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_translation_table::KernelTranslationTable;\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/memory/mmu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management Unit.\n//!\n//! In order to decouple `BSP` and `arch` parts of the MMU code (to keep them pluggable), this file\n//! provides types for composing an architecture-agnostic description of the kernel's virtual memory\n//! layout.\n//!\n//! The `BSP` provides such a description through the `bsp::memory::mmu::virt_mem_layout()`\n//! function.\n//!\n//! The `MMU` driver of the `arch` code uses `bsp::memory::mmu::virt_mem_layout()` to compile and\n//! install respective translation tables.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/memory/mmu.rs\"]\nmod arch_mmu;\n\nmod translation_table;\n\nuse crate::common;\nuse core::{fmt, ops::RangeInclusive};\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_mmu::mmu;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// MMU enable errors variants.\n#[allow(missing_docs)]\n#[derive(Debug)]\npub enum MMUEnableError {\n    AlreadyEnabled,\n    Other(&'static str),\n}\n\n/// Memory Management interfaces.\npub mod interface {\n    use super::*;\n\n    /// MMU functions.\n    pub trait MMU {\n        /// Called by the kernel during early init. Supposed to take the translation tables from the\n        /// `BSP`-supplied `virt_mem_layout()` and install/activate them for the respective MMU.\n        ///\n        /// # Safety\n        ///\n        /// - Changes the HW's global state.\n        unsafe fn enable_mmu_and_caching(&self) -> Result<(), MMUEnableError>;\n\n        /// Returns true if the MMU is enabled, false otherwise.\n        fn is_enabled(&self) -> bool;\n    }\n}\n\n/// Describes the characteristics of a translation granule.\npub struct TranslationGranule<const GRANULE_SIZE: usize>;\n\n/// Describes properties of an address space.\npub struct AddressSpace<const AS_SIZE: usize>;\n\n/// Architecture agnostic translation types.\n#[allow(missing_docs)]\n#[allow(dead_code)]\n#[derive(Copy, Clone)]\npub enum Translation {\n    Identity,\n    Offset(usize),\n}\n\n/// Architecture agnostic memory attributes.\n#[allow(missing_docs)]\n#[derive(Copy, Clone)]\npub enum MemAttributes {\n    CacheableDRAM,\n    Device,\n}\n\n/// Architecture agnostic access permissions.\n#[allow(missing_docs)]\n#[derive(Copy, Clone)]\npub enum AccessPermissions {\n    ReadOnly,\n    ReadWrite,\n}\n\n/// Collection of memory attributes.\n#[allow(missing_docs)]\n#[derive(Copy, Clone)]\npub struct AttributeFields {\n    pub mem_attributes: MemAttributes,\n    pub acc_perms: AccessPermissions,\n    pub execute_never: bool,\n}\n\n/// Architecture agnostic descriptor for a memory range.\n#[allow(missing_docs)]\npub struct TranslationDescriptor {\n    pub name: &'static str,\n    pub virtual_range: fn() -> RangeInclusive<usize>,\n    pub physical_range_translation: Translation,\n    pub attribute_fields: AttributeFields,\n}\n\n/// Type for expressing the kernel's virtual memory layout.\npub struct KernelVirtualLayout<const NUM_SPECIAL_RANGES: usize> {\n    /// The last (inclusive) address of the address space.\n    max_virt_addr_inclusive: usize,\n\n    /// Array of descriptors for non-standard (normal cacheable DRAM) memory regions.\n    inner: [TranslationDescriptor; NUM_SPECIAL_RANGES],\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl fmt::Display for MMUEnableError {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        match self {\n            MMUEnableError::AlreadyEnabled => write!(f, \"MMU is already enabled\"),\n            MMUEnableError::Other(x) => write!(f, \"{}\", x),\n        }\n    }\n}\n\nimpl<const GRANULE_SIZE: usize> TranslationGranule<GRANULE_SIZE> {\n    /// The granule's size.\n    pub const SIZE: usize = Self::size_checked();\n\n    /// The granule's shift, aka log2(size).\n    pub const SHIFT: usize = Self::SIZE.trailing_zeros() as usize;\n\n    const fn size_checked() -> usize {\n        assert!(GRANULE_SIZE.is_power_of_two());\n\n        GRANULE_SIZE\n    }\n}\n\nimpl<const AS_SIZE: usize> AddressSpace<AS_SIZE> {\n    /// The address space size.\n    pub const SIZE: usize = Self::size_checked();\n\n    /// The address space shift, aka log2(size).\n    pub const SIZE_SHIFT: usize = Self::SIZE.trailing_zeros() as usize;\n\n    const fn size_checked() -> usize {\n        assert!(AS_SIZE.is_power_of_two());\n\n        // Check for architectural restrictions as well.\n        Self::arch_address_space_size_sanity_checks();\n\n        AS_SIZE\n    }\n}\n\nimpl Default for AttributeFields {\n    fn default() -> AttributeFields {\n        AttributeFields {\n            mem_attributes: MemAttributes::CacheableDRAM,\n            acc_perms: AccessPermissions::ReadWrite,\n            execute_never: true,\n        }\n    }\n}\n\n/// Human-readable output of a TranslationDescriptor.\nimpl fmt::Display for TranslationDescriptor {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        // Call the function to which self.range points, and dereference the result, which causes\n        // Rust to copy the value.\n        let start = *(self.virtual_range)().start();\n        let end = *(self.virtual_range)().end();\n        let size = end - start + 1;\n\n        let (size, unit) = common::size_human_readable_ceil(size);\n\n        let attr = match self.attribute_fields.mem_attributes {\n            MemAttributes::CacheableDRAM => \"C\",\n            MemAttributes::Device => \"Dev\",\n        };\n\n        let acc_p = match self.attribute_fields.acc_perms {\n            AccessPermissions::ReadOnly => \"RO\",\n            AccessPermissions::ReadWrite => \"RW\",\n        };\n\n        let xn = if self.attribute_fields.execute_never {\n            \"PXN\"\n        } else {\n            \"PX\"\n        };\n\n        write!(\n            f,\n            \"      {:#010x} - {:#010x} | {: >3} {} | {: <3} {} {: <3} | {}\",\n            start, end, size, unit, attr, acc_p, xn, self.name\n        )\n    }\n}\n\nimpl<const NUM_SPECIAL_RANGES: usize> KernelVirtualLayout<{ NUM_SPECIAL_RANGES }> {\n    /// Create a new instance.\n    pub const fn new(max: usize, layout: [TranslationDescriptor; NUM_SPECIAL_RANGES]) -> Self {\n        Self {\n            max_virt_addr_inclusive: max,\n            inner: layout,\n        }\n    }\n\n    /// For a virtual address, find and return the physical output address and corresponding\n    /// attributes.\n    ///\n    /// If the address is not found in `inner`, return an identity mapped default with normal\n    /// cacheable DRAM attributes.\n    pub fn virt_addr_properties(\n        &self,\n        virt_addr: usize,\n    ) -> Result<(usize, AttributeFields), &'static str> {\n        if virt_addr > self.max_virt_addr_inclusive {\n            return Err(\"Address out of range\");\n        }\n\n        for i in self.inner.iter() {\n            if (i.virtual_range)().contains(&virt_addr) {\n                let output_addr = match i.physical_range_translation {\n                    Translation::Identity => virt_addr,\n                    Translation::Offset(a) => a + (virt_addr - (i.virtual_range)().start()),\n                };\n\n                return Ok((output_addr, i.attribute_fields));\n            }\n        }\n\n        Ok((virt_addr, AttributeFields::default()))\n    }\n\n    /// Print the memory layout.\n    pub fn print_layout(&self) {\n        use crate::info;\n\n        for i in self.inner.iter() {\n            info!(\"{}\", i);\n        }\n    }\n}\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/memory.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management.\n\npub mod mmu;\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/panic_wait.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A panic handler that infinitely waits.\n\nuse crate::{cpu, println};\nuse core::panic::PanicInfo;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Stop immediately if called a second time.\n///\n/// # Note\n///\n/// Using atomics here relieves us from needing to use `unsafe` for the static variable.\n///\n/// On `AArch64`, which is the only implemented architecture at the time of writing this,\n/// [`AtomicBool::load`] and [`AtomicBool::store`] are lowered to ordinary load and store\n/// instructions. They are therefore safe to use even with MMU + caching deactivated.\n///\n/// [`AtomicBool::load`]: core::sync::atomic::AtomicBool::load\n/// [`AtomicBool::store`]: core::sync::atomic::AtomicBool::store\nfn panic_prevent_reenter() {\n    use core::sync::atomic::{AtomicBool, Ordering};\n\n    #[cfg(not(target_arch = \"aarch64\"))]\n    compile_error!(\"Add the target_arch to above's check if the following code is safe to use\");\n\n    static PANIC_IN_PROGRESS: AtomicBool = AtomicBool::new(false);\n\n    if !PANIC_IN_PROGRESS.load(Ordering::Relaxed) {\n        PANIC_IN_PROGRESS.store(true, Ordering::Relaxed);\n\n        return;\n    }\n\n    cpu::wait_forever()\n}\n\n#[panic_handler]\nfn panic(info: &PanicInfo) -> ! {\n    // Protect against panic infinite loops if any of the following code panics itself.\n    panic_prevent_reenter();\n\n    let timestamp = crate::time::time_manager().uptime();\n    let (location, line, column) = match info.location() {\n        Some(loc) => (loc.file(), loc.line(), loc.column()),\n        _ => (\"???\", 0, 0),\n    };\n\n    println!(\n        \"[  {:>3}.{:06}] Kernel panic!\\n\\n\\\n        Panic location:\\n      File '{}', line {}, column {}\\n\\n\\\n        {}\",\n        timestamp.as_secs(),\n        timestamp.subsec_micros(),\n        location,\n        line,\n        column,\n        info.message().unwrap_or(&format_args!(\"\")),\n    );\n\n    cpu::wait_forever()\n}\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/print.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Printing.\n\nuse crate::console;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n#[doc(hidden)]\npub fn _print(args: fmt::Arguments) {\n    console::console().write_fmt(args).unwrap();\n}\n\n/// Prints without a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! print {\n    ($($arg:tt)*) => ($crate::print::_print(format_args!($($arg)*)));\n}\n\n/// Prints with a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! println {\n    () => ($crate::print!(\"\\n\"));\n    ($($arg:tt)*) => ({\n        $crate::print::_print(format_args_nl!($($arg)*));\n    })\n}\n\n/// Prints an info, with a newline.\n#[macro_export]\nmacro_rules! info {\n    ($string:expr) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[  {:>3}.{:06}] \", $string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n        ));\n    });\n    ($format_string:expr, $($arg:tt)*) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[  {:>3}.{:06}] \", $format_string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n            $($arg)*\n        ));\n    })\n}\n\n/// Prints a warning, with a newline.\n#[macro_export]\nmacro_rules! warn {\n    ($string:expr) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[W {:>3}.{:06}] \", $string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n        ));\n    });\n    ($format_string:expr, $($arg:tt)*) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[W {:>3}.{:06}] \", $format_string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n            $($arg)*\n        ));\n    })\n}\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/synchronization.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Synchronization primitives.\n//!\n//! # Resources\n//!\n//!   - <https://doc.rust-lang.org/book/ch16-04-extensible-concurrency-sync-and-send.html>\n//!   - <https://stackoverflow.com/questions/59428096/understanding-the-send-trait>\n//!   - <https://doc.rust-lang.org/std/cell/index.html>\n\nuse core::cell::UnsafeCell;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Synchronization interfaces.\npub mod interface {\n\n    /// Any object implementing this trait guarantees exclusive access to the data wrapped within\n    /// the Mutex for the duration of the provided closure.\n    pub trait Mutex {\n        /// The type of the data that is wrapped by this mutex.\n        type Data;\n\n        /// Locks the mutex and grants the closure temporary mutable access to the wrapped data.\n        fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R;\n    }\n}\n\n/// A pseudo-lock for teaching purposes.\n///\n/// In contrast to a real Mutex implementation, does not protect against concurrent access from\n/// other cores to the contained data. This part is preserved for later lessons.\n///\n/// The lock will only be used as long as it is safe to do so, i.e. as long as the kernel is\n/// executing single-threaded, aka only running on a single core with interrupts disabled.\npub struct NullLock<T>\nwhere\n    T: ?Sized,\n{\n    data: UnsafeCell<T>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nunsafe impl<T> Send for NullLock<T> where T: ?Sized + Send {}\nunsafe impl<T> Sync for NullLock<T> where T: ?Sized + Send {}\n\nimpl<T> NullLock<T> {\n    /// Create an instance.\n    pub const fn new(data: T) -> Self {\n        Self {\n            data: UnsafeCell::new(data),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\n\nimpl<T> interface::Mutex for NullLock<T> {\n    type Data = T;\n\n    fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R {\n        // In a real lock, there would be code encapsulating this line that ensures that this\n        // mutable reference will ever only be given out once at a time.\n        let data = unsafe { &mut *self.data.get() };\n\n        f(data)\n    }\n}\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/src/time.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Timer primitives.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/time.rs\"]\nmod arch_time;\n\nuse core::time::Duration;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Provides time management functions.\npub struct TimeManager;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic TIME_MANAGER: TimeManager = TimeManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the global TimeManager.\npub fn time_manager() -> &'static TimeManager {\n    &TIME_MANAGER\n}\n\nimpl TimeManager {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self\n    }\n\n    /// The timer's resolution.\n    pub fn resolution(&self) -> Duration {\n        arch_time::resolution()\n    }\n\n    /// The uptime since power-on of the device.\n    ///\n    /// This includes time consumed by firmware and bootloaders.\n    pub fn uptime(&self) -> Duration {\n        arch_time::uptime()\n    }\n\n    /// Spin for a given duration.\n    pub fn spin_for(&self, duration: Duration) {\n        arch_time::spin_for(duration)\n    }\n}\n"
  },
  {
    "path": "10_virtual_mem_part1_identity_mapping/tests/boot_test_string.rb",
    "content": "# frozen_string_literal: true\n\nEXPECTED_PRINT = 'Echoing input now'\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/.vscode/settings.json",
    "content": "{\n    \"editor.formatOnSave\": true,\n    \"editor.rulers\": [100],\n    \"rust-analyzer.cargo.target\": \"aarch64-unknown-none-softfloat\",\n    \"rust-analyzer.cargo.features\": [\"bsp_rpi3\"],\n    \"rust-analyzer.checkOnSave.allTargets\": false,\n    \"rust-analyzer.checkOnSave.extraArgs\": [\"--bins\"],\n    \"rust-analyzer.lens.debug\": false,\n    \"rust-analyzer.lens.run\": false\n}\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/Cargo.toml",
    "content": "[package]\nname = \"mingo\"\nversion = \"0.11.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n\n[profile.release]\nlto = true\n\n[features]\ndefault = []\nbsp_rpi3 = [\"tock-registers\"]\nbsp_rpi4 = [\"tock-registers\"]\n\n[[bin]]\nname = \"kernel\"\npath = \"src/main.rs\"\n\n##--------------------------------------------------------------------------------------------------\n## Dependencies\n##--------------------------------------------------------------------------------------------------\n\n[dependencies]\n\n# Optional dependencies\ntock-registers = { version = \"0.8.x\", default-features = false, features = [\"register_types\"], optional = true }\n\n# Platform specific dependencies\n[target.'cfg(target_arch = \"aarch64\")'.dependencies]\naarch64-cpu = { version = \"9.x.x\" }\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/Makefile",
    "content": "## SPDX-License-Identifier: MIT OR Apache-2.0\n##\n## Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\ninclude ../common/docker.mk\ninclude ../common/format.mk\ninclude ../common/operating_system.mk\n\n##--------------------------------------------------------------------------------------------------\n## Optional, user-provided configuration values\n##--------------------------------------------------------------------------------------------------\n\n# Default to the RPi3.\nBSP ?= rpi3\n\n# Default to a serial device name that is common in Linux.\nDEV_SERIAL ?= /dev/ttyUSB0\n\n\n\n##--------------------------------------------------------------------------------------------------\n## BSP-specific configuration values\n##--------------------------------------------------------------------------------------------------\nQEMU_MISSING_STRING = \"This board is not yet supported for QEMU.\"\n\nifeq ($(BSP),rpi3)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE = raspi3\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    OPENOCD_ARG       = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi3.cfg\n    JTAG_BOOT_IMAGE   = ../X1_JTAG_boot/jtag_boot_rpi3.img\n    LD_SCRIPT_PATH    = $(shell pwd)/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a53\nelse ifeq ($(BSP),rpi4)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE =\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    OPENOCD_ARG       = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi4.cfg\n    JTAG_BOOT_IMAGE   = ../X1_JTAG_boot/jtag_boot_rpi4.img\n    LD_SCRIPT_PATH    = $(shell pwd)/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a72\nendif\n\n# Export for build.rs.\nexport LD_SCRIPT_PATH\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets and Prerequisites\n##--------------------------------------------------------------------------------------------------\nKERNEL_MANIFEST      = Cargo.toml\nKERNEL_LINKER_SCRIPT = kernel.ld\nLAST_BUILD_CONFIG    = target/$(BSP).build_config\n\nKERNEL_ELF      = target/$(TARGET)/release/kernel\n# This parses cargo's dep-info file.\n# https://doc.rust-lang.org/cargo/guide/build-cache.html#dep-info-files\nKERNEL_ELF_DEPS = $(filter-out %: ,$(file < $(KERNEL_ELF).d)) $(KERNEL_MANIFEST) $(LAST_BUILD_CONFIG)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Command building blocks\n##--------------------------------------------------------------------------------------------------\nRUSTFLAGS = $(RUSTC_MISC_ARGS)                   \\\n    -C link-arg=--library-path=$(LD_SCRIPT_PATH) \\\n    -C link-arg=--script=$(KERNEL_LINKER_SCRIPT)\n\nRUSTFLAGS_PEDANTIC = $(RUSTFLAGS) \\\n    -D warnings                   \\\n    -D missing_docs\n\nFEATURES      = --features bsp_$(BSP)\nCOMPILER_ARGS = --target=$(TARGET) \\\n    $(FEATURES)                    \\\n    --release\n\nRUSTC_CMD   = cargo rustc $(COMPILER_ARGS)\nDOC_CMD     = cargo doc $(COMPILER_ARGS)\nCLIPPY_CMD  = cargo clippy $(COMPILER_ARGS)\nOBJCOPY_CMD = rust-objcopy \\\n    --strip-all            \\\n    -O binary\n\nEXEC_QEMU          = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE)\nEXEC_TEST_DISPATCH = ruby ../common/tests/dispatch.rb\nEXEC_MINIPUSH      = ruby ../common/serial/minipush.rb\n\n##------------------------------------------------------------------------------\n## Dockerization\n##------------------------------------------------------------------------------\nDOCKER_CMD            = docker run -t --rm -v $(shell pwd):/work/tutorial -w /work/tutorial\nDOCKER_CMD_INTERACT   = $(DOCKER_CMD) -i\nDOCKER_ARG_DIR_COMMON = -v $(shell pwd)/../common:/work/common\nDOCKER_ARG_DIR_JTAG   = -v $(shell pwd)/../X1_JTAG_boot:/work/X1_JTAG_boot\nDOCKER_ARG_DEV        = --privileged -v /dev:/dev\nDOCKER_ARG_NET        = --network host\n\n# DOCKER_IMAGE defined in include file (see top of this file).\nDOCKER_QEMU  = $(DOCKER_CMD_INTERACT) $(DOCKER_IMAGE)\nDOCKER_TOOLS = $(DOCKER_CMD) $(DOCKER_IMAGE)\nDOCKER_TEST  = $(DOCKER_CMD) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\nDOCKER_GDB   = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_NET) $(DOCKER_IMAGE)\n\n# Dockerize commands, which require USB device passthrough, only on Linux.\nifeq ($(shell uname -s),Linux)\n    DOCKER_CMD_DEV = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_DEV)\n\n    DOCKER_CHAINBOOT = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\n    DOCKER_JTAGBOOT  = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_ARG_DIR_JTAG) $(DOCKER_IMAGE)\n    DOCKER_OPENOCD   = $(DOCKER_CMD_DEV) $(DOCKER_ARG_NET) $(DOCKER_IMAGE)\nelse\n    DOCKER_OPENOCD   = echo \"Not yet supported on non-Linux systems.\"; \\#\nendif\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: all doc qemu chainboot clippy clean readelf objdump nm check\n\nall: $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Save the configuration as a file, so make understands if it changed.\n##------------------------------------------------------------------------------\n$(LAST_BUILD_CONFIG):\n\t@rm -f target/*.build_config\n\t@mkdir -p target\n\t@touch $(LAST_BUILD_CONFIG)\n\n##------------------------------------------------------------------------------\n## Compile the kernel ELF\n##------------------------------------------------------------------------------\n$(KERNEL_ELF): $(KERNEL_ELF_DEPS)\n\t$(call color_header, \"Compiling kernel ELF - $(BSP)\")\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(RUSTC_CMD)\n\n##------------------------------------------------------------------------------\n## Generate the stripped kernel binary\n##------------------------------------------------------------------------------\n$(KERNEL_BIN): $(KERNEL_ELF)\n\t$(call color_header, \"Generating stripped binary\")\n\t@$(OBJCOPY_CMD) $(KERNEL_ELF) $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Name\")\n\t@echo $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Size\")\n\t$(call disk_usage_KiB, $(KERNEL_BIN))\n\n##------------------------------------------------------------------------------\n## Generate the documentation\n##------------------------------------------------------------------------------\ndoc:\n\t$(call color_header, \"Generating docs\")\n\t@$(DOC_CMD) --document-private-items --open\n\n##------------------------------------------------------------------------------\n## Run the kernel in QEMU\n##------------------------------------------------------------------------------\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\nqemu:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\nqemu: $(KERNEL_BIN)\n\t$(call color_header, \"Launching QEMU\")\n\t@$(DOCKER_QEMU) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\nendif\n\n##------------------------------------------------------------------------------\n## Push the kernel to the real HW target\n##------------------------------------------------------------------------------\nchainboot: $(KERNEL_BIN)\n\t@$(DOCKER_CHAINBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run clippy\n##------------------------------------------------------------------------------\nclippy:\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(CLIPPY_CMD)\n\n##------------------------------------------------------------------------------\n## Clean\n##------------------------------------------------------------------------------\nclean:\n\trm -rf target $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run readelf\n##------------------------------------------------------------------------------\nreadelf: $(KERNEL_ELF)\n\t$(call color_header, \"Launching readelf\")\n\t@$(DOCKER_TOOLS) $(READELF_BINARY) --headers $(KERNEL_ELF)\n\n##------------------------------------------------------------------------------\n## Run objdump\n##------------------------------------------------------------------------------\nobjdump: $(KERNEL_ELF)\n\t$(call color_header, \"Launching objdump\")\n\t@$(DOCKER_TOOLS) $(OBJDUMP_BINARY) --disassemble --demangle \\\n                --section .text   \\\n                --section .rodata \\\n                $(KERNEL_ELF) | rustfilt\n\n##------------------------------------------------------------------------------\n## Run nm\n##------------------------------------------------------------------------------\nnm: $(KERNEL_ELF)\n\t$(call color_header, \"Launching nm\")\n\t@$(DOCKER_TOOLS) $(NM_BINARY) --demangle --print-size $(KERNEL_ELF) | sort | rustfilt\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Debugging targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: jtagboot openocd gdb gdb-opt0\n\n##------------------------------------------------------------------------------\n## Push the JTAG boot image to the real HW target\n##------------------------------------------------------------------------------\njtagboot:\n\t@$(DOCKER_JTAGBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(JTAG_BOOT_IMAGE)\n\n##------------------------------------------------------------------------------\n## Start OpenOCD session\n##------------------------------------------------------------------------------\nopenocd:\n\t$(call color_header, \"Launching OpenOCD\")\n\t@$(DOCKER_OPENOCD) openocd $(OPENOCD_ARG)\n\n##------------------------------------------------------------------------------\n## Start GDB session\n##------------------------------------------------------------------------------\ngdb: RUSTC_MISC_ARGS += -C debuginfo=2\ngdb-opt0: RUSTC_MISC_ARGS += -C debuginfo=2 -C opt-level=0\ngdb gdb-opt0: $(KERNEL_ELF)\n\t$(call color_header, \"Launching GDB\")\n\t@$(DOCKER_GDB) gdb-multiarch -q $(KERNEL_ELF)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Testing targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: test test_boot\n\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\ntest_boot test:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\n##------------------------------------------------------------------------------\n## Run boot test\n##------------------------------------------------------------------------------\ntest_boot: $(KERNEL_BIN)\n\t$(call color_header, \"Boot test - $(BSP)\")\n\t@$(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\ntest: test_boot\n\nendif\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/README.md",
    "content": "# Tutorial 11 - Exceptions Part 1: Groundwork\n\n## tl;dr\n\n- We lay the groundwork for all the architectural `CPU exceptions`.\n- For now, only print an elaborate system state through a `panic!` call, and halt execution\n- This will help finding bugs during development and runtime.\n- For demo purposes, MMU `page faults` are used to demonstrate (i) returning from an exception and\n  (ii) the default `panic!` behavior.\n\n## Table of Contents\n\n- [Introduction](#introduction)\n- [Exception Types](#exception-types)\n- [Exception entry](#exception-entry)\n  * [Exception Vectors](#exception-vectors)\n- [Handler Code and Offsets](#handler-code-and-offsets)\n- [Rust and Assembly Implementation](#rust-and-assembly-implementation)\n  * [Context Save and Restore](#context-save-and-restore)\n  * [Exception Vector Table](#exception-vector-table)\n  * [Implementing handlers](#implementing-handlers)\n- [Causing an Exception - Testing the Code](#causing-an-exception---testing-the-code)\n- [Test it](#test-it)\n- [Diff to previous](#diff-to-previous)\n\n## Introduction\n\nNow that we are executing in `EL1`, and have activated the `MMU`, time is due for implementing `CPU\nexceptions`. For now, we only set up a scaffold with very basic functionality that will help us to\nfind bugs along the way. A follow-up `Interrupt` tutorial later will continue the work we start\nhere.\n\nPlease note that this tutorial is specific to the `AArch64` architecture. It does not contain any\ngeneric exception handling code yet.\n\n## Exception Types\n\nIn `AArch64`, it is differentiated between four types of exceptions. These are:\n- Synchronous\n  - For example, a `data abort` (e.g. `page fault`) or a `system call`. They happen in direct\n    consequence of executing a certain CPU instruction, hence _synchronously_.\n- Interrupt Request (`IRQ`)\n  - For example, an external device, like a timer, is asserting a physical interrupt line. IRQs\n    happen _asynchronously_.\n- Fast Interrupt Request (`FIQ`)\n  - These are basically interrupts that take priority over normal IRQs and have some more traits\n    that make them suitable to implement super-fast processing. However, this is out of scope for\n    this tutorial. For the sake of keeping these tutorials compact and concise, we will more or less\n    ignore FIQs and only implement a dummy handler that would halt the CPU core.\n- System Error (`SError`)\n  - Like IRQs, SErrors happen asynchronously and are technically more or less the same. They are\n    intended to signal rather fatal errors in the system, e.g. if a transaction times out on the\n    `SoC` interconnect. They are very implementation specific and it is up to the SoC vendor to\n    decide which events are delivered as SErrors instead of normal IRQs.\n\n## Exception entry\n\nI recommend to read pages D1-5355 of the [ARMv8 Architecture Reference Manual][ARMv8_Manual Ja] to\nunderstand the mechanisms of taking an exception.\n\nHere's an excerpt of important features for this tutorial:\n- Exception entry moves the processor to the same or a higher `Exception Level`, but never to a\n  lower `EL`.\n- The program status is saved in the `SPSR_ELx` register at the target `EL`.\n- The preferred return address is saved in the `ELR_ELx` register.\n  - \"Preferred\" here means that `ELR_ELx` may hold the instruction address of the instructions that\n    caused the exception (`synchronous case`) or the first instruction that did not complete due to\n    an `asynchronous` exception. Details in pages D1-5357 of the [ARMv8 Architecture Reference\n    Manual][ARMv8_Manual Ja].\n- All kinds of exceptions are turned off upon taking an exception, so that by default, exception\n  handlers can not get interrupted themselves.\n- Taking an exception will select the dedicated stack pointer of the target `EL`.\n  - For example, if an exception in `EL0` is taken, the Stack Pointer Select register `SPSel` will\n    switch from `0` to `1`, meaning that `SP_EL1` will be used by the exception vector code unless\n    you explicitly change it back to `SP_EL0`.\n\n\n### Exception Vectors\n\n`AArch64` has a total of `16` exception vectors. There is one for each of the four kinds that were\nintroduced already, and additionally, it is taken into account _where_ the exception was taken from\nand what the circumstances were.\n\nHere is a copy of the decision table as shown in pages D1-5358 of the [ARMv8 Architecture\nReference Manual][ARMv8_Manual Ja]:\n\n[ARMv8_Manual Ja]: https://developer.arm.com/documentation/ddi0487/ja/\n\n<table>\n    <thead>\n        <tr>\n            <th rowspan=2>Exception taken from </th>\n            <th colspan=4>Offset for exception type</th>\n        </tr>\n        <tr>\n            <th>Synchronous</th>\n            <th>IRQ or vIRQ</th>\n            <th>FIQ or vFIQ</th>\n            <th>SError or vSError</th>\n        </tr>\n    </thead>\n    <tbody>\n        <tr>\n            <td width=\"40%\">Current Exception level with SP_EL0.</td>\n            <td align=\"center\">0x000</td>\n            <td align=\"center\">0x080</td>\n            <td align=\"center\">0x100</td>\n            <td align=\"center\">0x180</td>\n        </tr>\n        <tr>\n            <td>Current Exception level with SP_ELx, x>0.</td>\n            <td align=\"center\">0x200</td>\n            <td align=\"center\">0x280</td>\n            <td align=\"center\">0x300</td>\n            <td align=\"center\">0x380</td>\n        </tr>\n        <tr>\n            <td>Lower Exception level, where the implemented level immediately lower than the target level is using AArch64.</td>\n            <td align=\"center\">0x400</td>\n            <td align=\"center\">0x480</td>\n            <td align=\"center\">0x500</td>\n            <td align=\"center\">0x580</td>\n        </tr>\n        <tr>\n            <td>Lower Exception level, where the implemented level immediately lower than the target level is using AArch32.</td>\n            <td align=\"center\">0x600</td>\n            <td align=\"center\">0x680</td>\n            <td align=\"center\">0x700</td>\n            <td align=\"center\">0x780</td>\n        </tr>\n    </tbody>\n</table>\n\nSince our kernel runs in `EL1`, using `SP_EL1`, if we'd cause a synchronous exception, the exception\nvector at offset `0x200` would be executed. But what does that even mean?\n\n## Handler Code and Offsets\n\nIn many architectures, Operating Systems register their exception handlers (aka vectors) by\ncompiling an architecturally defined data structure that stores function pointers to the different\nhandlers. This can be as simple as an ordinary array of function pointers. The `base address` of\nthis data structure is then stored into a special purpose register so that the CPU can branch to the\nrespective handler function upon taking an exception. The classic `x86_64` architecture follows this\nprinciple, for example.\n\nIn `AArch64`, it is a bit different. Here, we have the special purpose register as well, called\n`VBAR_EL1`: Vector Base Address Register.\n\nHowever, it does not store the base address of an array of function pointers, but the base address\nof a **memory location that contains code** for the 16 handlers, one handler back-to-back after the\nother. Each handler can take a maximum space of `0x80` bytes, aka `128` bytes. That's why the table\nabove shows `offsets`: To indicate at which offset a certain handler starts.\n\nOf course, you are not obliged to cram all your handler code into only 128 bytes. You are free to\nbranch off to any other functions at any time. Actually, that is needed in most cases anyways,\nbecause the context-saving code alone would take up most of the available space (you'll learn what\ncontext saving is shortly).\n\nAdditionally, there is a requirement that the `Vector Base Address` is aligned to `0x800` aka `2048`\nbytes.\n\n## Rust and Assembly Implementation\n\nThe implementation uses a mix of `Rust` and `Assembly` code.\n\n### Context Save and Restore\n\nException vectors, just like any other code, use a bunch of commonly shared processor resources.\nMost of all, the set of `General Purpose Registers` (GPRs) that each core in `AArch64` provides\n(`x0`-`x30`).\n\nIn order to not taint these registers when executing exception vector code, it is general practice\nto save these shared resources in memory (the stack, to be precise) as the very first action. This\nis commonly described as *saving the context*. Exception vector code can then use the shared\nresources in its own code without bothering, and as a last action before returning from exception\nhandling code, restore the context, so that the processor can continue where it left off before\ntaking the exception.\n\nContext save and restore is one of the few places in system software where there is no way around\nsome hand-crafted assembly. Introducing `exception.s`:\n\n```asm\n/// Call the function provided by parameter `\\handler` after saving the exception context. Provide\n/// the context as the first parameter to '\\handler'.\n.macro CALL_WITH_CONTEXT handler\n__vector_\\handler:\n\t// Make room on the stack for the exception context.\n\tsub\tsp,  sp,  #16 * 17\n\n\t// Store all general purpose registers on the stack.\n\tstp\tx0,  x1,  [sp, #16 * 0]\n\tstp\tx2,  x3,  [sp, #16 * 1]\n\tstp\tx4,  x5,  [sp, #16 * 2]\n\tstp\tx6,  x7,  [sp, #16 * 3]\n\tstp\tx8,  x9,  [sp, #16 * 4]\n\tstp\tx10, x11, [sp, #16 * 5]\n\tstp\tx12, x13, [sp, #16 * 6]\n\tstp\tx14, x15, [sp, #16 * 7]\n\tstp\tx16, x17, [sp, #16 * 8]\n\tstp\tx18, x19, [sp, #16 * 9]\n\tstp\tx20, x21, [sp, #16 * 10]\n\tstp\tx22, x23, [sp, #16 * 11]\n\tstp\tx24, x25, [sp, #16 * 12]\n\tstp\tx26, x27, [sp, #16 * 13]\n\tstp\tx28, x29, [sp, #16 * 14]\n\n\t// Add the exception link register (ELR_EL1), saved program status (SPSR_EL1) and exception\n\t// syndrome register (ESR_EL1).\n\tmrs\tx1,  ELR_EL1\n\tmrs\tx2,  SPSR_EL1\n\tmrs\tx3,  ESR_EL1\n\n\tstp\tlr,  x1,  [sp, #16 * 15]\n\tstp\tx2,  x3,  [sp, #16 * 16]\n\n\t// x0 is the first argument for the function called through `\\handler`.\n\tmov\tx0,  sp\n\n\t// Call `\\handler`.\n\tbl\t\\handler\n\n\t// After returning from exception handling code, replay the saved context and return via\n\t// `eret`.\n\tb\t__exception_restore_context\n\n.size\t__vector_\\handler, . - __vector_\\handler\n.type\t__vector_\\handler, function\n.endm\n```\n\nFirst, a macro for saving the context is defined. It eventually jumps to follow-up handler code, and\nfinally restores the context. In advance, we reserve space on the stack for the context. That is,\nthe 30 `GPRs`, the `link register`, the `exception link register` (holding the preferred return\naddress), the `saved program status` and the `exception syndrome register`. Afterwards, we store\nthose registers, save the current stack address in `x0` and branch off to follow-up handler-code,\nwhose function name is supplied as an argument to the macro (`\\handler`).\n\nThe handler code will be written in Rust, but use the platform's `C` ABI. This way, we can define a\nfunction signature that has a pointer to the context-data on the stack as its first argument, and\nknow that this argument is expected to be in the `x0` register. We need to use the `C` ABI here\nbecause `Rust` has no stable convention ([yet](https://github.com/rust-lang/rfcs/issues/600)).\n\n### Exception Vector Table\n\nNext, we craft the exception vector table:\n\n```asm\n// Align by 2^11 bytes, as demanded by ARMv8-A. Same as ALIGN(2048) in an ld script.\n.align 11\n\n// Export a symbol for the Rust code to use.\n__exception_vector_start:\n\n// Current exception level with SP_EL0.\n//\n// .org sets the offset relative to section start.\n//\n// # Safety\n//\n// - It must be ensured that `CALL_WITH_CONTEXT` <= 0x80 bytes.\n.org 0x000\n\tCALL_WITH_CONTEXT current_el0_synchronous\n.org 0x080\n\tCALL_WITH_CONTEXT current_el0_irq\n.org 0x100\n\tFIQ_SUSPEND\n.org 0x180\n\tCALL_WITH_CONTEXT current_el0_serror\n\n// Current exception level with SP_ELx, x > 0.\n.org 0x200\n\tCALL_WITH_CONTEXT current_elx_synchronous\n.org 0x280\n\tCALL_WITH_CONTEXT current_elx_irq\n.org 0x300\n\tFIQ_SUSPEND\n.org 0x380\n\tCALL_WITH_CONTEXT current_elx_serror\n\n[...]\n```\n\nNote how each vector starts at the required offset from the section start using the `.org`\ndirective. Each macro call introduces an explicit handler function name, which is implemented in\n`Rust` in `exception.rs`.\n\n### Implementing handlers\n\nThe file `exception.rs` first defines a `struct` of the exception context that is stored on the\nstack by the assembly code:\n\n```rust\n/// The exception context as it is stored on the stack on exception entry.\n#[repr(C)]\nstruct ExceptionContext {\n    /// General Purpose Registers.\n    gpr: [u64; 30],\n\n    /// The link register, aka x30.\n    lr: u64,\n\n    /// Exception link register. The program counter at the time the exception happened.\n    elr_el1: u64,\n\n    /// Saved program status.\n    spsr_el1: SpsrEL1,\n\n    // Exception syndrome register.\n    esr_el1: EsrEL1,\n}\n```\n\nThe handlers take a `struct ExceptionContext` argument. Since we do not plan to implement handlers\nfor each exception yet, a default handler is provided:\n\n```rust\n/// Prints verbose information about the exception and then panics.\nfn default_exception_handler(exc: &ExceptionContext) {\n    panic!(\n        \"CPU Exception!\\n\\n\\\n        {}\",\n        exc\n    );\n}\n```\n\nThe actual handlers referenced from the assembly can now branch to it for the time being, e.g.:\n\n```rust\n#[no_mangle]\nextern \"C\" fn current_elx_irq(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n```\n\n## Causing an Exception - Testing the Code\n\nWe want to see two cases in action:\n1. How taking, handling and returning from an exception works.\n2. How the `panic!` print for unhandled exceptions looks like.\n\nSo after setting up exceptions in `main.rs` by calling\n\n```rust\nexception::handling_init();\n```\n\nwe cause a data abort exception by reading from memory address `8 GiB`:\n\n```rust\n// Cause an exception by accessing a virtual address for which no translation was set up. This\n// code accesses the address 8 GiB, which is outside the mapped address space.\n//\n// For demo purposes, the exception handler will catch the faulting 8 GiB address and allow\n// execution to continue.\ninfo!(\"\");\ninfo!(\"Trying to read from address 8 GiB...\");\nlet mut big_addr: u64 = 8 * 1024 * 1024 * 1024;\nunsafe { core::ptr::read_volatile(big_addr as *mut u64) };\n```\n\nThis triggers our exception code, because we try to read from a virtual address for which no mapping\nhas been installed. Remember, we only mapped up to `4 GiB` of address space in the previous\ntutorial.\n\nTo survive this exception, the respective handler has a special demo case:\n\n```rust\n#[no_mangle]\nextern \"C\" fn current_elx_synchronous(e: &mut ExceptionContext) {\n    if e.fault_address_valid() {\n        let far_el1 = FAR_EL1.get();\n\n        // This catches the demo case for this tutorial. If the fault address happens to be 8 GiB,\n        // advance the exception link register for one instruction, so that execution can continue.\n        if far_el1 == 8 * 1024 * 1024 * 1024 {\n            e.elr_el1 += 4;\n\n            return;\n        }\n    }\n\n    default_exception_handler(e);\n}\n```\n\nIt checks if the faulting address equals `8 GiB`, and if so, advances the copy of the `ELR` by 4,\nwhich makes it point to the next instruction after the instruction that caused the exception. When\nthis handler returns, execution continues in the assembly macro we introduced before. The macro has\nonly one more line left: `b __exception_restore_context`, which jumps to an assembly function that\nplays back our saved context before finally executing `eret` to return from the exception.\n\nThis will kick us back into `main.rs`. But we also want to see the `panic!` print.\n\nTherefore, a second read is done, this time from address `9 GiB`. A case which the handler will not\ncatch, eventually triggering the `panic!` call from the default handler.\n\n## Test it\n\n```console\n$ make chainboot\n[...]\nMinipush 1.0\n\n[MP] ⏳ Waiting for /dev/ttyUSB0\n[MP] ✅ Serial connected\n[MP] 🔌 Please power the target now\n\n __  __ _      _ _                 _\n|  \\/  (_)_ _ (_) |   ___  __ _ __| |\n| |\\/| | | ' \\| | |__/ _ \\/ _` / _` |\n|_|  |_|_|_||_|_|____\\___/\\__,_\\__,_|\n\n           Raspberry Pi 3\n\n[ML] Requesting binary\n[MP] ⏩ Pushing 64 KiB =========================================🦀 100% 0 KiB/s Time: 00:00:00\n[ML] Loaded! Executing the payload now\n\n[    0.798323] mingo version 0.11.0\n[    0.798530] Booting on: Raspberry Pi 3\n[    0.798985] MMU online. Special regions:\n[    0.799462]       0x00080000 - 0x0008ffff |  64 KiB | C   RO PX  | Kernel code and RO data\n[    0.800480]       0x3f000000 - 0x4000ffff |  17 MiB | Dev RW PXN | Device MMIO\n[    0.801369] Current privilege level: EL1\n[    0.801845] Exception handling state:\n[    0.802290]       Debug:  Masked\n[    0.802680]       SError: Masked\n[    0.803069]       IRQ:    Masked\n[    0.803459]       FIQ:    Masked\n[    0.803849] Architectural timer resolution: 52 ns\n[    0.804423] Drivers loaded:\n[    0.804759]       1. BCM PL011 UART\n[    0.805182]       2. BCM GPIO\n[    0.805539] Timer test, spinning for 1 second\n[    1.806070]\n[    1.806074] Trying to read from address 8 GiB...\n[    1.806624] ************************************************\n[    1.807316] Whoa! We recovered from a synchronous exception!\n[    1.808009] ************************************************\n[    1.808703]\n[    1.808876] Let's try again\n[    1.809212] Trying to read from address 9 GiB...\n[    1.809776] Kernel panic!\n\nPanic location:\n      File 'src/_arch/aarch64/exception.rs', line 58, column 5\n\nCPU Exception!\n\nESR_EL1: 0x96000004\n      Exception Class         (EC) : 0x25 - Data Abort, current EL\n      Instr Specific Syndrome (ISS): 0x4\nFAR_EL1: 0x0000000240000000\nSPSR_EL1: 0x600003c5\n      Flags:\n            Negative (N): Not set\n            Zero     (Z): Set\n            Carry    (C): Set\n            Overflow (V): Not set\n      Exception handling state:\n            Debug  (D): Masked\n            SError (A): Masked\n            IRQ    (I): Masked\n            FIQ    (F): Masked\n      Illegal Execution State (IL): Not set\nELR_EL1: 0x00000000000845f8\n\nGeneral purpose register:\n      x0 : 0x0000000000000000         x1 : 0x0000000000086187\n      x2 : 0x0000000000000027         x3 : 0x0000000000081280\n      x4 : 0x0000000000000006         x5 : 0x1e27329c00000000\n      x6 : 0x0000000000000000         x7 : 0xd3d18908028f0243\n      x8 : 0x0000000240000000         x9 : 0x0000000000086187\n      x10: 0x0000000000000443         x11: 0x000000003f201000\n      x12: 0x0000000000000019         x13: 0x00000000ffffd8f0\n      x14: 0x000000000000147b         x15: 0x00000000ffffff9c\n      x16: 0x000000000007fd38         x17: 0x0000000005f5e0ff\n      x18: 0x00000000000c58fc         x19: 0x0000000000090008\n      x20: 0x0000000000085fc0         x21: 0x000000003b9aca00\n      x22: 0x0000000000082238         x23: 0x00000000000813d4\n      x24: 0x0000000010624dd3         x25: 0xffffffffc4653600\n      x26: 0x0000000000086988         x27: 0x0000000000086080\n      x28: 0x0000000000085f10         x29: 0x0000000000085c00\n      lr : 0x00000000000845ec\n```\n\n## Diff to previous\n```diff\n\ndiff -uNr 10_virtual_mem_part1_identity_mapping/Cargo.toml 11_exceptions_part1_groundwork/Cargo.toml\n--- 10_virtual_mem_part1_identity_mapping/Cargo.toml\n+++ 11_exceptions_part1_groundwork/Cargo.toml\n@@ -1,6 +1,6 @@\n [package]\n name = \"mingo\"\n-version = \"0.10.0\"\n+version = \"0.11.0\"\n authors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\n edition = \"2021\"\n\n\ndiff -uNr 10_virtual_mem_part1_identity_mapping/src/_arch/aarch64/exception.rs 11_exceptions_part1_groundwork/src/_arch/aarch64/exception.rs\n--- 10_virtual_mem_part1_identity_mapping/src/_arch/aarch64/exception.rs\n+++ 11_exceptions_part1_groundwork/src/_arch/aarch64/exception.rs\n@@ -11,8 +11,263 @@\n //!\n //! crate::exception::arch_exception\n\n-use aarch64_cpu::registers::*;\n-use tock_registers::interfaces::Readable;\n+use aarch64_cpu::{asm::barrier, registers::*};\n+use core::{arch::global_asm, cell::UnsafeCell, fmt};\n+use tock_registers::{\n+    interfaces::{Readable, Writeable},\n+    registers::InMemoryRegister,\n+};\n+\n+// Assembly counterpart to this file.\n+global_asm!(include_str!(\"exception.s\"));\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Wrapper structs for memory copies of registers.\n+#[repr(transparent)]\n+struct SpsrEL1(InMemoryRegister<u64, SPSR_EL1::Register>);\n+struct EsrEL1(InMemoryRegister<u64, ESR_EL1::Register>);\n+\n+/// The exception context as it is stored on the stack on exception entry.\n+#[repr(C)]\n+struct ExceptionContext {\n+    /// General Purpose Registers.\n+    gpr: [u64; 30],\n+\n+    /// The link register, aka x30.\n+    lr: u64,\n+\n+    /// Exception link register. The program counter at the time the exception happened.\n+    elr_el1: u64,\n+\n+    /// Saved program status.\n+    spsr_el1: SpsrEL1,\n+\n+    /// Exception syndrome register.\n+    esr_el1: EsrEL1,\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Code\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Prints verbose information about the exception and then panics.\n+fn default_exception_handler(exc: &ExceptionContext) {\n+    panic!(\n+        \"CPU Exception!\\n\\n\\\n+        {}\",\n+        exc\n+    );\n+}\n+\n+//------------------------------------------------------------------------------\n+// Current, EL0\n+//------------------------------------------------------------------------------\n+\n+#[no_mangle]\n+extern \"C\" fn current_el0_synchronous(_e: &mut ExceptionContext) {\n+    panic!(\"Should not be here. Use of SP_EL0 in EL1 is not supported.\")\n+}\n+\n+#[no_mangle]\n+extern \"C\" fn current_el0_irq(_e: &mut ExceptionContext) {\n+    panic!(\"Should not be here. Use of SP_EL0 in EL1 is not supported.\")\n+}\n+\n+#[no_mangle]\n+extern \"C\" fn current_el0_serror(_e: &mut ExceptionContext) {\n+    panic!(\"Should not be here. Use of SP_EL0 in EL1 is not supported.\")\n+}\n+\n+//------------------------------------------------------------------------------\n+// Current, ELx\n+//------------------------------------------------------------------------------\n+\n+#[no_mangle]\n+extern \"C\" fn current_elx_synchronous(e: &mut ExceptionContext) {\n+    if e.fault_address_valid() {\n+        let far_el1 = FAR_EL1.get();\n+\n+        // This catches the demo case for this tutorial. If the fault address happens to be 8 GiB,\n+        // advance the exception link register for one instruction, so that execution can continue.\n+        if far_el1 == 8 * 1024 * 1024 * 1024 {\n+            e.elr_el1 += 4;\n+\n+            return;\n+        }\n+    }\n+\n+    default_exception_handler(e);\n+}\n+\n+#[no_mangle]\n+extern \"C\" fn current_elx_irq(e: &mut ExceptionContext) {\n+    default_exception_handler(e);\n+}\n+\n+#[no_mangle]\n+extern \"C\" fn current_elx_serror(e: &mut ExceptionContext) {\n+    default_exception_handler(e);\n+}\n+\n+//------------------------------------------------------------------------------\n+// Lower, AArch64\n+//------------------------------------------------------------------------------\n+\n+#[no_mangle]\n+extern \"C\" fn lower_aarch64_synchronous(e: &mut ExceptionContext) {\n+    default_exception_handler(e);\n+}\n+\n+#[no_mangle]\n+extern \"C\" fn lower_aarch64_irq(e: &mut ExceptionContext) {\n+    default_exception_handler(e);\n+}\n+\n+#[no_mangle]\n+extern \"C\" fn lower_aarch64_serror(e: &mut ExceptionContext) {\n+    default_exception_handler(e);\n+}\n+\n+//------------------------------------------------------------------------------\n+// Lower, AArch32\n+//------------------------------------------------------------------------------\n+\n+#[no_mangle]\n+extern \"C\" fn lower_aarch32_synchronous(e: &mut ExceptionContext) {\n+    default_exception_handler(e);\n+}\n+\n+#[no_mangle]\n+extern \"C\" fn lower_aarch32_irq(e: &mut ExceptionContext) {\n+    default_exception_handler(e);\n+}\n+\n+#[no_mangle]\n+extern \"C\" fn lower_aarch32_serror(e: &mut ExceptionContext) {\n+    default_exception_handler(e);\n+}\n+\n+//------------------------------------------------------------------------------\n+// Misc\n+//------------------------------------------------------------------------------\n+\n+/// Human readable SPSR_EL1.\n+#[rustfmt::skip]\n+impl fmt::Display for SpsrEL1 {\n+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n+        // Raw value.\n+        writeln!(f, \"SPSR_EL1: {:#010x}\", self.0.get())?;\n+\n+        let to_flag_str = |x| -> _ {\n+            if x { \"Set\" } else { \"Not set\" }\n+         };\n+\n+        writeln!(f, \"      Flags:\")?;\n+        writeln!(f, \"            Negative (N): {}\", to_flag_str(self.0.is_set(SPSR_EL1::N)))?;\n+        writeln!(f, \"            Zero     (Z): {}\", to_flag_str(self.0.is_set(SPSR_EL1::Z)))?;\n+        writeln!(f, \"            Carry    (C): {}\", to_flag_str(self.0.is_set(SPSR_EL1::C)))?;\n+        writeln!(f, \"            Overflow (V): {}\", to_flag_str(self.0.is_set(SPSR_EL1::V)))?;\n+\n+        let to_mask_str = |x| -> _ {\n+            if x { \"Masked\" } else { \"Unmasked\" }\n+        };\n+\n+        writeln!(f, \"      Exception handling state:\")?;\n+        writeln!(f, \"            Debug  (D): {}\", to_mask_str(self.0.is_set(SPSR_EL1::D)))?;\n+        writeln!(f, \"            SError (A): {}\", to_mask_str(self.0.is_set(SPSR_EL1::A)))?;\n+        writeln!(f, \"            IRQ    (I): {}\", to_mask_str(self.0.is_set(SPSR_EL1::I)))?;\n+        writeln!(f, \"            FIQ    (F): {}\", to_mask_str(self.0.is_set(SPSR_EL1::F)))?;\n+\n+        write!(f, \"      Illegal Execution State (IL): {}\",\n+            to_flag_str(self.0.is_set(SPSR_EL1::IL))\n+        )\n+    }\n+}\n+\n+impl EsrEL1 {\n+    #[inline(always)]\n+    fn exception_class(&self) -> Option<ESR_EL1::EC::Value> {\n+        self.0.read_as_enum(ESR_EL1::EC)\n+    }\n+}\n+\n+/// Human readable ESR_EL1.\n+#[rustfmt::skip]\n+impl fmt::Display for EsrEL1 {\n+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n+        // Raw print of whole register.\n+        writeln!(f, \"ESR_EL1: {:#010x}\", self.0.get())?;\n+\n+        // Raw print of exception class.\n+        write!(f, \"      Exception Class         (EC) : {:#x}\", self.0.read(ESR_EL1::EC))?;\n+\n+        // Exception class.\n+        let ec_translation = match self.exception_class() {\n+            Some(ESR_EL1::EC::Value::DataAbortCurrentEL) => \"Data Abort, current EL\",\n+            _ => \"N/A\",\n+        };\n+        writeln!(f, \" - {}\", ec_translation)?;\n+\n+        // Raw print of instruction specific syndrome.\n+        write!(f, \"      Instr Specific Syndrome (ISS): {:#x}\", self.0.read(ESR_EL1::ISS))\n+    }\n+}\n+\n+impl ExceptionContext {\n+    #[inline(always)]\n+    fn exception_class(&self) -> Option<ESR_EL1::EC::Value> {\n+        self.esr_el1.exception_class()\n+    }\n+\n+    #[inline(always)]\n+    fn fault_address_valid(&self) -> bool {\n+        use ESR_EL1::EC::Value::*;\n+\n+        match self.exception_class() {\n+            None => false,\n+            Some(ec) => matches!(\n+                ec,\n+                InstrAbortLowerEL\n+                    | InstrAbortCurrentEL\n+                    | PCAlignmentFault\n+                    | DataAbortLowerEL\n+                    | DataAbortCurrentEL\n+                    | WatchpointLowerEL\n+                    | WatchpointCurrentEL\n+            ),\n+        }\n+    }\n+}\n+\n+/// Human readable print of the exception context.\n+impl fmt::Display for ExceptionContext {\n+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n+        writeln!(f, \"{}\", self.esr_el1)?;\n+\n+        if self.fault_address_valid() {\n+            writeln!(f, \"FAR_EL1: {:#018x}\", FAR_EL1.get() as usize)?;\n+        }\n+\n+        writeln!(f, \"{}\", self.spsr_el1)?;\n+        writeln!(f, \"ELR_EL1: {:#018x}\", self.elr_el1)?;\n+        writeln!(f)?;\n+        writeln!(f, \"General purpose register:\")?;\n+\n+        #[rustfmt::skip]\n+        let alternating = |x| -> _ {\n+            if x modulo 2 == 0 { \"   \" } else { \"\\n\" }\n+        };\n+\n+        // Print two registers per line.\n+        for (i, reg) in self.gpr.iter().enumerate() {\n+            write!(f, \"      x{: <2}: {: >#018x}{}\", i, reg, alternating(i))?;\n+        }\n+        write!(f, \"      lr : {:#018x}\", self.lr)\n+    }\n+}\n\n //--------------------------------------------------------------------------------------------------\n // Public Code\n@@ -29,3 +284,23 @@\n         _ => (PrivilegeLevel::Unknown, \"Unknown\"),\n     }\n }\n+\n+/// Init exception handling by setting the exception vector base address register.\n+///\n+/// # Safety\n+///\n+/// - Changes the HW state of the executing core.\n+/// - The vector table and the symbol `__exception_vector_table_start` from the linker script must\n+///   adhere to the alignment and size constraints demanded by the ARMv8-A Architecture Reference\n+///   Manual.\n+pub unsafe fn handling_init() {\n+    // Provided by exception.S.\n+    extern \"Rust\" {\n+        static __exception_vector_start: UnsafeCell<()>;\n+    }\n+\n+    VBAR_EL1.set(__exception_vector_start.get() as u64);\n+\n+    // Force VBAR update to complete before next instruction.\n+    barrier::isb(barrier::SY);\n+}\n\ndiff -uNr 10_virtual_mem_part1_identity_mapping/src/_arch/aarch64/exception.s 11_exceptions_part1_groundwork/src/_arch/aarch64/exception.s\n--- 10_virtual_mem_part1_identity_mapping/src/_arch/aarch64/exception.s\n+++ 11_exceptions_part1_groundwork/src/_arch/aarch64/exception.s\n@@ -0,0 +1,154 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//--------------------------------------------------------------------------------------------------\n+// Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Call the function provided by parameter `\\handler` after saving the exception context. Provide\n+/// the context as the first parameter to '\\handler'.\n+.macro CALL_WITH_CONTEXT handler\n+__vector_\\handler:\n+\t// Make room on the stack for the exception context.\n+\tsub\tsp,  sp,  #16 * 17\n+\n+\t// Store all general purpose registers on the stack.\n+\tstp\tx0,  x1,  [sp, #16 * 0]\n+\tstp\tx2,  x3,  [sp, #16 * 1]\n+\tstp\tx4,  x5,  [sp, #16 * 2]\n+\tstp\tx6,  x7,  [sp, #16 * 3]\n+\tstp\tx8,  x9,  [sp, #16 * 4]\n+\tstp\tx10, x11, [sp, #16 * 5]\n+\tstp\tx12, x13, [sp, #16 * 6]\n+\tstp\tx14, x15, [sp, #16 * 7]\n+\tstp\tx16, x17, [sp, #16 * 8]\n+\tstp\tx18, x19, [sp, #16 * 9]\n+\tstp\tx20, x21, [sp, #16 * 10]\n+\tstp\tx22, x23, [sp, #16 * 11]\n+\tstp\tx24, x25, [sp, #16 * 12]\n+\tstp\tx26, x27, [sp, #16 * 13]\n+\tstp\tx28, x29, [sp, #16 * 14]\n+\n+\t// Add the exception link register (ELR_EL1), saved program status (SPSR_EL1) and exception\n+\t// syndrome register (ESR_EL1).\n+\tmrs\tx1,  ELR_EL1\n+\tmrs\tx2,  SPSR_EL1\n+\tmrs\tx3,  ESR_EL1\n+\n+\tstp\tlr,  x1,  [sp, #16 * 15]\n+\tstp\tx2,  x3,  [sp, #16 * 16]\n+\n+\t// x0 is the first argument for the function called through `\\handler`.\n+\tmov\tx0,  sp\n+\n+\t// Call `\\handler`.\n+\tbl\t\\handler\n+\n+\t// After returning from exception handling code, replay the saved context and return via\n+\t// `eret`.\n+\tb\t__exception_restore_context\n+\n+.size\t__vector_\\handler, . - __vector_\\handler\n+.type\t__vector_\\handler, function\n+.endm\n+\n+.macro FIQ_SUSPEND\n+1:\twfe\n+\tb\t1b\n+.endm\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Code\n+//--------------------------------------------------------------------------------------------------\n+.section .text\n+\n+//------------------------------------------------------------------------------\n+// The exception vector table.\n+//------------------------------------------------------------------------------\n+\n+// Align by 2^11 bytes, as demanded by ARMv8-A. Same as ALIGN(2048) in an ld script.\n+.align 11\n+\n+// Export a symbol for the Rust code to use.\n+__exception_vector_start:\n+\n+// Current exception level with SP_EL0.\n+//\n+// .org sets the offset relative to section start.\n+//\n+// # Safety\n+//\n+// - It must be ensured that `CALL_WITH_CONTEXT` <= 0x80 bytes.\n+.org 0x000\n+\tCALL_WITH_CONTEXT current_el0_synchronous\n+.org 0x080\n+\tCALL_WITH_CONTEXT current_el0_irq\n+.org 0x100\n+\tFIQ_SUSPEND\n+.org 0x180\n+\tCALL_WITH_CONTEXT current_el0_serror\n+\n+// Current exception level with SP_ELx, x > 0.\n+.org 0x200\n+\tCALL_WITH_CONTEXT current_elx_synchronous\n+.org 0x280\n+\tCALL_WITH_CONTEXT current_elx_irq\n+.org 0x300\n+\tFIQ_SUSPEND\n+.org 0x380\n+\tCALL_WITH_CONTEXT current_elx_serror\n+\n+// Lower exception level, AArch64\n+.org 0x400\n+\tCALL_WITH_CONTEXT lower_aarch64_synchronous\n+.org 0x480\n+\tCALL_WITH_CONTEXT lower_aarch64_irq\n+.org 0x500\n+\tFIQ_SUSPEND\n+.org 0x580\n+\tCALL_WITH_CONTEXT lower_aarch64_serror\n+\n+// Lower exception level, AArch32\n+.org 0x600\n+\tCALL_WITH_CONTEXT lower_aarch32_synchronous\n+.org 0x680\n+\tCALL_WITH_CONTEXT lower_aarch32_irq\n+.org 0x700\n+\tFIQ_SUSPEND\n+.org 0x780\n+\tCALL_WITH_CONTEXT lower_aarch32_serror\n+.org 0x800\n+\n+//------------------------------------------------------------------------------\n+// fn __exception_restore_context()\n+//------------------------------------------------------------------------------\n+__exception_restore_context:\n+\tldr\tw19,      [sp, #16 * 16]\n+\tldp\tlr,  x20, [sp, #16 * 15]\n+\n+\tmsr\tSPSR_EL1, x19\n+\tmsr\tELR_EL1,  x20\n+\n+\tldp\tx0,  x1,  [sp, #16 * 0]\n+\tldp\tx2,  x3,  [sp, #16 * 1]\n+\tldp\tx4,  x5,  [sp, #16 * 2]\n+\tldp\tx6,  x7,  [sp, #16 * 3]\n+\tldp\tx8,  x9,  [sp, #16 * 4]\n+\tldp\tx10, x11, [sp, #16 * 5]\n+\tldp\tx12, x13, [sp, #16 * 6]\n+\tldp\tx14, x15, [sp, #16 * 7]\n+\tldp\tx16, x17, [sp, #16 * 8]\n+\tldp\tx18, x19, [sp, #16 * 9]\n+\tldp\tx20, x21, [sp, #16 * 10]\n+\tldp\tx22, x23, [sp, #16 * 11]\n+\tldp\tx24, x25, [sp, #16 * 12]\n+\tldp\tx26, x27, [sp, #16 * 13]\n+\tldp\tx28, x29, [sp, #16 * 14]\n+\n+\tadd\tsp,  sp,  #16 * 17\n+\n+\teret\n+\n+.size\t__exception_restore_context, . - __exception_restore_context\n+.type\t__exception_restore_context, function\n\ndiff -uNr 10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory/mmu.rs 11_exceptions_part1_groundwork/src/bsp/raspberrypi/memory/mmu.rs\n--- 10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory/mmu.rs\n+++ 11_exceptions_part1_groundwork/src/bsp/raspberrypi/memory/mmu.rs\n@@ -15,7 +15,7 @@\n /// The kernel's address space defined by this BSP.\n pub type KernelAddrSpace = AddressSpace<{ memory_map::END_INCLUSIVE + 1 }>;\n\n-const NUM_MEM_RANGES: usize = 3;\n+const NUM_MEM_RANGES: usize = 2;\n\n /// The virtual memory layout.\n ///\n@@ -35,16 +35,6 @@\n             },\n         },\n         TranslationDescriptor {\n-            name: \"Remapped Device MMIO\",\n-            virtual_range: remapped_mmio_range_inclusive,\n-            physical_range_translation: Translation::Offset(memory_map::mmio::START + 0x20_0000),\n-            attribute_fields: AttributeFields {\n-                mem_attributes: MemAttributes::Device,\n-                acc_perms: AccessPermissions::ReadWrite,\n-                execute_never: true,\n-            },\n-        },\n-        TranslationDescriptor {\n             name: \"Device MMIO\",\n             virtual_range: mmio_range_inclusive,\n             physical_range_translation: Translation::Identity,\n@@ -67,11 +57,6 @@\n     RangeInclusive::new(super::code_start(), super::code_end_exclusive() - 1)\n }\n\n-fn remapped_mmio_range_inclusive() -> RangeInclusive<usize> {\n-    // The last 64 KiB slot in the first 512 MiB\n-    RangeInclusive::new(0x1FFF_0000, 0x1FFF_FFFF)\n-}\n-\n fn mmio_range_inclusive() -> RangeInclusive<usize> {\n     RangeInclusive::new(memory_map::mmio::START, memory_map::mmio::END_INCLUSIVE)\n }\n\ndiff -uNr 10_virtual_mem_part1_identity_mapping/src/bsp.rs 11_exceptions_part1_groundwork/src/bsp.rs\n--- 10_virtual_mem_part1_identity_mapping/src/bsp.rs\n+++ 11_exceptions_part1_groundwork/src/bsp.rs\n@@ -4,7 +4,7 @@\n\n //! Conditional reexporting of Board Support Packages.\n\n-pub mod device_driver;\n+mod device_driver;\n\n #[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\n mod raspberrypi;\n\ndiff -uNr 10_virtual_mem_part1_identity_mapping/src/exception.rs 11_exceptions_part1_groundwork/src/exception.rs\n--- 10_virtual_mem_part1_identity_mapping/src/exception.rs\n+++ 11_exceptions_part1_groundwork/src/exception.rs\n@@ -13,7 +13,7 @@\n //--------------------------------------------------------------------------------------------------\n // Architectural Public Reexports\n //--------------------------------------------------------------------------------------------------\n-pub use arch_exception::current_privilege_level;\n+pub use arch_exception::{current_privilege_level, handling_init};\n\n //--------------------------------------------------------------------------------------------------\n // Public Definitions\n\ndiff -uNr 10_virtual_mem_part1_identity_mapping/src/main.rs 11_exceptions_part1_groundwork/src/main.rs\n--- 10_virtual_mem_part1_identity_mapping/src/main.rs\n+++ 11_exceptions_part1_groundwork/src/main.rs\n@@ -144,6 +144,8 @@\n unsafe fn kernel_init() -> ! {\n     use memory::mmu::interface::MMU;\n\n+    exception::handling_init();\n+\n     if let Err(string) = memory::mmu::mmu().enable_mmu_and_caching() {\n         panic!(\"MMU: {}\", string);\n     }\n@@ -163,7 +165,7 @@\n\n /// The main function running after the early init.\n fn kernel_main() -> ! {\n-    use console::{console, interface::Write};\n+    use console::console;\n     use core::time::Duration;\n\n     info!(\n@@ -193,13 +195,28 @@\n     info!(\"Timer test, spinning for 1 second\");\n     time::time_manager().spin_for(Duration::from_secs(1));\n\n-    let remapped_uart = unsafe { bsp::device_driver::PL011Uart::new(0x1FFF_1000) };\n-    writeln!(\n-        remapped_uart,\n-        \"[     !!!    ] Writing through the remapped UART at 0x1FFF_1000\"\n-    )\n-    .unwrap();\n+    // Cause an exception by accessing a virtual address for which no translation was set up. This\n+    // code accesses the address 8 GiB, which is outside the mapped address space.\n+    //\n+    // For demo purposes, the exception handler will catch the faulting 8 GiB address and allow\n+    // execution to continue.\n+    info!(\"\");\n+    info!(\"Trying to read from address 8 GiB...\");\n+    let mut big_addr: u64 = 8 * 1024 * 1024 * 1024;\n+    unsafe { core::ptr::read_volatile(big_addr as *mut u64) };\n+\n+    info!(\"************************************************\");\n+    info!(\"Whoa! We recovered from a synchronous exception!\");\n+    info!(\"************************************************\");\n+    info!(\"\");\n+    info!(\"Let's try again\");\n+\n+    // Now use address 9 GiB. The exception handler won't forgive us this time.\n+    info!(\"Trying to read from address 9 GiB...\");\n+    big_addr = 9 * 1024 * 1024 * 1024;\n+    unsafe { core::ptr::read_volatile(big_addr as *mut u64) };\n\n+    // Will never reach here in this tutorial.\n     info!(\"Echoing input now\");\n\n     // Discard any spurious received characters before going into echo mode.\n\ndiff -uNr 10_virtual_mem_part1_identity_mapping/tests/boot_test_string.rb 11_exceptions_part1_groundwork/tests/boot_test_string.rb\n--- 10_virtual_mem_part1_identity_mapping/tests/boot_test_string.rb\n+++ 11_exceptions_part1_groundwork/tests/boot_test_string.rb\n@@ -1,3 +1,3 @@\n # frozen_string_literal: true\n\n-EXPECTED_PRINT = 'Echoing input now'\n+EXPECTED_PRINT = 'lr : 0x'\n\n```\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/build.rs",
    "content": "use std::{env, fs, process};\n\nfn main() {\n    let ld_script_path = match env::var(\"LD_SCRIPT_PATH\") {\n        Ok(var) => var,\n        _ => process::exit(0),\n    };\n\n    let files = fs::read_dir(ld_script_path).unwrap();\n    files\n        .filter_map(Result::ok)\n        .filter(|d| {\n            if let Some(e) = d.path().extension() {\n                e == \"ld\"\n            } else {\n                false\n            }\n        })\n        .for_each(|f| println!(\"cargo:rerun-if-changed={}\", f.path().display()));\n}\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/_arch/aarch64/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural boot code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::boot::arch_boot\n\nuse aarch64_cpu::{asm, registers::*};\nuse core::arch::global_asm;\nuse tock_registers::interfaces::Writeable;\n\n// Assembly counterpart to this file.\nglobal_asm!(\n    include_str!(\"boot.s\"),\n    CONST_CURRENTEL_EL2 = const 0x8,\n    CONST_CORE_ID_MASK = const 0b11\n);\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Prepares the transition from EL2 to EL1.\n///\n/// # Safety\n///\n/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.\n/// - The HW state of EL1 must be prepared in a sound way.\n#[inline(always)]\nunsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: u64) {\n    // Enable timer counter registers for EL1.\n    CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET);\n\n    // No offset for reading the counters.\n    CNTVOFF_EL2.set(0);\n\n    // Set EL1 execution state to AArch64.\n    HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64);\n\n    // Set up a simulated exception return.\n    //\n    // First, fake a saved program status where all interrupts were masked and SP_EL1 was used as a\n    // stack pointer.\n    SPSR_EL2.write(\n        SPSR_EL2::D::Masked\n            + SPSR_EL2::A::Masked\n            + SPSR_EL2::I::Masked\n            + SPSR_EL2::F::Masked\n            + SPSR_EL2::M::EL1h,\n    );\n\n    // Second, let the link register point to kernel_init().\n    ELR_EL2.set(crate::kernel_init as *const () as u64);\n\n    // Set up SP_EL1 (stack pointer), which will be used by EL1 once we \"return\" to it. Since there\n    // are no plans to ever return to EL2, just re-use the same stack.\n    SP_EL1.set(phys_boot_core_stack_end_exclusive_addr);\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The Rust entry of the `kernel` binary.\n///\n/// The function is called from the assembly `_start` function.\n///\n/// # Safety\n///\n/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`.\n#[no_mangle]\npub unsafe extern \"C\" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! {\n    prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr);\n\n    // Use `eret` to \"return\" to EL1. This results in execution of kernel_init() in EL1.\n    asm::eret()\n}\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/_arch/aarch64/cpu/boot.s",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2022 Andre Richter <andre.o.richter@gmail.com>\n\n//--------------------------------------------------------------------------------------------------\n// Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Load the address of a symbol into a register, PC-relative.\n//\n// The symbol must lie within +/- 4 GiB of the Program Counter.\n//\n// # Resources\n//\n// - https://sourceware.org/binutils/docs-2.36/as/AArch64_002dRelocations.html\n.macro ADR_REL register, symbol\n\tadrp\t\\register, \\symbol\n\tadd\t\\register, \\register, #:lo12:\\symbol\n.endm\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n.section .text._start\n\n//------------------------------------------------------------------------------\n// fn _start()\n//------------------------------------------------------------------------------\n_start:\n\t// Only proceed if the core executes in EL2. Park it otherwise.\n\tmrs\tx0, CurrentEL\n\tcmp\tx0, {CONST_CURRENTEL_EL2}\n\tb.ne\t.L_parking_loop\n\n\t// Only proceed on the boot core. Park it otherwise.\n\tmrs\tx1, MPIDR_EL1\n\tand\tx1, x1, {CONST_CORE_ID_MASK}\n\tldr\tx2, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs\n\tcmp\tx1, x2\n\tb.ne\t.L_parking_loop\n\n\t// If execution reaches here, it is the boot core.\n\n\t// Initialize DRAM.\n\tADR_REL\tx0, __bss_start\n\tADR_REL x1, __bss_end_exclusive\n\n.L_bss_init_loop:\n\tcmp\tx0, x1\n\tb.eq\t.L_prepare_rust\n\tstp\txzr, xzr, [x0], #16\n\tb\t.L_bss_init_loop\n\n\t// Prepare the jump to Rust code.\n.L_prepare_rust:\n\t// Set the stack pointer. This ensures that any code in EL2 that needs the stack will work.\n\tADR_REL\tx0, __boot_core_stack_end_exclusive\n\tmov\tsp, x0\n\n\t// Read the CPU's timer counter frequency and store it in ARCH_TIMER_COUNTER_FREQUENCY.\n\t// Abort if the frequency read back as 0.\n\tADR_REL\tx1, ARCH_TIMER_COUNTER_FREQUENCY // provided by aarch64/time.rs\n\tmrs\tx2, CNTFRQ_EL0\n\tcmp\tx2, xzr\n\tb.eq\t.L_parking_loop\n\tstr\tw2, [x1]\n\n\t// Jump to Rust code. x0 holds the function argument provided to _start_rust().\n\tb\t_start_rust\n\n\t// Infinitely wait for events (aka \"park the core\").\n.L_parking_loop:\n\twfe\n\tb\t.L_parking_loop\n\n.size\t_start, . - _start\n.type\t_start, function\n.global\t_start\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/_arch/aarch64/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural processor code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::arch_cpu\n\nuse aarch64_cpu::asm;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\npub use asm::nop;\n\n/// Pause execution on the core.\n#[inline(always)]\npub fn wait_forever() -> ! {\n    loop {\n        asm::wfe()\n    }\n}\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/_arch/aarch64/exception/asynchronous.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural asynchronous exception handling.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::exception::asynchronous::arch_asynchronous\n\nuse aarch64_cpu::registers::*;\nuse tock_registers::interfaces::Readable;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\ntrait DaifField {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register>;\n}\n\nstruct Debug;\nstruct SError;\nstruct IRQ;\nstruct FIQ;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl DaifField for Debug {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::D\n    }\n}\n\nimpl DaifField for SError {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::A\n    }\n}\n\nimpl DaifField for IRQ {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::I\n    }\n}\n\nimpl DaifField for FIQ {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::F\n    }\n}\n\nfn is_masked<T>() -> bool\nwhere\n    T: DaifField,\n{\n    DAIF.is_set(T::daif_field())\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Print the AArch64 exceptions status.\n#[rustfmt::skip]\npub fn print_state() {\n    use crate::info;\n\n    let to_mask_str = |x| -> _ {\n        if x { \"Masked\" } else { \"Unmasked\" }\n    };\n\n    info!(\"      Debug:  {}\", to_mask_str(is_masked::<Debug>()));\n    info!(\"      SError: {}\", to_mask_str(is_masked::<SError>()));\n    info!(\"      IRQ:    {}\", to_mask_str(is_masked::<IRQ>()));\n    info!(\"      FIQ:    {}\", to_mask_str(is_masked::<FIQ>()));\n}\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/_arch/aarch64/exception.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural synchronous and asynchronous exception handling.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::exception::arch_exception\n\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::{arch::global_asm, cell::UnsafeCell, fmt};\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    registers::InMemoryRegister,\n};\n\n// Assembly counterpart to this file.\nglobal_asm!(include_str!(\"exception.s\"));\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Wrapper structs for memory copies of registers.\n#[repr(transparent)]\nstruct SpsrEL1(InMemoryRegister<u64, SPSR_EL1::Register>);\nstruct EsrEL1(InMemoryRegister<u64, ESR_EL1::Register>);\n\n/// The exception context as it is stored on the stack on exception entry.\n#[repr(C)]\nstruct ExceptionContext {\n    /// General Purpose Registers.\n    gpr: [u64; 30],\n\n    /// The link register, aka x30.\n    lr: u64,\n\n    /// Exception link register. The program counter at the time the exception happened.\n    elr_el1: u64,\n\n    /// Saved program status.\n    spsr_el1: SpsrEL1,\n\n    /// Exception syndrome register.\n    esr_el1: EsrEL1,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Prints verbose information about the exception and then panics.\nfn default_exception_handler(exc: &ExceptionContext) {\n    panic!(\n        \"CPU Exception!\\n\\n\\\n        {}\",\n        exc\n    );\n}\n\n//------------------------------------------------------------------------------\n// Current, EL0\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn current_el0_synchronous(_e: &mut ExceptionContext) {\n    panic!(\"Should not be here. Use of SP_EL0 in EL1 is not supported.\")\n}\n\n#[no_mangle]\nextern \"C\" fn current_el0_irq(_e: &mut ExceptionContext) {\n    panic!(\"Should not be here. Use of SP_EL0 in EL1 is not supported.\")\n}\n\n#[no_mangle]\nextern \"C\" fn current_el0_serror(_e: &mut ExceptionContext) {\n    panic!(\"Should not be here. Use of SP_EL0 in EL1 is not supported.\")\n}\n\n//------------------------------------------------------------------------------\n// Current, ELx\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn current_elx_synchronous(e: &mut ExceptionContext) {\n    if e.fault_address_valid() {\n        let far_el1 = FAR_EL1.get();\n\n        // This catches the demo case for this tutorial. If the fault address happens to be 8 GiB,\n        // advance the exception link register for one instruction, so that execution can continue.\n        if far_el1 == 8 * 1024 * 1024 * 1024 {\n            e.elr_el1 += 4;\n\n            return;\n        }\n    }\n\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn current_elx_irq(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn current_elx_serror(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n//------------------------------------------------------------------------------\n// Lower, AArch64\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn lower_aarch64_synchronous(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch64_irq(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch64_serror(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n//------------------------------------------------------------------------------\n// Lower, AArch32\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn lower_aarch32_synchronous(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch32_irq(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch32_serror(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n//------------------------------------------------------------------------------\n// Misc\n//------------------------------------------------------------------------------\n\n/// Human readable SPSR_EL1.\n#[rustfmt::skip]\nimpl fmt::Display for SpsrEL1 {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        // Raw value.\n        writeln!(f, \"SPSR_EL1: {:#010x}\", self.0.get())?;\n\n        let to_flag_str = |x| -> _ {\n            if x { \"Set\" } else { \"Not set\" }\n         };\n\n        writeln!(f, \"      Flags:\")?;\n        writeln!(f, \"            Negative (N): {}\", to_flag_str(self.0.is_set(SPSR_EL1::N)))?;\n        writeln!(f, \"            Zero     (Z): {}\", to_flag_str(self.0.is_set(SPSR_EL1::Z)))?;\n        writeln!(f, \"            Carry    (C): {}\", to_flag_str(self.0.is_set(SPSR_EL1::C)))?;\n        writeln!(f, \"            Overflow (V): {}\", to_flag_str(self.0.is_set(SPSR_EL1::V)))?;\n\n        let to_mask_str = |x| -> _ {\n            if x { \"Masked\" } else { \"Unmasked\" }\n        };\n\n        writeln!(f, \"      Exception handling state:\")?;\n        writeln!(f, \"            Debug  (D): {}\", to_mask_str(self.0.is_set(SPSR_EL1::D)))?;\n        writeln!(f, \"            SError (A): {}\", to_mask_str(self.0.is_set(SPSR_EL1::A)))?;\n        writeln!(f, \"            IRQ    (I): {}\", to_mask_str(self.0.is_set(SPSR_EL1::I)))?;\n        writeln!(f, \"            FIQ    (F): {}\", to_mask_str(self.0.is_set(SPSR_EL1::F)))?;\n\n        write!(f, \"      Illegal Execution State (IL): {}\",\n            to_flag_str(self.0.is_set(SPSR_EL1::IL))\n        )\n    }\n}\n\nimpl EsrEL1 {\n    #[inline(always)]\n    fn exception_class(&self) -> Option<ESR_EL1::EC::Value> {\n        self.0.read_as_enum(ESR_EL1::EC)\n    }\n}\n\n/// Human readable ESR_EL1.\n#[rustfmt::skip]\nimpl fmt::Display for EsrEL1 {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        // Raw print of whole register.\n        writeln!(f, \"ESR_EL1: {:#010x}\", self.0.get())?;\n\n        // Raw print of exception class.\n        write!(f, \"      Exception Class         (EC) : {:#x}\", self.0.read(ESR_EL1::EC))?;\n\n        // Exception class.\n        let ec_translation = match self.exception_class() {\n            Some(ESR_EL1::EC::Value::DataAbortCurrentEL) => \"Data Abort, current EL\",\n            _ => \"N/A\",\n        };\n        writeln!(f, \" - {}\", ec_translation)?;\n\n        // Raw print of instruction specific syndrome.\n        write!(f, \"      Instr Specific Syndrome (ISS): {:#x}\", self.0.read(ESR_EL1::ISS))\n    }\n}\n\nimpl ExceptionContext {\n    #[inline(always)]\n    fn exception_class(&self) -> Option<ESR_EL1::EC::Value> {\n        self.esr_el1.exception_class()\n    }\n\n    #[inline(always)]\n    fn fault_address_valid(&self) -> bool {\n        use ESR_EL1::EC::Value::*;\n\n        match self.exception_class() {\n            None => false,\n            Some(ec) => matches!(\n                ec,\n                InstrAbortLowerEL\n                    | InstrAbortCurrentEL\n                    | PCAlignmentFault\n                    | DataAbortLowerEL\n                    | DataAbortCurrentEL\n                    | WatchpointLowerEL\n                    | WatchpointCurrentEL\n            ),\n        }\n    }\n}\n\n/// Human readable print of the exception context.\nimpl fmt::Display for ExceptionContext {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        writeln!(f, \"{}\", self.esr_el1)?;\n\n        if self.fault_address_valid() {\n            writeln!(f, \"FAR_EL1: {:#018x}\", FAR_EL1.get() as usize)?;\n        }\n\n        writeln!(f, \"{}\", self.spsr_el1)?;\n        writeln!(f, \"ELR_EL1: {:#018x}\", self.elr_el1)?;\n        writeln!(f)?;\n        writeln!(f, \"General purpose register:\")?;\n\n        #[rustfmt::skip]\n        let alternating = |x| -> _ {\n            if x % 2 == 0 { \"   \" } else { \"\\n\" }\n        };\n\n        // Print two registers per line.\n        for (i, reg) in self.gpr.iter().enumerate() {\n            write!(f, \"      x{: <2}: {: >#018x}{}\", i, reg, alternating(i))?;\n        }\n        write!(f, \"      lr : {:#018x}\", self.lr)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse crate::exception::PrivilegeLevel;\n\n/// The processing element's current privilege level.\npub fn current_privilege_level() -> (PrivilegeLevel, &'static str) {\n    let el = CurrentEL.read_as_enum(CurrentEL::EL);\n    match el {\n        Some(CurrentEL::EL::Value::EL2) => (PrivilegeLevel::Hypervisor, \"EL2\"),\n        Some(CurrentEL::EL::Value::EL1) => (PrivilegeLevel::Kernel, \"EL1\"),\n        Some(CurrentEL::EL::Value::EL0) => (PrivilegeLevel::User, \"EL0\"),\n        _ => (PrivilegeLevel::Unknown, \"Unknown\"),\n    }\n}\n\n/// Init exception handling by setting the exception vector base address register.\n///\n/// # Safety\n///\n/// - Changes the HW state of the executing core.\n/// - The vector table and the symbol `__exception_vector_table_start` from the linker script must\n///   adhere to the alignment and size constraints demanded by the ARMv8-A Architecture Reference\n///   Manual.\npub unsafe fn handling_init() {\n    // Provided by exception.S.\n    extern \"Rust\" {\n        static __exception_vector_start: UnsafeCell<()>;\n    }\n\n    VBAR_EL1.set(__exception_vector_start.get() as u64);\n\n    // Force VBAR update to complete before next instruction.\n    barrier::isb(barrier::SY);\n}\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/_arch/aarch64/exception.s",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2022 Andre Richter <andre.o.richter@gmail.com>\n\n//--------------------------------------------------------------------------------------------------\n// Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Call the function provided by parameter `\\handler` after saving the exception context. Provide\n/// the context as the first parameter to '\\handler'.\n.macro CALL_WITH_CONTEXT handler\n__vector_\\handler:\n\t// Make room on the stack for the exception context.\n\tsub\tsp,  sp,  #16 * 17\n\n\t// Store all general purpose registers on the stack.\n\tstp\tx0,  x1,  [sp, #16 * 0]\n\tstp\tx2,  x3,  [sp, #16 * 1]\n\tstp\tx4,  x5,  [sp, #16 * 2]\n\tstp\tx6,  x7,  [sp, #16 * 3]\n\tstp\tx8,  x9,  [sp, #16 * 4]\n\tstp\tx10, x11, [sp, #16 * 5]\n\tstp\tx12, x13, [sp, #16 * 6]\n\tstp\tx14, x15, [sp, #16 * 7]\n\tstp\tx16, x17, [sp, #16 * 8]\n\tstp\tx18, x19, [sp, #16 * 9]\n\tstp\tx20, x21, [sp, #16 * 10]\n\tstp\tx22, x23, [sp, #16 * 11]\n\tstp\tx24, x25, [sp, #16 * 12]\n\tstp\tx26, x27, [sp, #16 * 13]\n\tstp\tx28, x29, [sp, #16 * 14]\n\n\t// Add the exception link register (ELR_EL1), saved program status (SPSR_EL1) and exception\n\t// syndrome register (ESR_EL1).\n\tmrs\tx1,  ELR_EL1\n\tmrs\tx2,  SPSR_EL1\n\tmrs\tx3,  ESR_EL1\n\n\tstp\tlr,  x1,  [sp, #16 * 15]\n\tstp\tx2,  x3,  [sp, #16 * 16]\n\n\t// x0 is the first argument for the function called through `\\handler`.\n\tmov\tx0,  sp\n\n\t// Call `\\handler`.\n\tbl\t\\handler\n\n\t// After returning from exception handling code, replay the saved context and return via\n\t// `eret`.\n\tb\t__exception_restore_context\n\n.size\t__vector_\\handler, . - __vector_\\handler\n.type\t__vector_\\handler, function\n.endm\n\n.macro FIQ_SUSPEND\n1:\twfe\n\tb\t1b\n.endm\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n.section .text\n\n//------------------------------------------------------------------------------\n// The exception vector table.\n//------------------------------------------------------------------------------\n\n// Align by 2^11 bytes, as demanded by ARMv8-A. Same as ALIGN(2048) in an ld script.\n.align 11\n\n// Export a symbol for the Rust code to use.\n__exception_vector_start:\n\n// Current exception level with SP_EL0.\n//\n// .org sets the offset relative to section start.\n//\n// # Safety\n//\n// - It must be ensured that `CALL_WITH_CONTEXT` <= 0x80 bytes.\n.org 0x000\n\tCALL_WITH_CONTEXT current_el0_synchronous\n.org 0x080\n\tCALL_WITH_CONTEXT current_el0_irq\n.org 0x100\n\tFIQ_SUSPEND\n.org 0x180\n\tCALL_WITH_CONTEXT current_el0_serror\n\n// Current exception level with SP_ELx, x > 0.\n.org 0x200\n\tCALL_WITH_CONTEXT current_elx_synchronous\n.org 0x280\n\tCALL_WITH_CONTEXT current_elx_irq\n.org 0x300\n\tFIQ_SUSPEND\n.org 0x380\n\tCALL_WITH_CONTEXT current_elx_serror\n\n// Lower exception level, AArch64\n.org 0x400\n\tCALL_WITH_CONTEXT lower_aarch64_synchronous\n.org 0x480\n\tCALL_WITH_CONTEXT lower_aarch64_irq\n.org 0x500\n\tFIQ_SUSPEND\n.org 0x580\n\tCALL_WITH_CONTEXT lower_aarch64_serror\n\n// Lower exception level, AArch32\n.org 0x600\n\tCALL_WITH_CONTEXT lower_aarch32_synchronous\n.org 0x680\n\tCALL_WITH_CONTEXT lower_aarch32_irq\n.org 0x700\n\tFIQ_SUSPEND\n.org 0x780\n\tCALL_WITH_CONTEXT lower_aarch32_serror\n.org 0x800\n\n//------------------------------------------------------------------------------\n// fn __exception_restore_context()\n//------------------------------------------------------------------------------\n__exception_restore_context:\n\tldr\tw19,      [sp, #16 * 16]\n\tldp\tlr,  x20, [sp, #16 * 15]\n\n\tmsr\tSPSR_EL1, x19\n\tmsr\tELR_EL1,  x20\n\n\tldp\tx0,  x1,  [sp, #16 * 0]\n\tldp\tx2,  x3,  [sp, #16 * 1]\n\tldp\tx4,  x5,  [sp, #16 * 2]\n\tldp\tx6,  x7,  [sp, #16 * 3]\n\tldp\tx8,  x9,  [sp, #16 * 4]\n\tldp\tx10, x11, [sp, #16 * 5]\n\tldp\tx12, x13, [sp, #16 * 6]\n\tldp\tx14, x15, [sp, #16 * 7]\n\tldp\tx16, x17, [sp, #16 * 8]\n\tldp\tx18, x19, [sp, #16 * 9]\n\tldp\tx20, x21, [sp, #16 * 10]\n\tldp\tx22, x23, [sp, #16 * 11]\n\tldp\tx24, x25, [sp, #16 * 12]\n\tldp\tx26, x27, [sp, #16 * 13]\n\tldp\tx28, x29, [sp, #16 * 14]\n\n\tadd\tsp,  sp,  #16 * 17\n\n\teret\n\n.size\t__exception_restore_context, . - __exception_restore_context\n.type\t__exception_restore_context, function\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu/translation_table.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural translation table.\n//!\n//! Only 64 KiB granule is supported.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::memory::mmu::translation_table::arch_translation_table\n\nuse crate::{\n    bsp, memory,\n    memory::mmu::{\n        arch_mmu::{Granule512MiB, Granule64KiB},\n        AccessPermissions, AttributeFields, MemAttributes,\n    },\n};\nuse core::convert;\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields,\n    registers::InMemoryRegister,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// A table descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-15.\nregister_bitfields! {u64,\n    STAGE1_TABLE_DESCRIPTOR [\n        /// Physical address of the next descriptor.\n        NEXT_LEVEL_TABLE_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16]\n\n        TYPE  OFFSET(1) NUMBITS(1) [\n            Block = 0,\n            Table = 1\n        ],\n\n        VALID OFFSET(0) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ]\n    ]\n}\n\n// A level 3 page descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-17.\nregister_bitfields! {u64,\n    STAGE1_PAGE_DESCRIPTOR [\n        /// Unprivileged execute-never.\n        UXN      OFFSET(54) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ],\n\n        /// Privileged execute-never.\n        PXN      OFFSET(53) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ],\n\n        /// Physical address of the next table descriptor (lvl2) or the page descriptor (lvl3).\n        OUTPUT_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16]\n\n        /// Access flag.\n        AF       OFFSET(10) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ],\n\n        /// Shareability field.\n        SH       OFFSET(8) NUMBITS(2) [\n            OuterShareable = 0b10,\n            InnerShareable = 0b11\n        ],\n\n        /// Access Permissions.\n        AP       OFFSET(6) NUMBITS(2) [\n            RW_EL1 = 0b00,\n            RW_EL1_EL0 = 0b01,\n            RO_EL1 = 0b10,\n            RO_EL1_EL0 = 0b11\n        ],\n\n        /// Memory attributes index into the MAIR_EL1 register.\n        AttrIndx OFFSET(2) NUMBITS(3) [],\n\n        TYPE     OFFSET(1) NUMBITS(1) [\n            Reserved_Invalid = 0,\n            Page = 1\n        ],\n\n        VALID    OFFSET(0) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ]\n    ]\n}\n\n/// A table descriptor for 64 KiB aperture.\n///\n/// The output points to the next table.\n#[derive(Copy, Clone)]\n#[repr(C)]\nstruct TableDescriptor {\n    value: u64,\n}\n\n/// A page descriptor with 64 KiB aperture.\n///\n/// The output points to physical memory.\n#[derive(Copy, Clone)]\n#[repr(C)]\nstruct PageDescriptor {\n    value: u64,\n}\n\ntrait StartAddr {\n    fn phys_start_addr_u64(&self) -> u64;\n    fn phys_start_addr_usize(&self) -> usize;\n}\n\nconst NUM_LVL2_TABLES: usize = bsp::memory::mmu::KernelAddrSpace::SIZE >> Granule512MiB::SHIFT;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB\n/// aligned, so the lvl3 is put first.\n#[repr(C)]\n#[repr(align(65536))]\npub struct FixedSizeTranslationTable<const NUM_TABLES: usize> {\n    /// Page descriptors, covering 64 KiB windows per entry.\n    lvl3: [[PageDescriptor; 8192]; NUM_TABLES],\n\n    /// Table descriptors, covering 512 MiB windows.\n    lvl2: [TableDescriptor; NUM_TABLES],\n}\n\n/// A translation table type for the kernel space.\npub type KernelTranslationTable = FixedSizeTranslationTable<NUM_LVL2_TABLES>;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n// The binary is still identity mapped, so we don't need to convert here.\nimpl<T, const N: usize> StartAddr for [T; N] {\n    fn phys_start_addr_u64(&self) -> u64 {\n        self as *const T as u64\n    }\n\n    fn phys_start_addr_usize(&self) -> usize {\n        self as *const _ as usize\n    }\n}\n\nimpl TableDescriptor {\n    /// Create an instance.\n    ///\n    /// Descriptor is invalid by default.\n    pub const fn new_zeroed() -> Self {\n        Self { value: 0 }\n    }\n\n    /// Create an instance pointing to the supplied address.\n    pub fn from_next_lvl_table_addr(phys_next_lvl_table_addr: usize) -> Self {\n        let val = InMemoryRegister::<u64, STAGE1_TABLE_DESCRIPTOR::Register>::new(0);\n\n        let shifted = phys_next_lvl_table_addr >> Granule64KiB::SHIFT;\n        val.write(\n            STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64)\n                + STAGE1_TABLE_DESCRIPTOR::TYPE::Table\n                + STAGE1_TABLE_DESCRIPTOR::VALID::True,\n        );\n\n        TableDescriptor { value: val.get() }\n    }\n}\n\n/// Convert the kernel's generic memory attributes to HW-specific attributes of the MMU.\nimpl convert::From<AttributeFields>\n    for tock_registers::fields::FieldValue<u64, STAGE1_PAGE_DESCRIPTOR::Register>\n{\n    fn from(attribute_fields: AttributeFields) -> Self {\n        // Memory attributes.\n        let mut desc = match attribute_fields.mem_attributes {\n            MemAttributes::CacheableDRAM => {\n                STAGE1_PAGE_DESCRIPTOR::SH::InnerShareable\n                    + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::NORMAL)\n            }\n            MemAttributes::Device => {\n                STAGE1_PAGE_DESCRIPTOR::SH::OuterShareable\n                    + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::DEVICE)\n            }\n        };\n\n        // Access Permissions.\n        desc += match attribute_fields.acc_perms {\n            AccessPermissions::ReadOnly => STAGE1_PAGE_DESCRIPTOR::AP::RO_EL1,\n            AccessPermissions::ReadWrite => STAGE1_PAGE_DESCRIPTOR::AP::RW_EL1,\n        };\n\n        // The execute-never attribute is mapped to PXN in AArch64.\n        desc += if attribute_fields.execute_never {\n            STAGE1_PAGE_DESCRIPTOR::PXN::True\n        } else {\n            STAGE1_PAGE_DESCRIPTOR::PXN::False\n        };\n\n        // Always set unprivileged exectue-never as long as userspace is not implemented yet.\n        desc += STAGE1_PAGE_DESCRIPTOR::UXN::True;\n\n        desc\n    }\n}\n\nimpl PageDescriptor {\n    /// Create an instance.\n    ///\n    /// Descriptor is invalid by default.\n    pub const fn new_zeroed() -> Self {\n        Self { value: 0 }\n    }\n\n    /// Create an instance.\n    pub fn from_output_addr(phys_output_addr: usize, attribute_fields: &AttributeFields) -> Self {\n        let val = InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(0);\n\n        let shifted = phys_output_addr as u64 >> Granule64KiB::SHIFT;\n        val.write(\n            STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted)\n                + STAGE1_PAGE_DESCRIPTOR::AF::True\n                + STAGE1_PAGE_DESCRIPTOR::TYPE::Page\n                + STAGE1_PAGE_DESCRIPTOR::VALID::True\n                + (*attribute_fields).into(),\n        );\n\n        Self { value: val.get() }\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<const NUM_TABLES: usize> FixedSizeTranslationTable<NUM_TABLES> {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        // Can't have a zero-sized address space.\n        assert!(NUM_TABLES > 0);\n\n        Self {\n            lvl3: [[PageDescriptor::new_zeroed(); 8192]; NUM_TABLES],\n            lvl2: [TableDescriptor::new_zeroed(); NUM_TABLES],\n        }\n    }\n\n    /// Iterates over all static translation table entries and fills them at once.\n    ///\n    /// # Safety\n    ///\n    /// - Modifies a `static mut`. Ensure it only happens from here.\n    pub unsafe fn populate_tt_entries(&mut self) -> Result<(), &'static str> {\n        for (l2_nr, l2_entry) in self.lvl2.iter_mut().enumerate() {\n            *l2_entry =\n                TableDescriptor::from_next_lvl_table_addr(self.lvl3[l2_nr].phys_start_addr_usize());\n\n            for (l3_nr, l3_entry) in self.lvl3[l2_nr].iter_mut().enumerate() {\n                let virt_addr = (l2_nr << Granule512MiB::SHIFT) + (l3_nr << Granule64KiB::SHIFT);\n\n                let (phys_output_addr, attribute_fields) =\n                    bsp::memory::mmu::virt_mem_layout().virt_addr_properties(virt_addr)?;\n\n                *l3_entry = PageDescriptor::from_output_addr(phys_output_addr, &attribute_fields);\n            }\n        }\n\n        Ok(())\n    }\n\n    /// The translation table's base address to be used for programming the MMU.\n    pub fn phys_base_address(&self) -> u64 {\n        self.lvl2.phys_start_addr_u64()\n    }\n}\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management Unit Driver.\n//!\n//! Only 64 KiB granule is supported.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::memory::mmu::arch_mmu\n\nuse crate::{\n    bsp, memory,\n    memory::mmu::{translation_table::KernelTranslationTable, TranslationGranule},\n};\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::intrinsics::unlikely;\nuse tock_registers::interfaces::{ReadWriteable, Readable, Writeable};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Memory Management Unit type.\nstruct MemoryManagementUnit;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub type Granule512MiB = TranslationGranule<{ 512 * 1024 * 1024 }>;\npub type Granule64KiB = TranslationGranule<{ 64 * 1024 }>;\n\n/// Constants for indexing the MAIR_EL1.\n#[allow(dead_code)]\npub mod mair {\n    pub const DEVICE: u64 = 0;\n    pub const NORMAL: u64 = 1;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n/// The kernel translation tables.\n///\n/// # Safety\n///\n/// - Supposed to land in `.bss`. Therefore, ensure that all initial member values boil down to \"0\".\nstatic mut KERNEL_TABLES: KernelTranslationTable = KernelTranslationTable::new();\n\nstatic MMU: MemoryManagementUnit = MemoryManagementUnit;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<const AS_SIZE: usize> memory::mmu::AddressSpace<AS_SIZE> {\n    /// Checks for architectural restrictions.\n    pub const fn arch_address_space_size_sanity_checks() {\n        // Size must be at least one full 512 MiB table.\n        assert!((AS_SIZE % Granule512MiB::SIZE) == 0);\n\n        // Check for 48 bit virtual address size as maximum, which is supported by any ARMv8\n        // version.\n        assert!(AS_SIZE <= (1 << 48));\n    }\n}\n\nimpl MemoryManagementUnit {\n    /// Setup function for the MAIR_EL1 register.\n    fn set_up_mair(&self) {\n        // Define the memory types being mapped.\n        MAIR_EL1.write(\n            // Attribute 1 - Cacheable normal DRAM.\n            MAIR_EL1::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc +\n        MAIR_EL1::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc +\n\n        // Attribute 0 - Device.\n        MAIR_EL1::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck,\n        );\n    }\n\n    /// Configure various settings of stage 1 of the EL1 translation regime.\n    fn configure_translation_control(&self) {\n        let t0sz = (64 - bsp::memory::mmu::KernelAddrSpace::SIZE_SHIFT) as u64;\n\n        TCR_EL1.write(\n            TCR_EL1::TBI0::Used\n                + TCR_EL1::IPS::Bits_40\n                + TCR_EL1::TG0::KiB_64\n                + TCR_EL1::SH0::Inner\n                + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable\n                + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable\n                + TCR_EL1::EPD0::EnableTTBR0Walks\n                + TCR_EL1::A1::TTBR0\n                + TCR_EL1::T0SZ.val(t0sz)\n                + TCR_EL1::EPD1::DisableTTBR1Walks,\n        );\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the MMU instance.\npub fn mmu() -> &'static impl memory::mmu::interface::MMU {\n    &MMU\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse memory::mmu::MMUEnableError;\n\nimpl memory::mmu::interface::MMU for MemoryManagementUnit {\n    unsafe fn enable_mmu_and_caching(&self) -> Result<(), MMUEnableError> {\n        if unlikely(self.is_enabled()) {\n            return Err(MMUEnableError::AlreadyEnabled);\n        }\n\n        // Fail early if translation granule is not supported.\n        if unlikely(!ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran64::Supported)) {\n            return Err(MMUEnableError::Other(\n                \"Translation granule not supported in HW\",\n            ));\n        }\n\n        // Prepare the memory attribute indirection register.\n        self.set_up_mair();\n\n        // Populate translation tables.\n        KERNEL_TABLES\n            .populate_tt_entries()\n            .map_err(MMUEnableError::Other)?;\n\n        // Set the \"Translation Table Base Register\".\n        TTBR0_EL1.set_baddr(KERNEL_TABLES.phys_base_address());\n\n        self.configure_translation_control();\n\n        // Switch the MMU on.\n        //\n        // First, force all previous changes to be seen before the MMU is enabled.\n        barrier::isb(barrier::SY);\n\n        // Enable the MMU and turn on data and instruction caching.\n        SCTLR_EL1.modify(SCTLR_EL1::M::Enable + SCTLR_EL1::C::Cacheable + SCTLR_EL1::I::Cacheable);\n\n        // Force MMU init to complete before next instruction.\n        barrier::isb(barrier::SY);\n\n        Ok(())\n    }\n\n    #[inline(always)]\n    fn is_enabled(&self) -> bool {\n        SCTLR_EL1.matches_all(SCTLR_EL1::M::Enable)\n    }\n}\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/_arch/aarch64/time.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural timer primitives.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::time::arch_time\n\nuse crate::warn;\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::{\n    num::{NonZeroU128, NonZeroU32, NonZeroU64},\n    ops::{Add, Div},\n    time::Duration,\n};\nuse tock_registers::interfaces::Readable;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nconst NANOSEC_PER_SEC: NonZeroU64 = NonZeroU64::new(1_000_000_000).unwrap();\n\n#[derive(Copy, Clone, PartialOrd, PartialEq)]\nstruct GenericTimerCounterValue(u64);\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n/// Boot assembly code overwrites this value with the value of CNTFRQ_EL0 before any Rust code is\n/// executed. This given value here is just a (safe) dummy.\n#[no_mangle]\nstatic ARCH_TIMER_COUNTER_FREQUENCY: NonZeroU32 = NonZeroU32::MIN;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nfn arch_timer_counter_frequency() -> NonZeroU32 {\n    // Read volatile is needed here to prevent the compiler from optimizing\n    // ARCH_TIMER_COUNTER_FREQUENCY away.\n    //\n    // This is safe, because all the safety requirements as stated in read_volatile()'s\n    // documentation are fulfilled.\n    unsafe { core::ptr::read_volatile(&ARCH_TIMER_COUNTER_FREQUENCY) }\n}\n\nimpl GenericTimerCounterValue {\n    pub const MAX: Self = GenericTimerCounterValue(u64::MAX);\n}\n\nimpl Add for GenericTimerCounterValue {\n    type Output = Self;\n\n    fn add(self, other: Self) -> Self {\n        GenericTimerCounterValue(self.0.wrapping_add(other.0))\n    }\n}\n\nimpl From<GenericTimerCounterValue> for Duration {\n    fn from(counter_value: GenericTimerCounterValue) -> Self {\n        if counter_value.0 == 0 {\n            return Duration::ZERO;\n        }\n\n        let frequency: NonZeroU64 = arch_timer_counter_frequency().into();\n\n        // Div<NonZeroU64> implementation for u64 cannot panic.\n        let secs = counter_value.0.div(frequency);\n\n        // This is safe, because frequency can never be greater than u32::MAX, which means the\n        // largest theoretical value for sub_second_counter_value is (u32::MAX - 1). Therefore,\n        // (sub_second_counter_value * NANOSEC_PER_SEC) cannot overflow an u64.\n        //\n        // The subsequent division ensures the result fits into u32, since the max result is smaller\n        // than NANOSEC_PER_SEC. Therefore, just cast it to u32 using `as`.\n        let sub_second_counter_value = counter_value.0 % frequency;\n        let nanos = unsafe { sub_second_counter_value.unchecked_mul(u64::from(NANOSEC_PER_SEC)) }\n            .div(frequency) as u32;\n\n        Duration::new(secs, nanos)\n    }\n}\n\nfn max_duration() -> Duration {\n    Duration::from(GenericTimerCounterValue::MAX)\n}\n\nimpl TryFrom<Duration> for GenericTimerCounterValue {\n    type Error = &'static str;\n\n    fn try_from(duration: Duration) -> Result<Self, Self::Error> {\n        if duration < resolution() {\n            return Ok(GenericTimerCounterValue(0));\n        }\n\n        if duration > max_duration() {\n            return Err(\"Conversion error. Duration too big\");\n        }\n\n        let frequency: u128 = u32::from(arch_timer_counter_frequency()) as u128;\n        let duration: u128 = duration.as_nanos();\n\n        // This is safe, because frequency can never be greater than u32::MAX, and\n        // (Duration::MAX.as_nanos() * u32::MAX) < u128::MAX.\n        let counter_value =\n            unsafe { duration.unchecked_mul(frequency) }.div(NonZeroU128::from(NANOSEC_PER_SEC));\n\n        // Since we checked above that we are <= max_duration(), just cast to u64.\n        Ok(GenericTimerCounterValue(counter_value as u64))\n    }\n}\n\n#[inline(always)]\nfn read_cntpct() -> GenericTimerCounterValue {\n    // Prevent that the counter is read ahead of time due to out-of-order execution.\n    barrier::isb(barrier::SY);\n    let cnt = CNTPCT_EL0.get();\n\n    GenericTimerCounterValue(cnt)\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The timer's resolution.\npub fn resolution() -> Duration {\n    Duration::from(GenericTimerCounterValue(1))\n}\n\n/// The uptime since power-on of the device.\n///\n/// This includes time consumed by firmware and bootloaders.\npub fn uptime() -> Duration {\n    read_cntpct().into()\n}\n\n/// Spin for a given duration.\npub fn spin_for(duration: Duration) {\n    let curr_counter_value = read_cntpct();\n\n    let counter_value_delta: GenericTimerCounterValue = match duration.try_into() {\n        Err(msg) => {\n            warn!(\"spin_for: {}. Skipping\", msg);\n            return;\n        }\n        Ok(val) => val,\n    };\n    let counter_value_target = curr_counter_value + counter_value_delta;\n\n    // Busy wait.\n    //\n    // Read CNTPCT_EL0 directly to avoid the ISB that is part of [`read_cntpct`].\n    while GenericTimerCounterValue(CNTPCT_EL0.get()) < counter_value_target {}\n}\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GPIO Driver.\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper, driver, synchronization,\n    synchronization::NullLock,\n};\nuse tock_registers::{\n    interfaces::{ReadWriteable, Writeable},\n    register_bitfields, register_structs,\n    registers::ReadWrite,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// GPIO registers.\n//\n// Descriptions taken from\n// - https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf\n// - https://datasheets.raspberrypi.org/bcm2711/bcm2711-peripherals.pdf\nregister_bitfields! {\n    u32,\n\n    /// GPIO Function Select 1\n    GPFSEL1 [\n        /// Pin 15\n        FSEL15 OFFSET(15) NUMBITS(3) [\n            Input = 0b000,\n            Output = 0b001,\n            AltFunc0 = 0b100  // PL011 UART RX\n\n        ],\n\n        /// Pin 14\n        FSEL14 OFFSET(12) NUMBITS(3) [\n            Input = 0b000,\n            Output = 0b001,\n            AltFunc0 = 0b100  // PL011 UART TX\n        ]\n    ],\n\n    /// GPIO Pull-up/down Register\n    ///\n    /// BCM2837 only.\n    GPPUD [\n        /// Controls the actuation of the internal pull-up/down control line to ALL the GPIO pins.\n        PUD OFFSET(0) NUMBITS(2) [\n            Off = 0b00,\n            PullDown = 0b01,\n            PullUp = 0b10\n        ]\n    ],\n\n    /// GPIO Pull-up/down Clock Register 0\n    ///\n    /// BCM2837 only.\n    GPPUDCLK0 [\n        /// Pin 15\n        PUDCLK15 OFFSET(15) NUMBITS(1) [\n            NoEffect = 0,\n            AssertClock = 1\n        ],\n\n        /// Pin 14\n        PUDCLK14 OFFSET(14) NUMBITS(1) [\n            NoEffect = 0,\n            AssertClock = 1\n        ]\n    ],\n\n    /// GPIO Pull-up / Pull-down Register 0\n    ///\n    /// BCM2711 only.\n    GPIO_PUP_PDN_CNTRL_REG0 [\n        /// Pin 15\n        GPIO_PUP_PDN_CNTRL15 OFFSET(30) NUMBITS(2) [\n            NoResistor = 0b00,\n            PullUp = 0b01\n        ],\n\n        /// Pin 14\n        GPIO_PUP_PDN_CNTRL14 OFFSET(28) NUMBITS(2) [\n            NoResistor = 0b00,\n            PullUp = 0b01\n        ]\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    RegisterBlock {\n        (0x00 => _reserved1),\n        (0x04 => GPFSEL1: ReadWrite<u32, GPFSEL1::Register>),\n        (0x08 => _reserved2),\n        (0x94 => GPPUD: ReadWrite<u32, GPPUD::Register>),\n        (0x98 => GPPUDCLK0: ReadWrite<u32, GPPUDCLK0::Register>),\n        (0x9C => _reserved3),\n        (0xE4 => GPIO_PUP_PDN_CNTRL_REG0: ReadWrite<u32, GPIO_PUP_PDN_CNTRL_REG0::Register>),\n        (0xE8 => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\nstruct GPIOInner {\n    registers: Registers,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the GPIO HW.\npub struct GPIO {\n    inner: NullLock<GPIOInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GPIOInner {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n        }\n    }\n\n    /// Disable pull-up/down on pins 14 and 15.\n    #[cfg(feature = \"bsp_rpi3\")]\n    fn disable_pud_14_15_bcm2837(&mut self) {\n        use crate::time;\n        use core::time::Duration;\n\n        // The Linux 2837 GPIO driver waits 1 µs between the steps.\n        const DELAY: Duration = Duration::from_micros(1);\n\n        self.registers.GPPUD.write(GPPUD::PUD::Off);\n        time::time_manager().spin_for(DELAY);\n\n        self.registers\n            .GPPUDCLK0\n            .write(GPPUDCLK0::PUDCLK15::AssertClock + GPPUDCLK0::PUDCLK14::AssertClock);\n        time::time_manager().spin_for(DELAY);\n\n        self.registers.GPPUD.write(GPPUD::PUD::Off);\n        self.registers.GPPUDCLK0.set(0);\n    }\n\n    /// Disable pull-up/down on pins 14 and 15.\n    #[cfg(feature = \"bsp_rpi4\")]\n    fn disable_pud_14_15_bcm2711(&mut self) {\n        self.registers.GPIO_PUP_PDN_CNTRL_REG0.write(\n            GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL15::PullUp\n                + GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL14::PullUp,\n        );\n    }\n\n    /// Map PL011 UART as standard output.\n    ///\n    /// TX to pin 14\n    /// RX to pin 15\n    pub fn map_pl011_uart(&mut self) {\n        // Select the UART on pins 14 and 15.\n        self.registers\n            .GPFSEL1\n            .modify(GPFSEL1::FSEL15::AltFunc0 + GPFSEL1::FSEL14::AltFunc0);\n\n        // Disable pull-up/down on pins 14 and 15.\n        #[cfg(feature = \"bsp_rpi3\")]\n        self.disable_pud_14_15_bcm2837();\n\n        #[cfg(feature = \"bsp_rpi4\")]\n        self.disable_pud_14_15_bcm2711();\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GPIO {\n    pub const COMPATIBLE: &'static str = \"BCM GPIO\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            inner: NullLock::new(GPIOInner::new(mmio_start_addr)),\n        }\n    }\n\n    /// Concurrency safe version of `GPIOInner.map_pl011_uart()`\n    pub fn map_pl011_uart(&self) {\n        self.inner.lock(|inner| inner.map_pl011_uart())\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl driver::interface::DeviceDriver for GPIO {\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n}\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! PL011 UART driver.\n//!\n//! # Resources\n//!\n//! - <https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf>\n//! - <https://developer.arm.com/documentation/ddi0183/latest>\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper, console, cpu, driver, synchronization,\n    synchronization::NullLock,\n};\nuse core::fmt;\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields, register_structs,\n    registers::{ReadOnly, ReadWrite, WriteOnly},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// PL011 UART registers.\n//\n// Descriptions taken from \"PrimeCell UART (PL011) Technical Reference Manual\" r1p5.\nregister_bitfields! {\n    u32,\n\n    /// Flag Register.\n    FR [\n        /// Transmit FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n        /// Line Control Register, LCR_H.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the transmit holding register is empty.\n        /// - If the FIFO is enabled, the TXFE bit is set when the transmit FIFO is empty.\n        /// - This bit does not indicate if there is data in the transmit shift register.\n        TXFE OFFSET(7) NUMBITS(1) [],\n\n        /// Transmit FIFO full. The meaning of this bit depends on the state of the FEN bit in the\n        /// LCR_H Register.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the transmit holding register is full.\n        /// - If the FIFO is enabled, the TXFF bit is set when the transmit FIFO is full.\n        TXFF OFFSET(5) NUMBITS(1) [],\n\n        /// Receive FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n        /// LCR_H Register.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the receive holding register is empty.\n        /// - If the FIFO is enabled, the RXFE bit is set when the receive FIFO is empty.\n        RXFE OFFSET(4) NUMBITS(1) [],\n\n        /// UART busy. If this bit is set to 1, the UART is busy transmitting data. This bit remains\n        /// set until the complete byte, including all the stop bits, has been sent from the shift\n        /// register.\n        ///\n        /// This bit is set as soon as the transmit FIFO becomes non-empty, regardless of whether\n        /// the UART is enabled or not.\n        BUSY OFFSET(3) NUMBITS(1) []\n    ],\n\n    /// Integer Baud Rate Divisor.\n    IBRD [\n        /// The integer baud rate divisor.\n        BAUD_DIVINT OFFSET(0) NUMBITS(16) []\n    ],\n\n    /// Fractional Baud Rate Divisor.\n    FBRD [\n        ///  The fractional baud rate divisor.\n        BAUD_DIVFRAC OFFSET(0) NUMBITS(6) []\n    ],\n\n    /// Line Control Register.\n    LCR_H [\n        /// Word length. These bits indicate the number of data bits transmitted or received in a\n        /// frame.\n        #[allow(clippy::enum_variant_names)]\n        WLEN OFFSET(5) NUMBITS(2) [\n            FiveBit = 0b00,\n            SixBit = 0b01,\n            SevenBit = 0b10,\n            EightBit = 0b11\n        ],\n\n        /// Enable FIFOs:\n        ///\n        /// 0 = FIFOs are disabled (character mode) that is, the FIFOs become 1-byte-deep holding\n        /// registers.\n        ///\n        /// 1 = Transmit and receive FIFO buffers are enabled (FIFO mode).\n        FEN  OFFSET(4) NUMBITS(1) [\n            FifosDisabled = 0,\n            FifosEnabled = 1\n        ]\n    ],\n\n    /// Control Register.\n    CR [\n        /// Receive enable. If this bit is set to 1, the receive section of the UART is enabled.\n        /// Data reception occurs for either UART signals or SIR signals depending on the setting of\n        /// the SIREN bit. When the UART is disabled in the middle of reception, it completes the\n        /// current character before stopping.\n        RXE OFFSET(9) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// Transmit enable. If this bit is set to 1, the transmit section of the UART is enabled.\n        /// Data transmission occurs for either UART signals, or SIR signals depending on the\n        /// setting of the SIREN bit. When the UART is disabled in the middle of transmission, it\n        /// completes the current character before stopping.\n        TXE OFFSET(8) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// UART enable:\n        ///\n        /// 0 = UART is disabled. If the UART is disabled in the middle of transmission or\n        /// reception, it completes the current character before stopping.\n        ///\n        /// 1 = The UART is enabled. Data transmission and reception occurs for either UART signals\n        /// or SIR signals depending on the setting of the SIREN bit\n        UARTEN OFFSET(0) NUMBITS(1) [\n            /// If the UART is disabled in the middle of transmission or reception, it completes the\n            /// current character before stopping.\n            Disabled = 0,\n            Enabled = 1\n        ]\n    ],\n\n    /// Interrupt Clear Register.\n    ICR [\n        /// Meta field for all pending interrupts.\n        ALL OFFSET(0) NUMBITS(11) []\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    pub RegisterBlock {\n        (0x00 => DR: ReadWrite<u32>),\n        (0x04 => _reserved1),\n        (0x18 => FR: ReadOnly<u32, FR::Register>),\n        (0x1c => _reserved2),\n        (0x24 => IBRD: WriteOnly<u32, IBRD::Register>),\n        (0x28 => FBRD: WriteOnly<u32, FBRD::Register>),\n        (0x2c => LCR_H: WriteOnly<u32, LCR_H::Register>),\n        (0x30 => CR: WriteOnly<u32, CR::Register>),\n        (0x34 => _reserved3),\n        (0x44 => ICR: WriteOnly<u32, ICR::Register>),\n        (0x48 => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\n#[derive(PartialEq)]\nenum BlockingMode {\n    Blocking,\n    NonBlocking,\n}\n\nstruct PL011UartInner {\n    registers: Registers,\n    chars_written: usize,\n    chars_read: usize,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the UART.\npub struct PL011Uart {\n    inner: NullLock<PL011UartInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PL011UartInner {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n            chars_written: 0,\n            chars_read: 0,\n        }\n    }\n\n    /// Set up baud rate and characteristics.\n    ///\n    /// This results in 8N1 and 921_600 baud.\n    ///\n    /// The calculation for the BRD is (we set the clock to 48 MHz in config.txt):\n    /// `(48_000_000 / 16) / 921_600 = 3.2552083`.\n    ///\n    /// This means the integer part is `3` and goes into the `IBRD`.\n    /// The fractional part is `0.2552083`.\n    ///\n    /// `FBRD` calculation according to the PL011 Technical Reference Manual:\n    /// `INTEGER((0.2552083 * 64) + 0.5) = 16`.\n    ///\n    /// Therefore, the generated baud rate divider is: `3 + 16/64 = 3.25`. Which results in a\n    /// genrated baud rate of `48_000_000 / (16 * 3.25) = 923_077`.\n    ///\n    /// Error = `((923_077 - 921_600) / 921_600) * 100 = 0.16%`.\n    pub fn init(&mut self) {\n        // Execution can arrive here while there are still characters queued in the TX FIFO and\n        // actively being sent out by the UART hardware. If the UART is turned off in this case,\n        // those queued characters would be lost.\n        //\n        // For example, this can happen during runtime on a call to panic!(), because panic!()\n        // initializes its own UART instance and calls init().\n        //\n        // Hence, flush first to ensure all pending characters are transmitted.\n        self.flush();\n\n        // Turn the UART off temporarily.\n        self.registers.CR.set(0);\n\n        // Clear all pending interrupts.\n        self.registers.ICR.write(ICR::ALL::CLEAR);\n\n        // From the PL011 Technical Reference Manual:\n        //\n        // The LCR_H, IBRD, and FBRD registers form the single 30-bit wide LCR Register that is\n        // updated on a single write strobe generated by a LCR_H write. So, to internally update the\n        // contents of IBRD or FBRD, a LCR_H write must always be performed at the end.\n        //\n        // Set the baud rate, 8N1 and FIFO enabled.\n        self.registers.IBRD.write(IBRD::BAUD_DIVINT.val(3));\n        self.registers.FBRD.write(FBRD::BAUD_DIVFRAC.val(16));\n        self.registers\n            .LCR_H\n            .write(LCR_H::WLEN::EightBit + LCR_H::FEN::FifosEnabled);\n\n        // Turn the UART on.\n        self.registers\n            .CR\n            .write(CR::UARTEN::Enabled + CR::TXE::Enabled + CR::RXE::Enabled);\n    }\n\n    /// Send a character.\n    fn write_char(&mut self, c: char) {\n        // Spin while TX FIFO full is set, waiting for an empty slot.\n        while self.registers.FR.matches_all(FR::TXFF::SET) {\n            cpu::nop();\n        }\n\n        // Write the character to the buffer.\n        self.registers.DR.set(c as u32);\n\n        self.chars_written += 1;\n    }\n\n    /// Block execution until the last buffered character has been physically put on the TX wire.\n    fn flush(&self) {\n        // Spin until the busy bit is cleared.\n        while self.registers.FR.matches_all(FR::BUSY::SET) {\n            cpu::nop();\n        }\n    }\n\n    /// Retrieve a character.\n    fn read_char_converting(&mut self, blocking_mode: BlockingMode) -> Option<char> {\n        // If RX FIFO is empty,\n        if self.registers.FR.matches_all(FR::RXFE::SET) {\n            // immediately return in non-blocking mode.\n            if blocking_mode == BlockingMode::NonBlocking {\n                return None;\n            }\n\n            // Otherwise, wait until a char was received.\n            while self.registers.FR.matches_all(FR::RXFE::SET) {\n                cpu::nop();\n            }\n        }\n\n        // Read one character.\n        let mut ret = self.registers.DR.get() as u8 as char;\n\n        // Convert carrige return to newline.\n        if ret == '\\r' {\n            ret = '\\n'\n        }\n\n        // Update statistics.\n        self.chars_read += 1;\n\n        Some(ret)\n    }\n}\n\n/// Implementing `core::fmt::Write` enables usage of the `format_args!` macros, which in turn are\n/// used to implement the `kernel`'s `print!` and `println!` macros. By implementing `write_str()`,\n/// we get `write_fmt()` automatically.\n///\n/// The function takes an `&mut self`, so it must be implemented for the inner struct.\n///\n/// See [`src/print.rs`].\n///\n/// [`src/print.rs`]: ../../print/index.html\nimpl fmt::Write for PL011UartInner {\n    fn write_str(&mut self, s: &str) -> fmt::Result {\n        for c in s.chars() {\n            self.write_char(c);\n        }\n\n        Ok(())\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PL011Uart {\n    pub const COMPATIBLE: &'static str = \"BCM PL011 UART\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            inner: NullLock::new(PL011UartInner::new(mmio_start_addr)),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl driver::interface::DeviceDriver for PL011Uart {\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n\n    unsafe fn init(&self) -> Result<(), &'static str> {\n        self.inner.lock(|inner| inner.init());\n\n        Ok(())\n    }\n}\n\nimpl console::interface::Write for PL011Uart {\n    /// Passthrough of `args` to the `core::fmt::Write` implementation, but guarded by a Mutex to\n    /// serialize access.\n    fn write_char(&self, c: char) {\n        self.inner.lock(|inner| inner.write_char(c));\n    }\n\n    fn write_fmt(&self, args: core::fmt::Arguments) -> fmt::Result {\n        // Fully qualified syntax for the call to `core::fmt::Write::write_fmt()` to increase\n        // readability.\n        self.inner.lock(|inner| fmt::Write::write_fmt(inner, args))\n    }\n\n    fn flush(&self) {\n        // Spin until TX FIFO empty is set.\n        self.inner.lock(|inner| inner.flush());\n    }\n}\n\nimpl console::interface::Read for PL011Uart {\n    fn read_char(&self) -> char {\n        self.inner\n            .lock(|inner| inner.read_char_converting(BlockingMode::Blocking).unwrap())\n    }\n\n    fn clear_rx(&self) {\n        // Read from the RX FIFO until it is indicating empty.\n        while self\n            .inner\n            .lock(|inner| inner.read_char_converting(BlockingMode::NonBlocking))\n            .is_some()\n        {}\n    }\n}\n\nimpl console::interface::Statistics for PL011Uart {\n    fn chars_written(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_written)\n    }\n\n    fn chars_read(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_read)\n    }\n}\n\nimpl console::interface::All for PL011Uart {}\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/bsp/device_driver/bcm.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BCM driver top level.\n\nmod bcm2xxx_gpio;\nmod bcm2xxx_pl011_uart;\n\npub use bcm2xxx_gpio::*;\npub use bcm2xxx_pl011_uart::*;\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/bsp/device_driver/common.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Common device driver code.\n\nuse core::{marker::PhantomData, ops};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct MMIODerefWrapper<T> {\n    start_addr: usize,\n    phantom: PhantomData<fn() -> T>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T> MMIODerefWrapper<T> {\n    /// Create an instance.\n    pub const unsafe fn new(start_addr: usize) -> Self {\n        Self {\n            start_addr,\n            phantom: PhantomData,\n        }\n    }\n}\n\nimpl<T> ops::Deref for MMIODerefWrapper<T> {\n    type Target = T;\n\n    fn deref(&self) -> &Self::Target {\n        unsafe { &*(self.start_addr as *const _) }\n    }\n}\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/bsp/device_driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Device driver.\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod bcm;\nmod common;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use bcm::*;\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/bsp/raspberrypi/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Processor code.\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Used by `arch` code to find the early boot core.\n#[no_mangle]\n#[link_section = \".text._start_arguments\"]\npub static BOOT_CORE_ID: u64 = 0;\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/bsp/raspberrypi/driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP driver support.\n\nuse super::memory::map::mmio;\nuse crate::{bsp::device_driver, console, driver as generic_driver};\nuse core::sync::atomic::{AtomicBool, Ordering};\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic PL011_UART: device_driver::PL011Uart =\n    unsafe { device_driver::PL011Uart::new(mmio::PL011_UART_START) };\nstatic GPIO: device_driver::GPIO = unsafe { device_driver::GPIO::new(mmio::GPIO_START) };\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// This must be called only after successful init of the UART driver.\nfn post_init_uart() -> Result<(), &'static str> {\n    console::register_console(&PL011_UART);\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the GPIO driver.\nfn post_init_gpio() -> Result<(), &'static str> {\n    GPIO.map_pl011_uart();\n    Ok(())\n}\n\nfn driver_uart() -> Result<(), &'static str> {\n    let uart_descriptor =\n        generic_driver::DeviceDriverDescriptor::new(&PL011_UART, Some(post_init_uart));\n    generic_driver::driver_manager().register_driver(uart_descriptor);\n\n    Ok(())\n}\n\nfn driver_gpio() -> Result<(), &'static str> {\n    let gpio_descriptor = generic_driver::DeviceDriverDescriptor::new(&GPIO, Some(post_init_gpio));\n    generic_driver::driver_manager().register_driver(gpio_descriptor);\n\n    Ok(())\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Initialize the driver subsystem.\n///\n/// # Safety\n///\n/// See child function calls.\npub unsafe fn init() -> Result<(), &'static str> {\n    static INIT_DONE: AtomicBool = AtomicBool::new(false);\n    if INIT_DONE.load(Ordering::Relaxed) {\n        return Err(\"Init already done\");\n    }\n\n    driver_uart()?;\n    driver_gpio()?;\n\n    INIT_DONE.store(true, Ordering::Relaxed);\n    Ok(())\n}\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/bsp/raspberrypi/kernel.ld",
    "content": "/* SPDX-License-Identifier: MIT OR Apache-2.0\n *\n * Copyright (c) 2018-2022 Andre Richter <andre.o.richter@gmail.com>\n */\n\nPAGE_SIZE = 64K;\nPAGE_MASK = PAGE_SIZE - 1;\n\n__rpi_phys_dram_start_addr = 0;\n\n/* The physical address at which the the kernel binary will be loaded by the Raspberry's firmware */\n__rpi_phys_binary_load_addr = 0x80000;\n\n\nENTRY(__rpi_phys_binary_load_addr)\n\n/* Flags:\n *     4 == R\n *     5 == RX\n *     6 == RW\n *\n * Segments are marked PT_LOAD below so that the ELF file provides virtual and physical addresses.\n * It doesn't mean all of them need actually be loaded.\n */\nPHDRS\n{\n    segment_boot_core_stack PT_LOAD FLAGS(6);\n    segment_code            PT_LOAD FLAGS(5);\n    segment_data            PT_LOAD FLAGS(6);\n}\n\nSECTIONS\n{\n    . =  __rpi_phys_dram_start_addr;\n\n    /***********************************************************************************************\n    * Boot Core Stack\n    ***********************************************************************************************/\n    .boot_core_stack (NOLOAD) :\n    {\n                                             /*   ^             */\n                                             /*   | stack       */\n        . += __rpi_phys_binary_load_addr;    /*   | growth      */\n                                             /*   | direction   */\n        __boot_core_stack_end_exclusive = .; /*   |             */\n    } :segment_boot_core_stack\n\n    ASSERT((. & PAGE_MASK) == 0, \"End of boot core stack is not page aligned\")\n\n    /***********************************************************************************************\n    * Code + RO Data + Global Offset Table\n    ***********************************************************************************************/\n    __code_start = .;\n    .text :\n    {\n        KEEP(*(.text._start))\n        *(.text._start_arguments) /* Constants (or statics in Rust speak) read by _start(). */\n        *(.text._start_rust)      /* The Rust entry point */\n        *(.text*)                 /* Everything else */\n    } :segment_code\n\n    .rodata : ALIGN(8) { *(.rodata*) } :segment_code\n\n    . = ALIGN(PAGE_SIZE);\n    __code_end_exclusive = .;\n\n    /***********************************************************************************************\n    * Data + BSS\n    ***********************************************************************************************/\n    .data : { *(.data*) } :segment_data\n\n    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */\n    .bss (NOLOAD) : ALIGN(16)\n    {\n        __bss_start = .;\n        *(.bss*);\n        . = ALIGN(16);\n        __bss_end_exclusive = .;\n    } :segment_data\n\n    /***********************************************************************************************\n    * Misc\n    ***********************************************************************************************/\n    .got : { *(.got*) }\n    ASSERT(SIZEOF(.got) == 0, \"Relocation support not expected\")\n\n    /DISCARD/ : { *(.comment*) }\n}\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/bsp/raspberrypi/memory/mmu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Memory Management Unit.\n\nuse super::map as memory_map;\nuse crate::memory::mmu::*;\nuse core::ops::RangeInclusive;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// The kernel's address space defined by this BSP.\npub type KernelAddrSpace = AddressSpace<{ memory_map::END_INCLUSIVE + 1 }>;\n\nconst NUM_MEM_RANGES: usize = 2;\n\n/// The virtual memory layout.\n///\n/// The layout must contain only special ranges, aka anything that is _not_ normal cacheable DRAM.\n/// It is agnostic of the paging granularity that the architecture's MMU will use.\npub static LAYOUT: KernelVirtualLayout<NUM_MEM_RANGES> = KernelVirtualLayout::new(\n    memory_map::END_INCLUSIVE,\n    [\n        TranslationDescriptor {\n            name: \"Kernel code and RO data\",\n            virtual_range: code_range_inclusive,\n            physical_range_translation: Translation::Identity,\n            attribute_fields: AttributeFields {\n                mem_attributes: MemAttributes::CacheableDRAM,\n                acc_perms: AccessPermissions::ReadOnly,\n                execute_never: false,\n            },\n        },\n        TranslationDescriptor {\n            name: \"Device MMIO\",\n            virtual_range: mmio_range_inclusive,\n            physical_range_translation: Translation::Identity,\n            attribute_fields: AttributeFields {\n                mem_attributes: MemAttributes::Device,\n                acc_perms: AccessPermissions::ReadWrite,\n                execute_never: true,\n            },\n        },\n    ],\n);\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nfn code_range_inclusive() -> RangeInclusive<usize> {\n    // Notice the subtraction to turn the exclusive end into an inclusive end.\n    #[allow(clippy::range_minus_one)]\n    RangeInclusive::new(super::code_start(), super::code_end_exclusive() - 1)\n}\n\nfn mmio_range_inclusive() -> RangeInclusive<usize> {\n    RangeInclusive::new(memory_map::mmio::START, memory_map::mmio::END_INCLUSIVE)\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the virtual memory layout.\npub fn virt_mem_layout() -> &'static KernelVirtualLayout<NUM_MEM_RANGES> {\n    &LAYOUT\n}\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/bsp/raspberrypi/memory.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Memory Management.\n//!\n//! The physical memory layout.\n//!\n//! The Raspberry's firmware copies the kernel binary to 0x8_0000. The preceding region will be used\n//! as the boot core's stack.\n//!\n//! +---------------------------------------+\n//! |                                       | 0x0\n//! |                                       |                                ^\n//! | Boot-core Stack                       |                                | stack\n//! |                                       |                                | growth\n//! |                                       |                                | direction\n//! +---------------------------------------+\n//! |                                       | code_start @ 0x8_0000\n//! | .text                                 |\n//! | .rodata                               |\n//! | .got                                  |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       | code_end_exclusive\n//! | .data                                 |\n//! | .bss                                  |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       |\n//! |                                       |\npub mod mmu;\n\nuse core::cell::UnsafeCell;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Symbols from the linker script.\nextern \"Rust\" {\n    static __code_start: UnsafeCell<()>;\n    static __code_end_exclusive: UnsafeCell<()>;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// The board's physical memory map.\n#[rustfmt::skip]\npub(super) mod map {\n    /// The inclusive end address of the memory map.\n    ///\n    /// End address + 1 must be power of two.\n    ///\n    /// # Note\n    ///\n    /// RPi3 and RPi4 boards can have different amounts of RAM. To make our code lean for\n    /// educational purposes, we set the max size of the address space to 4 GiB regardless of board.\n    /// This way, we can map the entire range that we need (end of MMIO for RPi4) in one take.\n    ///\n    /// However, making this trade-off has the downside of making it possible for the CPU to assert a\n    /// physical address that is not backed by any DRAM (e.g. accessing an address close to 4 GiB on\n    /// an RPi3 that comes with 1 GiB of RAM). This would result in a crash or other kind of error.\n    pub const END_INCLUSIVE:       usize = 0xFFFF_FFFF;\n\n    pub const GPIO_OFFSET:         usize = 0x0020_0000;\n    pub const UART_OFFSET:         usize = 0x0020_1000;\n\n    /// Physical devices.\n    #[cfg(feature = \"bsp_rpi3\")]\n    pub mod mmio {\n        use super::*;\n\n        pub const START:            usize =         0x3F00_0000;\n        pub const GPIO_START:       usize = START + GPIO_OFFSET;\n        pub const PL011_UART_START: usize = START + UART_OFFSET;\n        pub const END_INCLUSIVE:    usize =         0x4000_FFFF;\n    }\n\n    /// Physical devices.\n    #[cfg(feature = \"bsp_rpi4\")]\n    pub mod mmio {\n        use super::*;\n\n        pub const START:            usize =         0xFE00_0000;\n        pub const GPIO_START:       usize = START + GPIO_OFFSET;\n        pub const PL011_UART_START: usize = START + UART_OFFSET;\n        pub const END_INCLUSIVE:    usize =         0xFF84_FFFF;\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Start page address of the code segment.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn code_start() -> usize {\n    unsafe { __code_start.get() as usize }\n}\n\n/// Exclusive end page address of the code segment.\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn code_end_exclusive() -> usize {\n    unsafe { __code_end_exclusive.get() as usize }\n}\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/bsp/raspberrypi.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Top-level BSP file for the Raspberry Pi 3 and 4.\n\npub mod cpu;\npub mod driver;\npub mod memory;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Board identification.\npub fn board_name() -> &'static str {\n    #[cfg(feature = \"bsp_rpi3\")]\n    {\n        \"Raspberry Pi 3\"\n    }\n\n    #[cfg(feature = \"bsp_rpi4\")]\n    {\n        \"Raspberry Pi 4\"\n    }\n}\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/bsp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Conditional reexporting of Board Support Packages.\n\nmod device_driver;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod raspberrypi;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use raspberrypi::*;\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/common.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! General purpose code.\n\n/// Convert a size into human readable format.\npub const fn size_human_readable_ceil(size: usize) -> (usize, &'static str) {\n    const KIB: usize = 1024;\n    const MIB: usize = 1024 * 1024;\n    const GIB: usize = 1024 * 1024 * 1024;\n\n    if (size / GIB) > 0 {\n        (size.div_ceil(GIB), \"GiB\")\n    } else if (size / MIB) > 0 {\n        (size.div_ceil(MIB), \"MiB\")\n    } else if (size / KIB) > 0 {\n        (size.div_ceil(KIB), \"KiB\")\n    } else {\n        (size, \"Byte\")\n    }\n}\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/console/null_console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Null console.\n\nuse super::interface;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct NullConsole;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\npub static NULL_CONSOLE: NullConsole = NullConsole {};\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl interface::Write for NullConsole {\n    fn write_char(&self, _c: char) {}\n\n    fn write_fmt(&self, _args: fmt::Arguments) -> fmt::Result {\n        fmt::Result::Ok(())\n    }\n\n    fn flush(&self) {}\n}\n\nimpl interface::Read for NullConsole {\n    fn clear_rx(&self) {}\n}\n\nimpl interface::Statistics for NullConsole {}\nimpl interface::All for NullConsole {}\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! System console.\n\nmod null_console;\n\nuse crate::synchronization::{self, NullLock};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Console interfaces.\npub mod interface {\n    use core::fmt;\n\n    /// Console write functions.\n    pub trait Write {\n        /// Write a single character.\n        fn write_char(&self, c: char);\n\n        /// Write a Rust format string.\n        fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result;\n\n        /// Block until the last buffered character has been physically put on the TX wire.\n        fn flush(&self);\n    }\n\n    /// Console read functions.\n    pub trait Read {\n        /// Read a single character.\n        fn read_char(&self) -> char {\n            ' '\n        }\n\n        /// Clear RX buffers, if any.\n        fn clear_rx(&self);\n    }\n\n    /// Console statistics.\n    pub trait Statistics {\n        /// Return the number of characters written.\n        fn chars_written(&self) -> usize {\n            0\n        }\n\n        /// Return the number of characters read.\n        fn chars_read(&self) -> usize {\n            0\n        }\n    }\n\n    /// Trait alias for a full-fledged console.\n    pub trait All: Write + Read + Statistics {}\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic CUR_CONSOLE: NullLock<&'static (dyn interface::All + Sync)> =\n    NullLock::new(&null_console::NULL_CONSOLE);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\n/// Register a new console.\npub fn register_console(new_console: &'static (dyn interface::All + Sync)) {\n    CUR_CONSOLE.lock(|con| *con = new_console);\n}\n\n/// Return a reference to the currently registered console.\n///\n/// This is the global console used by all printing macros.\npub fn console() -> &'static dyn interface::All {\n    CUR_CONSOLE.lock(|con| *con)\n}\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Boot code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/cpu/boot.rs\"]\nmod arch_boot;\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Processor code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/cpu.rs\"]\nmod arch_cpu;\n\nmod boot;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_cpu::{nop, wait_forever};\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Driver support.\n\nuse crate::{\n    info,\n    synchronization::{interface::Mutex, NullLock},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nconst NUM_DRIVERS: usize = 5;\n\nstruct DriverManagerInner {\n    next_index: usize,\n    descriptors: [Option<DeviceDriverDescriptor>; NUM_DRIVERS],\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Driver interfaces.\npub mod interface {\n    /// Device Driver functions.\n    pub trait DeviceDriver {\n        /// Return a compatibility string for identifying the driver.\n        fn compatible(&self) -> &'static str;\n\n        /// Called by the kernel to bring up the device.\n        ///\n        /// # Safety\n        ///\n        /// - During init, drivers might do stuff with system-wide impact.\n        unsafe fn init(&self) -> Result<(), &'static str> {\n            Ok(())\n        }\n    }\n}\n\n/// Tpye to be used as an optional callback after a driver's init() has run.\npub type DeviceDriverPostInitCallback = unsafe fn() -> Result<(), &'static str>;\n\n/// A descriptor for device drivers.\n#[derive(Copy, Clone)]\npub struct DeviceDriverDescriptor {\n    device_driver: &'static (dyn interface::DeviceDriver + Sync),\n    post_init_callback: Option<DeviceDriverPostInitCallback>,\n}\n\n/// Provides device driver management functions.\npub struct DriverManager {\n    inner: NullLock<DriverManagerInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic DRIVER_MANAGER: DriverManager = DriverManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl DriverManagerInner {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            next_index: 0,\n            descriptors: [None; NUM_DRIVERS],\n        }\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl DeviceDriverDescriptor {\n    /// Create an instance.\n    pub fn new(\n        device_driver: &'static (dyn interface::DeviceDriver + Sync),\n        post_init_callback: Option<DeviceDriverPostInitCallback>,\n    ) -> Self {\n        Self {\n            device_driver,\n            post_init_callback,\n        }\n    }\n}\n\n/// Return a reference to the global DriverManager.\npub fn driver_manager() -> &'static DriverManager {\n    &DRIVER_MANAGER\n}\n\nimpl DriverManager {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            inner: NullLock::new(DriverManagerInner::new()),\n        }\n    }\n\n    /// Register a device driver with the kernel.\n    pub fn register_driver(&self, descriptor: DeviceDriverDescriptor) {\n        self.inner.lock(|inner| {\n            inner.descriptors[inner.next_index] = Some(descriptor);\n            inner.next_index += 1;\n        })\n    }\n\n    /// Helper for iterating over registered drivers.\n    fn for_each_descriptor<'a>(&'a self, f: impl FnMut(&'a DeviceDriverDescriptor)) {\n        self.inner.lock(|inner| {\n            inner\n                .descriptors\n                .iter()\n                .filter_map(|x| x.as_ref())\n                .for_each(f)\n        })\n    }\n\n    /// Fully initialize all drivers.\n    ///\n    /// # Safety\n    ///\n    /// - During init, drivers might do stuff with system-wide impact.\n    pub unsafe fn init_drivers(&self) {\n        self.for_each_descriptor(|descriptor| {\n            // 1. Initialize driver.\n            if let Err(x) = descriptor.device_driver.init() {\n                panic!(\n                    \"Error initializing driver: {}: {}\",\n                    descriptor.device_driver.compatible(),\n                    x\n                );\n            }\n\n            // 2. Call corresponding post init callback.\n            if let Some(callback) = &descriptor.post_init_callback {\n                if let Err(x) = callback() {\n                    panic!(\n                        \"Error during driver post-init callback: {}: {}\",\n                        descriptor.device_driver.compatible(),\n                        x\n                    );\n                }\n            }\n        });\n    }\n\n    /// Enumerate all registered device drivers.\n    pub fn enumerate(&self) {\n        let mut i: usize = 1;\n        self.for_each_descriptor(|descriptor| {\n            info!(\"      {}. {}\", i, descriptor.device_driver.compatible());\n\n            i += 1;\n        });\n    }\n}\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/exception/asynchronous.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Asynchronous exception handling.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/exception/asynchronous.rs\"]\nmod arch_asynchronous;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_asynchronous::print_state;\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/exception.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Synchronous and asynchronous exception handling.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/exception.rs\"]\nmod arch_exception;\n\npub mod asynchronous;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_exception::{current_privilege_level, handling_init};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Kernel privilege levels.\n#[allow(missing_docs)]\n#[derive(Eq, PartialEq)]\npub enum PrivilegeLevel {\n    User,\n    Kernel,\n    Hypervisor,\n    Unknown,\n}\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/main.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n// Rust embedded logo for `make doc`.\n#![doc(\n    html_logo_url = \"https://raw.githubusercontent.com/rust-embedded/wg/master/assets/logo/ewg-logo-blue-white-on-transparent.png\"\n)]\n\n//! The `kernel` binary.\n//!\n//! # Code organization and architecture\n//!\n//! The code is divided into different *modules*, each representing a typical **subsystem** of the\n//! `kernel`. Top-level module files of subsystems reside directly in the `src` folder. For example,\n//! `src/memory.rs` contains code that is concerned with all things memory management.\n//!\n//! ## Visibility of processor architecture code\n//!\n//! Some of the `kernel`'s subsystems depend on low-level code that is specific to the target\n//! processor architecture. For each supported processor architecture, there exists a subfolder in\n//! `src/_arch`, for example, `src/_arch/aarch64`.\n//!\n//! The architecture folders mirror the subsystem modules laid out in `src`. For example,\n//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go\n//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in\n//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic\n//! module's name prefixed with `arch_`.\n//!\n//! For example, this is the top of `src/memory/mmu.rs`:\n//!\n//! ```\n//! #[cfg(target_arch = \"aarch64\")]\n//! #[path = \"../_arch/aarch64/memory/mmu.rs\"]\n//! mod arch_mmu;\n//! ```\n//!\n//! Often times, items from the `arch_ module` will be publicly reexported by the parent module.\n//! This way, each architecture specific module can provide its implementation of an item, while the\n//! caller must not be concerned which architecture has been conditionally compiled.\n//!\n//! ## BSP code\n//!\n//! `BSP` stands for Board Support Package. `BSP` code is organized under `src/bsp.rs` and contains\n//! target board specific definitions and functions. These are things such as the board's memory map\n//! or instances of drivers for devices that are featured on the respective board.\n//!\n//! Just like processor architecture code, the `BSP` code's module structure tries to mirror the\n//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is\n//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`.\n//!\n//! ## Kernel interfaces\n//!\n//! Both `arch` and `bsp` contain code that is conditionally compiled depending on the actual target\n//! and board for which the kernel is compiled. For example, the `interrupt controller` hardware of\n//! the `Raspberry Pi 3` and the `Raspberry Pi 4` is different, but we want the rest of the `kernel`\n//! code to play nicely with any of the two without much hassle.\n//!\n//! In order to provide a clean abstraction between `arch`, `bsp` and `generic kernel code`,\n//! `interface` traits are provided *whenever possible* and *where it makes sense*. They are defined\n//! in the respective subsystem module and help to enforce the idiom of *program to an interface,\n//! not an implementation*. For example, there will be a common IRQ handling interface which the two\n//! different interrupt controller `drivers` of both Raspberrys will implement, and only export the\n//! interface to the rest of the `kernel`.\n//!\n//! ```\n//!         +-------------------+\n//!         | Interface (Trait) |\n//!         |                   |\n//!         +--+-------------+--+\n//!            ^             ^\n//!            |             |\n//!            |             |\n//! +----------+--+       +--+----------+\n//! | kernel code |       |  bsp code   |\n//! |             |       |  arch code  |\n//! +-------------+       +-------------+\n//! ```\n//!\n//! # Summary\n//!\n//! For a logical `kernel` subsystem, corresponding code can be distributed over several physical\n//! locations. Here is an example for the **memory** subsystem:\n//!\n//! - `src/memory.rs` and `src/memory/**/*`\n//!   - Common code that is agnostic of target processor architecture and `BSP` characteristics.\n//!     - Example: A function to zero a chunk of memory.\n//!   - Interfaces for the memory subsystem that are implemented by `arch` or `BSP` code.\n//!     - Example: An `MMU` interface that defines `MMU` function prototypes.\n//! - `src/bsp/__board_name__/memory.rs` and `src/bsp/__board_name__/memory/**/*`\n//!   - `BSP` specific code.\n//!   - Example: The board's memory map (physical addresses of DRAM and MMIO devices).\n//! - `src/_arch/__arch_name__/memory.rs` and `src/_arch/__arch_name__/memory/**/*`\n//!   - Processor architecture specific code.\n//!   - Example: Implementation of the `MMU` interface for the `__arch_name__` processor\n//!     architecture.\n//!\n//! From a namespace perspective, **memory** subsystem code lives in:\n//!\n//! - `crate::memory::*`\n//! - `crate::bsp::memory::*`\n//!\n//! # Boot flow\n//!\n//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.\n//!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.\n//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.\n\n#![allow(clippy::upper_case_acronyms)]\n#![allow(incomplete_features)]\n#![feature(asm_const)]\n#![feature(const_option)]\n#![feature(core_intrinsics)]\n#![feature(format_args_nl)]\n#![feature(int_roundings)]\n#![feature(nonzero_min_max)]\n#![feature(panic_info_message)]\n#![feature(trait_alias)]\n#![feature(unchecked_math)]\n#![no_main]\n#![no_std]\n\nmod bsp;\nmod common;\nmod console;\nmod cpu;\nmod driver;\nmod exception;\nmod memory;\nmod panic_wait;\nmod print;\nmod synchronization;\nmod time;\n\n/// Early init code.\n///\n/// # Safety\n///\n/// - Only a single core must be active and running this function.\n/// - The init calls in this function must appear in the correct order:\n///     - MMU + Data caching must be activated at the earliest. Without it, any atomic operations,\n///       e.g. the yet-to-be-introduced spinlocks in the device drivers (which currently employ\n///       NullLocks instead of spinlocks), will fail to work (properly) on the RPi SoCs.\nunsafe fn kernel_init() -> ! {\n    use memory::mmu::interface::MMU;\n\n    exception::handling_init();\n\n    if let Err(string) = memory::mmu::mmu().enable_mmu_and_caching() {\n        panic!(\"MMU: {}\", string);\n    }\n\n    // Initialize the BSP driver subsystem.\n    if let Err(x) = bsp::driver::init() {\n        panic!(\"Error initializing BSP driver subsystem: {}\", x);\n    }\n\n    // Initialize all device drivers.\n    driver::driver_manager().init_drivers();\n    // println! is usable from here on.\n\n    // Transition from unsafe to safe.\n    kernel_main()\n}\n\n/// The main function running after the early init.\nfn kernel_main() -> ! {\n    use console::console;\n    use core::time::Duration;\n\n    info!(\n        \"{} version {}\",\n        env!(\"CARGO_PKG_NAME\"),\n        env!(\"CARGO_PKG_VERSION\")\n    );\n    info!(\"Booting on: {}\", bsp::board_name());\n\n    info!(\"MMU online. Special regions:\");\n    bsp::memory::mmu::virt_mem_layout().print_layout();\n\n    let (_, privilege_level) = exception::current_privilege_level();\n    info!(\"Current privilege level: {}\", privilege_level);\n\n    info!(\"Exception handling state:\");\n    exception::asynchronous::print_state();\n\n    info!(\n        \"Architectural timer resolution: {} ns\",\n        time::time_manager().resolution().as_nanos()\n    );\n\n    info!(\"Drivers loaded:\");\n    driver::driver_manager().enumerate();\n\n    info!(\"Timer test, spinning for 1 second\");\n    time::time_manager().spin_for(Duration::from_secs(1));\n\n    // Cause an exception by accessing a virtual address for which no translation was set up. This\n    // code accesses the address 8 GiB, which is outside the mapped address space.\n    //\n    // For demo purposes, the exception handler will catch the faulting 8 GiB address and allow\n    // execution to continue.\n    info!(\"\");\n    info!(\"Trying to read from address 8 GiB...\");\n    let mut big_addr: u64 = 8 * 1024 * 1024 * 1024;\n    unsafe { core::ptr::read_volatile(big_addr as *mut u64) };\n\n    info!(\"************************************************\");\n    info!(\"Whoa! We recovered from a synchronous exception!\");\n    info!(\"************************************************\");\n    info!(\"\");\n    info!(\"Let's try again\");\n\n    // Now use address 9 GiB. The exception handler won't forgive us this time.\n    info!(\"Trying to read from address 9 GiB...\");\n    big_addr = 9 * 1024 * 1024 * 1024;\n    unsafe { core::ptr::read_volatile(big_addr as *mut u64) };\n\n    // Will never reach here in this tutorial.\n    info!(\"Echoing input now\");\n\n    // Discard any spurious received characters before going into echo mode.\n    console().clear_rx();\n    loop {\n        let c = console().read_char();\n        console().write_char(c);\n    }\n}\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/memory/mmu/translation_table.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Translation table.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../../_arch/aarch64/memory/mmu/translation_table.rs\"]\nmod arch_translation_table;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_translation_table::KernelTranslationTable;\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/memory/mmu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management Unit.\n//!\n//! In order to decouple `BSP` and `arch` parts of the MMU code (to keep them pluggable), this file\n//! provides types for composing an architecture-agnostic description of the kernel's virtual memory\n//! layout.\n//!\n//! The `BSP` provides such a description through the `bsp::memory::mmu::virt_mem_layout()`\n//! function.\n//!\n//! The `MMU` driver of the `arch` code uses `bsp::memory::mmu::virt_mem_layout()` to compile and\n//! install respective translation tables.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/memory/mmu.rs\"]\nmod arch_mmu;\n\nmod translation_table;\n\nuse crate::common;\nuse core::{fmt, ops::RangeInclusive};\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_mmu::mmu;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// MMU enable errors variants.\n#[allow(missing_docs)]\n#[derive(Debug)]\npub enum MMUEnableError {\n    AlreadyEnabled,\n    Other(&'static str),\n}\n\n/// Memory Management interfaces.\npub mod interface {\n    use super::*;\n\n    /// MMU functions.\n    pub trait MMU {\n        /// Called by the kernel during early init. Supposed to take the translation tables from the\n        /// `BSP`-supplied `virt_mem_layout()` and install/activate them for the respective MMU.\n        ///\n        /// # Safety\n        ///\n        /// - Changes the HW's global state.\n        unsafe fn enable_mmu_and_caching(&self) -> Result<(), MMUEnableError>;\n\n        /// Returns true if the MMU is enabled, false otherwise.\n        fn is_enabled(&self) -> bool;\n    }\n}\n\n/// Describes the characteristics of a translation granule.\npub struct TranslationGranule<const GRANULE_SIZE: usize>;\n\n/// Describes properties of an address space.\npub struct AddressSpace<const AS_SIZE: usize>;\n\n/// Architecture agnostic translation types.\n#[allow(missing_docs)]\n#[allow(dead_code)]\n#[derive(Copy, Clone)]\npub enum Translation {\n    Identity,\n    Offset(usize),\n}\n\n/// Architecture agnostic memory attributes.\n#[allow(missing_docs)]\n#[derive(Copy, Clone)]\npub enum MemAttributes {\n    CacheableDRAM,\n    Device,\n}\n\n/// Architecture agnostic access permissions.\n#[allow(missing_docs)]\n#[derive(Copy, Clone)]\npub enum AccessPermissions {\n    ReadOnly,\n    ReadWrite,\n}\n\n/// Collection of memory attributes.\n#[allow(missing_docs)]\n#[derive(Copy, Clone)]\npub struct AttributeFields {\n    pub mem_attributes: MemAttributes,\n    pub acc_perms: AccessPermissions,\n    pub execute_never: bool,\n}\n\n/// Architecture agnostic descriptor for a memory range.\n#[allow(missing_docs)]\npub struct TranslationDescriptor {\n    pub name: &'static str,\n    pub virtual_range: fn() -> RangeInclusive<usize>,\n    pub physical_range_translation: Translation,\n    pub attribute_fields: AttributeFields,\n}\n\n/// Type for expressing the kernel's virtual memory layout.\npub struct KernelVirtualLayout<const NUM_SPECIAL_RANGES: usize> {\n    /// The last (inclusive) address of the address space.\n    max_virt_addr_inclusive: usize,\n\n    /// Array of descriptors for non-standard (normal cacheable DRAM) memory regions.\n    inner: [TranslationDescriptor; NUM_SPECIAL_RANGES],\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl fmt::Display for MMUEnableError {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        match self {\n            MMUEnableError::AlreadyEnabled => write!(f, \"MMU is already enabled\"),\n            MMUEnableError::Other(x) => write!(f, \"{}\", x),\n        }\n    }\n}\n\nimpl<const GRANULE_SIZE: usize> TranslationGranule<GRANULE_SIZE> {\n    /// The granule's size.\n    pub const SIZE: usize = Self::size_checked();\n\n    /// The granule's shift, aka log2(size).\n    pub const SHIFT: usize = Self::SIZE.trailing_zeros() as usize;\n\n    const fn size_checked() -> usize {\n        assert!(GRANULE_SIZE.is_power_of_two());\n\n        GRANULE_SIZE\n    }\n}\n\nimpl<const AS_SIZE: usize> AddressSpace<AS_SIZE> {\n    /// The address space size.\n    pub const SIZE: usize = Self::size_checked();\n\n    /// The address space shift, aka log2(size).\n    pub const SIZE_SHIFT: usize = Self::SIZE.trailing_zeros() as usize;\n\n    const fn size_checked() -> usize {\n        assert!(AS_SIZE.is_power_of_two());\n\n        // Check for architectural restrictions as well.\n        Self::arch_address_space_size_sanity_checks();\n\n        AS_SIZE\n    }\n}\n\nimpl Default for AttributeFields {\n    fn default() -> AttributeFields {\n        AttributeFields {\n            mem_attributes: MemAttributes::CacheableDRAM,\n            acc_perms: AccessPermissions::ReadWrite,\n            execute_never: true,\n        }\n    }\n}\n\n/// Human-readable output of a TranslationDescriptor.\nimpl fmt::Display for TranslationDescriptor {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        // Call the function to which self.range points, and dereference the result, which causes\n        // Rust to copy the value.\n        let start = *(self.virtual_range)().start();\n        let end = *(self.virtual_range)().end();\n        let size = end - start + 1;\n\n        let (size, unit) = common::size_human_readable_ceil(size);\n\n        let attr = match self.attribute_fields.mem_attributes {\n            MemAttributes::CacheableDRAM => \"C\",\n            MemAttributes::Device => \"Dev\",\n        };\n\n        let acc_p = match self.attribute_fields.acc_perms {\n            AccessPermissions::ReadOnly => \"RO\",\n            AccessPermissions::ReadWrite => \"RW\",\n        };\n\n        let xn = if self.attribute_fields.execute_never {\n            \"PXN\"\n        } else {\n            \"PX\"\n        };\n\n        write!(\n            f,\n            \"      {:#010x} - {:#010x} | {: >3} {} | {: <3} {} {: <3} | {}\",\n            start, end, size, unit, attr, acc_p, xn, self.name\n        )\n    }\n}\n\nimpl<const NUM_SPECIAL_RANGES: usize> KernelVirtualLayout<{ NUM_SPECIAL_RANGES }> {\n    /// Create a new instance.\n    pub const fn new(max: usize, layout: [TranslationDescriptor; NUM_SPECIAL_RANGES]) -> Self {\n        Self {\n            max_virt_addr_inclusive: max,\n            inner: layout,\n        }\n    }\n\n    /// For a virtual address, find and return the physical output address and corresponding\n    /// attributes.\n    ///\n    /// If the address is not found in `inner`, return an identity mapped default with normal\n    /// cacheable DRAM attributes.\n    pub fn virt_addr_properties(\n        &self,\n        virt_addr: usize,\n    ) -> Result<(usize, AttributeFields), &'static str> {\n        if virt_addr > self.max_virt_addr_inclusive {\n            return Err(\"Address out of range\");\n        }\n\n        for i in self.inner.iter() {\n            if (i.virtual_range)().contains(&virt_addr) {\n                let output_addr = match i.physical_range_translation {\n                    Translation::Identity => virt_addr,\n                    Translation::Offset(a) => a + (virt_addr - (i.virtual_range)().start()),\n                };\n\n                return Ok((output_addr, i.attribute_fields));\n            }\n        }\n\n        Ok((virt_addr, AttributeFields::default()))\n    }\n\n    /// Print the memory layout.\n    pub fn print_layout(&self) {\n        use crate::info;\n\n        for i in self.inner.iter() {\n            info!(\"{}\", i);\n        }\n    }\n}\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/memory.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management.\n\npub mod mmu;\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/panic_wait.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A panic handler that infinitely waits.\n\nuse crate::{cpu, println};\nuse core::panic::PanicInfo;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Stop immediately if called a second time.\n///\n/// # Note\n///\n/// Using atomics here relieves us from needing to use `unsafe` for the static variable.\n///\n/// On `AArch64`, which is the only implemented architecture at the time of writing this,\n/// [`AtomicBool::load`] and [`AtomicBool::store`] are lowered to ordinary load and store\n/// instructions. They are therefore safe to use even with MMU + caching deactivated.\n///\n/// [`AtomicBool::load`]: core::sync::atomic::AtomicBool::load\n/// [`AtomicBool::store`]: core::sync::atomic::AtomicBool::store\nfn panic_prevent_reenter() {\n    use core::sync::atomic::{AtomicBool, Ordering};\n\n    #[cfg(not(target_arch = \"aarch64\"))]\n    compile_error!(\"Add the target_arch to above's check if the following code is safe to use\");\n\n    static PANIC_IN_PROGRESS: AtomicBool = AtomicBool::new(false);\n\n    if !PANIC_IN_PROGRESS.load(Ordering::Relaxed) {\n        PANIC_IN_PROGRESS.store(true, Ordering::Relaxed);\n\n        return;\n    }\n\n    cpu::wait_forever()\n}\n\n#[panic_handler]\nfn panic(info: &PanicInfo) -> ! {\n    // Protect against panic infinite loops if any of the following code panics itself.\n    panic_prevent_reenter();\n\n    let timestamp = crate::time::time_manager().uptime();\n    let (location, line, column) = match info.location() {\n        Some(loc) => (loc.file(), loc.line(), loc.column()),\n        _ => (\"???\", 0, 0),\n    };\n\n    println!(\n        \"[  {:>3}.{:06}] Kernel panic!\\n\\n\\\n        Panic location:\\n      File '{}', line {}, column {}\\n\\n\\\n        {}\",\n        timestamp.as_secs(),\n        timestamp.subsec_micros(),\n        location,\n        line,\n        column,\n        info.message().unwrap_or(&format_args!(\"\")),\n    );\n\n    cpu::wait_forever()\n}\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/print.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Printing.\n\nuse crate::console;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n#[doc(hidden)]\npub fn _print(args: fmt::Arguments) {\n    console::console().write_fmt(args).unwrap();\n}\n\n/// Prints without a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! print {\n    ($($arg:tt)*) => ($crate::print::_print(format_args!($($arg)*)));\n}\n\n/// Prints with a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! println {\n    () => ($crate::print!(\"\\n\"));\n    ($($arg:tt)*) => ({\n        $crate::print::_print(format_args_nl!($($arg)*));\n    })\n}\n\n/// Prints an info, with a newline.\n#[macro_export]\nmacro_rules! info {\n    ($string:expr) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[  {:>3}.{:06}] \", $string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n        ));\n    });\n    ($format_string:expr, $($arg:tt)*) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[  {:>3}.{:06}] \", $format_string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n            $($arg)*\n        ));\n    })\n}\n\n/// Prints a warning, with a newline.\n#[macro_export]\nmacro_rules! warn {\n    ($string:expr) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[W {:>3}.{:06}] \", $string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n        ));\n    });\n    ($format_string:expr, $($arg:tt)*) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[W {:>3}.{:06}] \", $format_string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n            $($arg)*\n        ));\n    })\n}\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/synchronization.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Synchronization primitives.\n//!\n//! # Resources\n//!\n//!   - <https://doc.rust-lang.org/book/ch16-04-extensible-concurrency-sync-and-send.html>\n//!   - <https://stackoverflow.com/questions/59428096/understanding-the-send-trait>\n//!   - <https://doc.rust-lang.org/std/cell/index.html>\n\nuse core::cell::UnsafeCell;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Synchronization interfaces.\npub mod interface {\n\n    /// Any object implementing this trait guarantees exclusive access to the data wrapped within\n    /// the Mutex for the duration of the provided closure.\n    pub trait Mutex {\n        /// The type of the data that is wrapped by this mutex.\n        type Data;\n\n        /// Locks the mutex and grants the closure temporary mutable access to the wrapped data.\n        fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R;\n    }\n}\n\n/// A pseudo-lock for teaching purposes.\n///\n/// In contrast to a real Mutex implementation, does not protect against concurrent access from\n/// other cores to the contained data. This part is preserved for later lessons.\n///\n/// The lock will only be used as long as it is safe to do so, i.e. as long as the kernel is\n/// executing single-threaded, aka only running on a single core with interrupts disabled.\npub struct NullLock<T>\nwhere\n    T: ?Sized,\n{\n    data: UnsafeCell<T>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nunsafe impl<T> Send for NullLock<T> where T: ?Sized + Send {}\nunsafe impl<T> Sync for NullLock<T> where T: ?Sized + Send {}\n\nimpl<T> NullLock<T> {\n    /// Create an instance.\n    pub const fn new(data: T) -> Self {\n        Self {\n            data: UnsafeCell::new(data),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\n\nimpl<T> interface::Mutex for NullLock<T> {\n    type Data = T;\n\n    fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R {\n        // In a real lock, there would be code encapsulating this line that ensures that this\n        // mutable reference will ever only be given out once at a time.\n        let data = unsafe { &mut *self.data.get() };\n\n        f(data)\n    }\n}\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/src/time.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Timer primitives.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/time.rs\"]\nmod arch_time;\n\nuse core::time::Duration;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Provides time management functions.\npub struct TimeManager;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic TIME_MANAGER: TimeManager = TimeManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the global TimeManager.\npub fn time_manager() -> &'static TimeManager {\n    &TIME_MANAGER\n}\n\nimpl TimeManager {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self\n    }\n\n    /// The timer's resolution.\n    pub fn resolution(&self) -> Duration {\n        arch_time::resolution()\n    }\n\n    /// The uptime since power-on of the device.\n    ///\n    /// This includes time consumed by firmware and bootloaders.\n    pub fn uptime(&self) -> Duration {\n        arch_time::uptime()\n    }\n\n    /// Spin for a given duration.\n    pub fn spin_for(&self, duration: Duration) {\n        arch_time::spin_for(duration)\n    }\n}\n"
  },
  {
    "path": "11_exceptions_part1_groundwork/tests/boot_test_string.rb",
    "content": "# frozen_string_literal: true\n\nEXPECTED_PRINT = 'lr : 0x'\n"
  },
  {
    "path": "12_integrated_testing/.cargo/config.toml",
    "content": "[target.'cfg(target_os = \"none\")']\nrunner = \"target/kernel_test_runner.sh\"\n"
  },
  {
    "path": "12_integrated_testing/.vscode/settings.json",
    "content": "{\n    \"editor.formatOnSave\": true,\n    \"editor.rulers\": [100],\n    \"rust-analyzer.cargo.target\": \"aarch64-unknown-none-softfloat\",\n    \"rust-analyzer.cargo.features\": [\"bsp_rpi3\"],\n    \"rust-analyzer.checkOnSave.allTargets\": false,\n    \"rust-analyzer.checkOnSave.extraArgs\": [\"--lib\", \"--bins\"],\n    \"rust-analyzer.lens.debug\": false,\n    \"rust-analyzer.lens.run\": false\n}\n"
  },
  {
    "path": "12_integrated_testing/Cargo.toml",
    "content": "[workspace]\n\nmembers = [\n        \"libraries/*\",\n        \"kernel\"\n]\n\n[profile.release]\nlto = true\n"
  },
  {
    "path": "12_integrated_testing/Makefile",
    "content": "## SPDX-License-Identifier: MIT OR Apache-2.0\n##\n## Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\ninclude ../common/docker.mk\ninclude ../common/format.mk\ninclude ../common/operating_system.mk\n\n##--------------------------------------------------------------------------------------------------\n## Optional, user-provided configuration values\n##--------------------------------------------------------------------------------------------------\n\n# Default to the RPi3.\nBSP ?= rpi3\n\n# Default to a serial device name that is common in Linux.\nDEV_SERIAL ?= /dev/ttyUSB0\n\n# Optional integration test name.\nifdef TEST\n    TEST_ARG = --test $(TEST)\nelse\n    TEST_ARG = --test '*'\nendif\n\n\n\n##--------------------------------------------------------------------------------------------------\n## BSP-specific configuration values\n##--------------------------------------------------------------------------------------------------\nQEMU_MISSING_STRING = \"This board is not yet supported for QEMU.\"\n\nifeq ($(BSP),rpi3)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE = raspi3\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    QEMU_TEST_ARGS    = $(QEMU_RELEASE_ARGS) -semihosting\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    OPENOCD_ARG       = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi3.cfg\n    JTAG_BOOT_IMAGE   = ../X1_JTAG_boot/jtag_boot_rpi3.img\n    LD_SCRIPT_PATH    = $(shell pwd)/kernel/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a53\nelse ifeq ($(BSP),rpi4)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE =\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    QEMU_TEST_ARGS    = $(QEMU_RELEASE_ARGS) -semihosting\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    OPENOCD_ARG       = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi4.cfg\n    JTAG_BOOT_IMAGE   = ../X1_JTAG_boot/jtag_boot_rpi4.img\n    LD_SCRIPT_PATH    = $(shell pwd)/kernel/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a72\nendif\n\n# Export for build.rs.\nexport LD_SCRIPT_PATH\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets and Prerequisites\n##--------------------------------------------------------------------------------------------------\nKERNEL_MANIFEST      = kernel/Cargo.toml\nKERNEL_LINKER_SCRIPT = kernel.ld\nLAST_BUILD_CONFIG    = target/$(BSP).build_config\n\nKERNEL_ELF      = target/$(TARGET)/release/kernel\n# This parses cargo's dep-info file.\n# https://doc.rust-lang.org/cargo/guide/build-cache.html#dep-info-files\nKERNEL_ELF_DEPS = $(filter-out %: ,$(file < $(KERNEL_ELF).d)) $(KERNEL_MANIFEST) $(LAST_BUILD_CONFIG)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Command building blocks\n##--------------------------------------------------------------------------------------------------\nRUSTFLAGS = $(RUSTC_MISC_ARGS)                   \\\n    -C link-arg=--library-path=$(LD_SCRIPT_PATH) \\\n    -C link-arg=--script=$(KERNEL_LINKER_SCRIPT)\n\nRUSTFLAGS_PEDANTIC = $(RUSTFLAGS) \\\n    -D warnings                   \\\n    -D missing_docs\n\nFEATURES      = --features bsp_$(BSP)\nCOMPILER_ARGS = --target=$(TARGET) \\\n    $(FEATURES)                    \\\n    --release\n\nRUSTC_CMD   = cargo rustc $(COMPILER_ARGS) --manifest-path $(KERNEL_MANIFEST)\nDOC_CMD     = cargo doc $(COMPILER_ARGS)\nCLIPPY_CMD  = cargo clippy $(COMPILER_ARGS)\nTEST_CMD    = cargo test $(COMPILER_ARGS) --manifest-path $(KERNEL_MANIFEST)\nOBJCOPY_CMD = rust-objcopy \\\n    --strip-all            \\\n    -O binary\n\nEXEC_QEMU          = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE)\nEXEC_TEST_DISPATCH = ruby ../common/tests/dispatch.rb\nEXEC_MINIPUSH      = ruby ../common/serial/minipush.rb\n\n##------------------------------------------------------------------------------\n## Dockerization\n##------------------------------------------------------------------------------\nDOCKER_CMD            = docker run -t --rm -v $(shell pwd):/work/tutorial -w /work/tutorial\nDOCKER_CMD_INTERACT   = $(DOCKER_CMD) -i\nDOCKER_ARG_DIR_COMMON = -v $(shell pwd)/../common:/work/common\nDOCKER_ARG_DIR_JTAG   = -v $(shell pwd)/../X1_JTAG_boot:/work/X1_JTAG_boot\nDOCKER_ARG_DEV        = --privileged -v /dev:/dev\nDOCKER_ARG_NET        = --network host\n\n# DOCKER_IMAGE defined in include file (see top of this file).\nDOCKER_QEMU  = $(DOCKER_CMD_INTERACT) $(DOCKER_IMAGE)\nDOCKER_TOOLS = $(DOCKER_CMD) $(DOCKER_IMAGE)\nDOCKER_TEST  = $(DOCKER_CMD) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\nDOCKER_GDB   = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_NET) $(DOCKER_IMAGE)\n\n# Dockerize commands, which require USB device passthrough, only on Linux.\nifeq ($(shell uname -s),Linux)\n    DOCKER_CMD_DEV = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_DEV)\n\n    DOCKER_CHAINBOOT = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\n    DOCKER_JTAGBOOT  = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_ARG_DIR_JTAG) $(DOCKER_IMAGE)\n    DOCKER_OPENOCD   = $(DOCKER_CMD_DEV) $(DOCKER_ARG_NET) $(DOCKER_IMAGE)\nelse\n    DOCKER_OPENOCD   = echo \"Not yet supported on non-Linux systems.\"; \\#\nendif\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: all doc qemu chainboot clippy clean readelf objdump nm check\n\nall: $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Save the configuration as a file, so make understands if it changed.\n##------------------------------------------------------------------------------\n$(LAST_BUILD_CONFIG):\n\t@rm -f target/*.build_config\n\t@mkdir -p target\n\t@touch $(LAST_BUILD_CONFIG)\n\n##------------------------------------------------------------------------------\n## Compile the kernel ELF\n##------------------------------------------------------------------------------\n$(KERNEL_ELF): $(KERNEL_ELF_DEPS)\n\t$(call color_header, \"Compiling kernel ELF - $(BSP)\")\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(RUSTC_CMD)\n\n##------------------------------------------------------------------------------\n## Generate the stripped kernel binary\n##------------------------------------------------------------------------------\n$(KERNEL_BIN): $(KERNEL_ELF)\n\t$(call color_header, \"Generating stripped binary\")\n\t@$(OBJCOPY_CMD) $(KERNEL_ELF) $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Name\")\n\t@echo $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Size\")\n\t$(call disk_usage_KiB, $(KERNEL_BIN))\n\n##------------------------------------------------------------------------------\n## Generate the documentation\n##------------------------------------------------------------------------------\ndoc:\n\t$(call color_header, \"Generating docs\")\n\t@$(DOC_CMD) --document-private-items --open\n\n##------------------------------------------------------------------------------\n## Run the kernel in QEMU\n##------------------------------------------------------------------------------\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\nqemu:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\nqemu: $(KERNEL_BIN)\n\t$(call color_header, \"Launching QEMU\")\n\t@$(DOCKER_QEMU) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\nendif\n\n##------------------------------------------------------------------------------\n## Push the kernel to the real HW target\n##------------------------------------------------------------------------------\nchainboot: $(KERNEL_BIN)\n\t@$(DOCKER_CHAINBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run clippy\n##------------------------------------------------------------------------------\nclippy:\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(CLIPPY_CMD)\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(CLIPPY_CMD) --features test_build --tests \\\n                --manifest-path $(KERNEL_MANIFEST)\n\n##------------------------------------------------------------------------------\n## Clean\n##------------------------------------------------------------------------------\nclean:\n\trm -rf target $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run readelf\n##------------------------------------------------------------------------------\nreadelf: $(KERNEL_ELF)\n\t$(call color_header, \"Launching readelf\")\n\t@$(DOCKER_TOOLS) $(READELF_BINARY) --headers $(KERNEL_ELF)\n\n##------------------------------------------------------------------------------\n## Run objdump\n##------------------------------------------------------------------------------\nobjdump: $(KERNEL_ELF)\n\t$(call color_header, \"Launching objdump\")\n\t@$(DOCKER_TOOLS) $(OBJDUMP_BINARY) --disassemble --demangle \\\n                --section .text   \\\n                --section .rodata \\\n                $(KERNEL_ELF) | rustfilt\n\n##------------------------------------------------------------------------------\n## Run nm\n##------------------------------------------------------------------------------\nnm: $(KERNEL_ELF)\n\t$(call color_header, \"Launching nm\")\n\t@$(DOCKER_TOOLS) $(NM_BINARY) --demangle --print-size $(KERNEL_ELF) | sort | rustfilt\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Debugging targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: jtagboot openocd gdb gdb-opt0\n\n##------------------------------------------------------------------------------\n## Push the JTAG boot image to the real HW target\n##------------------------------------------------------------------------------\njtagboot:\n\t@$(DOCKER_JTAGBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(JTAG_BOOT_IMAGE)\n\n##------------------------------------------------------------------------------\n## Start OpenOCD session\n##------------------------------------------------------------------------------\nopenocd:\n\t$(call color_header, \"Launching OpenOCD\")\n\t@$(DOCKER_OPENOCD) openocd $(OPENOCD_ARG)\n\n##------------------------------------------------------------------------------\n## Start GDB session\n##------------------------------------------------------------------------------\ngdb: RUSTC_MISC_ARGS += -C debuginfo=2\ngdb-opt0: RUSTC_MISC_ARGS += -C debuginfo=2 -C opt-level=0\ngdb gdb-opt0: $(KERNEL_ELF)\n\t$(call color_header, \"Launching GDB\")\n\t@$(DOCKER_GDB) gdb-multiarch -q $(KERNEL_ELF)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Testing targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: test test_boot test_unit test_integration\n\ntest_unit test_integration: FEATURES += --features test_build\n\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\ntest_boot test_unit test_integration test:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\n##------------------------------------------------------------------------------\n## Run boot test\n##------------------------------------------------------------------------------\ntest_boot: $(KERNEL_BIN)\n\t$(call color_header, \"Boot test - $(BSP)\")\n\t@$(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Helpers for unit and integration test targets\n##------------------------------------------------------------------------------\ndefine KERNEL_TEST_RUNNER\n#!/usr/bin/env bash\n\n    # The cargo test runner seems to change into the crate under test's directory. Therefore, ensure\n    # this script executes from the root.\n    cd $(shell pwd)\n\n    TEST_ELF=$$(echo $$1 | sed -e 's/.*target/target/g')\n    TEST_BINARY=$$(echo $$1.img | sed -e 's/.*target/target/g')\n\n    $(OBJCOPY_CMD) $$TEST_ELF $$TEST_BINARY\n    $(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_TEST_ARGS) -kernel $$TEST_BINARY\nendef\n\nexport KERNEL_TEST_RUNNER\n\ndefine test_prepare\n    @mkdir -p target\n    @echo \"$$KERNEL_TEST_RUNNER\" > target/kernel_test_runner.sh\n    @chmod +x target/kernel_test_runner.sh\nendef\n\n##------------------------------------------------------------------------------\n## Run unit test(s)\n##------------------------------------------------------------------------------\ntest_unit:\n\t$(call color_header, \"Compiling unit test(s) - $(BSP)\")\n\t$(call test_prepare)\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(TEST_CMD) --lib\n\n##------------------------------------------------------------------------------\n## Run integration test(s)\n##------------------------------------------------------------------------------\ntest_integration:\n\t$(call color_header, \"Compiling integration test(s) - $(BSP)\")\n\t$(call test_prepare)\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(TEST_CMD) $(TEST_ARG)\n\ntest: test_boot test_unit test_integration\n\nendif\n"
  },
  {
    "path": "12_integrated_testing/README.md",
    "content": "# Tutorial 12 - Integrated Testing\n\n## tl;dr\n\n- We implement our own integrated test framework using `Rust`'s [custom_test_frameworks] feature by\n  enabling `Unit Tests` and `Integration Tests` using `QEMU`.\n- It is also possible to have test automation for I/O with the kernel's `console` (provided over\n  `UART` in our case). That is, sending strings/characters to the console and expecting specific\n  answers in return.\n- The already existing basic `boot test` remains unchanged.\n\n<img src=\"../doc/12_demo.gif\" width=\"880\">\n\n## Table of Contents\n\n- [Introduction](#introduction)\n- [Challenges](#challenges)\n  * [Acknowledgements](#acknowledgements)\n- [Folder Restructuring](#folder-restructuring)\n- [Implementation](#implementation)\n  * [Test Organization](#test-organization)\n  * [Enabling `custom_test_frameworks` for Unit Tests](#enabling-custom_test_frameworks-for-unit-tests)\n    + [The Unit Test Runner](#the-unit-test-runner)\n    + [Calling the Test `main()` Function](#calling-the-test-main-function)\n  * [Quitting QEMU with user-defined Exit Codes](#quitting-qemu-with-user-defined-exit-codes)\n    + [Exiting Unit Tests](#exiting-unit-tests)\n  * [Controlling Test Kernel Execution](#controlling-test-kernel-execution)\n    + [Wrapping QEMU Test Execution](#wrapping-qemu-test-execution)\n  * [Writing Unit Tests](#writing-unit-tests)\n  * [Integration Tests](#integration-tests)\n    + [Test Harness](#test-harness)\n    + [No Test Harness](#no-test-harness)\n    + [Overriding Panic Behavior](#overriding-panic-behavior)\n  * [Console Tests](#console-tests)\n- [Test it](#test-it)\n- [Diff to previous](#diff-to-previous)\n\n## Introduction\n\nThrough the course of the previous tutorials, we silently started to adopt a kind of anti-pattern:\nUsing the kernel's main function to not only boot the target, but also test or showcase\nfunctionality. For example:\n  - Stalling execution during boot to test the kernel's timekeeping code by spinning for 1 second.\n  - Willingly causing exceptions to see the exception handler running.\n\nThe feature set of the kernel is now rich enough so that it makes sense to introduce proper\nintegrated testing modeled after Rust's [native testing framework]. This tutorial extends our single\nexisting kernel test with three new testing facilities:\n  - Classic `Unit Tests`.\n  - [Integration Tests] (self-contained tests stored in the `$CRATE/tests/` directory).\n  - `Console I/O Tests`. These are integration tests acting on external stimuli - aka `console`\n    input. Sending strings/characters to the console and expecting specific answers in return.\n\n[native testing framework]: https://doc.rust-lang.org/book/ch11-00-testing.html\n\n## Challenges\n\nTesting Rust `#![no_std]` code like our kernel is, at the point of writing this tutorial, not an\neasy endeavor. The short version is: We cannot use Rust's [native testing framework] straight away.\nUtilizing the `#[test]` attribute macro and running `cargo test` would throw compilation errors,\nbecause there are dependencies on the standard library.\n\n[native testing framework]: https://doc.rust-lang.org/book/ch11-00-testing.html\n\nWe have to fall back to Rust's unstable [custom_test_frameworks] feature. It relieves us from\ndependencies on the standard library, but comes at the cost of having a reduced feature set. Instead\nof annotating functions with `#[test]`, the `#[test_case]` attribute must be used. Additionally, we\nneed to write a `test_runner` function, which is supposed to execute all the functions annotated\nwith `#[test_case]`. This is barely enough to get `Unit Tests` running, though. There will be some\nmore challenges that need be solved for getting `Integration Tests` running as well.\n\nPlease note that for automation purposes, all testing will be done in `QEMU` and not on real\nhardware.\n\n[custom_test_frameworks]: https://doc.rust-lang.org/unstable-book/language-features/custom-test-frameworks.html\n[Integration Tests]: https://doc.rust-lang.org/book/ch11-03-test-organization.html#integration-tests\n\n### Acknowledgements\n\nOn this occasion, kudos to [@phil-opp] for his x86-based [testing] article. It helped a lot in\nputting together this tutorial. Please go ahead and read it for a different perspective and\nadditional insights.\n\n[testing]: https://os.phil-opp.com/testing\n\n## Folder Restructuring\n\nFor reasons explained later, in this tutorial, we need to add two support crates next to our main\nkernel crate. To keep everything organized in separate directories, we are switching to what `cargo`\ncalls a [virtual manifest]. The kernel crate moves to `$ROOT/kernel`, and the support crates will go\ninto `$ROOT/libraries/`. The `Cargo.toml` in the `$ROOT` folder desribes this layout:\n\n```toml\n[workspace]\n\nmembers = [\n        \"libraries/*\",\n        \"kernel\"\n]\n```\n\n[virtual manifest]: https://doc.rust-lang.org/cargo/reference/workspaces.html#virtual-manifest\n\n## Implementation\n\nWe introduce two new `Makefile` targets:\n\n```console\n$ make test_unit\n$ make test_integration\n```\n\nIn essence, the `make test_*` targets will execute `cargo test` instead of `cargo rustc`. The\ndetails will be explained in due course. The rest of the tutorial will explain as chronologically as\npossible what happens when `make test_*` aka `cargo test` runs.\n\nPlease note that the new targets are added to the existing `make test` target, so this is now your\none-stop target to execute all possible tests for the kernel:\n\n```Makefile\ntest: test_boot test_unit test_integration\n```\n\n### Test Organization\n\nUntil now, our kernel crate was a so-called `binary crate`. As [explained in the official Rust\nbook], this crate type disallows having `integration tests`. Quoting the book:\n\n[explained in the official Rust book]: https://doc.rust-lang.org/book/ch11-03-test-organization.html#integration-tests-for-binary-crates\n\n> If our project is a binary crate that only contains a _src/main.rs_ file and doesn’t have a\n> _src/lib.rs_ file, we can’t create integration tests in the _tests_ directory and bring functions\n> defined in the _src/main.rs_ file into scope with a `use` statement. Only library crates expose\n> functions that other crates can use; binary crates are meant to be run on their own.\n\n> This is one of the reasons Rust projects that provide a binary have a straightforward\n> _src/main.rs_ file that calls logic that lives in the _src/lib.rs_ file. Using that structure,\n> integration tests _can_ test the library crate with `use` to make the important functionality\n> available. If the important functionality works, the small amount of code in the _src/main.rs_\n> file will work as well, and that small amount of code doesn’t need to be tested.\n\nSo let's do that first: We add a `lib.rs` to our crate that aggregates and exports the lion's share\nof the kernel code. The `main.rs` file is stripped down to the minimum. It only keeps the\n`kernel_init() -> !` and `kernel_main() -> !` functions, everything else is brought into scope with\n`use` statements.\n\nSince it is not possible to use `kernel` as the name for both the library and the binary part of the\ncrate, new entries in `$ROOT/kernel/Cargo.toml` are needed to differentiate the names. What's more,\n`cargo test` would try to compile and run `unit tests` for both. In our case, it will be sufficient\nto have all the unit test code in `lib.rs`, so test generation for `main.rs` can be disabled in\n`Cargo.toml` as well through the `test` flag:\n\n```toml\n[lib]\nname = \"libkernel\"\ntest = true\n\n[[bin]]\nname = \"kernel\"\ntest = false\n```\n\n### Enabling `custom_test_frameworks` for Unit Tests\n\nIn `lib.rs`, we add the following headers to get started with `custom_test_frameworks`:\n\n```rust\n// Testing\n#![cfg_attr(test, no_main)]\n#![feature(custom_test_frameworks)]\n#![reexport_test_harness_main = \"test_main\"]\n#![test_runner(crate::test_runner)]\n```\n\nSince this is a library now, we do not keep the `#![no_main]` inner attribute that `main.rs` has,\nbecause a library has no `main()` entry function, so the attribute does not apply. When compiling\nfor testing, though, it is still needed. The reason is that `cargo test` basically turns `lib.rs`\ninto a binary again by inserting a generated `main()` function (which is then calling a function\nthat runs all the unit tests, but more about that in a second...).\n\nHowever, since  our kernel code [overrides the compiler-inserted `main` shim] by way of using\n`#![no_main]`, we need the same when `cargo test` is producing its test kernel binary. After all,\nwhat we want is a minimal kernel that boots on the target and runs its own unit tests. Therefore, we\nconditionally set this attribute (`#![cfg_attr(test, no_main)]`) when the `test` flag is set, which\nit is when `cargo test` runs.\n\n[overrides the compiler-inserted `main` shim]: https://doc.rust-lang.org/unstable-book/language-features/lang-items.html?highlight=no_main#writing-an-executable-without-stdlib\n\n#### The Unit Test Runner\n\nThe `#![test_runner(crate::test_runner)]` attribute declares the path of the test runner function\nthat we are supposed to provide. This is the one that will be called by the `cargo test` generated\n`main()` function. Here is the implementation in `lib.rs`:\n\n```rust\n/// The default runner for unit tests.\npub fn test_runner(tests: &[&test_types::UnitTest]) {\n    // This line will be printed as the test header.\n    println!(\"Running {} tests\", tests.len());\n\n    for (i, test) in tests.iter().enumerate() {\n        print!(\"{:>3}. {:.<58}\", i + 1, test.name);\n\n        // Run the actual test.\n        (test.test_func)();\n\n        // Failed tests call panic!(). Execution reaches here only if the test has passed.\n        println!(\"[ok]\")\n    }\n}\n```\n\nThe function signature shows that `test_runner` takes one argument: A slice of\n`test_types::UnitTest` references. This type definition lives in an external crate stored at\n`$ROOT/libraries/test_types`. It is external because the type is also needed for a self-made\n[procedural macro] that we'll use to write unit tests, and procedural macros _have_ to live in their\nown crate. So to avoid a circular dependency between kernel and proc-macro, this split was needed.\nAnyways, here is the type definition:\n\n[procedural macro]: https://doc.rust-lang.org/reference/procedural-macros.html\n\n```rust\n/// Unit test container.\npub struct UnitTest {\n    /// Name of the test.\n    pub name: &'static str,\n\n    /// Function pointer to the test.\n    pub test_func: fn(),\n}\n```\n\nA `UnitTest` provides a name and a classic function pointer to the unit test function. The\n`test_runner` just iterates over the slice, prints the respective test's name and calls the test\nfunction.\n\nThe convetion is that as long as the test function does not `panic!`, the test was successful.\n\n#### Calling the Test `main()` Function\n\nThe last of the attributes we added is `#![reexport_test_harness_main = \"test_main\"]`. Remember that\nour kernel uses the `no_main` attribute, and that we also set it for the test compilation. We did\nthat because we wrote our own `_start()` function, which kicks off the following call chain during\nkernel boot:\n\n| | Function  | File |\n| - | - | - |\n| 1. | `_start()` | The library's `boot.s` |\n| 2. | (some more aarch64 code) | The library's `boot.rs` |\n| 3. | `kernel_init()` | `main.rs` |\n| 4. | `kernel_main()` | `main.rs` |\n\nA function named `main` is never called. Hence, the `main()` function generated by `cargo test`\nwould be silently dropped, and therefore the tests would never be executed. As you can see, the\nfirst function getting called in our carved-out `main.rs` is `kernel_init()`. So in order to get the\ntests to execute, we add a test-environment version of `kernel_init()` to `lib.rs` as well\n(conditional compilation ensures it is only present when the test flag is set), and call the `cargo\ntest` generated `main()` function from there.\n\nThis is where `#![reexport_test_harness_main = \"test_main\"]` finally comes into picture. It declares\nthe name of the generated main function so that we can manually call it. Here is the final\nimplementation in `lib.rs`:\n\n```rust\n/// The `kernel_init()` for unit tests.\n#[cfg(test)]\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    bsp::driver::qemu_bring_up_console();\n\n    test_main();\n\n    cpu::qemu_exit_success()\n}\n```\n\nNote the call to `bsp::driver::qemu_bring_up_console()`. Since we are running all our tests inside\n`QEMU`, we need to ensure that whatever peripheral implements the kernel's `console` interface is\ninitialized, so that we can print from our tests. If you recall [tutorial 03], bringing up\nperipherals in `QEMU` might not need the full initialization as is needed on real hardware (setting\nclocks, config registers, etc...) due to the abstractions in `QEMU`'s emulation code. So this is an\nopportunity to cut down on setup code.\n\n[tutorial 03]: ../03_hacky_hello_world\n\nAs a matter of fact, for the `Raspberrys`, nothing needs to be done, so the function is empy. But\nthis might be different for other hardware emulated by `QEMU`, so it makes sense to introduce the\nfunction now to make it easier in case new `BSPs` are added to the kernel in the future.\n\nNext, the reexported `test_main()` is called, which will call our `test_runner()` which finally\nprints the unit test names and executes them.\n\n### Quitting QEMU with user-defined Exit Codes\n\nLet's recap where we are right now:\n\nWe've enabled `custom_test_frameworks` in `lib.rs` to a point where, when using a `make test_unit`\ntarget, the code gets compiled to a test kernel binary that eventually executes all the\n(yet-to-be-defined) `UnitTest` instances by executing all the way from `_start()` to our\n`test_runner()` function.\n\nThrough mechanisms that are explained later, `cargo` will now instantiate a `QEMU` process that\nexectues this test kernel. The question now is: How is test success/failure communicated to `cargo`?\nAnswer: `cargo` inspects `QEMU`'s [exit status]:\n\n  - `0` translates to testing was successful.\n  - `non-0` means failure.\n\nHence, we need a clever trick now so that our Rust kernel code can get `QEMU` to exit itself with an\nexit status that the kernel code supplies. In [@phil-opp]'s testing article, you [learned how to do\nthis] for `x86 QEMU` systems by using a special `ISA` debug-exit device. Unfortunately, we can't\nhave that one for our `aarch64` system because it is not compatible.\n\nIn our case, we can leverage the ARM [semihosting] emulation of `QEMU` and do a `SYS_EXIT`\nsemihosting call with an additional parameter for the exit code. I've written a separate crate,\n[qemu-exit], to do this. So let us import it and utilize it in `_arch/aarch64/cpu.rs` to provide the\nfollowing exit calls for the kernel:\n\n```rust\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n#[cfg(feature = \"test_build\")]\nuse qemu_exit::QEMUExit;\n\n#[cfg(feature = \"test_build\")]\nconst QEMU_EXIT_HANDLE: qemu_exit::AArch64 = qemu_exit::AArch64::new();\n\n/// Make the host QEMU binary execute `exit(1)`.\n#[cfg(feature = \"test_build\")]\npub fn qemu_exit_failure() -> ! {\n    QEMU_EXIT_HANDLE.exit_failure()\n}\n\n/// Make the host QEMU binary execute `exit(0)`.\n#[cfg(feature = \"test_build\")]\npub fn qemu_exit_success() -> ! {\n    QEMU_EXIT_HANDLE.exit_success()\n}\n```\n\n[Click here] in case you are interested in the implementation. Note that for the functions to work,\nthe `-semihosting` flag must be added to the `QEMU` invocation.\n\nYou might have also noted the `#[cfg(feature = \"test_build\")]`. In the `Makefile`, we ensure that\nthis feature is only enabled when `cargo test` runs. This way, it is ensured that testing-specific\ncode is conditionally compiled only for testing.\n\n[exit status]: https://en.wikipedia.org/wiki/Exit_status\n[@phil-opp]: https://github.com/phil-opp\n[learned how to do this]: https://os.phil-opp.com/testing/#exiting-qemu\n[semihosting]: https://static.docs.arm.com/100863/0200/semihosting.pdf\n[qemu-exit]: https://github.com/andre-richter/qemu-exit\n[Click here]: https://github.com/andre-richter/qemu-exit/blob/master/src/aarch64.rs\n\n#### Exiting Unit Tests\n\nUnit test failure shall be triggered by the `panic!` macro, either directly or by way of using\n`assert!` macros. Until now, our `panic!` implementation finally called `cpu::wait_forever()` to\nsafely park the panicked CPU core in a busy loop. This can't be used for the unit tests, because\n`cargo` would wait forever for `QEMU` to exit and stall the whole test run. Again, conditional\ncompilation is used to differentiate between a release and testing version of how a `panic!`\nconcludes:\n\n```rust\n/// The point of exit for `libkernel`.\n///\n/// It is linked weakly, so that the integration tests can overload its standard behavior.\n#[linkage = \"weak\"]\n#[no_mangle]\nfn _panic_exit() -> ! {\n    #[cfg(not(feature = \"test_build\"))]\n    {\n        cpu::wait_forever()\n    }\n\n    #[cfg(feature = \"test_build\")]\n    {\n        cpu::qemu_exit_failure()\n    }\n}\n```\n\nIn case _none_ of the unit tests panicked, `lib.rs`'s `kernel_init()` calls\n`cpu::qemu_exit_success()` to successfully conclude the unit test run.\n\n### Controlling Test Kernel Execution\n\nNow is a good time to catch up on how the test kernel binary is actually being executed. Normally,\n`cargo test` would try to execute the compiled binary as a normal child process. This would fail\nhorribly because we build a kernel, and not a userspace process. Also, chances are high that you sit\nin front of an `x86` machine, whereas the RPi kernel is `AArch64`.\n\nTherefore, we need to install some hooks that make sure the test kernel gets executed inside `QEMU`,\nquite like it is done for the existing `make qemu` target that is in place since `tutorial 1`. The\nfirst step is to add a new file to the project, `.cargo/config.toml`:\n\n```toml\n[target.'cfg(target_os = \"none\")']\nrunner = \"target/kernel_test_runner.sh\"\n```\n\nInstead of executing a compilation result directly, the `runner` flag will instruct `cargo` to\ndelegate the execution. Using the setting depicted above, `target/kernel_test_runner.sh` will be\nexecuted and given the full path to the compiled test kernel as the first command line argument.\n\nThe file `kernel_test_runner.sh` does not exist by default. We generate it on demand when one of the\n`make test_*` targets is called:\n\n```Makefile\n##------------------------------------------------------------------------------\n## Helpers for unit and integration test targets\n##------------------------------------------------------------------------------\ndefine KERNEL_TEST_RUNNER\n    #!/usr/bin/env bash\n\n    # The cargo test runner seems to change into the crate under test's directory. Therefore, ensure\n    # this script executes from the root.\n    cd $(shell pwd)\n\n    TEST_ELF=$$(echo $$1 | sed -e 's/.*target/target/g')\n    TEST_BINARY=$$(echo $$1.img | sed -e 's/.*target/target/g')\n\n    $(OBJCOPY_CMD) $$TEST_ELF $$TEST_BINARY\n    $(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_TEST_ARGS) -kernel $$TEST_BINARY\nendef\n\nexport KERNEL_TEST_RUNNER\n\ndefine test_prepare\n    @mkdir -p target\n    @echo \"$$KERNEL_TEST_RUNNER\" > target/kernel_test_runner.sh\n    @chmod +x target/kernel_test_runner.sh\nendef\n\n##------------------------------------------------------------------------------\n## Run unit test(s)\n##------------------------------------------------------------------------------\ntest_unit:\n\t$(call color_header, \"Compiling unit test(s) - $(BSP)\")\n\t$(call test_prepare)\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(TEST_CMD) --lib\n```\n\nIt first does the standard `objcopy` step to strip the `ELF` down to a raw binary. Just like in all\nthe other Makefile targets. Next, the script generates a relative path from the absolute path\nprovided to it by `cargo`, and finally compiles a `docker` command to execute the test kernel. For\nreference, here it is fully resolved for an `RPi3 BSP`:\n\n```bash\ndocker run -t --rm -v /opt/rust-raspberrypi-OS-tutorials/12_integrated_testing:/work/tutorial -w /work/tutorial -v /opt/rust-raspberrypi-OS-tutorials/12_integrated_testing/../common:/work/common rustembedded/osdev-utils:2021.12 ruby ../common/tests/dispatch.rb qemu-system-aarch64 -M raspi3 -serial stdio -display none -semihosting -kernel $TEST_BINARY\n```\n\nThis command is quite similar to the one used in the `make test_boot` target that we have since\n`tutorial 3`. However, we never bothered explaining it, so lets take a closer look this time. One of\nthe key ingredients is that we execute this script: `ruby ../common/tests/dispatch.rb`.\n\n#### Wrapping QEMU Test Execution\n\n`dispatch.rb` is a [Ruby] script which first determines what kind of test is due by inspecting the\n`QEMU`-command that was given to it. In case of `unit tests`, we are only interested if they all\nexecuted successfully, which can be checked by inspecting `QEMU`'s exit code. So the script takes\nthe provided qemu command it got from `ARGV`, and creates and runs an instance of `ExitCodeTest`:\n\n```ruby\nqemu_cmd = ARGV.join(' ')\nbinary = ARGV.last\ntest_name = binary.gsub(%r{.*deps/}, '').split('-')[0]\n\n# Check if virtual manifest (tutorial 12 or later) or not\npath_prefix = File.exist?('kernel/Cargo.toml') ? 'kernel/' : ''\n\ncase test_name\nwhen 'kernel8.img'\n    load \"#{path_prefix}tests/boot_test_string.rb\" # provides 'EXPECTED_PRINT'\n    BootTest.new(qemu_cmd, EXPECTED_PRINT).run # Doesn't return\n\nwhen 'libkernel'\n    ExitCodeTest.new(qemu_cmd, 'Kernel library unit tests').run # Doesn't return\n```\n\nThe easy case is `QEMU` exiting by itself by means of `aarch64::exit_success()` or\n`aarch64::exit_failure()`. But the script can also catch the case of a test that gets stuck, e.g. in\nan unintentional busy loop or a crash. If `ExitCodeTest` does not observe any output of the test\nkernel for `MAX_WAIT_SECS`, it cancels the execution and marks the test as failed. Test success or\nfailure is finally reported back to `cargo`.\n\nHere is the essential part happening in `class ExitCodeTest` (If `QEMU` exits itself, an `EOFError`\nis thrown):\n\n```ruby\ndef run_concrete_test\n    Timeout.timeout(MAX_WAIT_SECS) do\n        @test_output << @qemu_serial.read_nonblock(1024) while @qemu_serial.wait_readable\n    end\nrescue EOFError\n    @qemu_serial.close\n    @test_error = $CHILD_STATUS.to_i.zero? ? false : 'QEMU exit status != 0'\nrescue Timeout::Error\n    @test_error = 'Timed out waiting for test'\nrescue StandardError => e\n    @test_error = e.inspect\nend\n```\n\nPlease note that `dispatch.rb` and all its dependencies live in the shared folder\n`../common/tests/`.\n\n[Ruby]: https://www.ruby-lang.org/\n\n### Writing Unit Tests\n\nAlright, that's a wrap for the whole chain from `make test_unit` all the way to reporting the test\nexit status back to `cargo test`. It is a lot to digest already, but we haven't even learned to\nwrite `Unit Tests` yet.\n\nIn essence, it is almost like in `std` environments, with the difference that `#[test]` can't be\nused, because it is part of the standard library. The `no_std` replacement attribute provided by\n`custom_test_frameworks` is `#[test_case]`. You can put `#[test_case]` before functions, constants\nor statics (you have to decide for one and stick with it). Each attributed item is added to the\n\"list\" that is then passed to the `test_runner` function.\n\nAs you learned earlier, we decided that our tests shall be instances of `test_types::UnitTest`. Here\nis the type definition again:\n\n```rust\n/// Unit test container.\npub struct UnitTest {\n    /// Name of the test.\n    pub name: &'static str,\n\n    /// Function pointer to the test.\n    pub test_func: fn(),\n}\n```\n\nSo what we could do now is write something like:\n\n```rust\n#[cfg(test)]\nmod tests {\n    use super::*;\n\n    #[test_case]\n    const TEST1: test_types::UnitTest = test_types::UnitTest {\n            name: \"test_runner_executes_in_kernel_mode\",\n            test_func: || {\n                let (level, _) = current_privilege_level();\n\n                assert!(level == PrivilegeLevel::Kernel)\n            },\n        };\n}\n```\n\nSince this is a bit boiler-platy with the const and name definition, let's write a [procedural\nmacro] named `#[kernel_test]` to simplify this. It should work this way:\n\n  1. Must be put before functions that take no arguments and return nothing.\n  1. Automatically constructs a `const UnitTest` from attributed functions like shown above by:\n      1. Converting the function name to the `name` member of the `UnitTest` struct.\n      1. Populating the `test_func` member with a closure that executes the body of the attributed\n         function.\n\nFor the sake of brevity, we're not going to discuss the macro implementation. [The source is in the\ntest-macros crate] if you're interested in it. Using the macro, the example shown before now boils\ndown to this (this is now an actual example from [exception.rs]:\n\n[procedural macro]: https://doc.rust-lang.org/reference/procedural-macros.html\n[The source is in the test-macros crate]: test-macros/src/lib.rs\n[exception.rs]: src/exception.rs\n\n```rust\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// Libkernel unit tests must execute in kernel mode.\n    #[kernel_test]\n    fn test_runner_executes_in_kernel_mode() {\n        let (level, _) = current_privilege_level();\n\n        assert!(level == PrivilegeLevel::Kernel)\n    }\n}\n```\n\nNote that since proc macros need to live in their own crates, we need to create a new one at\n`$ROOT/libraries/test-macros` and save it there.\n\nAaaaaand that's how you write unit tests. We're finished with that part for good now :raised_hands:.\n\n### Integration Tests\n\nWe are still not done with the tutorial, though :scream:.\n\nIntegration tests need some special attention here and there too. As you already learned, they live\nin `$CRATE/tests/`. Each `.rs` file in there gets compiled into its own test kernel binary and\nexecuted separately by `cargo test`. The code in the integration tests includes the library part of\nour kernel (`libkernel`) through `use` statements.\n\nAlso note that the entry point for each `integration test` must be the `kernel_init()` function\nagain, just like in the `unit test` case.\n\n#### Test Harness\n\nBy default, `cargo test` will pull in the test harness (that's the official name for the generated\n`main()` function) into integration tests as well. This gives you a further means of partitioning\nyour test code into individual chunks. For example, take a look at `tests/01_timer_sanity.rs`:\n\n```rust\n//! Timer sanity tests.\n\n#![feature(custom_test_frameworks)]\n#![no_main]\n#![no_std]\n#![reexport_test_harness_main = \"test_main\"]\n#![test_runner(libkernel::test_runner)]\n\nuse core::time::Duration;\nuse libkernel::{bsp, cpu, exception, time};\nuse test_macros::kernel_test;\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    bsp::driver::qemu_bring_up_console();\n\n    // Depending on CPU arch, some timer bring-up code could go here. Not needed for the RPi.\n\n    test_main();\n\n    cpu::qemu_exit_success()\n}\n\n/// Simple check that the timer is running.\n#[kernel_test]\nfn timer_is_counting() {\n    assert!(time::time_manager().uptime().as_nanos() > 0)\n}\n\n/// Timer resolution must be sufficient.\n#[kernel_test]\nfn timer_resolution_is_sufficient() {\n    assert!(time::time_manager().resolution().as_nanos() > 0);\n    assert!(time::time_manager().resolution().as_nanos() < 100)\n}\n```\n\nNote how the `test_runner` from `libkernel` is pulled in through\n`#![test_runner(libkernel::test_runner)]`.\n\n#### No Test Harness\n\nFor some tests, however, it is not needed to have the harness, because there is no need or\npossibility to partition the test into individual pieces. In this case, all the test code can live\nin `kernel_init()`, and harness generation can be turned off through `$ROOT/kernel/Cargo.toml`. This\ntutorial introduces two tests that don't need a harness. Here is how harness generation is turned\noff for them:\n\n```toml\n# List of tests without harness.\n[[test]]\nname = \"00_console_sanity\"\nharness = false\n\n[[test]]\nname = \"02_exception_sync_page_fault\"\nharness = false\n\n[[test]]\nname = \"03_exception_restore_sanity\"\nharness = false\n```\n\n#### Overriding Panic Behavior\n\nDid you notice the `#[linkage = \"weak\"]` attribute some chapters earlier at the `_panic_exit()`\nfunction? This marks the function in `lib.rs` as a [weak symbol]. Let's look at it again:\n\n```rust\n/// The point of exit for `libkernel`.\n///\n/// It is linked weakly, so that the integration tests can overload its standard behavior.\n#[linkage = \"weak\"]\n#[no_mangle]\nfn _panic_exit() -> ! {\n    #[cfg(not(feature = \"test_build\"))]\n    {\n        cpu::wait_forever()\n    }\n\n    #[cfg(feature = \"test_build\")]\n    {\n        cpu::qemu_exit_failure()\n    }\n}\n```\n\n[weak symbol]: https://en.wikipedia.org/wiki/Weak_symbol\n\nThis enables integration tests in `$CRATE/tests/` to override this function according to their\nneeds. This is useful, because depending on the kind of test, a `panic!` could mean success or\nfailure. For example, `tests/02_exception_sync_page_fault.rs` is intentionally causing a page fault,\nso the wanted outcome is a `panic!`. Here is the whole test (minus some inline comments):\n\n```rust\n//! Page faults must result in synchronous exceptions.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\nmod panic_exit_success;\n\nuse libkernel::{bsp, cpu, exception, info, memory, println};\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    use memory::mmu::interface::MMU;\n\n    exception::handling_init();\n    bsp::driver::qemu_bring_up_console();\n\n    // This line will be printed as the test header.\n    println!(\"Testing synchronous exception handling by causing a page fault\");\n\n    if let Err(string) = memory::mmu::mmu().enable_mmu_and_caching() {\n        info!(\"MMU: {}\", string);\n        cpu::qemu_exit_failure()\n    }\n\n    info!(\"Writing beyond mapped area to address 9 GiB...\");\n    let big_addr: u64 = 9 * 1024 * 1024 * 1024;\n    core::ptr::read_volatile(big_addr as *mut u64);\n\n    // If execution reaches here, the memory access above did not cause a page fault exception.\n    cpu::qemu_exit_failure()\n}\n```\n\nThe `_panic_exit()` version that makes `QEMU` return `0` (indicating test success) is pulled in by\n`mod panic_exit_success;`, and it will take precedence over the `weak` version from `lib.rs`.\n\n### Console Tests\n\nAs the kernel or OS grows, it will be more and more interesting to test user/kernel interaction\nthrough the serial console. That is, sending strings/characters to the console and expecting\nspecific answers in return. The `dispatch.rb` wrapper script provides infrastructure to recognize\nand dispatch console I/O tests with little overhead. It basically works like this:\n\n  1. For each integration test, check if a companion file to the `.rs` test file exists.\n      - A companion file has the same name, but ends in `.rb`.\n      - The companion file contains one or more console I/O subtests.\n  1. If it exists, load the file to dynamically import the console subtests.\n  1. Create a `ConsoleIOTest` instance and run it.\n      - This first spawns `QEMU` and attaches to `QEMU`'s serial console emulation.\n      - Then it runs all console subtests on it.\n\nHere is an excerpt from `00_console_sanity.rb` showing a subtest that does a handshake with the\nkernel over the console:\n\n```ruby\nrequire 'console_io_test'\n\n# Verify sending and receiving works as expected.\nclass TxRxHandshakeTest < SubtestBase\n    def name\n        'Transmit and Receive handshake'\n    end\n\n    def run(qemu_out, qemu_in)\n        qemu_in.write_nonblock('ABC')\n        expect_or_raise(qemu_out, 'OK1234')\n    end\nend\n```\n\nThe subtest first sends `\"ABC\"` over the console to the kernel, and then expects to receive\n`\"OK1234\"` back. On the kernel side, it looks like this in `00_console_sanity.rs`:\n\n```rust\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Console tests should time out on the I/O harness in case of panic.\nmod panic_wait_forever;\n\nuse libkernel::{bsp, console, cpu, exception, print};\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    use console::console;\n\n    exception::handling_init();\n    bsp::driver::qemu_bring_up_console();\n\n    // Handshake\n    assert_eq!(console().read_char(), 'A');\n    assert_eq!(console().read_char(), 'B');\n    assert_eq!(console().read_char(), 'C');\n    print!(\"OK1234\");\n```\n\n## Test it\n\nBelieve it or not, that is all. There are four ways you can run tests now:\n\n  1. `make test` will run all tests back-to-back. That is, the ever existing `boot test` first, then\n     `unit tests`, then `integration tests`.\n  1. `make test_unit` will run `libkernel`'s unit tests.\n  1. `make test_integration` will run all integration tests back-to-back.\n  1. `TEST=TEST_NAME make test_integration` will run a specficic integration test.\n      - For example, `TEST=01_timer_sanity make test_integration`\n\n```console\n$ make test\n[...]\n\n     Running unittests (target/aarch64-unknown-none-softfloat/release/deps/libkernel-142a8d94bc9c615a)\n         -------------------------------------------------------------------\n         🦀 Running 6 tests\n         -------------------------------------------------------------------\n\n           1. virt_mem_layout_sections_are_64KiB_aligned................[ok]\n           2. virt_mem_layout_has_no_overlaps...........................[ok]\n           3. test_runner_executes_in_kernel_mode.......................[ok]\n           4. kernel_tables_in_bss......................................[ok]\n           5. size_of_tabledescriptor_equals_64_bit.....................[ok]\n           6. size_of_pagedescriptor_equals_64_bit......................[ok]\n\n         -------------------------------------------------------------------\n         ✅ Success: Kernel library unit tests\n         -------------------------------------------------------------------\n\n\n\nCompiling integration test(s) - rpi3\n    Finished release [optimized] target(s) in 0.00s\n     Running tests/00_console_sanity.rs (target/aarch64-unknown-none-softfloat/release/deps/00_console_sanity-c06130838f14dbff)\n         -------------------------------------------------------------------\n         🦀 Running 3 console I/O tests\n         -------------------------------------------------------------------\n\n           1. Transmit and Receive handshake............................[ok]\n           2. Transmit statistics.......................................[ok]\n           3. Receive statistics........................................[ok]\n\n         Console log:\n           ABCOK123463\n\n         -------------------------------------------------------------------\n         ✅ Success: 00_console_sanity.rs\n         -------------------------------------------------------------------\n\n\n     Running tests/01_timer_sanity.rs (target/aarch64-unknown-none-softfloat/release/deps/01_timer_sanity-62a954d22239d1a3)\n         -------------------------------------------------------------------\n         🦀 Running 3 tests\n         -------------------------------------------------------------------\n\n           1. timer_is_counting.........................................[ok]\n           2. timer_resolution_is_sufficient............................[ok]\n           3. spin_accuracy_check_1_second..............................[ok]\n\n         -------------------------------------------------------------------\n         ✅ Success: 01_timer_sanity.rs\n         -------------------------------------------------------------------\n\n\n     Running tests/02_exception_sync_page_fault.rs (target/aarch64-unknown-none-softfloat/release/deps/02_exception_sync_page_fault-2d8ec603ef1c4d8e)\n         -------------------------------------------------------------------\n         🦀 Testing synchronous exception handling by causing a page fault\n         -------------------------------------------------------------------\n\n         [    0.132792] Writing beyond mapped area to address 9 GiB...\n         [    0.134563] Kernel panic!\n\n         Panic location:\n               File 'src/_arch/aarch64/exception.rs', line 58, column 5\n\n         CPU Exception!\n\n         ESR_EL1: 0x96000004\n               Exception Class         (EC) : 0x25 - Data Abort, current EL\n         [...]\n\n         -------------------------------------------------------------------\n         ✅ Success: 02_exception_sync_page_fault.rs\n         -------------------------------------------------------------------\n\n\n     Running tests/03_exception_restore_sanity.rs (target/aarch64-unknown-none-softfloat/release/deps/03_exception_restore_sanity-a56e14285bb26e0e)\n         -------------------------------------------------------------------\n         🦀 Running 1 console I/O tests\n         -------------------------------------------------------------------\n\n           1. Exception restore.........................................[ok]\n\n         Console log:\n           Testing exception restore\n           [    0.130757] Making a dummy system call\n           [    0.132592] Back from system call!\n\n         -------------------------------------------------------------------\n         ✅ Success: 03_exception_restore_sanity.rs\n         -------------------------------------------------------------------\n\n```\n\n## Diff to previous\n\nThe diff in this tutorial is skipped, because due to the changes in top-level folder structure, it\nbecomes unreadable. This might be fixed in the future. For now, consider using a diff tool like\n`meld` to diff between the previous and the `kernel` folder of this tutorial to see the lion's share\nof changes:\n\n```console\nmeld 11_exceptions_part1_groundwork 12_integrated_testing/kernel\n```\n"
  },
  {
    "path": "12_integrated_testing/kernel/Cargo.toml",
    "content": "[package]\nname = \"mingo\"\nversion = \"0.12.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n\n[features]\ndefault = []\nbsp_rpi3 = [\"tock-registers\"]\nbsp_rpi4 = [\"tock-registers\"]\ntest_build = [\"qemu-exit\"]\n\n##--------------------------------------------------------------------------------------------------\n## Dependencies\n##--------------------------------------------------------------------------------------------------\n\n[dependencies]\ntest-types = { path = \"../libraries/test-types\" }\n\n# Optional dependencies\ntock-registers = { version = \"0.8.x\", default-features = false, features = [\"register_types\"], optional = true }\nqemu-exit = { version = \"3.x.x\", optional = true }\n\n# Platform specific dependencies\n[target.'cfg(target_arch = \"aarch64\")'.dependencies]\naarch64-cpu = { version = \"9.x.x\" }\n\n##--------------------------------------------------------------------------------------------------\n## Testing\n##--------------------------------------------------------------------------------------------------\n\n[dev-dependencies]\ntest-macros = { path = \"../libraries/test-macros\" }\n\n# Unit tests are done in the library part of the kernel.\n[lib]\nname = \"libkernel\"\ntest = true\n\n# Disable unit tests for the kernel binary.\n[[bin]]\nname = \"kernel\"\npath = \"src/main.rs\"\ntest = false\n\n# List of tests without harness.\n[[test]]\nname = \"00_console_sanity\"\nharness = false\n\n[[test]]\nname = \"02_exception_sync_page_fault\"\nharness = false\n\n[[test]]\nname = \"03_exception_restore_sanity\"\nharness = false\n"
  },
  {
    "path": "12_integrated_testing/kernel/build.rs",
    "content": "use std::{env, fs, process};\n\nfn main() {\n    let ld_script_path = match env::var(\"LD_SCRIPT_PATH\") {\n        Ok(var) => var,\n        _ => process::exit(0),\n    };\n\n    let files = fs::read_dir(ld_script_path).unwrap();\n    files\n        .filter_map(Result::ok)\n        .filter(|d| {\n            if let Some(e) = d.path().extension() {\n                e == \"ld\"\n            } else {\n                false\n            }\n        })\n        .for_each(|f| println!(\"cargo:rerun-if-changed={}\", f.path().display()));\n}\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/_arch/aarch64/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural boot code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::boot::arch_boot\n\nuse aarch64_cpu::{asm, registers::*};\nuse core::arch::global_asm;\nuse tock_registers::interfaces::Writeable;\n\n// Assembly counterpart to this file.\nglobal_asm!(\n    include_str!(\"boot.s\"),\n    CONST_CURRENTEL_EL2 = const 0x8,\n    CONST_CORE_ID_MASK = const 0b11\n);\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Prepares the transition from EL2 to EL1.\n///\n/// # Safety\n///\n/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.\n/// - The HW state of EL1 must be prepared in a sound way.\n#[inline(always)]\nunsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: u64) {\n    // Enable timer counter registers for EL1.\n    CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET);\n\n    // No offset for reading the counters.\n    CNTVOFF_EL2.set(0);\n\n    // Set EL1 execution state to AArch64.\n    HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64);\n\n    // Set up a simulated exception return.\n    //\n    // First, fake a saved program status where all interrupts were masked and SP_EL1 was used as a\n    // stack pointer.\n    SPSR_EL2.write(\n        SPSR_EL2::D::Masked\n            + SPSR_EL2::A::Masked\n            + SPSR_EL2::I::Masked\n            + SPSR_EL2::F::Masked\n            + SPSR_EL2::M::EL1h,\n    );\n\n    // Second, let the link register point to kernel_init().\n    ELR_EL2.set(crate::kernel_init as *const () as u64);\n\n    // Set up SP_EL1 (stack pointer), which will be used by EL1 once we \"return\" to it. Since there\n    // are no plans to ever return to EL2, just re-use the same stack.\n    SP_EL1.set(phys_boot_core_stack_end_exclusive_addr);\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The Rust entry of the `kernel` binary.\n///\n/// The function is called from the assembly `_start` function.\n///\n/// # Safety\n///\n/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`.\n#[no_mangle]\npub unsafe extern \"C\" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! {\n    prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr);\n\n    // Use `eret` to \"return\" to EL1. This results in execution of kernel_init() in EL1.\n    asm::eret()\n}\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/_arch/aarch64/cpu/boot.s",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2022 Andre Richter <andre.o.richter@gmail.com>\n\n//--------------------------------------------------------------------------------------------------\n// Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Load the address of a symbol into a register, PC-relative.\n//\n// The symbol must lie within +/- 4 GiB of the Program Counter.\n//\n// # Resources\n//\n// - https://sourceware.org/binutils/docs-2.36/as/AArch64_002dRelocations.html\n.macro ADR_REL register, symbol\n\tadrp\t\\register, \\symbol\n\tadd\t\\register, \\register, #:lo12:\\symbol\n.endm\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n.section .text._start\n\n//------------------------------------------------------------------------------\n// fn _start()\n//------------------------------------------------------------------------------\n_start:\n\t// Only proceed if the core executes in EL2. Park it otherwise.\n\tmrs\tx0, CurrentEL\n\tcmp\tx0, {CONST_CURRENTEL_EL2}\n\tb.ne\t.L_parking_loop\n\n\t// Only proceed on the boot core. Park it otherwise.\n\tmrs\tx1, MPIDR_EL1\n\tand\tx1, x1, {CONST_CORE_ID_MASK}\n\tldr\tx2, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs\n\tcmp\tx1, x2\n\tb.ne\t.L_parking_loop\n\n\t// If execution reaches here, it is the boot core.\n\n\t// Initialize DRAM.\n\tADR_REL\tx0, __bss_start\n\tADR_REL x1, __bss_end_exclusive\n\n.L_bss_init_loop:\n\tcmp\tx0, x1\n\tb.eq\t.L_prepare_rust\n\tstp\txzr, xzr, [x0], #16\n\tb\t.L_bss_init_loop\n\n\t// Prepare the jump to Rust code.\n.L_prepare_rust:\n\t// Set the stack pointer. This ensures that any code in EL2 that needs the stack will work.\n\tADR_REL\tx0, __boot_core_stack_end_exclusive\n\tmov\tsp, x0\n\n\t// Read the CPU's timer counter frequency and store it in ARCH_TIMER_COUNTER_FREQUENCY.\n\t// Abort if the frequency read back as 0.\n\tADR_REL\tx1, ARCH_TIMER_COUNTER_FREQUENCY // provided by aarch64/time.rs\n\tmrs\tx2, CNTFRQ_EL0\n\tcmp\tx2, xzr\n\tb.eq\t.L_parking_loop\n\tstr\tw2, [x1]\n\n\t// Jump to Rust code. x0 holds the function argument provided to _start_rust().\n\tb\t_start_rust\n\n\t// Infinitely wait for events (aka \"park the core\").\n.L_parking_loop:\n\twfe\n\tb\t.L_parking_loop\n\n.size\t_start, . - _start\n.type\t_start, function\n.global\t_start\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/_arch/aarch64/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural processor code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::arch_cpu\n\nuse aarch64_cpu::asm;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\npub use asm::nop;\n\n/// Pause execution on the core.\n#[inline(always)]\npub fn wait_forever() -> ! {\n    loop {\n        asm::wfe()\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n#[cfg(feature = \"test_build\")]\nuse qemu_exit::QEMUExit;\n\n#[cfg(feature = \"test_build\")]\nconst QEMU_EXIT_HANDLE: qemu_exit::AArch64 = qemu_exit::AArch64::new();\n\n/// Make the host QEMU binary execute `exit(1)`.\n#[cfg(feature = \"test_build\")]\npub fn qemu_exit_failure() -> ! {\n    QEMU_EXIT_HANDLE.exit_failure()\n}\n\n/// Make the host QEMU binary execute `exit(0)`.\n#[cfg(feature = \"test_build\")]\npub fn qemu_exit_success() -> ! {\n    QEMU_EXIT_HANDLE.exit_success()\n}\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/_arch/aarch64/exception/asynchronous.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural asynchronous exception handling.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::exception::asynchronous::arch_asynchronous\n\nuse aarch64_cpu::registers::*;\nuse tock_registers::interfaces::Readable;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\ntrait DaifField {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register>;\n}\n\nstruct Debug;\nstruct SError;\nstruct IRQ;\nstruct FIQ;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl DaifField for Debug {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::D\n    }\n}\n\nimpl DaifField for SError {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::A\n    }\n}\n\nimpl DaifField for IRQ {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::I\n    }\n}\n\nimpl DaifField for FIQ {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::F\n    }\n}\n\nfn is_masked<T>() -> bool\nwhere\n    T: DaifField,\n{\n    DAIF.is_set(T::daif_field())\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Print the AArch64 exceptions status.\n#[rustfmt::skip]\npub fn print_state() {\n    use crate::info;\n\n    let to_mask_str = |x| -> _ {\n        if x { \"Masked\" } else { \"Unmasked\" }\n    };\n\n    info!(\"      Debug:  {}\", to_mask_str(is_masked::<Debug>()));\n    info!(\"      SError: {}\", to_mask_str(is_masked::<SError>()));\n    info!(\"      IRQ:    {}\", to_mask_str(is_masked::<IRQ>()));\n    info!(\"      FIQ:    {}\", to_mask_str(is_masked::<FIQ>()));\n}\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/_arch/aarch64/exception.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural synchronous and asynchronous exception handling.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::exception::arch_exception\n\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::{arch::global_asm, cell::UnsafeCell, fmt};\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    registers::InMemoryRegister,\n};\n\n// Assembly counterpart to this file.\nglobal_asm!(include_str!(\"exception.s\"));\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Wrapper structs for memory copies of registers.\n#[repr(transparent)]\nstruct SpsrEL1(InMemoryRegister<u64, SPSR_EL1::Register>);\nstruct EsrEL1(InMemoryRegister<u64, ESR_EL1::Register>);\n\n/// The exception context as it is stored on the stack on exception entry.\n#[repr(C)]\nstruct ExceptionContext {\n    /// General Purpose Registers.\n    gpr: [u64; 30],\n\n    /// The link register, aka x30.\n    lr: u64,\n\n    /// Exception link register. The program counter at the time the exception happened.\n    elr_el1: u64,\n\n    /// Saved program status.\n    spsr_el1: SpsrEL1,\n\n    /// Exception syndrome register.\n    esr_el1: EsrEL1,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Prints verbose information about the exception and then panics.\nfn default_exception_handler(exc: &ExceptionContext) {\n    panic!(\n        \"CPU Exception!\\n\\n\\\n        {}\",\n        exc\n    );\n}\n\n//------------------------------------------------------------------------------\n// Current, EL0\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn current_el0_synchronous(_e: &mut ExceptionContext) {\n    panic!(\"Should not be here. Use of SP_EL0 in EL1 is not supported.\")\n}\n\n#[no_mangle]\nextern \"C\" fn current_el0_irq(_e: &mut ExceptionContext) {\n    panic!(\"Should not be here. Use of SP_EL0 in EL1 is not supported.\")\n}\n\n#[no_mangle]\nextern \"C\" fn current_el0_serror(_e: &mut ExceptionContext) {\n    panic!(\"Should not be here. Use of SP_EL0 in EL1 is not supported.\")\n}\n\n//------------------------------------------------------------------------------\n// Current, ELx\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn current_elx_synchronous(e: &mut ExceptionContext) {\n    #[cfg(feature = \"test_build\")]\n    {\n        const TEST_SVC_ID: u64 = 0x1337;\n\n        if let Some(ESR_EL1::EC::Value::SVC64) = e.esr_el1.exception_class() {\n            if e.esr_el1.iss() == TEST_SVC_ID {\n                return;\n            }\n        }\n    }\n\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn current_elx_irq(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn current_elx_serror(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n//------------------------------------------------------------------------------\n// Lower, AArch64\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn lower_aarch64_synchronous(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch64_irq(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch64_serror(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n//------------------------------------------------------------------------------\n// Lower, AArch32\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn lower_aarch32_synchronous(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch32_irq(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch32_serror(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n//------------------------------------------------------------------------------\n// Misc\n//------------------------------------------------------------------------------\n\n/// Human readable SPSR_EL1.\n#[rustfmt::skip]\nimpl fmt::Display for SpsrEL1 {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        // Raw value.\n        writeln!(f, \"SPSR_EL1: {:#010x}\", self.0.get())?;\n\n        let to_flag_str = |x| -> _ {\n            if x { \"Set\" } else { \"Not set\" }\n         };\n\n        writeln!(f, \"      Flags:\")?;\n        writeln!(f, \"            Negative (N): {}\", to_flag_str(self.0.is_set(SPSR_EL1::N)))?;\n        writeln!(f, \"            Zero     (Z): {}\", to_flag_str(self.0.is_set(SPSR_EL1::Z)))?;\n        writeln!(f, \"            Carry    (C): {}\", to_flag_str(self.0.is_set(SPSR_EL1::C)))?;\n        writeln!(f, \"            Overflow (V): {}\", to_flag_str(self.0.is_set(SPSR_EL1::V)))?;\n\n        let to_mask_str = |x| -> _ {\n            if x { \"Masked\" } else { \"Unmasked\" }\n        };\n\n        writeln!(f, \"      Exception handling state:\")?;\n        writeln!(f, \"            Debug  (D): {}\", to_mask_str(self.0.is_set(SPSR_EL1::D)))?;\n        writeln!(f, \"            SError (A): {}\", to_mask_str(self.0.is_set(SPSR_EL1::A)))?;\n        writeln!(f, \"            IRQ    (I): {}\", to_mask_str(self.0.is_set(SPSR_EL1::I)))?;\n        writeln!(f, \"            FIQ    (F): {}\", to_mask_str(self.0.is_set(SPSR_EL1::F)))?;\n\n        write!(f, \"      Illegal Execution State (IL): {}\",\n            to_flag_str(self.0.is_set(SPSR_EL1::IL))\n        )\n    }\n}\n\nimpl EsrEL1 {\n    #[inline(always)]\n    fn exception_class(&self) -> Option<ESR_EL1::EC::Value> {\n        self.0.read_as_enum(ESR_EL1::EC)\n    }\n\n    #[cfg(feature = \"test_build\")]\n    #[inline(always)]\n    fn iss(&self) -> u64 {\n        self.0.read(ESR_EL1::ISS)\n    }\n}\n\n/// Human readable ESR_EL1.\n#[rustfmt::skip]\nimpl fmt::Display for EsrEL1 {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        // Raw print of whole register.\n        writeln!(f, \"ESR_EL1: {:#010x}\", self.0.get())?;\n\n        // Raw print of exception class.\n        write!(f, \"      Exception Class         (EC) : {:#x}\", self.0.read(ESR_EL1::EC))?;\n\n        // Exception class.\n        let ec_translation = match self.exception_class() {\n            Some(ESR_EL1::EC::Value::DataAbortCurrentEL) => \"Data Abort, current EL\",\n            _ => \"N/A\",\n        };\n        writeln!(f, \" - {}\", ec_translation)?;\n\n        // Raw print of instruction specific syndrome.\n        write!(f, \"      Instr Specific Syndrome (ISS): {:#x}\", self.0.read(ESR_EL1::ISS))\n    }\n}\n\nimpl ExceptionContext {\n    #[inline(always)]\n    fn exception_class(&self) -> Option<ESR_EL1::EC::Value> {\n        self.esr_el1.exception_class()\n    }\n\n    #[inline(always)]\n    fn fault_address_valid(&self) -> bool {\n        use ESR_EL1::EC::Value::*;\n\n        match self.exception_class() {\n            None => false,\n            Some(ec) => matches!(\n                ec,\n                InstrAbortLowerEL\n                    | InstrAbortCurrentEL\n                    | PCAlignmentFault\n                    | DataAbortLowerEL\n                    | DataAbortCurrentEL\n                    | WatchpointLowerEL\n                    | WatchpointCurrentEL\n            ),\n        }\n    }\n}\n\n/// Human readable print of the exception context.\nimpl fmt::Display for ExceptionContext {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        writeln!(f, \"{}\", self.esr_el1)?;\n\n        if self.fault_address_valid() {\n            writeln!(f, \"FAR_EL1: {:#018x}\", FAR_EL1.get() as usize)?;\n        }\n\n        writeln!(f, \"{}\", self.spsr_el1)?;\n        writeln!(f, \"ELR_EL1: {:#018x}\", self.elr_el1)?;\n        writeln!(f)?;\n        writeln!(f, \"General purpose register:\")?;\n\n        #[rustfmt::skip]\n        let alternating = |x| -> _ {\n            if x % 2 == 0 { \"   \" } else { \"\\n\" }\n        };\n\n        // Print two registers per line.\n        for (i, reg) in self.gpr.iter().enumerate() {\n            write!(f, \"      x{: <2}: {: >#018x}{}\", i, reg, alternating(i))?;\n        }\n        write!(f, \"      lr : {:#018x}\", self.lr)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse crate::exception::PrivilegeLevel;\n\n/// The processing element's current privilege level.\npub fn current_privilege_level() -> (PrivilegeLevel, &'static str) {\n    let el = CurrentEL.read_as_enum(CurrentEL::EL);\n    match el {\n        Some(CurrentEL::EL::Value::EL2) => (PrivilegeLevel::Hypervisor, \"EL2\"),\n        Some(CurrentEL::EL::Value::EL1) => (PrivilegeLevel::Kernel, \"EL1\"),\n        Some(CurrentEL::EL::Value::EL0) => (PrivilegeLevel::User, \"EL0\"),\n        _ => (PrivilegeLevel::Unknown, \"Unknown\"),\n    }\n}\n\n/// Init exception handling by setting the exception vector base address register.\n///\n/// # Safety\n///\n/// - Changes the HW state of the executing core.\n/// - The vector table and the symbol `__exception_vector_table_start` from the linker script must\n///   adhere to the alignment and size constraints demanded by the ARMv8-A Architecture Reference\n///   Manual.\npub unsafe fn handling_init() {\n    // Provided by exception.S.\n    extern \"Rust\" {\n        static __exception_vector_start: UnsafeCell<()>;\n    }\n\n    VBAR_EL1.set(__exception_vector_start.get() as u64);\n\n    // Force VBAR update to complete before next instruction.\n    barrier::isb(barrier::SY);\n}\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/_arch/aarch64/exception.s",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2022 Andre Richter <andre.o.richter@gmail.com>\n\n//--------------------------------------------------------------------------------------------------\n// Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Call the function provided by parameter `\\handler` after saving the exception context. Provide\n/// the context as the first parameter to '\\handler'.\n.macro CALL_WITH_CONTEXT handler\n__vector_\\handler:\n\t// Make room on the stack for the exception context.\n\tsub\tsp,  sp,  #16 * 17\n\n\t// Store all general purpose registers on the stack.\n\tstp\tx0,  x1,  [sp, #16 * 0]\n\tstp\tx2,  x3,  [sp, #16 * 1]\n\tstp\tx4,  x5,  [sp, #16 * 2]\n\tstp\tx6,  x7,  [sp, #16 * 3]\n\tstp\tx8,  x9,  [sp, #16 * 4]\n\tstp\tx10, x11, [sp, #16 * 5]\n\tstp\tx12, x13, [sp, #16 * 6]\n\tstp\tx14, x15, [sp, #16 * 7]\n\tstp\tx16, x17, [sp, #16 * 8]\n\tstp\tx18, x19, [sp, #16 * 9]\n\tstp\tx20, x21, [sp, #16 * 10]\n\tstp\tx22, x23, [sp, #16 * 11]\n\tstp\tx24, x25, [sp, #16 * 12]\n\tstp\tx26, x27, [sp, #16 * 13]\n\tstp\tx28, x29, [sp, #16 * 14]\n\n\t// Add the exception link register (ELR_EL1), saved program status (SPSR_EL1) and exception\n\t// syndrome register (ESR_EL1).\n\tmrs\tx1,  ELR_EL1\n\tmrs\tx2,  SPSR_EL1\n\tmrs\tx3,  ESR_EL1\n\n\tstp\tlr,  x1,  [sp, #16 * 15]\n\tstp\tx2,  x3,  [sp, #16 * 16]\n\n\t// x0 is the first argument for the function called through `\\handler`.\n\tmov\tx0,  sp\n\n\t// Call `\\handler`.\n\tbl\t\\handler\n\n\t// After returning from exception handling code, replay the saved context and return via\n\t// `eret`.\n\tb\t__exception_restore_context\n\n.size\t__vector_\\handler, . - __vector_\\handler\n.type\t__vector_\\handler, function\n.endm\n\n.macro FIQ_SUSPEND\n1:\twfe\n\tb\t1b\n.endm\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n.section .text\n\n//------------------------------------------------------------------------------\n// The exception vector table.\n//------------------------------------------------------------------------------\n\n// Align by 2^11 bytes, as demanded by ARMv8-A. Same as ALIGN(2048) in an ld script.\n.align 11\n\n// Export a symbol for the Rust code to use.\n__exception_vector_start:\n\n// Current exception level with SP_EL0.\n//\n// .org sets the offset relative to section start.\n//\n// # Safety\n//\n// - It must be ensured that `CALL_WITH_CONTEXT` <= 0x80 bytes.\n.org 0x000\n\tCALL_WITH_CONTEXT current_el0_synchronous\n.org 0x080\n\tCALL_WITH_CONTEXT current_el0_irq\n.org 0x100\n\tFIQ_SUSPEND\n.org 0x180\n\tCALL_WITH_CONTEXT current_el0_serror\n\n// Current exception level with SP_ELx, x > 0.\n.org 0x200\n\tCALL_WITH_CONTEXT current_elx_synchronous\n.org 0x280\n\tCALL_WITH_CONTEXT current_elx_irq\n.org 0x300\n\tFIQ_SUSPEND\n.org 0x380\n\tCALL_WITH_CONTEXT current_elx_serror\n\n// Lower exception level, AArch64\n.org 0x400\n\tCALL_WITH_CONTEXT lower_aarch64_synchronous\n.org 0x480\n\tCALL_WITH_CONTEXT lower_aarch64_irq\n.org 0x500\n\tFIQ_SUSPEND\n.org 0x580\n\tCALL_WITH_CONTEXT lower_aarch64_serror\n\n// Lower exception level, AArch32\n.org 0x600\n\tCALL_WITH_CONTEXT lower_aarch32_synchronous\n.org 0x680\n\tCALL_WITH_CONTEXT lower_aarch32_irq\n.org 0x700\n\tFIQ_SUSPEND\n.org 0x780\n\tCALL_WITH_CONTEXT lower_aarch32_serror\n.org 0x800\n\n//------------------------------------------------------------------------------\n// fn __exception_restore_context()\n//------------------------------------------------------------------------------\n__exception_restore_context:\n\tldr\tw19,      [sp, #16 * 16]\n\tldp\tlr,  x20, [sp, #16 * 15]\n\n\tmsr\tSPSR_EL1, x19\n\tmsr\tELR_EL1,  x20\n\n\tldp\tx0,  x1,  [sp, #16 * 0]\n\tldp\tx2,  x3,  [sp, #16 * 1]\n\tldp\tx4,  x5,  [sp, #16 * 2]\n\tldp\tx6,  x7,  [sp, #16 * 3]\n\tldp\tx8,  x9,  [sp, #16 * 4]\n\tldp\tx10, x11, [sp, #16 * 5]\n\tldp\tx12, x13, [sp, #16 * 6]\n\tldp\tx14, x15, [sp, #16 * 7]\n\tldp\tx16, x17, [sp, #16 * 8]\n\tldp\tx18, x19, [sp, #16 * 9]\n\tldp\tx20, x21, [sp, #16 * 10]\n\tldp\tx22, x23, [sp, #16 * 11]\n\tldp\tx24, x25, [sp, #16 * 12]\n\tldp\tx26, x27, [sp, #16 * 13]\n\tldp\tx28, x29, [sp, #16 * 14]\n\n\tadd\tsp,  sp,  #16 * 17\n\n\teret\n\n.size\t__exception_restore_context, . - __exception_restore_context\n.type\t__exception_restore_context, function\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/_arch/aarch64/memory/mmu/translation_table.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural translation table.\n//!\n//! Only 64 KiB granule is supported.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::memory::mmu::translation_table::arch_translation_table\n\nuse crate::{\n    bsp, memory,\n    memory::mmu::{\n        arch_mmu::{Granule512MiB, Granule64KiB},\n        AccessPermissions, AttributeFields, MemAttributes,\n    },\n};\nuse core::convert;\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields,\n    registers::InMemoryRegister,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// A table descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-15.\nregister_bitfields! {u64,\n    STAGE1_TABLE_DESCRIPTOR [\n        /// Physical address of the next descriptor.\n        NEXT_LEVEL_TABLE_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16]\n\n        TYPE  OFFSET(1) NUMBITS(1) [\n            Block = 0,\n            Table = 1\n        ],\n\n        VALID OFFSET(0) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ]\n    ]\n}\n\n// A level 3 page descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-17.\nregister_bitfields! {u64,\n    STAGE1_PAGE_DESCRIPTOR [\n        /// Unprivileged execute-never.\n        UXN      OFFSET(54) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ],\n\n        /// Privileged execute-never.\n        PXN      OFFSET(53) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ],\n\n        /// Physical address of the next table descriptor (lvl2) or the page descriptor (lvl3).\n        OUTPUT_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16]\n\n        /// Access flag.\n        AF       OFFSET(10) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ],\n\n        /// Shareability field.\n        SH       OFFSET(8) NUMBITS(2) [\n            OuterShareable = 0b10,\n            InnerShareable = 0b11\n        ],\n\n        /// Access Permissions.\n        AP       OFFSET(6) NUMBITS(2) [\n            RW_EL1 = 0b00,\n            RW_EL1_EL0 = 0b01,\n            RO_EL1 = 0b10,\n            RO_EL1_EL0 = 0b11\n        ],\n\n        /// Memory attributes index into the MAIR_EL1 register.\n        AttrIndx OFFSET(2) NUMBITS(3) [],\n\n        TYPE     OFFSET(1) NUMBITS(1) [\n            Reserved_Invalid = 0,\n            Page = 1\n        ],\n\n        VALID    OFFSET(0) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ]\n    ]\n}\n\n/// A table descriptor for 64 KiB aperture.\n///\n/// The output points to the next table.\n#[derive(Copy, Clone)]\n#[repr(C)]\nstruct TableDescriptor {\n    value: u64,\n}\n\n/// A page descriptor with 64 KiB aperture.\n///\n/// The output points to physical memory.\n#[derive(Copy, Clone)]\n#[repr(C)]\nstruct PageDescriptor {\n    value: u64,\n}\n\ntrait StartAddr {\n    fn phys_start_addr_u64(&self) -> u64;\n    fn phys_start_addr_usize(&self) -> usize;\n}\n\nconst NUM_LVL2_TABLES: usize = bsp::memory::mmu::KernelAddrSpace::SIZE >> Granule512MiB::SHIFT;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB\n/// aligned, so the lvl3 is put first.\n#[repr(C)]\n#[repr(align(65536))]\npub struct FixedSizeTranslationTable<const NUM_TABLES: usize> {\n    /// Page descriptors, covering 64 KiB windows per entry.\n    lvl3: [[PageDescriptor; 8192]; NUM_TABLES],\n\n    /// Table descriptors, covering 512 MiB windows.\n    lvl2: [TableDescriptor; NUM_TABLES],\n}\n\n/// A translation table type for the kernel space.\npub type KernelTranslationTable = FixedSizeTranslationTable<NUM_LVL2_TABLES>;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n// The binary is still identity mapped, so we don't need to convert here.\nimpl<T, const N: usize> StartAddr for [T; N] {\n    fn phys_start_addr_u64(&self) -> u64 {\n        self as *const T as u64\n    }\n\n    fn phys_start_addr_usize(&self) -> usize {\n        self as *const _ as usize\n    }\n}\n\nimpl TableDescriptor {\n    /// Create an instance.\n    ///\n    /// Descriptor is invalid by default.\n    pub const fn new_zeroed() -> Self {\n        Self { value: 0 }\n    }\n\n    /// Create an instance pointing to the supplied address.\n    pub fn from_next_lvl_table_addr(phys_next_lvl_table_addr: usize) -> Self {\n        let val = InMemoryRegister::<u64, STAGE1_TABLE_DESCRIPTOR::Register>::new(0);\n\n        let shifted = phys_next_lvl_table_addr >> Granule64KiB::SHIFT;\n        val.write(\n            STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64)\n                + STAGE1_TABLE_DESCRIPTOR::TYPE::Table\n                + STAGE1_TABLE_DESCRIPTOR::VALID::True,\n        );\n\n        TableDescriptor { value: val.get() }\n    }\n}\n\n/// Convert the kernel's generic memory attributes to HW-specific attributes of the MMU.\nimpl convert::From<AttributeFields>\n    for tock_registers::fields::FieldValue<u64, STAGE1_PAGE_DESCRIPTOR::Register>\n{\n    fn from(attribute_fields: AttributeFields) -> Self {\n        // Memory attributes.\n        let mut desc = match attribute_fields.mem_attributes {\n            MemAttributes::CacheableDRAM => {\n                STAGE1_PAGE_DESCRIPTOR::SH::InnerShareable\n                    + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::NORMAL)\n            }\n            MemAttributes::Device => {\n                STAGE1_PAGE_DESCRIPTOR::SH::OuterShareable\n                    + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::DEVICE)\n            }\n        };\n\n        // Access Permissions.\n        desc += match attribute_fields.acc_perms {\n            AccessPermissions::ReadOnly => STAGE1_PAGE_DESCRIPTOR::AP::RO_EL1,\n            AccessPermissions::ReadWrite => STAGE1_PAGE_DESCRIPTOR::AP::RW_EL1,\n        };\n\n        // The execute-never attribute is mapped to PXN in AArch64.\n        desc += if attribute_fields.execute_never {\n            STAGE1_PAGE_DESCRIPTOR::PXN::True\n        } else {\n            STAGE1_PAGE_DESCRIPTOR::PXN::False\n        };\n\n        // Always set unprivileged exectue-never as long as userspace is not implemented yet.\n        desc += STAGE1_PAGE_DESCRIPTOR::UXN::True;\n\n        desc\n    }\n}\n\nimpl PageDescriptor {\n    /// Create an instance.\n    ///\n    /// Descriptor is invalid by default.\n    pub const fn new_zeroed() -> Self {\n        Self { value: 0 }\n    }\n\n    /// Create an instance.\n    pub fn from_output_addr(phys_output_addr: usize, attribute_fields: &AttributeFields) -> Self {\n        let val = InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(0);\n\n        let shifted = phys_output_addr as u64 >> Granule64KiB::SHIFT;\n        val.write(\n            STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted)\n                + STAGE1_PAGE_DESCRIPTOR::AF::True\n                + STAGE1_PAGE_DESCRIPTOR::TYPE::Page\n                + STAGE1_PAGE_DESCRIPTOR::VALID::True\n                + (*attribute_fields).into(),\n        );\n\n        Self { value: val.get() }\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<const NUM_TABLES: usize> FixedSizeTranslationTable<NUM_TABLES> {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        // Can't have a zero-sized address space.\n        assert!(NUM_TABLES > 0);\n\n        Self {\n            lvl3: [[PageDescriptor::new_zeroed(); 8192]; NUM_TABLES],\n            lvl2: [TableDescriptor::new_zeroed(); NUM_TABLES],\n        }\n    }\n\n    /// Iterates over all static translation table entries and fills them at once.\n    ///\n    /// # Safety\n    ///\n    /// - Modifies a `static mut`. Ensure it only happens from here.\n    pub unsafe fn populate_tt_entries(&mut self) -> Result<(), &'static str> {\n        for (l2_nr, l2_entry) in self.lvl2.iter_mut().enumerate() {\n            *l2_entry =\n                TableDescriptor::from_next_lvl_table_addr(self.lvl3[l2_nr].phys_start_addr_usize());\n\n            for (l3_nr, l3_entry) in self.lvl3[l2_nr].iter_mut().enumerate() {\n                let virt_addr = (l2_nr << Granule512MiB::SHIFT) + (l3_nr << Granule64KiB::SHIFT);\n\n                let (phys_output_addr, attribute_fields) =\n                    bsp::memory::mmu::virt_mem_layout().virt_addr_properties(virt_addr)?;\n\n                *l3_entry = PageDescriptor::from_output_addr(phys_output_addr, &attribute_fields);\n            }\n        }\n\n        Ok(())\n    }\n\n    /// The translation table's base address to be used for programming the MMU.\n    pub fn phys_base_address(&self) -> u64 {\n        self.lvl2.phys_start_addr_u64()\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// Check if the size of `struct TableDescriptor` is as expected.\n    #[kernel_test]\n    fn size_of_tabledescriptor_equals_64_bit() {\n        assert_eq!(\n            core::mem::size_of::<TableDescriptor>(),\n            core::mem::size_of::<u64>()\n        );\n    }\n\n    /// Check if the size of `struct PageDescriptor` is as expected.\n    #[kernel_test]\n    fn size_of_pagedescriptor_equals_64_bit() {\n        assert_eq!(\n            core::mem::size_of::<PageDescriptor>(),\n            core::mem::size_of::<u64>()\n        );\n    }\n}\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/_arch/aarch64/memory/mmu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management Unit Driver.\n//!\n//! Only 64 KiB granule is supported.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::memory::mmu::arch_mmu\n\nuse crate::{\n    bsp, memory,\n    memory::mmu::{translation_table::KernelTranslationTable, TranslationGranule},\n};\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::intrinsics::unlikely;\nuse tock_registers::interfaces::{ReadWriteable, Readable, Writeable};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Memory Management Unit type.\nstruct MemoryManagementUnit;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub type Granule512MiB = TranslationGranule<{ 512 * 1024 * 1024 }>;\npub type Granule64KiB = TranslationGranule<{ 64 * 1024 }>;\n\n/// Constants for indexing the MAIR_EL1.\n#[allow(dead_code)]\npub mod mair {\n    pub const DEVICE: u64 = 0;\n    pub const NORMAL: u64 = 1;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n/// The kernel translation tables.\n///\n/// # Safety\n///\n/// - Supposed to land in `.bss`. Therefore, ensure that all initial member values boil down to \"0\".\nstatic mut KERNEL_TABLES: KernelTranslationTable = KernelTranslationTable::new();\n\nstatic MMU: MemoryManagementUnit = MemoryManagementUnit;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<const AS_SIZE: usize> memory::mmu::AddressSpace<AS_SIZE> {\n    /// Checks for architectural restrictions.\n    pub const fn arch_address_space_size_sanity_checks() {\n        // Size must be at least one full 512 MiB table.\n        assert!((AS_SIZE % Granule512MiB::SIZE) == 0);\n\n        // Check for 48 bit virtual address size as maximum, which is supported by any ARMv8\n        // version.\n        assert!(AS_SIZE <= (1 << 48));\n    }\n}\n\nimpl MemoryManagementUnit {\n    /// Setup function for the MAIR_EL1 register.\n    fn set_up_mair(&self) {\n        // Define the memory types being mapped.\n        MAIR_EL1.write(\n            // Attribute 1 - Cacheable normal DRAM.\n            MAIR_EL1::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc +\n        MAIR_EL1::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc +\n\n        // Attribute 0 - Device.\n        MAIR_EL1::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck,\n        );\n    }\n\n    /// Configure various settings of stage 1 of the EL1 translation regime.\n    fn configure_translation_control(&self) {\n        let t0sz = (64 - bsp::memory::mmu::KernelAddrSpace::SIZE_SHIFT) as u64;\n\n        TCR_EL1.write(\n            TCR_EL1::TBI0::Used\n                + TCR_EL1::IPS::Bits_40\n                + TCR_EL1::TG0::KiB_64\n                + TCR_EL1::SH0::Inner\n                + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable\n                + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable\n                + TCR_EL1::EPD0::EnableTTBR0Walks\n                + TCR_EL1::A1::TTBR0\n                + TCR_EL1::T0SZ.val(t0sz)\n                + TCR_EL1::EPD1::DisableTTBR1Walks,\n        );\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the MMU instance.\npub fn mmu() -> &'static impl memory::mmu::interface::MMU {\n    &MMU\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse memory::mmu::MMUEnableError;\n\nimpl memory::mmu::interface::MMU for MemoryManagementUnit {\n    unsafe fn enable_mmu_and_caching(&self) -> Result<(), MMUEnableError> {\n        if unlikely(self.is_enabled()) {\n            return Err(MMUEnableError::AlreadyEnabled);\n        }\n\n        // Fail early if translation granule is not supported.\n        if unlikely(!ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran64::Supported)) {\n            return Err(MMUEnableError::Other(\n                \"Translation granule not supported in HW\",\n            ));\n        }\n\n        // Prepare the memory attribute indirection register.\n        self.set_up_mair();\n\n        // Populate translation tables.\n        KERNEL_TABLES\n            .populate_tt_entries()\n            .map_err(MMUEnableError::Other)?;\n\n        // Set the \"Translation Table Base Register\".\n        TTBR0_EL1.set_baddr(KERNEL_TABLES.phys_base_address());\n\n        self.configure_translation_control();\n\n        // Switch the MMU on.\n        //\n        // First, force all previous changes to be seen before the MMU is enabled.\n        barrier::isb(barrier::SY);\n\n        // Enable the MMU and turn on data and instruction caching.\n        SCTLR_EL1.modify(SCTLR_EL1::M::Enable + SCTLR_EL1::C::Cacheable + SCTLR_EL1::I::Cacheable);\n\n        // Force MMU init to complete before next instruction.\n        barrier::isb(barrier::SY);\n\n        Ok(())\n    }\n\n    #[inline(always)]\n    fn is_enabled(&self) -> bool {\n        SCTLR_EL1.matches_all(SCTLR_EL1::M::Enable)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use core::{cell::UnsafeCell, ops::Range};\n    use test_macros::kernel_test;\n\n    /// Check if KERNEL_TABLES is in .bss.\n    #[kernel_test]\n    fn kernel_tables_in_bss() {\n        extern \"Rust\" {\n            static __bss_start: UnsafeCell<u64>;\n            static __bss_end_exclusive: UnsafeCell<u64>;\n        }\n\n        let bss_range = unsafe {\n            Range {\n                start: __bss_start.get(),\n                end: __bss_end_exclusive.get(),\n            }\n        };\n        let kernel_tables_addr = unsafe { &KERNEL_TABLES as *const _ as usize as *mut u64 };\n\n        assert!(bss_range.contains(&kernel_tables_addr));\n    }\n}\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/_arch/aarch64/time.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural timer primitives.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::time::arch_time\n\nuse crate::warn;\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::{\n    num::{NonZeroU128, NonZeroU32, NonZeroU64},\n    ops::{Add, Div},\n    time::Duration,\n};\nuse tock_registers::interfaces::Readable;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nconst NANOSEC_PER_SEC: NonZeroU64 = NonZeroU64::new(1_000_000_000).unwrap();\n\n#[derive(Copy, Clone, PartialOrd, PartialEq)]\nstruct GenericTimerCounterValue(u64);\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n/// Boot assembly code overwrites this value with the value of CNTFRQ_EL0 before any Rust code is\n/// executed. This given value here is just a (safe) dummy.\n#[no_mangle]\nstatic ARCH_TIMER_COUNTER_FREQUENCY: NonZeroU32 = NonZeroU32::MIN;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nfn arch_timer_counter_frequency() -> NonZeroU32 {\n    // Read volatile is needed here to prevent the compiler from optimizing\n    // ARCH_TIMER_COUNTER_FREQUENCY away.\n    //\n    // This is safe, because all the safety requirements as stated in read_volatile()'s\n    // documentation are fulfilled.\n    unsafe { core::ptr::read_volatile(&ARCH_TIMER_COUNTER_FREQUENCY) }\n}\n\nimpl GenericTimerCounterValue {\n    pub const MAX: Self = GenericTimerCounterValue(u64::MAX);\n}\n\nimpl Add for GenericTimerCounterValue {\n    type Output = Self;\n\n    fn add(self, other: Self) -> Self {\n        GenericTimerCounterValue(self.0.wrapping_add(other.0))\n    }\n}\n\nimpl From<GenericTimerCounterValue> for Duration {\n    fn from(counter_value: GenericTimerCounterValue) -> Self {\n        if counter_value.0 == 0 {\n            return Duration::ZERO;\n        }\n\n        let frequency: NonZeroU64 = arch_timer_counter_frequency().into();\n\n        // Div<NonZeroU64> implementation for u64 cannot panic.\n        let secs = counter_value.0.div(frequency);\n\n        // This is safe, because frequency can never be greater than u32::MAX, which means the\n        // largest theoretical value for sub_second_counter_value is (u32::MAX - 1). Therefore,\n        // (sub_second_counter_value * NANOSEC_PER_SEC) cannot overflow an u64.\n        //\n        // The subsequent division ensures the result fits into u32, since the max result is smaller\n        // than NANOSEC_PER_SEC. Therefore, just cast it to u32 using `as`.\n        let sub_second_counter_value = counter_value.0 % frequency;\n        let nanos = unsafe { sub_second_counter_value.unchecked_mul(u64::from(NANOSEC_PER_SEC)) }\n            .div(frequency) as u32;\n\n        Duration::new(secs, nanos)\n    }\n}\n\nfn max_duration() -> Duration {\n    Duration::from(GenericTimerCounterValue::MAX)\n}\n\nimpl TryFrom<Duration> for GenericTimerCounterValue {\n    type Error = &'static str;\n\n    fn try_from(duration: Duration) -> Result<Self, Self::Error> {\n        if duration < resolution() {\n            return Ok(GenericTimerCounterValue(0));\n        }\n\n        if duration > max_duration() {\n            return Err(\"Conversion error. Duration too big\");\n        }\n\n        let frequency: u128 = u32::from(arch_timer_counter_frequency()) as u128;\n        let duration: u128 = duration.as_nanos();\n\n        // This is safe, because frequency can never be greater than u32::MAX, and\n        // (Duration::MAX.as_nanos() * u32::MAX) < u128::MAX.\n        let counter_value =\n            unsafe { duration.unchecked_mul(frequency) }.div(NonZeroU128::from(NANOSEC_PER_SEC));\n\n        // Since we checked above that we are <= max_duration(), just cast to u64.\n        Ok(GenericTimerCounterValue(counter_value as u64))\n    }\n}\n\n#[inline(always)]\nfn read_cntpct() -> GenericTimerCounterValue {\n    // Prevent that the counter is read ahead of time due to out-of-order execution.\n    barrier::isb(barrier::SY);\n    let cnt = CNTPCT_EL0.get();\n\n    GenericTimerCounterValue(cnt)\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The timer's resolution.\npub fn resolution() -> Duration {\n    Duration::from(GenericTimerCounterValue(1))\n}\n\n/// The uptime since power-on of the device.\n///\n/// This includes time consumed by firmware and bootloaders.\npub fn uptime() -> Duration {\n    read_cntpct().into()\n}\n\n/// Spin for a given duration.\npub fn spin_for(duration: Duration) {\n    let curr_counter_value = read_cntpct();\n\n    let counter_value_delta: GenericTimerCounterValue = match duration.try_into() {\n        Err(msg) => {\n            warn!(\"spin_for: {}. Skipping\", msg);\n            return;\n        }\n        Ok(val) => val,\n    };\n    let counter_value_target = curr_counter_value + counter_value_delta;\n\n    // Busy wait.\n    //\n    // Read CNTPCT_EL0 directly to avoid the ISB that is part of [`read_cntpct`].\n    while GenericTimerCounterValue(CNTPCT_EL0.get()) < counter_value_target {}\n}\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GPIO Driver.\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper, driver, synchronization,\n    synchronization::NullLock,\n};\nuse tock_registers::{\n    interfaces::{ReadWriteable, Writeable},\n    register_bitfields, register_structs,\n    registers::ReadWrite,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// GPIO registers.\n//\n// Descriptions taken from\n// - https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf\n// - https://datasheets.raspberrypi.org/bcm2711/bcm2711-peripherals.pdf\nregister_bitfields! {\n    u32,\n\n    /// GPIO Function Select 1\n    GPFSEL1 [\n        /// Pin 15\n        FSEL15 OFFSET(15) NUMBITS(3) [\n            Input = 0b000,\n            Output = 0b001,\n            AltFunc0 = 0b100  // PL011 UART RX\n\n        ],\n\n        /// Pin 14\n        FSEL14 OFFSET(12) NUMBITS(3) [\n            Input = 0b000,\n            Output = 0b001,\n            AltFunc0 = 0b100  // PL011 UART TX\n        ]\n    ],\n\n    /// GPIO Pull-up/down Register\n    ///\n    /// BCM2837 only.\n    GPPUD [\n        /// Controls the actuation of the internal pull-up/down control line to ALL the GPIO pins.\n        PUD OFFSET(0) NUMBITS(2) [\n            Off = 0b00,\n            PullDown = 0b01,\n            PullUp = 0b10\n        ]\n    ],\n\n    /// GPIO Pull-up/down Clock Register 0\n    ///\n    /// BCM2837 only.\n    GPPUDCLK0 [\n        /// Pin 15\n        PUDCLK15 OFFSET(15) NUMBITS(1) [\n            NoEffect = 0,\n            AssertClock = 1\n        ],\n\n        /// Pin 14\n        PUDCLK14 OFFSET(14) NUMBITS(1) [\n            NoEffect = 0,\n            AssertClock = 1\n        ]\n    ],\n\n    /// GPIO Pull-up / Pull-down Register 0\n    ///\n    /// BCM2711 only.\n    GPIO_PUP_PDN_CNTRL_REG0 [\n        /// Pin 15\n        GPIO_PUP_PDN_CNTRL15 OFFSET(30) NUMBITS(2) [\n            NoResistor = 0b00,\n            PullUp = 0b01\n        ],\n\n        /// Pin 14\n        GPIO_PUP_PDN_CNTRL14 OFFSET(28) NUMBITS(2) [\n            NoResistor = 0b00,\n            PullUp = 0b01\n        ]\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    RegisterBlock {\n        (0x00 => _reserved1),\n        (0x04 => GPFSEL1: ReadWrite<u32, GPFSEL1::Register>),\n        (0x08 => _reserved2),\n        (0x94 => GPPUD: ReadWrite<u32, GPPUD::Register>),\n        (0x98 => GPPUDCLK0: ReadWrite<u32, GPPUDCLK0::Register>),\n        (0x9C => _reserved3),\n        (0xE4 => GPIO_PUP_PDN_CNTRL_REG0: ReadWrite<u32, GPIO_PUP_PDN_CNTRL_REG0::Register>),\n        (0xE8 => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\nstruct GPIOInner {\n    registers: Registers,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the GPIO HW.\npub struct GPIO {\n    inner: NullLock<GPIOInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GPIOInner {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n        }\n    }\n\n    /// Disable pull-up/down on pins 14 and 15.\n    #[cfg(feature = \"bsp_rpi3\")]\n    fn disable_pud_14_15_bcm2837(&mut self) {\n        use crate::time;\n        use core::time::Duration;\n\n        // The Linux 2837 GPIO driver waits 1 µs between the steps.\n        const DELAY: Duration = Duration::from_micros(1);\n\n        self.registers.GPPUD.write(GPPUD::PUD::Off);\n        time::time_manager().spin_for(DELAY);\n\n        self.registers\n            .GPPUDCLK0\n            .write(GPPUDCLK0::PUDCLK15::AssertClock + GPPUDCLK0::PUDCLK14::AssertClock);\n        time::time_manager().spin_for(DELAY);\n\n        self.registers.GPPUD.write(GPPUD::PUD::Off);\n        self.registers.GPPUDCLK0.set(0);\n    }\n\n    /// Disable pull-up/down on pins 14 and 15.\n    #[cfg(feature = \"bsp_rpi4\")]\n    fn disable_pud_14_15_bcm2711(&mut self) {\n        self.registers.GPIO_PUP_PDN_CNTRL_REG0.write(\n            GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL15::PullUp\n                + GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL14::PullUp,\n        );\n    }\n\n    /// Map PL011 UART as standard output.\n    ///\n    /// TX to pin 14\n    /// RX to pin 15\n    pub fn map_pl011_uart(&mut self) {\n        // Select the UART on pins 14 and 15.\n        self.registers\n            .GPFSEL1\n            .modify(GPFSEL1::FSEL15::AltFunc0 + GPFSEL1::FSEL14::AltFunc0);\n\n        // Disable pull-up/down on pins 14 and 15.\n        #[cfg(feature = \"bsp_rpi3\")]\n        self.disable_pud_14_15_bcm2837();\n\n        #[cfg(feature = \"bsp_rpi4\")]\n        self.disable_pud_14_15_bcm2711();\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GPIO {\n    pub const COMPATIBLE: &'static str = \"BCM GPIO\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            inner: NullLock::new(GPIOInner::new(mmio_start_addr)),\n        }\n    }\n\n    /// Concurrency safe version of `GPIOInner.map_pl011_uart()`\n    pub fn map_pl011_uart(&self) {\n        self.inner.lock(|inner| inner.map_pl011_uart())\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl driver::interface::DeviceDriver for GPIO {\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n}\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! PL011 UART driver.\n//!\n//! # Resources\n//!\n//! - <https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf>\n//! - <https://developer.arm.com/documentation/ddi0183/latest>\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper, console, cpu, driver, synchronization,\n    synchronization::NullLock,\n};\nuse core::fmt;\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields, register_structs,\n    registers::{ReadOnly, ReadWrite, WriteOnly},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// PL011 UART registers.\n//\n// Descriptions taken from \"PrimeCell UART (PL011) Technical Reference Manual\" r1p5.\nregister_bitfields! {\n    u32,\n\n    /// Flag Register.\n    FR [\n        /// Transmit FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n        /// Line Control Register, LCR_H.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the transmit holding register is empty.\n        /// - If the FIFO is enabled, the TXFE bit is set when the transmit FIFO is empty.\n        /// - This bit does not indicate if there is data in the transmit shift register.\n        TXFE OFFSET(7) NUMBITS(1) [],\n\n        /// Transmit FIFO full. The meaning of this bit depends on the state of the FEN bit in the\n        /// LCR_H Register.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the transmit holding register is full.\n        /// - If the FIFO is enabled, the TXFF bit is set when the transmit FIFO is full.\n        TXFF OFFSET(5) NUMBITS(1) [],\n\n        /// Receive FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n        /// LCR_H Register.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the receive holding register is empty.\n        /// - If the FIFO is enabled, the RXFE bit is set when the receive FIFO is empty.\n        RXFE OFFSET(4) NUMBITS(1) [],\n\n        /// UART busy. If this bit is set to 1, the UART is busy transmitting data. This bit remains\n        /// set until the complete byte, including all the stop bits, has been sent from the shift\n        /// register.\n        ///\n        /// This bit is set as soon as the transmit FIFO becomes non-empty, regardless of whether\n        /// the UART is enabled or not.\n        BUSY OFFSET(3) NUMBITS(1) []\n    ],\n\n    /// Integer Baud Rate Divisor.\n    IBRD [\n        /// The integer baud rate divisor.\n        BAUD_DIVINT OFFSET(0) NUMBITS(16) []\n    ],\n\n    /// Fractional Baud Rate Divisor.\n    FBRD [\n        ///  The fractional baud rate divisor.\n        BAUD_DIVFRAC OFFSET(0) NUMBITS(6) []\n    ],\n\n    /// Line Control Register.\n    LCR_H [\n        /// Word length. These bits indicate the number of data bits transmitted or received in a\n        /// frame.\n        #[allow(clippy::enum_variant_names)]\n        WLEN OFFSET(5) NUMBITS(2) [\n            FiveBit = 0b00,\n            SixBit = 0b01,\n            SevenBit = 0b10,\n            EightBit = 0b11\n        ],\n\n        /// Enable FIFOs:\n        ///\n        /// 0 = FIFOs are disabled (character mode) that is, the FIFOs become 1-byte-deep holding\n        /// registers.\n        ///\n        /// 1 = Transmit and receive FIFO buffers are enabled (FIFO mode).\n        FEN  OFFSET(4) NUMBITS(1) [\n            FifosDisabled = 0,\n            FifosEnabled = 1\n        ]\n    ],\n\n    /// Control Register.\n    CR [\n        /// Receive enable. If this bit is set to 1, the receive section of the UART is enabled.\n        /// Data reception occurs for either UART signals or SIR signals depending on the setting of\n        /// the SIREN bit. When the UART is disabled in the middle of reception, it completes the\n        /// current character before stopping.\n        RXE OFFSET(9) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// Transmit enable. If this bit is set to 1, the transmit section of the UART is enabled.\n        /// Data transmission occurs for either UART signals, or SIR signals depending on the\n        /// setting of the SIREN bit. When the UART is disabled in the middle of transmission, it\n        /// completes the current character before stopping.\n        TXE OFFSET(8) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// UART enable:\n        ///\n        /// 0 = UART is disabled. If the UART is disabled in the middle of transmission or\n        /// reception, it completes the current character before stopping.\n        ///\n        /// 1 = The UART is enabled. Data transmission and reception occurs for either UART signals\n        /// or SIR signals depending on the setting of the SIREN bit\n        UARTEN OFFSET(0) NUMBITS(1) [\n            /// If the UART is disabled in the middle of transmission or reception, it completes the\n            /// current character before stopping.\n            Disabled = 0,\n            Enabled = 1\n        ]\n    ],\n\n    /// Interrupt Clear Register.\n    ICR [\n        /// Meta field for all pending interrupts.\n        ALL OFFSET(0) NUMBITS(11) []\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    pub RegisterBlock {\n        (0x00 => DR: ReadWrite<u32>),\n        (0x04 => _reserved1),\n        (0x18 => FR: ReadOnly<u32, FR::Register>),\n        (0x1c => _reserved2),\n        (0x24 => IBRD: WriteOnly<u32, IBRD::Register>),\n        (0x28 => FBRD: WriteOnly<u32, FBRD::Register>),\n        (0x2c => LCR_H: WriteOnly<u32, LCR_H::Register>),\n        (0x30 => CR: WriteOnly<u32, CR::Register>),\n        (0x34 => _reserved3),\n        (0x44 => ICR: WriteOnly<u32, ICR::Register>),\n        (0x48 => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\n#[derive(PartialEq)]\nenum BlockingMode {\n    Blocking,\n    NonBlocking,\n}\n\nstruct PL011UartInner {\n    registers: Registers,\n    chars_written: usize,\n    chars_read: usize,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the UART.\npub struct PL011Uart {\n    inner: NullLock<PL011UartInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PL011UartInner {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n            chars_written: 0,\n            chars_read: 0,\n        }\n    }\n\n    /// Set up baud rate and characteristics.\n    ///\n    /// This results in 8N1 and 921_600 baud.\n    ///\n    /// The calculation for the BRD is (we set the clock to 48 MHz in config.txt):\n    /// `(48_000_000 / 16) / 921_600 = 3.2552083`.\n    ///\n    /// This means the integer part is `3` and goes into the `IBRD`.\n    /// The fractional part is `0.2552083`.\n    ///\n    /// `FBRD` calculation according to the PL011 Technical Reference Manual:\n    /// `INTEGER((0.2552083 * 64) + 0.5) = 16`.\n    ///\n    /// Therefore, the generated baud rate divider is: `3 + 16/64 = 3.25`. Which results in a\n    /// genrated baud rate of `48_000_000 / (16 * 3.25) = 923_077`.\n    ///\n    /// Error = `((923_077 - 921_600) / 921_600) * 100 = 0.16%`.\n    pub fn init(&mut self) {\n        // Execution can arrive here while there are still characters queued in the TX FIFO and\n        // actively being sent out by the UART hardware. If the UART is turned off in this case,\n        // those queued characters would be lost.\n        //\n        // For example, this can happen during runtime on a call to panic!(), because panic!()\n        // initializes its own UART instance and calls init().\n        //\n        // Hence, flush first to ensure all pending characters are transmitted.\n        self.flush();\n\n        // Turn the UART off temporarily.\n        self.registers.CR.set(0);\n\n        // Clear all pending interrupts.\n        self.registers.ICR.write(ICR::ALL::CLEAR);\n\n        // From the PL011 Technical Reference Manual:\n        //\n        // The LCR_H, IBRD, and FBRD registers form the single 30-bit wide LCR Register that is\n        // updated on a single write strobe generated by a LCR_H write. So, to internally update the\n        // contents of IBRD or FBRD, a LCR_H write must always be performed at the end.\n        //\n        // Set the baud rate, 8N1 and FIFO enabled.\n        self.registers.IBRD.write(IBRD::BAUD_DIVINT.val(3));\n        self.registers.FBRD.write(FBRD::BAUD_DIVFRAC.val(16));\n        self.registers\n            .LCR_H\n            .write(LCR_H::WLEN::EightBit + LCR_H::FEN::FifosEnabled);\n\n        // Turn the UART on.\n        self.registers\n            .CR\n            .write(CR::UARTEN::Enabled + CR::TXE::Enabled + CR::RXE::Enabled);\n    }\n\n    /// Send a character.\n    fn write_char(&mut self, c: char) {\n        // Spin while TX FIFO full is set, waiting for an empty slot.\n        while self.registers.FR.matches_all(FR::TXFF::SET) {\n            cpu::nop();\n        }\n\n        // Write the character to the buffer.\n        self.registers.DR.set(c as u32);\n\n        self.chars_written += 1;\n    }\n\n    /// Block execution until the last buffered character has been physically put on the TX wire.\n    fn flush(&self) {\n        // Spin until the busy bit is cleared.\n        while self.registers.FR.matches_all(FR::BUSY::SET) {\n            cpu::nop();\n        }\n    }\n\n    /// Retrieve a character.\n    fn read_char_converting(&mut self, blocking_mode: BlockingMode) -> Option<char> {\n        // If RX FIFO is empty,\n        if self.registers.FR.matches_all(FR::RXFE::SET) {\n            // immediately return in non-blocking mode.\n            if blocking_mode == BlockingMode::NonBlocking {\n                return None;\n            }\n\n            // Otherwise, wait until a char was received.\n            while self.registers.FR.matches_all(FR::RXFE::SET) {\n                cpu::nop();\n            }\n        }\n\n        // Read one character.\n        let mut ret = self.registers.DR.get() as u8 as char;\n\n        // Convert carrige return to newline.\n        if ret == '\\r' {\n            ret = '\\n'\n        }\n\n        // Update statistics.\n        self.chars_read += 1;\n\n        Some(ret)\n    }\n}\n\n/// Implementing `core::fmt::Write` enables usage of the `format_args!` macros, which in turn are\n/// used to implement the `kernel`'s `print!` and `println!` macros. By implementing `write_str()`,\n/// we get `write_fmt()` automatically.\n///\n/// The function takes an `&mut self`, so it must be implemented for the inner struct.\n///\n/// See [`src/print.rs`].\n///\n/// [`src/print.rs`]: ../../print/index.html\nimpl fmt::Write for PL011UartInner {\n    fn write_str(&mut self, s: &str) -> fmt::Result {\n        for c in s.chars() {\n            self.write_char(c);\n        }\n\n        Ok(())\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PL011Uart {\n    pub const COMPATIBLE: &'static str = \"BCM PL011 UART\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            inner: NullLock::new(PL011UartInner::new(mmio_start_addr)),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl driver::interface::DeviceDriver for PL011Uart {\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n\n    unsafe fn init(&self) -> Result<(), &'static str> {\n        self.inner.lock(|inner| inner.init());\n\n        Ok(())\n    }\n}\n\nimpl console::interface::Write for PL011Uart {\n    /// Passthrough of `args` to the `core::fmt::Write` implementation, but guarded by a Mutex to\n    /// serialize access.\n    fn write_char(&self, c: char) {\n        self.inner.lock(|inner| inner.write_char(c));\n    }\n\n    fn write_fmt(&self, args: core::fmt::Arguments) -> fmt::Result {\n        // Fully qualified syntax for the call to `core::fmt::Write::write_fmt()` to increase\n        // readability.\n        self.inner.lock(|inner| fmt::Write::write_fmt(inner, args))\n    }\n\n    fn flush(&self) {\n        // Spin until TX FIFO empty is set.\n        self.inner.lock(|inner| inner.flush());\n    }\n}\n\nimpl console::interface::Read for PL011Uart {\n    fn read_char(&self) -> char {\n        self.inner\n            .lock(|inner| inner.read_char_converting(BlockingMode::Blocking).unwrap())\n    }\n\n    fn clear_rx(&self) {\n        // Read from the RX FIFO until it is indicating empty.\n        while self\n            .inner\n            .lock(|inner| inner.read_char_converting(BlockingMode::NonBlocking))\n            .is_some()\n        {}\n    }\n}\n\nimpl console::interface::Statistics for PL011Uart {\n    fn chars_written(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_written)\n    }\n\n    fn chars_read(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_read)\n    }\n}\n\nimpl console::interface::All for PL011Uart {}\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/bsp/device_driver/bcm.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BCM driver top level.\n\nmod bcm2xxx_gpio;\nmod bcm2xxx_pl011_uart;\n\npub use bcm2xxx_gpio::*;\npub use bcm2xxx_pl011_uart::*;\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/bsp/device_driver/common.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Common device driver code.\n\nuse core::{marker::PhantomData, ops};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct MMIODerefWrapper<T> {\n    start_addr: usize,\n    phantom: PhantomData<fn() -> T>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T> MMIODerefWrapper<T> {\n    /// Create an instance.\n    pub const unsafe fn new(start_addr: usize) -> Self {\n        Self {\n            start_addr,\n            phantom: PhantomData,\n        }\n    }\n}\n\nimpl<T> ops::Deref for MMIODerefWrapper<T> {\n    type Target = T;\n\n    fn deref(&self) -> &Self::Target {\n        unsafe { &*(self.start_addr as *const _) }\n    }\n}\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/bsp/device_driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Device driver.\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod bcm;\nmod common;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use bcm::*;\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/bsp/raspberrypi/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Processor code.\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Used by `arch` code to find the early boot core.\n#[no_mangle]\n#[link_section = \".text._start_arguments\"]\npub static BOOT_CORE_ID: u64 = 0;\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/bsp/raspberrypi/driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP driver support.\n\nuse super::memory::map::mmio;\nuse crate::{bsp::device_driver, console, driver as generic_driver};\nuse core::sync::atomic::{AtomicBool, Ordering};\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic PL011_UART: device_driver::PL011Uart =\n    unsafe { device_driver::PL011Uart::new(mmio::PL011_UART_START) };\nstatic GPIO: device_driver::GPIO = unsafe { device_driver::GPIO::new(mmio::GPIO_START) };\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// This must be called only after successful init of the UART driver.\nfn post_init_uart() -> Result<(), &'static str> {\n    console::register_console(&PL011_UART);\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the GPIO driver.\nfn post_init_gpio() -> Result<(), &'static str> {\n    GPIO.map_pl011_uart();\n    Ok(())\n}\n\nfn driver_uart() -> Result<(), &'static str> {\n    let uart_descriptor =\n        generic_driver::DeviceDriverDescriptor::new(&PL011_UART, Some(post_init_uart));\n    generic_driver::driver_manager().register_driver(uart_descriptor);\n\n    Ok(())\n}\n\nfn driver_gpio() -> Result<(), &'static str> {\n    let gpio_descriptor = generic_driver::DeviceDriverDescriptor::new(&GPIO, Some(post_init_gpio));\n    generic_driver::driver_manager().register_driver(gpio_descriptor);\n\n    Ok(())\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Initialize the driver subsystem.\n///\n/// # Safety\n///\n/// See child function calls.\npub unsafe fn init() -> Result<(), &'static str> {\n    static INIT_DONE: AtomicBool = AtomicBool::new(false);\n    if INIT_DONE.load(Ordering::Relaxed) {\n        return Err(\"Init already done\");\n    }\n\n    driver_uart()?;\n    driver_gpio()?;\n\n    INIT_DONE.store(true, Ordering::Relaxed);\n    Ok(())\n}\n\n/// Minimal code needed to bring up the console in QEMU (for testing only). This is often less steps\n/// than on real hardware due to QEMU's abstractions.\n#[cfg(feature = \"test_build\")]\npub fn qemu_bring_up_console() {\n    console::register_console(&PL011_UART);\n}\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/bsp/raspberrypi/kernel.ld",
    "content": "/* SPDX-License-Identifier: MIT OR Apache-2.0\n *\n * Copyright (c) 2018-2022 Andre Richter <andre.o.richter@gmail.com>\n */\n\nPAGE_SIZE = 64K;\nPAGE_MASK = PAGE_SIZE - 1;\n\n__rpi_phys_dram_start_addr = 0;\n\n/* The physical address at which the the kernel binary will be loaded by the Raspberry's firmware */\n__rpi_phys_binary_load_addr = 0x80000;\n\n\nENTRY(__rpi_phys_binary_load_addr)\n\n/* Flags:\n *     4 == R\n *     5 == RX\n *     6 == RW\n *\n * Segments are marked PT_LOAD below so that the ELF file provides virtual and physical addresses.\n * It doesn't mean all of them need actually be loaded.\n */\nPHDRS\n{\n    segment_boot_core_stack PT_LOAD FLAGS(6);\n    segment_code            PT_LOAD FLAGS(5);\n    segment_data            PT_LOAD FLAGS(6);\n}\n\nSECTIONS\n{\n    . =  __rpi_phys_dram_start_addr;\n\n    /***********************************************************************************************\n    * Boot Core Stack\n    ***********************************************************************************************/\n    .boot_core_stack (NOLOAD) :\n    {\n                                             /*   ^             */\n                                             /*   | stack       */\n        . += __rpi_phys_binary_load_addr;    /*   | growth      */\n                                             /*   | direction   */\n        __boot_core_stack_end_exclusive = .; /*   |             */\n    } :segment_boot_core_stack\n\n    ASSERT((. & PAGE_MASK) == 0, \"End of boot core stack is not page aligned\")\n\n    /***********************************************************************************************\n    * Code + RO Data + Global Offset Table\n    ***********************************************************************************************/\n    __code_start = .;\n    .text :\n    {\n        KEEP(*(.text._start))\n        *(.text._start_arguments) /* Constants (or statics in Rust speak) read by _start(). */\n        *(.text._start_rust)      /* The Rust entry point */\n        *(.text*)                 /* Everything else */\n    } :segment_code\n\n    .rodata : ALIGN(8) { *(.rodata*) } :segment_code\n\n    . = ALIGN(PAGE_SIZE);\n    __code_end_exclusive = .;\n\n    /***********************************************************************************************\n    * Data + BSS\n    ***********************************************************************************************/\n    .data : { *(.data*) } :segment_data\n\n    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */\n    .bss (NOLOAD) : ALIGN(16)\n    {\n        __bss_start = .;\n        *(.bss*);\n        . = ALIGN(16);\n        __bss_end_exclusive = .;\n    } :segment_data\n\n    /***********************************************************************************************\n    * Misc\n    ***********************************************************************************************/\n    .got : { *(.got*) }\n    ASSERT(SIZEOF(.got) == 0, \"Relocation support not expected\")\n\n    /DISCARD/ : { *(.comment*) }\n}\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/bsp/raspberrypi/memory/mmu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Memory Management Unit.\n\nuse super::map as memory_map;\nuse crate::memory::mmu::*;\nuse core::ops::RangeInclusive;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// The kernel's address space defined by this BSP.\npub type KernelAddrSpace = AddressSpace<{ memory_map::END_INCLUSIVE + 1 }>;\n\nconst NUM_MEM_RANGES: usize = 2;\n\n/// The virtual memory layout.\n///\n/// The layout must contain only special ranges, aka anything that is _not_ normal cacheable DRAM.\n/// It is agnostic of the paging granularity that the architecture's MMU will use.\npub static LAYOUT: KernelVirtualLayout<NUM_MEM_RANGES> = KernelVirtualLayout::new(\n    memory_map::END_INCLUSIVE,\n    [\n        TranslationDescriptor {\n            name: \"Kernel code and RO data\",\n            virtual_range: code_range_inclusive,\n            physical_range_translation: Translation::Identity,\n            attribute_fields: AttributeFields {\n                mem_attributes: MemAttributes::CacheableDRAM,\n                acc_perms: AccessPermissions::ReadOnly,\n                execute_never: false,\n            },\n        },\n        TranslationDescriptor {\n            name: \"Device MMIO\",\n            virtual_range: mmio_range_inclusive,\n            physical_range_translation: Translation::Identity,\n            attribute_fields: AttributeFields {\n                mem_attributes: MemAttributes::Device,\n                acc_perms: AccessPermissions::ReadWrite,\n                execute_never: true,\n            },\n        },\n    ],\n);\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nfn code_range_inclusive() -> RangeInclusive<usize> {\n    // Notice the subtraction to turn the exclusive end into an inclusive end.\n    #[allow(clippy::range_minus_one)]\n    RangeInclusive::new(super::code_start(), super::code_end_exclusive() - 1)\n}\n\nfn mmio_range_inclusive() -> RangeInclusive<usize> {\n    RangeInclusive::new(memory_map::mmio::START, memory_map::mmio::END_INCLUSIVE)\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the virtual memory layout.\npub fn virt_mem_layout() -> &'static KernelVirtualLayout<NUM_MEM_RANGES> {\n    &LAYOUT\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// Check alignment of the kernel's virtual memory layout sections.\n    #[kernel_test]\n    fn virt_mem_layout_sections_are_64KiB_aligned() {\n        const SIXTYFOUR_KIB: usize = 65536;\n\n        for i in LAYOUT.inner().iter() {\n            let start: usize = *(i.virtual_range)().start();\n            let end: usize = *(i.virtual_range)().end() + 1;\n\n            assert_eq!(start % SIXTYFOUR_KIB, 0);\n            assert_eq!(end % SIXTYFOUR_KIB, 0);\n            assert!(end >= start);\n        }\n    }\n\n    /// Ensure the kernel's virtual memory layout is free of overlaps.\n    #[kernel_test]\n    fn virt_mem_layout_has_no_overlaps() {\n        let layout = virt_mem_layout().inner();\n\n        for (i, first) in layout.iter().enumerate() {\n            for second in layout.iter().skip(i + 1) {\n                let first_range = first.virtual_range;\n                let second_range = second.virtual_range;\n\n                assert!(!first_range().contains(second_range().start()));\n                assert!(!first_range().contains(second_range().end()));\n                assert!(!second_range().contains(first_range().start()));\n                assert!(!second_range().contains(first_range().end()));\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/bsp/raspberrypi/memory.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Memory Management.\n//!\n//! The physical memory layout.\n//!\n//! The Raspberry's firmware copies the kernel binary to 0x8_0000. The preceding region will be used\n//! as the boot core's stack.\n//!\n//! +---------------------------------------+\n//! |                                       | 0x0\n//! |                                       |                                ^\n//! | Boot-core Stack                       |                                | stack\n//! |                                       |                                | growth\n//! |                                       |                                | direction\n//! +---------------------------------------+\n//! |                                       | code_start @ 0x8_0000\n//! | .text                                 |\n//! | .rodata                               |\n//! | .got                                  |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       | code_end_exclusive\n//! | .data                                 |\n//! | .bss                                  |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       |\n//! |                                       |\npub mod mmu;\n\nuse core::cell::UnsafeCell;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Symbols from the linker script.\nextern \"Rust\" {\n    static __code_start: UnsafeCell<()>;\n    static __code_end_exclusive: UnsafeCell<()>;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// The board's physical memory map.\n#[rustfmt::skip]\npub(super) mod map {\n    /// The inclusive end address of the memory map.\n    ///\n    /// End address + 1 must be power of two.\n    ///\n    /// # Note\n    ///\n    /// RPi3 and RPi4 boards can have different amounts of RAM. To make our code lean for\n    /// educational purposes, we set the max size of the address space to 4 GiB regardless of board.\n    /// This way, we can map the entire range that we need (end of MMIO for RPi4) in one take.\n    ///\n    /// However, making this trade-off has the downside of making it possible for the CPU to assert a\n    /// physical address that is not backed by any DRAM (e.g. accessing an address close to 4 GiB on\n    /// an RPi3 that comes with 1 GiB of RAM). This would result in a crash or other kind of error.\n    pub const END_INCLUSIVE:       usize = 0xFFFF_FFFF;\n\n    pub const GPIO_OFFSET:         usize = 0x0020_0000;\n    pub const UART_OFFSET:         usize = 0x0020_1000;\n\n    /// Physical devices.\n    #[cfg(feature = \"bsp_rpi3\")]\n    pub mod mmio {\n        use super::*;\n\n        pub const START:            usize =         0x3F00_0000;\n        pub const GPIO_START:       usize = START + GPIO_OFFSET;\n        pub const PL011_UART_START: usize = START + UART_OFFSET;\n        pub const END_INCLUSIVE:    usize =         0x4000_FFFF;\n    }\n\n    /// Physical devices.\n    #[cfg(feature = \"bsp_rpi4\")]\n    pub mod mmio {\n        use super::*;\n\n        pub const START:            usize =         0xFE00_0000;\n        pub const GPIO_START:       usize = START + GPIO_OFFSET;\n        pub const PL011_UART_START: usize = START + UART_OFFSET;\n        pub const END_INCLUSIVE:    usize =         0xFF84_FFFF;\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Start page address of the code segment.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn code_start() -> usize {\n    unsafe { __code_start.get() as usize }\n}\n\n/// Exclusive end page address of the code segment.\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn code_end_exclusive() -> usize {\n    unsafe { __code_end_exclusive.get() as usize }\n}\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/bsp/raspberrypi.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Top-level BSP file for the Raspberry Pi 3 and 4.\n\npub mod cpu;\npub mod driver;\npub mod memory;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Board identification.\npub fn board_name() -> &'static str {\n    #[cfg(feature = \"bsp_rpi3\")]\n    {\n        \"Raspberry Pi 3\"\n    }\n\n    #[cfg(feature = \"bsp_rpi4\")]\n    {\n        \"Raspberry Pi 4\"\n    }\n}\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/bsp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Conditional reexporting of Board Support Packages.\n\nmod device_driver;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod raspberrypi;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use raspberrypi::*;\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/common.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! General purpose code.\n\n/// Convert a size into human readable format.\npub const fn size_human_readable_ceil(size: usize) -> (usize, &'static str) {\n    const KIB: usize = 1024;\n    const MIB: usize = 1024 * 1024;\n    const GIB: usize = 1024 * 1024 * 1024;\n\n    if (size / GIB) > 0 {\n        (size.div_ceil(GIB), \"GiB\")\n    } else if (size / MIB) > 0 {\n        (size.div_ceil(MIB), \"MiB\")\n    } else if (size / KIB) > 0 {\n        (size.div_ceil(KIB), \"KiB\")\n    } else {\n        (size, \"Byte\")\n    }\n}\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/console/null_console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Null console.\n\nuse super::interface;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct NullConsole;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\npub static NULL_CONSOLE: NullConsole = NullConsole {};\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl interface::Write for NullConsole {\n    fn write_char(&self, _c: char) {}\n\n    fn write_fmt(&self, _args: fmt::Arguments) -> fmt::Result {\n        fmt::Result::Ok(())\n    }\n\n    fn flush(&self) {}\n}\n\nimpl interface::Read for NullConsole {\n    fn clear_rx(&self) {}\n}\n\nimpl interface::Statistics for NullConsole {}\nimpl interface::All for NullConsole {}\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! System console.\n\nmod null_console;\n\nuse crate::synchronization::{self, NullLock};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Console interfaces.\npub mod interface {\n    use core::fmt;\n\n    /// Console write functions.\n    pub trait Write {\n        /// Write a single character.\n        fn write_char(&self, c: char);\n\n        /// Write a Rust format string.\n        fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result;\n\n        /// Block until the last buffered character has been physically put on the TX wire.\n        fn flush(&self);\n    }\n\n    /// Console read functions.\n    pub trait Read {\n        /// Read a single character.\n        fn read_char(&self) -> char {\n            ' '\n        }\n\n        /// Clear RX buffers, if any.\n        fn clear_rx(&self);\n    }\n\n    /// Console statistics.\n    pub trait Statistics {\n        /// Return the number of characters written.\n        fn chars_written(&self) -> usize {\n            0\n        }\n\n        /// Return the number of characters read.\n        fn chars_read(&self) -> usize {\n            0\n        }\n    }\n\n    /// Trait alias for a full-fledged console.\n    pub trait All: Write + Read + Statistics {}\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic CUR_CONSOLE: NullLock<&'static (dyn interface::All + Sync)> =\n    NullLock::new(&null_console::NULL_CONSOLE);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\n/// Register a new console.\npub fn register_console(new_console: &'static (dyn interface::All + Sync)) {\n    CUR_CONSOLE.lock(|con| *con = new_console);\n}\n\n/// Return a reference to the currently registered console.\n///\n/// This is the global console used by all printing macros.\npub fn console() -> &'static dyn interface::All {\n    CUR_CONSOLE.lock(|con| *con)\n}\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Boot code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/cpu/boot.rs\"]\nmod arch_boot;\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Processor code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/cpu.rs\"]\nmod arch_cpu;\n\nmod boot;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_cpu::{nop, wait_forever};\n\n#[cfg(feature = \"test_build\")]\npub use arch_cpu::{qemu_exit_failure, qemu_exit_success};\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Driver support.\n\nuse crate::{\n    info,\n    synchronization::{interface::Mutex, NullLock},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nconst NUM_DRIVERS: usize = 5;\n\nstruct DriverManagerInner {\n    next_index: usize,\n    descriptors: [Option<DeviceDriverDescriptor>; NUM_DRIVERS],\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Driver interfaces.\npub mod interface {\n    /// Device Driver functions.\n    pub trait DeviceDriver {\n        /// Return a compatibility string for identifying the driver.\n        fn compatible(&self) -> &'static str;\n\n        /// Called by the kernel to bring up the device.\n        ///\n        /// # Safety\n        ///\n        /// - During init, drivers might do stuff with system-wide impact.\n        unsafe fn init(&self) -> Result<(), &'static str> {\n            Ok(())\n        }\n    }\n}\n\n/// Tpye to be used as an optional callback after a driver's init() has run.\npub type DeviceDriverPostInitCallback = unsafe fn() -> Result<(), &'static str>;\n\n/// A descriptor for device drivers.\n#[derive(Copy, Clone)]\npub struct DeviceDriverDescriptor {\n    device_driver: &'static (dyn interface::DeviceDriver + Sync),\n    post_init_callback: Option<DeviceDriverPostInitCallback>,\n}\n\n/// Provides device driver management functions.\npub struct DriverManager {\n    inner: NullLock<DriverManagerInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic DRIVER_MANAGER: DriverManager = DriverManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl DriverManagerInner {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            next_index: 0,\n            descriptors: [None; NUM_DRIVERS],\n        }\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl DeviceDriverDescriptor {\n    /// Create an instance.\n    pub fn new(\n        device_driver: &'static (dyn interface::DeviceDriver + Sync),\n        post_init_callback: Option<DeviceDriverPostInitCallback>,\n    ) -> Self {\n        Self {\n            device_driver,\n            post_init_callback,\n        }\n    }\n}\n\n/// Return a reference to the global DriverManager.\npub fn driver_manager() -> &'static DriverManager {\n    &DRIVER_MANAGER\n}\n\nimpl DriverManager {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            inner: NullLock::new(DriverManagerInner::new()),\n        }\n    }\n\n    /// Register a device driver with the kernel.\n    pub fn register_driver(&self, descriptor: DeviceDriverDescriptor) {\n        self.inner.lock(|inner| {\n            inner.descriptors[inner.next_index] = Some(descriptor);\n            inner.next_index += 1;\n        })\n    }\n\n    /// Helper for iterating over registered drivers.\n    fn for_each_descriptor<'a>(&'a self, f: impl FnMut(&'a DeviceDriverDescriptor)) {\n        self.inner.lock(|inner| {\n            inner\n                .descriptors\n                .iter()\n                .filter_map(|x| x.as_ref())\n                .for_each(f)\n        })\n    }\n\n    /// Fully initialize all drivers.\n    ///\n    /// # Safety\n    ///\n    /// - During init, drivers might do stuff with system-wide impact.\n    pub unsafe fn init_drivers(&self) {\n        self.for_each_descriptor(|descriptor| {\n            // 1. Initialize driver.\n            if let Err(x) = descriptor.device_driver.init() {\n                panic!(\n                    \"Error initializing driver: {}: {}\",\n                    descriptor.device_driver.compatible(),\n                    x\n                );\n            }\n\n            // 2. Call corresponding post init callback.\n            if let Some(callback) = &descriptor.post_init_callback {\n                if let Err(x) = callback() {\n                    panic!(\n                        \"Error during driver post-init callback: {}: {}\",\n                        descriptor.device_driver.compatible(),\n                        x\n                    );\n                }\n            }\n        });\n    }\n\n    /// Enumerate all registered device drivers.\n    pub fn enumerate(&self) {\n        let mut i: usize = 1;\n        self.for_each_descriptor(|descriptor| {\n            info!(\"      {}. {}\", i, descriptor.device_driver.compatible());\n\n            i += 1;\n        });\n    }\n}\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/exception/asynchronous.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Asynchronous exception handling.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/exception/asynchronous.rs\"]\nmod arch_asynchronous;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_asynchronous::print_state;\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/exception.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Synchronous and asynchronous exception handling.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/exception.rs\"]\nmod arch_exception;\n\npub mod asynchronous;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_exception::{current_privilege_level, handling_init};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Kernel privilege levels.\n#[allow(missing_docs)]\n#[derive(Eq, PartialEq)]\npub enum PrivilegeLevel {\n    User,\n    Kernel,\n    Hypervisor,\n    Unknown,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// Libkernel unit tests must execute in kernel mode.\n    #[kernel_test]\n    fn test_runner_executes_in_kernel_mode() {\n        let (level, _) = current_privilege_level();\n\n        assert!(level == PrivilegeLevel::Kernel)\n    }\n}\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/lib.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n// Rust embedded logo for `make doc`.\n#![doc(\n    html_logo_url = \"https://raw.githubusercontent.com/rust-embedded/wg/master/assets/logo/ewg-logo-blue-white-on-transparent.png\"\n)]\n\n//! The `kernel` library.\n//!\n//! Used to compose the final kernel binary.\n//!\n//! # Code organization and architecture\n//!\n//! The code is divided into different *modules*, each representing a typical **subsystem** of the\n//! `kernel`. Top-level module files of subsystems reside directly in the `src` folder. For example,\n//! `src/memory.rs` contains code that is concerned with all things memory management.\n//!\n//! ## Visibility of processor architecture code\n//!\n//! Some of the `kernel`'s subsystems depend on low-level code that is specific to the target\n//! processor architecture. For each supported processor architecture, there exists a subfolder in\n//! `src/_arch`, for example, `src/_arch/aarch64`.\n//!\n//! The architecture folders mirror the subsystem modules laid out in `src`. For example,\n//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go\n//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in\n//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic\n//! module's name prefixed with `arch_`.\n//!\n//! For example, this is the top of `src/memory/mmu.rs`:\n//!\n//! ```\n//! #[cfg(target_arch = \"aarch64\")]\n//! #[path = \"../_arch/aarch64/memory/mmu.rs\"]\n//! mod arch_mmu;\n//! ```\n//!\n//! Often times, items from the `arch_ module` will be publicly reexported by the parent module.\n//! This way, each architecture specific module can provide its implementation of an item, while the\n//! caller must not be concerned which architecture has been conditionally compiled.\n//!\n//! ## BSP code\n//!\n//! `BSP` stands for Board Support Package. `BSP` code is organized under `src/bsp.rs` and contains\n//! target board specific definitions and functions. These are things such as the board's memory map\n//! or instances of drivers for devices that are featured on the respective board.\n//!\n//! Just like processor architecture code, the `BSP` code's module structure tries to mirror the\n//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is\n//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`.\n//!\n//! ## Kernel interfaces\n//!\n//! Both `arch` and `bsp` contain code that is conditionally compiled depending on the actual target\n//! and board for which the kernel is compiled. For example, the `interrupt controller` hardware of\n//! the `Raspberry Pi 3` and the `Raspberry Pi 4` is different, but we want the rest of the `kernel`\n//! code to play nicely with any of the two without much hassle.\n//!\n//! In order to provide a clean abstraction between `arch`, `bsp` and `generic kernel code`,\n//! `interface` traits are provided *whenever possible* and *where it makes sense*. They are defined\n//! in the respective subsystem module and help to enforce the idiom of *program to an interface,\n//! not an implementation*. For example, there will be a common IRQ handling interface which the two\n//! different interrupt controller `drivers` of both Raspberrys will implement, and only export the\n//! interface to the rest of the `kernel`.\n//!\n//! ```\n//!         +-------------------+\n//!         | Interface (Trait) |\n//!         |                   |\n//!         +--+-------------+--+\n//!            ^             ^\n//!            |             |\n//!            |             |\n//! +----------+--+       +--+----------+\n//! | kernel code |       |  bsp code   |\n//! |             |       |  arch code  |\n//! +-------------+       +-------------+\n//! ```\n//!\n//! # Summary\n//!\n//! For a logical `kernel` subsystem, corresponding code can be distributed over several physical\n//! locations. Here is an example for the **memory** subsystem:\n//!\n//! - `src/memory.rs` and `src/memory/**/*`\n//!   - Common code that is agnostic of target processor architecture and `BSP` characteristics.\n//!     - Example: A function to zero a chunk of memory.\n//!   - Interfaces for the memory subsystem that are implemented by `arch` or `BSP` code.\n//!     - Example: An `MMU` interface that defines `MMU` function prototypes.\n//! - `src/bsp/__board_name__/memory.rs` and `src/bsp/__board_name__/memory/**/*`\n//!   - `BSP` specific code.\n//!   - Example: The board's memory map (physical addresses of DRAM and MMIO devices).\n//! - `src/_arch/__arch_name__/memory.rs` and `src/_arch/__arch_name__/memory/**/*`\n//!   - Processor architecture specific code.\n//!   - Example: Implementation of the `MMU` interface for the `__arch_name__` processor\n//!     architecture.\n//!\n//! From a namespace perspective, **memory** subsystem code lives in:\n//!\n//! - `crate::memory::*`\n//! - `crate::bsp::memory::*`\n//!\n//! # Boot flow\n//!\n//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.\n//!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.\n//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.\n\n#![allow(clippy::upper_case_acronyms)]\n#![allow(incomplete_features)]\n#![feature(asm_const)]\n#![feature(const_option)]\n#![feature(core_intrinsics)]\n#![feature(format_args_nl)]\n#![feature(int_roundings)]\n#![feature(linkage)]\n#![feature(nonzero_min_max)]\n#![feature(panic_info_message)]\n#![feature(trait_alias)]\n#![feature(unchecked_math)]\n#![no_std]\n// Testing\n#![cfg_attr(test, no_main)]\n#![feature(custom_test_frameworks)]\n#![reexport_test_harness_main = \"test_main\"]\n#![test_runner(crate::test_runner)]\n\nmod panic_wait;\nmod synchronization;\n\npub mod bsp;\npub mod common;\npub mod console;\npub mod cpu;\npub mod driver;\npub mod exception;\npub mod memory;\npub mod print;\npub mod time;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Version string.\npub fn version() -> &'static str {\n    concat!(\n        env!(\"CARGO_PKG_NAME\"),\n        \" version \",\n        env!(\"CARGO_PKG_VERSION\")\n    )\n}\n\n#[cfg(not(test))]\nextern \"Rust\" {\n    fn kernel_init() -> !;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n/// The default runner for unit tests.\npub fn test_runner(tests: &[&test_types::UnitTest]) {\n    // This line will be printed as the test header.\n    println!(\"Running {} tests\", tests.len());\n\n    for (i, test) in tests.iter().enumerate() {\n        print!(\"{:>3}. {:.<58}\", i + 1, test.name);\n\n        // Run the actual test.\n        (test.test_func)();\n\n        // Failed tests call panic!(). Execution reaches here only if the test has passed.\n        println!(\"[ok]\")\n    }\n}\n\n/// The `kernel_init()` for unit tests.\n#[cfg(test)]\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    bsp::driver::qemu_bring_up_console();\n\n    test_main();\n\n    cpu::qemu_exit_success()\n}\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/main.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n// Rust embedded logo for `make doc`.\n#![doc(\n    html_logo_url = \"https://raw.githubusercontent.com/rust-embedded/wg/master/assets/logo/ewg-logo-blue-white-on-transparent.png\"\n)]\n\n//! The `kernel` binary.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\nuse libkernel::{bsp, console, driver, exception, info, memory, time};\n\n/// Early init code.\n///\n/// # Safety\n///\n/// - Only a single core must be active and running this function.\n/// - The init calls in this function must appear in the correct order:\n///     - MMU + Data caching must be activated at the earliest. Without it, any atomic operations,\n///       e.g. the yet-to-be-introduced spinlocks in the device drivers (which currently employ\n///       NullLocks instead of spinlocks), will fail to work (properly) on the RPi SoCs.\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    use memory::mmu::interface::MMU;\n\n    exception::handling_init();\n\n    if let Err(string) = memory::mmu::mmu().enable_mmu_and_caching() {\n        panic!(\"MMU: {}\", string);\n    }\n\n    // Initialize the BSP driver subsystem.\n    if let Err(x) = bsp::driver::init() {\n        panic!(\"Error initializing BSP driver subsystem: {}\", x);\n    }\n\n    // Initialize all device drivers.\n    driver::driver_manager().init_drivers();\n    // println! is usable from here on.\n\n    // Transition from unsafe to safe.\n    kernel_main()\n}\n\n/// The main function running after the early init.\nfn kernel_main() -> ! {\n    use console::console;\n\n    info!(\"{}\", libkernel::version());\n    info!(\"Booting on: {}\", bsp::board_name());\n\n    info!(\"MMU online. Special regions:\");\n    bsp::memory::mmu::virt_mem_layout().print_layout();\n\n    let (_, privilege_level) = exception::current_privilege_level();\n    info!(\"Current privilege level: {}\", privilege_level);\n\n    info!(\"Exception handling state:\");\n    exception::asynchronous::print_state();\n\n    info!(\n        \"Architectural timer resolution: {} ns\",\n        time::time_manager().resolution().as_nanos()\n    );\n\n    info!(\"Drivers loaded:\");\n    driver::driver_manager().enumerate();\n\n    info!(\"Echoing input now\");\n\n    // Discard any spurious received characters before going into echo mode.\n    console().clear_rx();\n    loop {\n        let c = console().read_char();\n        console().write_char(c);\n    }\n}\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/memory/mmu/translation_table.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Translation table.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../../_arch/aarch64/memory/mmu/translation_table.rs\"]\nmod arch_translation_table;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_translation_table::KernelTranslationTable;\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/memory/mmu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management Unit.\n//!\n//! In order to decouple `BSP` and `arch` parts of the MMU code (to keep them pluggable), this file\n//! provides types for composing an architecture-agnostic description of the kernel's virtual memory\n//! layout.\n//!\n//! The `BSP` provides such a description through the `bsp::memory::mmu::virt_mem_layout()`\n//! function.\n//!\n//! The `MMU` driver of the `arch` code uses `bsp::memory::mmu::virt_mem_layout()` to compile and\n//! install respective translation tables.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/memory/mmu.rs\"]\nmod arch_mmu;\n\nmod translation_table;\n\nuse crate::common;\nuse core::{fmt, ops::RangeInclusive};\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_mmu::mmu;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// MMU enable errors variants.\n#[allow(missing_docs)]\n#[derive(Debug)]\npub enum MMUEnableError {\n    AlreadyEnabled,\n    Other(&'static str),\n}\n\n/// Memory Management interfaces.\npub mod interface {\n    use super::*;\n\n    /// MMU functions.\n    pub trait MMU {\n        /// Called by the kernel during early init. Supposed to take the translation tables from the\n        /// `BSP`-supplied `virt_mem_layout()` and install/activate them for the respective MMU.\n        ///\n        /// # Safety\n        ///\n        /// - Changes the HW's global state.\n        unsafe fn enable_mmu_and_caching(&self) -> Result<(), MMUEnableError>;\n\n        /// Returns true if the MMU is enabled, false otherwise.\n        fn is_enabled(&self) -> bool;\n    }\n}\n\n/// Describes the characteristics of a translation granule.\npub struct TranslationGranule<const GRANULE_SIZE: usize>;\n\n/// Describes properties of an address space.\npub struct AddressSpace<const AS_SIZE: usize>;\n\n/// Architecture agnostic translation types.\n#[allow(missing_docs)]\n#[derive(Copy, Clone)]\npub enum Translation {\n    Identity,\n    Offset(usize),\n}\n\n/// Architecture agnostic memory attributes.\n#[allow(missing_docs)]\n#[derive(Copy, Clone)]\npub enum MemAttributes {\n    CacheableDRAM,\n    Device,\n}\n\n/// Architecture agnostic access permissions.\n#[allow(missing_docs)]\n#[derive(Copy, Clone)]\npub enum AccessPermissions {\n    ReadOnly,\n    ReadWrite,\n}\n\n/// Collection of memory attributes.\n#[allow(missing_docs)]\n#[derive(Copy, Clone)]\npub struct AttributeFields {\n    pub mem_attributes: MemAttributes,\n    pub acc_perms: AccessPermissions,\n    pub execute_never: bool,\n}\n\n/// Architecture agnostic descriptor for a memory range.\n#[allow(missing_docs)]\npub struct TranslationDescriptor {\n    pub name: &'static str,\n    pub virtual_range: fn() -> RangeInclusive<usize>,\n    pub physical_range_translation: Translation,\n    pub attribute_fields: AttributeFields,\n}\n\n/// Type for expressing the kernel's virtual memory layout.\npub struct KernelVirtualLayout<const NUM_SPECIAL_RANGES: usize> {\n    /// The last (inclusive) address of the address space.\n    max_virt_addr_inclusive: usize,\n\n    /// Array of descriptors for non-standard (normal cacheable DRAM) memory regions.\n    inner: [TranslationDescriptor; NUM_SPECIAL_RANGES],\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl fmt::Display for MMUEnableError {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        match self {\n            MMUEnableError::AlreadyEnabled => write!(f, \"MMU is already enabled\"),\n            MMUEnableError::Other(x) => write!(f, \"{}\", x),\n        }\n    }\n}\n\nimpl<const GRANULE_SIZE: usize> TranslationGranule<GRANULE_SIZE> {\n    /// The granule's size.\n    pub const SIZE: usize = Self::size_checked();\n\n    /// The granule's shift, aka log2(size).\n    pub const SHIFT: usize = Self::SIZE.trailing_zeros() as usize;\n\n    const fn size_checked() -> usize {\n        assert!(GRANULE_SIZE.is_power_of_two());\n\n        GRANULE_SIZE\n    }\n}\n\nimpl<const AS_SIZE: usize> AddressSpace<AS_SIZE> {\n    /// The address space size.\n    pub const SIZE: usize = Self::size_checked();\n\n    /// The address space shift, aka log2(size).\n    pub const SIZE_SHIFT: usize = Self::SIZE.trailing_zeros() as usize;\n\n    const fn size_checked() -> usize {\n        assert!(AS_SIZE.is_power_of_two());\n\n        // Check for architectural restrictions as well.\n        Self::arch_address_space_size_sanity_checks();\n\n        AS_SIZE\n    }\n}\n\nimpl Default for AttributeFields {\n    fn default() -> AttributeFields {\n        AttributeFields {\n            mem_attributes: MemAttributes::CacheableDRAM,\n            acc_perms: AccessPermissions::ReadWrite,\n            execute_never: true,\n        }\n    }\n}\n\n/// Human-readable output of a TranslationDescriptor.\nimpl fmt::Display for TranslationDescriptor {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        // Call the function to which self.range points, and dereference the result, which causes\n        // Rust to copy the value.\n        let start = *(self.virtual_range)().start();\n        let end = *(self.virtual_range)().end();\n        let size = end - start + 1;\n\n        let (size, unit) = common::size_human_readable_ceil(size);\n\n        let attr = match self.attribute_fields.mem_attributes {\n            MemAttributes::CacheableDRAM => \"C\",\n            MemAttributes::Device => \"Dev\",\n        };\n\n        let acc_p = match self.attribute_fields.acc_perms {\n            AccessPermissions::ReadOnly => \"RO\",\n            AccessPermissions::ReadWrite => \"RW\",\n        };\n\n        let xn = if self.attribute_fields.execute_never {\n            \"PXN\"\n        } else {\n            \"PX\"\n        };\n\n        write!(\n            f,\n            \"      {:#010x} - {:#010x} | {: >3} {} | {: <3} {} {: <3} | {}\",\n            start, end, size, unit, attr, acc_p, xn, self.name\n        )\n    }\n}\n\nimpl<const NUM_SPECIAL_RANGES: usize> KernelVirtualLayout<{ NUM_SPECIAL_RANGES }> {\n    /// Create a new instance.\n    pub const fn new(max: usize, layout: [TranslationDescriptor; NUM_SPECIAL_RANGES]) -> Self {\n        Self {\n            max_virt_addr_inclusive: max,\n            inner: layout,\n        }\n    }\n\n    /// For a virtual address, find and return the physical output address and corresponding\n    /// attributes.\n    ///\n    /// If the address is not found in `inner`, return an identity mapped default with normal\n    /// cacheable DRAM attributes.\n    pub fn virt_addr_properties(\n        &self,\n        virt_addr: usize,\n    ) -> Result<(usize, AttributeFields), &'static str> {\n        if virt_addr > self.max_virt_addr_inclusive {\n            return Err(\"Address out of range\");\n        }\n\n        for i in self.inner.iter() {\n            if (i.virtual_range)().contains(&virt_addr) {\n                let output_addr = match i.physical_range_translation {\n                    Translation::Identity => virt_addr,\n                    Translation::Offset(a) => a + (virt_addr - (i.virtual_range)().start()),\n                };\n\n                return Ok((output_addr, i.attribute_fields));\n            }\n        }\n\n        Ok((virt_addr, AttributeFields::default()))\n    }\n\n    /// Print the memory layout.\n    pub fn print_layout(&self) {\n        use crate::info;\n\n        for i in self.inner.iter() {\n            info!(\"{}\", i);\n        }\n    }\n\n    #[cfg(test)]\n    pub fn inner(&self) -> &[TranslationDescriptor; NUM_SPECIAL_RANGES] {\n        &self.inner\n    }\n}\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/memory.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management.\n\npub mod mmu;\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/panic_wait.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A panic handler that infinitely waits.\n\nuse crate::{cpu, println};\nuse core::panic::PanicInfo;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// The point of exit for `libkernel`.\n///\n/// It is linked weakly, so that the integration tests can overload its standard behavior.\n#[linkage = \"weak\"]\n#[no_mangle]\nfn _panic_exit() -> ! {\n    #[cfg(not(feature = \"test_build\"))]\n    {\n        cpu::wait_forever()\n    }\n\n    #[cfg(feature = \"test_build\")]\n    {\n        cpu::qemu_exit_failure()\n    }\n}\n\n/// Stop immediately if called a second time.\n///\n/// # Note\n///\n/// Using atomics here relieves us from needing to use `unsafe` for the static variable.\n///\n/// On `AArch64`, which is the only implemented architecture at the time of writing this,\n/// [`AtomicBool::load`] and [`AtomicBool::store`] are lowered to ordinary load and store\n/// instructions. They are therefore safe to use even with MMU + caching deactivated.\n///\n/// [`AtomicBool::load`]: core::sync::atomic::AtomicBool::load\n/// [`AtomicBool::store`]: core::sync::atomic::AtomicBool::store\nfn panic_prevent_reenter() {\n    use core::sync::atomic::{AtomicBool, Ordering};\n\n    #[cfg(not(target_arch = \"aarch64\"))]\n    compile_error!(\"Add the target_arch to above's check if the following code is safe to use\");\n\n    static PANIC_IN_PROGRESS: AtomicBool = AtomicBool::new(false);\n\n    if !PANIC_IN_PROGRESS.load(Ordering::Relaxed) {\n        PANIC_IN_PROGRESS.store(true, Ordering::Relaxed);\n\n        return;\n    }\n\n    _panic_exit()\n}\n\n#[panic_handler]\nfn panic(info: &PanicInfo) -> ! {\n    // Protect against panic infinite loops if any of the following code panics itself.\n    panic_prevent_reenter();\n\n    let timestamp = crate::time::time_manager().uptime();\n    let (location, line, column) = match info.location() {\n        Some(loc) => (loc.file(), loc.line(), loc.column()),\n        _ => (\"???\", 0, 0),\n    };\n\n    println!(\n        \"[  {:>3}.{:06}] Kernel panic!\\n\\n\\\n        Panic location:\\n      File '{}', line {}, column {}\\n\\n\\\n        {}\",\n        timestamp.as_secs(),\n        timestamp.subsec_micros(),\n        location,\n        line,\n        column,\n        info.message().unwrap_or(&format_args!(\"\")),\n    );\n\n    _panic_exit()\n}\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/print.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Printing.\n\nuse crate::console;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n#[doc(hidden)]\npub fn _print(args: fmt::Arguments) {\n    console::console().write_fmt(args).unwrap();\n}\n\n/// Prints without a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! print {\n    ($($arg:tt)*) => ($crate::print::_print(format_args!($($arg)*)));\n}\n\n/// Prints with a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! println {\n    () => ($crate::print!(\"\\n\"));\n    ($($arg:tt)*) => ({\n        $crate::print::_print(format_args_nl!($($arg)*));\n    })\n}\n\n/// Prints an info, with a newline.\n#[macro_export]\nmacro_rules! info {\n    ($string:expr) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[  {:>3}.{:06}] \", $string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n        ));\n    });\n    ($format_string:expr, $($arg:tt)*) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[  {:>3}.{:06}] \", $format_string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n            $($arg)*\n        ));\n    })\n}\n\n/// Prints a warning, with a newline.\n#[macro_export]\nmacro_rules! warn {\n    ($string:expr) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[W {:>3}.{:06}] \", $string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n        ));\n    });\n    ($format_string:expr, $($arg:tt)*) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[W {:>3}.{:06}] \", $format_string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n            $($arg)*\n        ));\n    })\n}\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/synchronization.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Synchronization primitives.\n//!\n//! # Resources\n//!\n//!   - <https://doc.rust-lang.org/book/ch16-04-extensible-concurrency-sync-and-send.html>\n//!   - <https://stackoverflow.com/questions/59428096/understanding-the-send-trait>\n//!   - <https://doc.rust-lang.org/std/cell/index.html>\n\nuse core::cell::UnsafeCell;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Synchronization interfaces.\npub mod interface {\n\n    /// Any object implementing this trait guarantees exclusive access to the data wrapped within\n    /// the Mutex for the duration of the provided closure.\n    pub trait Mutex {\n        /// The type of the data that is wrapped by this mutex.\n        type Data;\n\n        /// Locks the mutex and grants the closure temporary mutable access to the wrapped data.\n        fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R;\n    }\n}\n\n/// A pseudo-lock for teaching purposes.\n///\n/// In contrast to a real Mutex implementation, does not protect against concurrent access from\n/// other cores to the contained data. This part is preserved for later lessons.\n///\n/// The lock will only be used as long as it is safe to do so, i.e. as long as the kernel is\n/// executing single-threaded, aka only running on a single core with interrupts disabled.\npub struct NullLock<T>\nwhere\n    T: ?Sized,\n{\n    data: UnsafeCell<T>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nunsafe impl<T> Send for NullLock<T> where T: ?Sized + Send {}\nunsafe impl<T> Sync for NullLock<T> where T: ?Sized + Send {}\n\nimpl<T> NullLock<T> {\n    /// Create an instance.\n    pub const fn new(data: T) -> Self {\n        Self {\n            data: UnsafeCell::new(data),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\n\nimpl<T> interface::Mutex for NullLock<T> {\n    type Data = T;\n\n    fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R {\n        // In a real lock, there would be code encapsulating this line that ensures that this\n        // mutable reference will ever only be given out once at a time.\n        let data = unsafe { &mut *self.data.get() };\n\n        f(data)\n    }\n}\n"
  },
  {
    "path": "12_integrated_testing/kernel/src/time.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Timer primitives.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/time.rs\"]\nmod arch_time;\n\nuse core::time::Duration;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Provides time management functions.\npub struct TimeManager;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic TIME_MANAGER: TimeManager = TimeManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the global TimeManager.\npub fn time_manager() -> &'static TimeManager {\n    &TIME_MANAGER\n}\n\nimpl TimeManager {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self\n    }\n\n    /// The timer's resolution.\n    pub fn resolution(&self) -> Duration {\n        arch_time::resolution()\n    }\n\n    /// The uptime since power-on of the device.\n    ///\n    /// This includes time consumed by firmware and bootloaders.\n    pub fn uptime(&self) -> Duration {\n        arch_time::uptime()\n    }\n\n    /// Spin for a given duration.\n    pub fn spin_for(&self, duration: Duration) {\n        arch_time::spin_for(duration)\n    }\n}\n"
  },
  {
    "path": "12_integrated_testing/kernel/tests/00_console_sanity.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'console_io_test'\n\n# Verify sending and receiving works as expected.\nclass TxRxHandshakeTest < SubtestBase\n    def name\n        'Transmit and Receive handshake'\n    end\n\n    def run(qemu_out, qemu_in)\n        qemu_in.write_nonblock('ABC')\n        expect_or_raise(qemu_out, 'OK1234')\n    end\nend\n\n# Check for correct TX statistics implementation. Depends on test 1 being run first.\nclass TxStatisticsTest < SubtestBase\n    def name\n        'Transmit statistics'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, '6')\n    end\nend\n\n# Check for correct RX statistics implementation. Depends on test 1 being run first.\nclass RxStatisticsTest < SubtestBase\n    def name\n        'Receive statistics'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, '3')\n    end\nend\n\n## -------------------------------------------------------------------------------------------------\n## Test registration\n## -------------------------------------------------------------------------------------------------\ndef subtest_collection\n    [TxRxHandshakeTest.new, TxStatisticsTest.new, RxStatisticsTest.new]\nend\n"
  },
  {
    "path": "12_integrated_testing/kernel/tests/00_console_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Console sanity tests - RX, TX and statistics.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Console tests should time out on the I/O harness in case of panic.\nmod panic_wait_forever;\n\nuse libkernel::{bsp, console, cpu, exception, print};\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    use console::console;\n\n    exception::handling_init();\n    bsp::driver::qemu_bring_up_console();\n\n    // Handshake\n    assert_eq!(console().read_char(), 'A');\n    assert_eq!(console().read_char(), 'B');\n    assert_eq!(console().read_char(), 'C');\n    print!(\"OK1234\");\n\n    // 6\n    print!(\"{}\", console().chars_written());\n\n    // 3\n    print!(\"{}\", console().chars_read());\n\n    // The QEMU process running this test will be closed by the I/O test harness.\n    cpu::wait_forever();\n}\n"
  },
  {
    "path": "12_integrated_testing/kernel/tests/01_timer_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Timer sanity tests.\n\n#![feature(custom_test_frameworks)]\n#![no_main]\n#![no_std]\n#![reexport_test_harness_main = \"test_main\"]\n#![test_runner(libkernel::test_runner)]\n\nuse core::time::Duration;\nuse libkernel::{bsp, cpu, exception, time};\nuse test_macros::kernel_test;\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    bsp::driver::qemu_bring_up_console();\n\n    // Depending on CPU arch, some timer bring-up code could go here. Not needed for the RPi.\n\n    test_main();\n\n    cpu::qemu_exit_success()\n}\n\n/// Simple check that the timer is running.\n#[kernel_test]\nfn timer_is_counting() {\n    assert!(time::time_manager().uptime().as_nanos() > 0)\n}\n\n/// Timer resolution must be sufficient.\n#[kernel_test]\nfn timer_resolution_is_sufficient() {\n    assert!(time::time_manager().resolution().as_nanos() > 0);\n    assert!(time::time_manager().resolution().as_nanos() < 100)\n}\n\n/// Sanity check spin_for() implementation.\n#[kernel_test]\nfn spin_accuracy_check_1_second() {\n    let t1 = time::time_manager().uptime();\n    time::time_manager().spin_for(Duration::from_secs(1));\n    let t2 = time::time_manager().uptime();\n\n    assert_eq!((t2 - t1).as_secs(), 1)\n}\n"
  },
  {
    "path": "12_integrated_testing/kernel/tests/02_exception_sync_page_fault.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Page faults must result in synchronous exceptions.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Overwrites libkernel's `panic_wait::_panic_exit()` so that it returns a \"success\" code.\n///\n/// In this test, reaching the panic is a success, because it is called from the synchronous\n/// exception handler, which is what this test wants to achieve.\n///\n/// It also means that this integration test can not use any other code that calls panic!() directly\n/// or indirectly.\nmod panic_exit_success;\n\nuse libkernel::{bsp, cpu, exception, info, memory, println};\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    use memory::mmu::interface::MMU;\n\n    exception::handling_init();\n    bsp::driver::qemu_bring_up_console();\n\n    // This line will be printed as the test header.\n    println!(\"Testing synchronous exception handling by causing a page fault\");\n\n    if let Err(string) = memory::mmu::mmu().enable_mmu_and_caching() {\n        info!(\"MMU: {}\", string);\n        cpu::qemu_exit_failure()\n    }\n\n    info!(\"Writing beyond mapped area to address 9 GiB...\");\n    let big_addr: u64 = 9 * 1024 * 1024 * 1024;\n    core::ptr::read_volatile(big_addr as *mut u64);\n\n    // If execution reaches here, the memory access above did not cause a page fault exception.\n    cpu::qemu_exit_failure()\n}\n"
  },
  {
    "path": "12_integrated_testing/kernel/tests/03_exception_restore_sanity.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'console_io_test'\n\n# Verify that exception restore works.\nclass ExceptionRestoreTest < SubtestBase\n    def name\n        'Exception restore'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, 'Back from system call!')\n    end\nend\n\n## -------------------------------------------------------------------------------------------------\n## Test registration\n## -------------------------------------------------------------------------------------------------\ndef subtest_collection\n    [ExceptionRestoreTest.new]\nend\n"
  },
  {
    "path": "12_integrated_testing/kernel/tests/03_exception_restore_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A simple sanity test to see if exception restore code works.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Console tests should time out on the I/O harness in case of panic.\nmod panic_wait_forever;\n\nuse core::arch::asm;\nuse libkernel::{bsp, cpu, exception, info, memory, println};\n\n#[inline(never)]\nfn nested_system_call() {\n    #[cfg(target_arch = \"aarch64\")]\n    unsafe {\n        asm!(\"svc #0x1337\", options(nomem, nostack, preserves_flags));\n    }\n\n    #[cfg(not(target_arch = \"aarch64\"))]\n    {\n        info!(\"Not supported yet\");\n        cpu::wait_forever();\n    }\n}\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    use memory::mmu::interface::MMU;\n\n    exception::handling_init();\n    bsp::driver::qemu_bring_up_console();\n\n    // This line will be printed as the test header.\n    println!(\"Testing exception restore\");\n\n    if let Err(string) = memory::mmu::mmu().enable_mmu_and_caching() {\n        info!(\"MMU: {}\", string);\n        cpu::qemu_exit_failure()\n    }\n\n    info!(\"Making a dummy system call\");\n\n    // Calling this inside a function indirectly tests if the link register is restored properly.\n    nested_system_call();\n\n    info!(\"Back from system call!\");\n\n    // The QEMU process running this test will be closed by the I/O test harness.\n    cpu::wait_forever();\n}\n"
  },
  {
    "path": "12_integrated_testing/kernel/tests/boot_test_string.rb",
    "content": "# frozen_string_literal: true\n\nEXPECTED_PRINT = 'Echoing input now'\n"
  },
  {
    "path": "12_integrated_testing/kernel/tests/panic_exit_success/mod.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n/// Overwrites libkernel's `panic_wait::_panic_exit()` with the QEMU-exit version.\n#[no_mangle]\nfn _panic_exit() -> ! {\n    libkernel::cpu::qemu_exit_success()\n}\n"
  },
  {
    "path": "12_integrated_testing/kernel/tests/panic_wait_forever/mod.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n/// Overwrites libkernel's `panic_wait::_panic_exit()` with wait_forever.\n#[no_mangle]\nfn _panic_exit() -> ! {\n    libkernel::cpu::wait_forever()\n}\n"
  },
  {
    "path": "12_integrated_testing/libraries/test-macros/Cargo.toml",
    "content": "[package]\nname = \"test-macros\"\nversion = \"0.1.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n\n[lib]\nproc-macro = true\n\n[dependencies]\nproc-macro2 = \"1.x\"\nquote = \"1.x\"\nsyn = { version = \"1.x\", features = [\"full\"] }\ntest-types = { path = \"../test-types\" }\n"
  },
  {
    "path": "12_integrated_testing/libraries/test-macros/src/lib.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\nuse proc_macro::TokenStream;\nuse proc_macro2::Span;\nuse quote::quote;\nuse syn::{parse_macro_input, Ident, ItemFn};\n\n#[proc_macro_attribute]\npub fn kernel_test(_attr: TokenStream, input: TokenStream) -> TokenStream {\n    let f = parse_macro_input!(input as ItemFn);\n\n    let test_name = &format!(\"{}\", f.sig.ident);\n    let test_ident = Ident::new(\n        &format!(\"{}_TEST_CONTAINER\", f.sig.ident.to_string().to_uppercase()),\n        Span::call_site(),\n    );\n    let test_code_block = f.block;\n\n    quote!(\n        #[test_case]\n        const #test_ident: test_types::UnitTest = test_types::UnitTest {\n            name: #test_name,\n            test_func: || #test_code_block,\n        };\n    )\n    .into()\n}\n"
  },
  {
    "path": "12_integrated_testing/libraries/test-types/Cargo.toml",
    "content": "[package]\nname = \"test-types\"\nversion = \"0.1.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n"
  },
  {
    "path": "12_integrated_testing/libraries/test-types/src/lib.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Types for the `custom_test_frameworks` implementation.\n\n#![no_std]\n\n/// Unit test container.\npub struct UnitTest {\n    /// Name of the test.\n    pub name: &'static str,\n\n    /// Function pointer to the test.\n    pub test_func: fn(),\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/.cargo/config.toml",
    "content": "[target.'cfg(target_os = \"none\")']\nrunner = \"target/kernel_test_runner.sh\"\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/.vscode/settings.json",
    "content": "{\n    \"editor.formatOnSave\": true,\n    \"editor.rulers\": [100],\n    \"rust-analyzer.cargo.target\": \"aarch64-unknown-none-softfloat\",\n    \"rust-analyzer.cargo.features\": [\"bsp_rpi3\"],\n    \"rust-analyzer.checkOnSave.allTargets\": false,\n    \"rust-analyzer.checkOnSave.extraArgs\": [\"--lib\", \"--bins\"],\n    \"rust-analyzer.lens.debug\": false,\n    \"rust-analyzer.lens.run\": false\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/Cargo.toml",
    "content": "[workspace]\n\nmembers = [\n        \"libraries/*\",\n        \"kernel\"\n]\n\n[profile.release]\nlto = true\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/Makefile",
    "content": "## SPDX-License-Identifier: MIT OR Apache-2.0\n##\n## Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\ninclude ../common/docker.mk\ninclude ../common/format.mk\ninclude ../common/operating_system.mk\n\n##--------------------------------------------------------------------------------------------------\n## Optional, user-provided configuration values\n##--------------------------------------------------------------------------------------------------\n\n# Default to the RPi3.\nBSP ?= rpi3\n\n# Default to a serial device name that is common in Linux.\nDEV_SERIAL ?= /dev/ttyUSB0\n\n# Optional integration test name.\nifdef TEST\n    TEST_ARG = --test $(TEST)\nelse\n    TEST_ARG = --test '*'\nendif\n\n\n\n##--------------------------------------------------------------------------------------------------\n## BSP-specific configuration values\n##--------------------------------------------------------------------------------------------------\nQEMU_MISSING_STRING = \"This board is not yet supported for QEMU.\"\n\nifeq ($(BSP),rpi3)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE = raspi3\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    QEMU_TEST_ARGS    = $(QEMU_RELEASE_ARGS) -semihosting\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    OPENOCD_ARG       = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi3.cfg\n    JTAG_BOOT_IMAGE   = ../X1_JTAG_boot/jtag_boot_rpi3.img\n    LD_SCRIPT_PATH    = $(shell pwd)/kernel/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a53\nelse ifeq ($(BSP),rpi4)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE =\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    QEMU_TEST_ARGS    = $(QEMU_RELEASE_ARGS) -semihosting\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    OPENOCD_ARG       = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi4.cfg\n    JTAG_BOOT_IMAGE   = ../X1_JTAG_boot/jtag_boot_rpi4.img\n    LD_SCRIPT_PATH    = $(shell pwd)/kernel/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a72\nendif\n\n# Export for build.rs.\nexport LD_SCRIPT_PATH\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets and Prerequisites\n##--------------------------------------------------------------------------------------------------\nKERNEL_MANIFEST      = kernel/Cargo.toml\nKERNEL_LINKER_SCRIPT = kernel.ld\nLAST_BUILD_CONFIG    = target/$(BSP).build_config\n\nKERNEL_ELF      = target/$(TARGET)/release/kernel\n# This parses cargo's dep-info file.\n# https://doc.rust-lang.org/cargo/guide/build-cache.html#dep-info-files\nKERNEL_ELF_DEPS = $(filter-out %: ,$(file < $(KERNEL_ELF).d)) $(KERNEL_MANIFEST) $(LAST_BUILD_CONFIG)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Command building blocks\n##--------------------------------------------------------------------------------------------------\nRUSTFLAGS = $(RUSTC_MISC_ARGS)                   \\\n    -C link-arg=--library-path=$(LD_SCRIPT_PATH) \\\n    -C link-arg=--script=$(KERNEL_LINKER_SCRIPT)\n\nRUSTFLAGS_PEDANTIC = $(RUSTFLAGS) \\\n    -D warnings                   \\\n    -D missing_docs\n\nFEATURES      = --features bsp_$(BSP)\nCOMPILER_ARGS = --target=$(TARGET) \\\n    $(FEATURES)                    \\\n    --release\n\nRUSTC_CMD   = cargo rustc $(COMPILER_ARGS) --manifest-path $(KERNEL_MANIFEST)\nDOC_CMD     = cargo doc $(COMPILER_ARGS)\nCLIPPY_CMD  = cargo clippy $(COMPILER_ARGS)\nTEST_CMD    = cargo test $(COMPILER_ARGS) --manifest-path $(KERNEL_MANIFEST)\nOBJCOPY_CMD = rust-objcopy \\\n    --strip-all            \\\n    -O binary\n\nEXEC_QEMU          = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE)\nEXEC_TEST_DISPATCH = ruby ../common/tests/dispatch.rb\nEXEC_MINIPUSH      = ruby ../common/serial/minipush.rb\n\n##------------------------------------------------------------------------------\n## Dockerization\n##------------------------------------------------------------------------------\nDOCKER_CMD            = docker run -t --rm -v $(shell pwd):/work/tutorial -w /work/tutorial\nDOCKER_CMD_INTERACT   = $(DOCKER_CMD) -i\nDOCKER_ARG_DIR_COMMON = -v $(shell pwd)/../common:/work/common\nDOCKER_ARG_DIR_JTAG   = -v $(shell pwd)/../X1_JTAG_boot:/work/X1_JTAG_boot\nDOCKER_ARG_DEV        = --privileged -v /dev:/dev\nDOCKER_ARG_NET        = --network host\n\n# DOCKER_IMAGE defined in include file (see top of this file).\nDOCKER_QEMU  = $(DOCKER_CMD_INTERACT) $(DOCKER_IMAGE)\nDOCKER_TOOLS = $(DOCKER_CMD) $(DOCKER_IMAGE)\nDOCKER_TEST  = $(DOCKER_CMD) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\nDOCKER_GDB   = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_NET) $(DOCKER_IMAGE)\n\n# Dockerize commands, which require USB device passthrough, only on Linux.\nifeq ($(shell uname -s),Linux)\n    DOCKER_CMD_DEV = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_DEV)\n\n    DOCKER_CHAINBOOT = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\n    DOCKER_JTAGBOOT  = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_ARG_DIR_JTAG) $(DOCKER_IMAGE)\n    DOCKER_OPENOCD   = $(DOCKER_CMD_DEV) $(DOCKER_ARG_NET) $(DOCKER_IMAGE)\nelse\n    DOCKER_OPENOCD   = echo \"Not yet supported on non-Linux systems.\"; \\#\nendif\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: all doc qemu chainboot clippy clean readelf objdump nm check\n\nall: $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Save the configuration as a file, so make understands if it changed.\n##------------------------------------------------------------------------------\n$(LAST_BUILD_CONFIG):\n\t@rm -f target/*.build_config\n\t@mkdir -p target\n\t@touch $(LAST_BUILD_CONFIG)\n\n##------------------------------------------------------------------------------\n## Compile the kernel ELF\n##------------------------------------------------------------------------------\n$(KERNEL_ELF): $(KERNEL_ELF_DEPS)\n\t$(call color_header, \"Compiling kernel ELF - $(BSP)\")\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(RUSTC_CMD)\n\n##------------------------------------------------------------------------------\n## Generate the stripped kernel binary\n##------------------------------------------------------------------------------\n$(KERNEL_BIN): $(KERNEL_ELF)\n\t$(call color_header, \"Generating stripped binary\")\n\t@$(OBJCOPY_CMD) $(KERNEL_ELF) $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Name\")\n\t@echo $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Size\")\n\t$(call disk_usage_KiB, $(KERNEL_BIN))\n\n##------------------------------------------------------------------------------\n## Generate the documentation\n##------------------------------------------------------------------------------\ndoc:\n\t$(call color_header, \"Generating docs\")\n\t@$(DOC_CMD) --document-private-items --open\n\n##------------------------------------------------------------------------------\n## Run the kernel in QEMU\n##------------------------------------------------------------------------------\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\nqemu:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\nqemu: $(KERNEL_BIN)\n\t$(call color_header, \"Launching QEMU\")\n\t@$(DOCKER_QEMU) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\nendif\n\n##------------------------------------------------------------------------------\n## Push the kernel to the real HW target\n##------------------------------------------------------------------------------\nchainboot: $(KERNEL_BIN)\n\t@$(DOCKER_CHAINBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run clippy\n##------------------------------------------------------------------------------\nclippy:\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(CLIPPY_CMD)\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(CLIPPY_CMD) --features test_build --tests \\\n                --manifest-path $(KERNEL_MANIFEST)\n\n##------------------------------------------------------------------------------\n## Clean\n##------------------------------------------------------------------------------\nclean:\n\trm -rf target $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run readelf\n##------------------------------------------------------------------------------\nreadelf: $(KERNEL_ELF)\n\t$(call color_header, \"Launching readelf\")\n\t@$(DOCKER_TOOLS) $(READELF_BINARY) --headers $(KERNEL_ELF)\n\n##------------------------------------------------------------------------------\n## Run objdump\n##------------------------------------------------------------------------------\nobjdump: $(KERNEL_ELF)\n\t$(call color_header, \"Launching objdump\")\n\t@$(DOCKER_TOOLS) $(OBJDUMP_BINARY) --disassemble --demangle \\\n                --section .text   \\\n                --section .rodata \\\n                $(KERNEL_ELF) | rustfilt\n\n##------------------------------------------------------------------------------\n## Run nm\n##------------------------------------------------------------------------------\nnm: $(KERNEL_ELF)\n\t$(call color_header, \"Launching nm\")\n\t@$(DOCKER_TOOLS) $(NM_BINARY) --demangle --print-size $(KERNEL_ELF) | sort | rustfilt\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Debugging targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: jtagboot openocd gdb gdb-opt0\n\n##------------------------------------------------------------------------------\n## Push the JTAG boot image to the real HW target\n##------------------------------------------------------------------------------\njtagboot:\n\t@$(DOCKER_JTAGBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(JTAG_BOOT_IMAGE)\n\n##------------------------------------------------------------------------------\n## Start OpenOCD session\n##------------------------------------------------------------------------------\nopenocd:\n\t$(call color_header, \"Launching OpenOCD\")\n\t@$(DOCKER_OPENOCD) openocd $(OPENOCD_ARG)\n\n##------------------------------------------------------------------------------\n## Start GDB session\n##------------------------------------------------------------------------------\ngdb: RUSTC_MISC_ARGS += -C debuginfo=2\ngdb-opt0: RUSTC_MISC_ARGS += -C debuginfo=2 -C opt-level=0\ngdb gdb-opt0: $(KERNEL_ELF)\n\t$(call color_header, \"Launching GDB\")\n\t@$(DOCKER_GDB) gdb-multiarch -q $(KERNEL_ELF)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Testing targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: test test_boot test_unit test_integration\n\ntest_unit test_integration: FEATURES += --features test_build\n\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\ntest_boot test_unit test_integration test:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\n##------------------------------------------------------------------------------\n## Run boot test\n##------------------------------------------------------------------------------\ntest_boot: $(KERNEL_BIN)\n\t$(call color_header, \"Boot test - $(BSP)\")\n\t@$(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Helpers for unit and integration test targets\n##------------------------------------------------------------------------------\ndefine KERNEL_TEST_RUNNER\n#!/usr/bin/env bash\n\n    # The cargo test runner seems to change into the crate under test's directory. Therefore, ensure\n    # this script executes from the root.\n    cd $(shell pwd)\n\n    TEST_ELF=$$(echo $$1 | sed -e 's/.*target/target/g')\n    TEST_BINARY=$$(echo $$1.img | sed -e 's/.*target/target/g')\n\n    $(OBJCOPY_CMD) $$TEST_ELF $$TEST_BINARY\n    $(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_TEST_ARGS) -kernel $$TEST_BINARY\nendef\n\nexport KERNEL_TEST_RUNNER\n\ndefine test_prepare\n    @mkdir -p target\n    @echo \"$$KERNEL_TEST_RUNNER\" > target/kernel_test_runner.sh\n    @chmod +x target/kernel_test_runner.sh\nendef\n\n##------------------------------------------------------------------------------\n## Run unit test(s)\n##------------------------------------------------------------------------------\ntest_unit:\n\t$(call color_header, \"Compiling unit test(s) - $(BSP)\")\n\t$(call test_prepare)\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(TEST_CMD) --lib\n\n##------------------------------------------------------------------------------\n## Run integration test(s)\n##------------------------------------------------------------------------------\ntest_integration:\n\t$(call color_header, \"Compiling integration test(s) - $(BSP)\")\n\t$(call test_prepare)\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(TEST_CMD) $(TEST_ARG)\n\ntest: test_boot test_unit test_integration\n\nendif\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/README.md",
    "content": "# Tutorial 13 - Exceptions Part 2: Peripheral IRQs\n\n## tl;dr\n\n- We write `device drivers` for the two interrupt controllers on the **Raspberry Pi 3** (`Broadcom`\n  custom controller) and **Pi 4** (`ARM` Generic Interrupt Controller v2, `GICv2`).\n- Modularity is ensured by interfacing everything through a trait named `IRQManager`.\n- Handling for our first peripheral IRQs is implemented: The `UART`'s receive IRQs.\n\n![Header](../doc/14_header.png)\n\n## Table of Contents\n\n- [Introduction](#introduction)\n- [Different Controllers: A Usecase for Abstraction](#different-controllers-a-usecase-for-abstraction)\n- [New Challenges: Reentrancy](#new-challenges-reentrancy)\n- [Implementation](#implementation)\n  * [The Kernel's Interfaces for Interrupt Handling](#the-kernels-interfaces-for-interrupt-handling)\n    + [Uniquely Identifying an IRQ](#uniquely-identifying-an-irq)\n      - [The BCM IRQ Number Scheme](#the-bcm-irq-number-scheme)\n      - [The GICv2 IRQ Number Scheme](#the-gicv2-irq-number-scheme)\n    + [Registering IRQ Handlers](#registering-irq-handlers)\n    + [Handling Pending IRQs](#handling-pending-irqs)\n  * [Reentrancy: What to protect?](#reentrancy-what-to-protect)\n  * [The Interrupt Controller Device Drivers](#the-interrupt-controller-device-drivers)\n    + [The BCM Driver (Pi 3)](#the-bcm-driver-pi-3)\n      - [Peripheral Controller Register Access](#peripheral-controller-register-access)\n      - [The IRQ Handler Table](#the-irq-handler-table)\n    + [The GICv2 Driver (Pi 4)](#the-gicv2-driver-pi-4)\n      - [GICC Details](#gicc-details)\n      - [GICD Details](#gicd-details)\n- [Test it](#test-it)\n- [Diff to previous](#diff-to-previous)\n\n## Introduction\n\nIn [tutorial 11], we laid the groundwork for exception handling from the processor architecture\nside. Handler stubs for the different exception types were set up, and a first glimpse at exception\nhandling was presented by causing a `synchronous` exception by means of a `page fault`.\n\n[tutorial 11]: ../11_exceptions_part1_groundwork\n\nIn this tutorial, we will add a first level of support for one of the three types of `asynchronous`\nexceptions that are defined for `AArch64`: `IRQs`. The overall goal for this tutorial is to get rid\nof the  busy-loop at the end of our current `kernel_main()` function, which actively polls the\n`UART` for newly received characters. Instead, we will let the processor idle and wait for the\n`UART`'s RX IRQs, which indicate that new characters were received. A respective `IRQ` service\nroutine, provided by the `UART` driver, will run in response to the `IRQ` and print the characters.\n\n## Different Controllers: A Usecase for Abstraction\n\nOne very exciting aspect of this tutorial is that the `Pi 3` and the `Pi 4` feature completely\ndifferent interrupt controllers. This is also a first in all of the tutorial series. Until now, both\nRaspberrys did not need differentiation with respect to their devices.\n\nThe `Pi 3` has a very simple custom controller made by Broadcom (BCM), the manufacturer of the Pi's\n`System-on-Chip`. The `Pi 4` features an implementation of `ARM`'s Generic Interrupt Controller\nversion 2 (`GICv2`). Since ARM's GIC controllers are the prevalent interrupt controllers in ARM\napplication procesors, it is very beneficial to finally have it on the Raspberry Pi. It will enable\npeople to learn about one of the most common building blocks in ARM-based embedded computing.\n\nThis also means that we can finally make full use of all the infrastructure for abstraction that we\nprepared already. We will design an `IRQManager` interface trait and implement it in both controller\ndrivers. The generic part of our `kernel` code will only be exposed to this trait (compare to the\ndiagram in the [tl;dr] section). This common idiom of *program to an interface, not an\nimplementation* enables a clean abstraction and makes the code modular and pluggable.\n\n[tl;dr]: #tldr\n\n## New Challenges: Reentrancy\n\nEnabling interrupts also poses new challenges with respect to protecting certain code sections in\nthe kernel from being [re-entered]. Please read the linked article for background on that topic.\n\n[re-entered]: https://en.wikipedia.org/wiki/Reentrancy_(computing)\n\nOur `kernel` is still running on a single core. For this reason, we are still using our `NullLock`\npseudo-locks for `Critical Sections` or `shared resources`, instead of real `Spinlocks`. Hence,\ninterrupt handling at this point in time does not put us at risk of running into one of those\ndreaded `deadlocks`, which is one of several side-effects that reentrancy can cause. For example, a\n`deadlock` because of interrupts can happen happen when the executing CPU core has locked a\n`Spinlock` at the beginning of a function, an IRQ happens, and the IRQ service routine is trying to\nexecute the same function. Since the lock is already locked, the core would spin forever waiting for\nit to be released.\n\nThere is no straight-forward way to tell if a function is `reentrantcy`-safe or not. It usually\nneeds careful manual checking to conclude. Even though it might be technically safe to `re-enter` a\nfunction, sometimes you don't want that to happen for functional reasons. For example, printing of a\nstring should not be interrupted by a an interrupt service routine that starts printing another\nstring, so that the output mixes. In the course of this tutorial, we will check and see where we\nwant to protect against `reentrancy`.\n\n## Implementation\n\nOkay, let's start. The following sections cover the the implementation in a top-down fashion,\nstarting with the trait that interfaces all the `kernel` components to each other.\n\n### The Kernel's Interfaces for Interrupt Handling\n\nFirst, we design the `IRQManager` trait that interrupt controller drivers must implement. The\nminimal set of functionality that we need for starters is:\n\n1. Registering an IRQ `handler` for a given IRQ `number`.\n2. Enabling an IRQ (from the controller side).\n3. Handling pending IRQs.\n4. Printing the list of registered IRQ handlers.\n\nThe trait is defined as `exception::asynchronous::interface::IRQManager`:\n\n```rust\npub trait IRQManager {\n    /// The IRQ number type depends on the implementation.\n    type IRQNumberType: Copy;\n\n    /// Register a handler.\n    fn register_handler(\n        &self,\n        irq_handler_descriptor: super::IRQHandlerDescriptor<Self::IRQNumberType>,\n    ) -> Result<(), &'static str>;\n\n    /// Enable an interrupt in the controller.\n    fn enable(&self, irq_number: &Self::IRQNumberType);\n\n    /// Handle pending interrupts.\n    ///\n    /// This function is called directly from the CPU's IRQ exception vector. On AArch64,\n    /// this means that the respective CPU core has disabled exception handling.\n    /// This function can therefore not be preempted and runs start to finish.\n    ///\n    /// Takes an IRQContext token to ensure it can only be called from IRQ context.\n    #[allow(clippy::trivially_copy_pass_by_ref)]\n    fn handle_pending_irqs<'irq_context>(\n        &'irq_context self,\n        ic: &super::IRQContext<'irq_context>,\n    );\n\n    /// Print list of registered handlers.\n    fn print_handler(&self) {}\n}\n```\n\n#### Uniquely Identifying an IRQ\n\nThe first member of the trait is the [associated type] `IRQNumberType`. The following explains why\nwe make it customizable for the implementor and do not define the type as a plain integer right\naway.\n\nInterrupts can generally be characterizied with the following properties:\n\n[associated type]: https://doc.rust-lang.org/book/ch19-03-advanced-traits.html#specifying-placeholder-types-in-trait-definitions-with-associated-types\n\n1. Software-generated vs hardware-generated.\n2. Private vs shared.\n\nDifferent interrupt controllers take different approaches at categorizing and numbering IRQs that\nhave one or the other property. Often times, this leads to situations where a plain integer does not\nsuffice to uniquely identify an IRQ, and makes it necessary to encode additional information in the\nused type. Letting the respective interrupt controller driver define `IRQManager::IRQNumberType`\nitself addresses this issue. The rest of the `BSP` must then conditionally use this type.\n\n##### The BCM IRQ Number Scheme\n\nThe `BCM` controller of the `Raspberry Pi 3`, for example, is composed of two functional parts: A\n**local** controller and a **peripheral** controller. The BCM's **local controller** handles all\n`private` IRQs, which means private SW-generated IRQs and IRQs of private HW devices. An example for\nthe latter would be the `ARMv8` timer. Each  CPU core has its own private instance of it. The BCM's\n**peripheral controller** handles all IRQs of `non-private` HW devices such as the `UART` (if those\nIRQs can be declared as `shared` according to our taxonomy above is a different discussion, because\nthe BCM controller allows these HW interrupts to be routed to _only one CPU core at a time_).\n\nThe IRQ numbers of the BCM **local controller** range from `0..11`. The numbers of the **peripheral\ncontroller** range from `0..63`. This demonstrates why a primitive integer type would not be\nsufficient to uniquely encode the IRQs, because their ranges overlap. In the driver for the `BCM`\ncontroller, we therefore define the associated type as follows:\n\n```rust\npub type LocalIRQ = BoundedUsize<{ InterruptController::MAX_LOCAL_IRQ_NUMBER }>;\npub type PeripheralIRQ = BoundedUsize<{ InterruptController::MAX_PERIPHERAL_IRQ_NUMBER }>;\n\n/// Used for the associated type of trait [`exception::asynchronous::interface::IRQManager`].\n#[derive(Copy, Clone)]\n#[allow(missing_docs)]\npub enum IRQNumber {\n    Local(LocalIRQ),\n    Peripheral(PeripheralIRQ),\n}\n```\n\nThe type `BoundedUsize` is a newtype around an `usize` that uses a [const generic] to ensure that\nthe value of the encapsulated IRQ number is in the allowed range (e.g. `0..MAX_LOCAL_IRQ_NUMBER` for\n`LocalIRQ`, with `MAX_LOCAL_IRQ_NUMBER == 11`).\n\n[const generic]: https://github.com/rust-lang/rfcs/blob/master/text/2000-const-generics.md\n\n##### The GICv2 IRQ Number Scheme\n\nThe `GICv2` in the `Raspberry Pi 4`, on the other hand, uses a different scheme. IRQ numbers `0..31`\nare for `private` IRQs. Those are further subdivided into `SW-generated` (SGIs, `0..15`) and\n`HW-generated` (PPIs, Private Peripheral Interrupts, `16..31`). Numbers `32..1019` are for `shared\nhardware-generated` interrupts (SPI, Shared Peripheral Interrupts).\n\nThere are no overlaps, so this scheme enables us to actually have a plain integer as a unique\nidentifier for the IRQs. We define the type as follows:\n\n```rust\n/// Used for the associated type of trait [`exception::asynchronous::interface::IRQManager`].\npub type IRQNumber = BoundedUsize<{ GICv2::MAX_IRQ_NUMBER }>;\n```\n\n#### Registering IRQ Handlers\n\nTo enable the controller driver to manage interrupt handling, it must know where to find respective\nhandlers, and it must know how to call them. For the latter, we define an `IRQHandler` trait in\n`exception::asynchronous` that must be implemented by any SW entity that wants to handle IRQs:\n\n```rust\n/// Implemented by types that handle IRQs.\npub trait IRQHandler {\n    /// Called when the corresponding interrupt is asserted.\n    fn handle(&self) -> Result<(), &'static str>;\n}\n```\n\nThe `PL011Uart` driver gets the honors for being our first driver to ever implement this trait. In\nthis tutorial, the `RX IRQ` and the `RX Timeout IRQ` will be configured. This means that the\n`PL011Uart` will assert it's interrupt line when one of following conditions is met:\n\n1. `RX IRQ`: The RX FIFO fill level is equal or more than the configured trigger level (which will be 1/8 of\n   the total FIFO size in our case).\n1. `RX Timeout IRQ`: The RX FIFO fill level is greater than zero, but less than the configured fill\n   level, and the characters have not been pulled for a certain amount of time. The exact time is\n   not documented in the respective `PL011Uart` datasheet. Usually, it is a single-digit multiple of\n   the time it takes to receive or transmit one character on the serial line.\n\n In the handler, our standard scheme of echoing any received characters back to the host is used:\n\n```rust\nimpl exception::asynchronous::interface::IRQHandler for PL011Uart {\n    fn handle(&self) -> Result<(), &'static str> {\n        self.inner.lock(|inner| {\n            let pending = inner.registers.MIS.extract();\n\n            // Clear all pending IRQs.\n            inner.registers.ICR.write(ICR::ALL::CLEAR);\n\n            // Check for any kind of RX interrupt.\n            if pending.matches_any(MIS::RXMIS::SET + MIS::RTMIS::SET) {\n                // Echo any received characters.\n                while let Some(c) = inner.read_char_converting(BlockingMode::NonBlocking) {\n                    inner.write_char(c)\n                }\n            }\n        });\n\n        Ok(())\n    }\n}\n```\n\nRegistering and enabling handlers in the interrupt controller is supposed to be done by the\nrespective drivers themselves. Therefore, we added a new function to the standard device driver\ntrait in `driver::interface::DeviceDriver` that must be implemented if IRQ handling is supported:\n\n```rust\n/// Called by the kernel to register and enable the device's IRQ handler.\n///\n/// Rust's type system will prevent a call to this function unless the calling instance\n/// itself has static lifetime.\nfn register_and_enable_irq_handler(\n    &'static self,\n    irq_number: &Self::IRQNumberType,\n) -> Result<(), &'static str> {\n    panic!(\n        \"Attempt to enable IRQ {} for device {}, but driver does not support this\",\n        irq_number,\n        self.compatible()\n    )\n}\n```\n\nHere is the implementation for the `PL011Uart`:\n\n```rust\nfn register_and_enable_irq_handler(\n    &'static self,\n    irq_number: &Self::IRQNumberType,\n) -> Result<(), &'static str> {\n    use exception::asynchronous::{irq_manager, IRQHandlerDescriptor};\n\n    let descriptor = IRQHandlerDescriptor::new(*irq_number, Self::COMPATIBLE, self);\n\n    irq_manager().register_handler(descriptor)?;\n    irq_manager().enable(irq_number);\n\n    Ok(())\n}\n```\n\nThe `exception::asynchronous::irq_manager()` function used here returns a reference to an\nimplementor of the `IRQManager` trait. Since the implementation is supposed to be done by the\nplatform's interrupt controller, this call will redirect to the `kernel`'s instance of either the\ndriver for the `BCM` controller (`Raspberry Pi 3`) or the driver for the `GICv2` (`Pi 4`). We will\nlook into the  implementation of the `register_handler()` function from the driver's perspective\nlater. The gist here is that the calls on `irq_manager()` will make the platform's interrupt\ncontroller aware that the `UART` driver (i) wants to handle its interrupt and (ii) which function it\nprovides to do so.\n\nAlso note how `irq_number` is supplied as a function argument and not hardcoded. The reason is that\nthe `UART` driver code is agnostic about the **IRQ numbers** that are associated to it. This is\nvendor-supplied information and as such typically part of the Board Support Package (`BSP`). It can\nvary from `BSP` to `BSP`, same like the board's memory map, which provides the `UART`'s MMIO\nregister addresses.\n\nWith all this in place, we can finally let drivers register and enable their IRQ handlers with the\ninterrupt controller, and unmask IRQ reception on the boot CPU core during the kernel init phase.\nThe global `driver_manager` takes care of this in the function `init_drivers_and_irqs()` (before\nthis tutorial, the function's name was `init_drivers()`), where this happens as the third and last\nstep of initializing all registered device drivers:\n\n```rust\npub unsafe fn init_drivers_and_irqs(&self) {\n    self.for_each_descriptor(|descriptor| {\n        // 1. Initialize driver.\n        if let Err(x) = descriptor.device_driver.init() {\n            // omitted for brevity\n        }\n\n        // 2. Call corresponding post init callback.\n        if let Some(callback) = &descriptor.post_init_callback {\n            // omitted for brevity\n        }\n    });\n\n    // 3. After all post-init callbacks were done, the interrupt controller should be\n    //    registered and functional. So let drivers register with it now.\n    self.for_each_descriptor(|descriptor| {\n        if let Some(irq_number) = &descriptor.irq_number {\n            if let Err(x) = descriptor\n                .device_driver\n                .register_and_enable_irq_handler(irq_number)\n            {\n                panic!(\n                    \"Error during driver interrupt handler registration: {}: {}\",\n                    descriptor.device_driver.compatible(),\n                    x\n                );\n            }\n        }\n    });\n}\n```\n\n\nIn `main.rs`, IRQs are unmasked right afterwards, after which point IRQ handling is live:\n\n```rust\n// Initialize all device drivers.\ndriver::driver_manager().init_drivers_and_irqs();\n\n// Unmask interrupts on the boot CPU core.\nexception::asynchronous::local_irq_unmask();\n```\n\n#### Handling Pending IRQs\n\nNow that interrupts can happen, the `kernel` needs a way of requesting the interrupt controller\ndriver to handle pending interrupts. Therefore, implementors of the trait `IRQManager` must also\nsupply the following function:\n\n```rust\nfn handle_pending_irqs<'irq_context>(\n    &'irq_context self,\n    ic: &super::IRQContext<'irq_context>,\n);\n```\n\nAn important aspect of this function signature is that we want to ensure that IRQ handling is only\npossible from IRQ context. Part of the reason is that this invariant allows us to make some implicit\nassumptions (which might depend on the target architecture, though). For example, as we have learned\nin [tutorial 11], in `AArch64`, _\"all kinds of exceptions are turned off upon taking an exception,\nso that by default, exception handlers can not get interrupted themselves\"_ (note that an IRQ is an\nexception). This is a useful property that relieves us from explicitly protecting IRQ handling from\nbeing interrupted itself. Another reason would be that calling IRQ handling functions from arbitrary\nexecution contexts just doesn't make a lot of sense.\n\n[tutorial 11]: ../11_exceptions_part1_groundwork/\n\nSo in order to ensure that this function is only being called from IRQ context, we borrow a\ntechnique that I first saw in the [Rust embedded WG]'s [bare-metal crate]. It uses Rust's type\nsystem to create a \"token\" that is only valid for the duration of the IRQ context. We create it\ndirectly at the top of the IRQ vector function in `_arch/aarch64/exception.rs`, and pass it on to\nthe the implementation of the trait's handling function:\n\n[Rust embedded WG]: https://github.com/rust-embedded/bare-metal\n[bare-metal crate]: https://github.com/rust-embedded/bare-metal/blob/master/src/lib.rs#L20\n\n```rust\n#[no_mangle]\nextern \"C\" fn current_elx_irq(_e: &mut ExceptionContext) {\n    let token = unsafe { &exception::asynchronous::IRQContext::new() };\n    exception::asynchronous::irq_manager().handle_pending_irqs(token);\n}\n```\n\nBy requiring the caller of the function `handle_pending_irqs()` to provide this `IRQContext` token,\nwe can prevent that the same function is accidentally being called from somewhere else. It is\nevident, though, that for this to work, it is the _user's responsibility_ to only ever create this\ntoken from within an IRQ context. If you want to circumvent this on purpose, you can do it.\n\n### Reentrancy: What to protect?\n\nNow that interrupt handling is live, we need to think about `reentrancy`. At [the beginning of this\ntutorial], we mused about the need to protect certain functions from being re-entered, and that it\nis not straight-forward to identify all the places that need protection.\n\n[the beginning of this tutorial]: #new-challenges-reentrancy\n\nIn this tutorial, we will keep this part short nonetheless by taking a better-safe-than-sorry\napproach. In the past, we already made efforts to prepare parts of `shared resources` (e.g. global\ndevice driver instances) to be protected against parallel access. We did so by wrapping them into\n`NullLocks`, which we will upgrade to real `Spinlocks` once we boot secondary CPU cores.\n\nWe can hook on that previous work and reason that anything that we wanted protected against parallel\naccess so far, we also want it protected against reentrancy now. Therefore, we upgrade all\n`NullLocks` to `IRQSafeNullocks`:\n\n```rust\nimpl<T> interface::Mutex for IRQSafeNullLock<T> {\n    type Data = T;\n\n    fn lock<R>(&self, f: impl FnOnce(&mut Self::Data) -> R) -> R {\n        // In a real lock, there would be code encapsulating this line that ensures that this\n        // mutable reference will ever only be given out once at a time.\n        let data = unsafe { &mut *self.data.get() };\n\n        // Execute the closure while IRQs are masked.\n        exception::asynchronous::exec_with_irq_masked(|| f(data))\n    }\n}\n```\n\nThe new part is that the call to `f(data)` is executed as a closure in\n`exception::asynchronous::exec_with_irq_masked()`. Inside that function, IRQs on the executing CPU\ncore are masked before the `f(data)` is being executed, and restored afterwards:\n\n```rust\n/// Executes the provided closure while IRQs are masked on the executing core.\n///\n/// While the function temporarily changes the HW state of the executing core, it restores it to the\n/// previous state before returning, so this is deemed safe.\n#[inline(always)]\npub fn exec_with_irq_masked<T>(f: impl FnOnce() -> T) -> T {\n    let saved = local_irq_mask_save();\n    let ret = f();\n    local_irq_restore(saved);\n\n    ret\n}\n```\n\nThe helper functions used here are defined in `src/_arch/aarch64/exception/asynchronous.rs`.\n\n### The Interrupt Controller Device Drivers\n\nThe previous sections explained how the `kernel` uses the `IRQManager` trait. Now, let's have a look\nat the driver-side of it in the Raspberry Pi `BSP`. We start with the Broadcom interrupt controller\nfeatured in the `Pi 3`.\n\n#### The BCM Driver (Pi 3)\n\nAs mentioned earlier, the `BCM` driver consists of two subcomponents, a **local** and a\n**peripheral** controller. The local controller owns a bunch of configuration registers, among\nothers, the `routing` configuration for peripheral IRQs such as those from the `UART`. Peripheral\nIRQs can be routed to _one core only_. In our case, we leave the default unchanged, which means\neverything is routed to the boot CPU core. The image below depicts the `struct diagram` of the\ndriver implementation.\n\n![BCM Driver](../doc/14_BCM_driver.png)\n\nWe have a top-level driver, which implements the `IRQManager` trait. _Only the top-level driver_ is\nexposed to the rest of the `kernel`. The top-level itself has two members, representing the local\nand the peripheral controller, respectively, which implement the `IRQManager` trait as well. This\ndesign allows for easy forwarding of function calls from the top-level driver to one of the\nsubcontrollers.\n\nFor this tutorial, we leave out implementation of the local controller, because we will only be\nconcerned with the peripheral  `UART` IRQ.\n\n##### Peripheral Controller Register Access\n\nWhen writing a device driver for a kernel with exception handling and multi-core support, it is\nalways important to analyze what parts of the driver will need protection against reentrancy (we\ntalked about this earlier in this tutorial) and/or parallel execution of other driver parts. If a\ndriver function needs to follow a vendor-defined sequence of multiple register operations that\ninclude `write operations`, this is usually a good hint that protection might be needed. But that is\nonly one of many examples.\n\nFor the driver implementation in this tutorial, we are following a simple rule: Register read access\nis deemed always safe. Write access is guarded by an `IRQSafeNullLock`, which means that we are safe\nagainst `reentrancy` issues, and also in the future when the kernel will be running on multiple\ncores, we can easily upgrade to a real spinlock, which serializes register write operations from\ndifferent CPU cores.\n\nIn fact, for this tutorial, we probably would not have needed any protection yet, because all the\ndriver does is read from the `PENDING_*` registers for the `handle_pending_irqs()` implementation,\nand writing to the `ENABLE_*` registers for the `enable()` implementation. However, the chosen\narchitecture will have us set up for future extensions, when more complex register manipulation\nsequences might be needed.\n\nSince nothing complex is happening in the implementation, it is not covered in detail here. Please\nrefer to [the source of the **peripheral** controller] to check it out.\n\n[the source of the **peripheral** controller]: kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs\n\n##### The IRQ Handler Table\n\nCalls to `register_handler()` result in the driver inserting the provided handler reference in a\nspecific table (the handler reference is a member of `IRQDescriptor`):\n\n```rust\ntype HandlerTable = [Option<exception::asynchronous::IRQHandlerDescriptor<PeripheralIRQ>>;\n    PeripheralIRQ::MAX_INCLUSIVE + 1];\n```\n\nOne of the requirements for safe operation of the `kernel` is that those handlers are not\nregistered, removed or exchanged in the middle of an IRQ handling situation. This, again, is a\nmulti-core scenario where one core might look up a handler entry while another core is modifying the\nsame in parallel.\n\nWhile we want to allow drivers to take the decision of registering or not registering a handler at\nruntime, there is no need to allow it for the _whole_ runtime of the kernel. It is fine to restrict\nthis option to the kernel `init phase`, at which only a single boot core runs and IRQs are masked.\n\nWe introduce the so called `InitStateLock` for cases like that. From an API-perspective, it is a\nspecial variant of a `Read/Write exclusion synchronization primitive`. RWLocks in the Rust standard\nlibrary [are characterized] as allowing _\"a number of readers or at most one writer at any point in\ntime\"_. For the `InitStateLock`, we only implement the `read()` and `write()` functions:\n\n[are characterized]: https://doc.rust-lang.org/std/sync/struct.RwLock.html\n\n```rust\nimpl<T> interface::ReadWriteEx for InitStateLock<T> {\n    type Data = T;\n\n    fn write<R>(&self, f: impl FnOnce(&mut Self::Data) -> R) -> R {\n        assert!(\n            state::state_manager().is_init(),\n            \"InitStateLock::write called after kernel init phase\"\n        );\n        assert!(\n            !exception::asynchronous::is_local_irq_masked(),\n            \"InitStateLock::write called with IRQs unmasked\"\n        );\n\n        let data = unsafe { &mut *self.data.get() };\n\n        f(data)\n    }\n\n    fn read<R>(&self, f: impl FnOnce(&Self::Data) -> R) -> R {\n        let data = unsafe { &*self.data.get() };\n\n        f(data)\n    }\n}\n```\n\nThe `write()` function is guarded by two `assertions`. One ensures that IRQs are masked, the other\nchecks the `state::state_manager()` if the kernel is still in the init phase. The `State Manager` is\nnew since this tutorial, and implemented in `src/state.rs`. It provides atomic state transition and\nreporting functions that are called when the kernel enters a new phase. In the current kernel, the\nonly call is happening before the transition from `kernel_init()` to `kernel_main()`:\n\n```rust\n// Announce conclusion of the kernel_init() phase.\nstate::state_manager().transition_to_single_core_main();\n```\n\nP.S.: Since the use case for the `InitStateLock` also applies to a few other places in the kernel\n(for example, registering the system-wide console during early boot), `InitStateLock`s have been\nincorporated in those other places as well.\n\n#### The GICv2 Driver (Pi 4)\n\nAs we learned earlier, the ARM `GICv2` in the `Raspberry Pi 4` features a continuous interrupt\nnumber range:\n- IRQ numbers `0..31` represent IRQs that are private (aka local) to the respective processor core.\n- IRQ numbers `32..1019` are for shared IRQs.\n\nThe `GIC` has a so-called `Distributor`, the `GICD`, and a `CPU Interface`, the `GICC`. The `GICD`,\namong other things, is used to enable IRQs and route them to one or more CPU cores. The `GICC` is\nused by CPU cores to check which IRQs are pending, and to acknowledge them once they were handled.\nThere is one dedicated `GICC` for _each CPU core_.\n\nOne neat thing about the `GICv2` is that any MMIO registers that are associated to core-private IRQs\nare `banked`. That means that different CPU cores can assert the same MMIO address, but they will\nend up accessing a core-private copy of the referenced register. This makes it very comfortable to\nprogram the `GIC`, because this hardware design ensures that each core only ever gets access to its\nown resources. Preventing one core to accidentally or willfully fiddle with the IRQ state of another\ncore must therefore not be enforced in software.\n\nIn summary, this means that any registers in the `GICD` that deal with the core-private IRQ range\nare banked. Since there is one `GICC` per CPU core, the whole thing is banked. This allows us to\ndesign the following `struct diagram` for our driver implementation:\n\n![GICv2 Driver](../doc/14_GICv2_driver.png)\n\nThe top-level struct is composed of a `GICD`, a `GICC` and a `HandlerTable`. The latter is\nimplemented identically as in the `Pi 3`.\n\n##### GICC Details\n\nSince the `GICC` is banked wholly, the top-level driver can directly forward any requests to it,\nwithout worrying about concurrency issues for now. Note that this only works as long as the `GICC`\nimplementation is only accessing the banked `GICC` registers, and does not save any state in member\nvariables that are stored in `DRAM`. The two main duties of the `GICC` struct are to read the `IAR`\n(Interrupt Acknowledge) register, which returns the number of the highest-priority pending IRQ, and\nwriting to the `EOIR` (End Of Interrupt) register, which tells the hardware that handling of an\ninterrupt is now concluded.\n\n##### GICD Details\n\nThe `GICD` hardware block differentiates between `shared` and `banked` registers. As with the\n`GICC`, we don't have to protect the banked registers against concurrent access. The shared\nregisters are wrapped into an `IRQSafeNullLock` again. The important parts of the `GICD` for this\ntutorial are the `ITARGETSR[256]` and `ISENABLER[32]` register arrays.\n\nEach `ITARGETSR` is subdivided into four _bytes_. Each byte represents one IRQ, and stores a bitmask\nthat encodes all the `GICCs` to which the respective IRQ is forwarded. For example,\n`ITARGETSR[0].byte0` would represent IRQ number 0, and `ITARGETSR[0].byte3` IRQ number 3. In the\n`ISENABLER`, each _bit_ represents an IRQ. For example, `ISENABLER[0].bit3` is IRQ number 3.\n\nIn summary, this means that `ITARGETSR[0..7]` and `ISENABLER[0]` represent the first 32 IRQs (the\nbanked ones), and as such, we split the register block into `shared` and `banked` parts accordingly\nin `gicd.rs`:\n\n```rust\nregister_structs! {\n    #[allow(non_snake_case)]\n    SharedRegisterBlock {\n        (0x000 => CTLR: ReadWrite<u32, CTLR::Register>),\n        (0x004 => TYPER: ReadOnly<u32, TYPER::Register>),\n        (0x008 => _reserved1),\n        (0x104 => ISENABLER: [ReadWrite<u32>; 31]),\n        (0x180 => _reserved2),\n        (0x820 => ITARGETSR: [ReadWrite<u32, ITARGETSR::Register>; 248]),\n        (0xC00 => @END),\n    }\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    BankedRegisterBlock {\n        (0x000 => _reserved1),\n        (0x100 => ISENABLER: ReadWrite<u32>),\n        (0x104 => _reserved2),\n        (0x800 => ITARGETSR: [ReadOnly<u32, ITARGETSR::Register>; 8]),\n        (0x820 => @END),\n    }\n}\n```\n\nAs with the implementation of the BCM interrupt controller driver, we won't cover the remaining\nparts in exhaustive detail. For that, please refer to [this folder] folder which contains all the\nsources.\n\n[this folder]: kernel/src/bsp/device_driver/arm\n\n## Test it\n\nWhen you load the kernel, any keystroke results in echoing back the character by way of IRQ\nhandling. There is no more polling done at the end of `kernel_main()`, just waiting for events such\nas IRQs:\n\n```rust\nfn kernel_main() -> ! {\n\n    // omitted for brevity\n\n    info!(\"Echoing input now\");\n    cpu::wait_forever();\n}\n```\n\nRaspberry Pi 3:\n\n```console\n$ make chainboot\n[...]\nMinipush 1.0\n\n[MP] ⏳ Waiting for /dev/ttyUSB0\n[MP] ✅ Serial connected\n[MP] 🔌 Please power the target now\n\n __  __ _      _ _                 _\n|  \\/  (_)_ _ (_) |   ___  __ _ __| |\n| |\\/| | | ' \\| | |__/ _ \\/ _` / _` |\n|_|  |_|_|_||_|_|____\\___/\\__,_\\__,_|\n\n           Raspberry Pi 3\n\n[ML] Requesting binary\n[MP] ⏩ Pushing 66 KiB =========================================🦀 100% 0 KiB/s Time: 00:00:00\n[ML] Loaded! Executing the payload now\n\n[    0.822492] mingo version 0.13.0\n[    0.822700] Booting on: Raspberry Pi 3\n[    0.823155] MMU online. Special regions:\n[    0.823632]       0x00080000 - 0x0008ffff |  64 KiB | C   RO PX  | Kernel code and RO data\n[    0.824650]       0x3f000000 - 0x4000ffff |  17 MiB | Dev RW PXN | Device MMIO\n[    0.825539] Current privilege level: EL1\n[    0.826015] Exception handling state:\n[    0.826459]       Debug:  Masked\n[    0.826849]       SError: Masked\n[    0.827239]       IRQ:    Unmasked\n[    0.827651]       FIQ:    Masked\n[    0.828041] Architectural timer resolution: 52 ns\n[    0.828615] Drivers loaded:\n[    0.828951]       1. BCM PL011 UART\n[    0.829373]       2. BCM GPIO\n[    0.829731]       3. BCM Interrupt Controller\n[    0.830262] Registered IRQ handlers:\n[    0.830695]       Peripheral handler:\n[    0.831141]              57. BCM PL011 UART\n[    0.831649] Echoing input now\n```\n\nRaspberry Pi 4:\n\n```console\n$ BSP=rpi4 make chainboot\n[...]\nMinipush 1.0\n\n[MP] ⏳ Waiting for /dev/ttyUSB0\n[MP] ✅ Serial connected\n[MP] 🔌 Please power the target now\n\n __  __ _      _ _                 _\n|  \\/  (_)_ _ (_) |   ___  __ _ __| |\n| |\\/| | | ' \\| | |__/ _ \\/ _` / _` |\n|_|  |_|_|_||_|_|____\\___/\\__,_\\__,_|\n\n           Raspberry Pi 4\n\n[ML] Requesting binary\n[MP] ⏩ Pushing 73 KiB =========================================🦀 100% 0 KiB/s Time: 00:00:00\n[ML] Loaded! Executing the payload now\n\n[    0.886853] mingo version 0.13.0\n[    0.886886] Booting on: Raspberry Pi 4\n[    0.887341] MMU online. Special regions:\n[    0.887818]       0x00080000 - 0x0008ffff |  64 KiB | C   RO PX  | Kernel code and RO data\n[    0.888836]       0xfe000000 - 0xff84ffff |  25 MiB | Dev RW PXN | Device MMIO\n[    0.889725] Current privilege level: EL1\n[    0.890201] Exception handling state:\n[    0.890645]       Debug:  Masked\n[    0.891035]       SError: Masked\n[    0.891425]       IRQ:    Unmasked\n[    0.891837]       FIQ:    Masked\n[    0.892227] Architectural timer resolution: 18 ns\n[    0.892801] Drivers loaded:\n[    0.893137]       1. BCM PL011 UART\n[    0.893560]       2. BCM GPIO\n[    0.893917]       3. GICv2 (ARM Generic Interrupt Controller v2)\n[    0.894654] Registered IRQ handlers:\n[    0.895087]       Peripheral handler:\n[    0.895534]             153. BCM PL011 UART\n[    0.896042] Echoing input now\n```\n\n## Diff to previous\n```diff\n\ndiff -uNr 12_integrated_testing/kernel/Cargo.toml 13_exceptions_part2_peripheral_IRQs/kernel/Cargo.toml\n--- 12_integrated_testing/kernel/Cargo.toml\n+++ 13_exceptions_part2_peripheral_IRQs/kernel/Cargo.toml\n@@ -1,6 +1,6 @@\n [package]\n name = \"mingo\"\n-version = \"0.12.0\"\n+version = \"0.13.0\"\n authors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\n edition = \"2021\"\n\n\ndiff -uNr 12_integrated_testing/kernel/src/_arch/aarch64/cpu/smp.rs 13_exceptions_part2_peripheral_IRQs/kernel/src/_arch/aarch64/cpu/smp.rs\n--- 12_integrated_testing/kernel/src/_arch/aarch64/cpu/smp.rs\n+++ 13_exceptions_part2_peripheral_IRQs/kernel/src/_arch/aarch64/cpu/smp.rs\n@@ -0,0 +1,30 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Architectural symmetric multiprocessing.\n+//!\n+//! # Orientation\n+//!\n+//! Since arch modules are imported into generic modules using the path attribute, the path of this\n+//! file is:\n+//!\n+//! crate::cpu::smp::arch_smp\n+\n+use aarch64_cpu::registers::*;\n+use tock_registers::interfaces::Readable;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Return the executing core's id.\n+#[inline(always)]\n+pub fn core_id<T>() -> T\n+where\n+    T: From<u8>,\n+{\n+    const CORE_MASK: u64 = 0b11;\n+\n+    T::from((MPIDR_EL1.get() & CORE_MASK) as u8)\n+}\n\ndiff -uNr 12_integrated_testing/kernel/src/_arch/aarch64/exception/asynchronous.rs 13_exceptions_part2_peripheral_IRQs/kernel/src/_arch/aarch64/exception/asynchronous.rs\n--- 12_integrated_testing/kernel/src/_arch/aarch64/exception/asynchronous.rs\n+++ 13_exceptions_part2_peripheral_IRQs/kernel/src/_arch/aarch64/exception/asynchronous.rs\n@@ -12,12 +12,17 @@\n //! crate::exception::asynchronous::arch_asynchronous\n\n use aarch64_cpu::registers::*;\n-use tock_registers::interfaces::Readable;\n+use core::arch::asm;\n+use tock_registers::interfaces::{Readable, Writeable};\n\n //--------------------------------------------------------------------------------------------------\n // Private Definitions\n //--------------------------------------------------------------------------------------------------\n\n+mod daif_bits {\n+    pub const IRQ: u8 = 0b0010;\n+}\n+\n trait DaifField {\n     fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register>;\n }\n@@ -66,6 +71,60 @@\n // Public Code\n //--------------------------------------------------------------------------------------------------\n\n+/// Returns whether IRQs are masked on the executing core.\n+pub fn is_local_irq_masked() -> bool {\n+    !is_masked::<IRQ>()\n+}\n+\n+/// Unmask IRQs on the executing core.\n+///\n+/// It is not needed to place an explicit instruction synchronization barrier after the `msr`.\n+/// Quoting the Architecture Reference Manual for ARMv8-A, section C5.1.3:\n+///\n+/// \"Writes to PSTATE.{PAN, D, A, I, F} occur in program order without the need for additional\n+/// synchronization.\"\n+#[inline(always)]\n+pub fn local_irq_unmask() {\n+    unsafe {\n+        asm!(\n+            \"msr DAIFClr, {arg}\",\n+            arg = const daif_bits::IRQ,\n+            options(nomem, nostack, preserves_flags)\n+        );\n+    }\n+}\n+\n+/// Mask IRQs on the executing core.\n+#[inline(always)]\n+pub fn local_irq_mask() {\n+    unsafe {\n+        asm!(\n+            \"msr DAIFSet, {arg}\",\n+            arg = const daif_bits::IRQ,\n+            options(nomem, nostack, preserves_flags)\n+        );\n+    }\n+}\n+\n+/// Mask IRQs on the executing core and return the previously saved interrupt mask bits (DAIF).\n+#[inline(always)]\n+pub fn local_irq_mask_save() -> u64 {\n+    let saved = DAIF.get();\n+    local_irq_mask();\n+\n+    saved\n+}\n+\n+/// Restore the interrupt mask bits (DAIF) using the callee's argument.\n+///\n+/// # Invariant\n+///\n+/// - No sanity checks on the input.\n+#[inline(always)]\n+pub fn local_irq_restore(saved: u64) {\n+    DAIF.set(saved);\n+}\n+\n /// Print the AArch64 exceptions status.\n #[rustfmt::skip]\n pub fn print_state() {\n\ndiff -uNr 12_integrated_testing/kernel/src/_arch/aarch64/exception.rs 13_exceptions_part2_peripheral_IRQs/kernel/src/_arch/aarch64/exception.rs\n--- 12_integrated_testing/kernel/src/_arch/aarch64/exception.rs\n+++ 13_exceptions_part2_peripheral_IRQs/kernel/src/_arch/aarch64/exception.rs\n@@ -11,6 +11,7 @@\n //!\n //! crate::exception::arch_exception\n\n+use crate::exception;\n use aarch64_cpu::{asm::barrier, registers::*};\n use core::{arch::global_asm, cell::UnsafeCell, fmt};\n use tock_registers::{\n@@ -102,8 +103,9 @@\n }\n\n #[no_mangle]\n-extern \"C\" fn current_elx_irq(e: &mut ExceptionContext) {\n-    default_exception_handler(e);\n+extern \"C\" fn current_elx_irq(_e: &mut ExceptionContext) {\n+    let token = unsafe { &exception::asynchronous::IRQContext::new() };\n+    exception::asynchronous::irq_manager().handle_pending_irqs(token);\n }\n\n #[no_mangle]\n\ndiff -uNr 12_integrated_testing/kernel/src/bsp/device_driver/arm/gicv2/gicc.rs 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/arm/gicv2/gicc.rs\n--- 12_integrated_testing/kernel/src/bsp/device_driver/arm/gicv2/gicc.rs\n+++ 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/arm/gicv2/gicc.rs\n@@ -0,0 +1,141 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! GICC Driver - GIC CPU interface.\n+\n+use crate::{bsp::device_driver::common::MMIODerefWrapper, exception};\n+use tock_registers::{\n+    interfaces::{Readable, Writeable},\n+    register_bitfields, register_structs,\n+    registers::ReadWrite,\n+};\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+register_bitfields! {\n+    u32,\n+\n+    /// CPU Interface Control Register\n+    CTLR [\n+        Enable OFFSET(0) NUMBITS(1) []\n+    ],\n+\n+    /// Interrupt Priority Mask Register\n+    PMR [\n+        Priority OFFSET(0) NUMBITS(8) []\n+    ],\n+\n+    /// Interrupt Acknowledge Register\n+    IAR [\n+        InterruptID OFFSET(0) NUMBITS(10) []\n+    ],\n+\n+    /// End of Interrupt Register\n+    EOIR [\n+        EOIINTID OFFSET(0) NUMBITS(10) []\n+    ]\n+}\n+\n+register_structs! {\n+    #[allow(non_snake_case)]\n+    pub RegisterBlock {\n+        (0x000 => CTLR: ReadWrite<u32, CTLR::Register>),\n+        (0x004 => PMR: ReadWrite<u32, PMR::Register>),\n+        (0x008 => _reserved1),\n+        (0x00C => IAR: ReadWrite<u32, IAR::Register>),\n+        (0x010 => EOIR: ReadWrite<u32, EOIR::Register>),\n+        (0x014  => @END),\n+    }\n+}\n+\n+/// Abstraction for the associated MMIO registers.\n+type Registers = MMIODerefWrapper<RegisterBlock>;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Representation of the GIC CPU interface.\n+pub struct GICC {\n+    registers: Registers,\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+impl GICC {\n+    /// Create an instance.\n+    ///\n+    /// # Safety\n+    ///\n+    /// - The user must ensure to provide a correct MMIO start address.\n+    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n+        Self {\n+            registers: Registers::new(mmio_start_addr),\n+        }\n+    }\n+\n+    /// Accept interrupts of any priority.\n+    ///\n+    /// Quoting the GICv2 Architecture Specification:\n+    ///\n+    ///   \"Writing 255 to the GICC_PMR always sets it to the largest supported priority field\n+    ///    value.\"\n+    ///\n+    /// # Safety\n+    ///\n+    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n+    ///   of `&mut self`.\n+    pub fn priority_accept_all(&self) {\n+        self.registers.PMR.write(PMR::Priority.val(255)); // Comment in arch spec.\n+    }\n+\n+    /// Enable the interface - start accepting IRQs.\n+    ///\n+    /// # Safety\n+    ///\n+    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n+    ///   of `&mut self`.\n+    pub fn enable(&self) {\n+        self.registers.CTLR.write(CTLR::Enable::SET);\n+    }\n+\n+    /// Extract the number of the highest-priority pending IRQ.\n+    ///\n+    /// Can only be called from IRQ context, which is ensured by taking an `IRQContext` token.\n+    ///\n+    /// # Safety\n+    ///\n+    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n+    ///   of `&mut self`.\n+    #[allow(clippy::trivially_copy_pass_by_ref)]\n+    pub fn pending_irq_number<'irq_context>(\n+        &self,\n+        _ic: &exception::asynchronous::IRQContext<'irq_context>,\n+    ) -> usize {\n+        self.registers.IAR.read(IAR::InterruptID) as usize\n+    }\n+\n+    /// Complete handling of the currently active IRQ.\n+    ///\n+    /// Can only be called from IRQ context, which is ensured by taking an `IRQContext` token.\n+    ///\n+    /// To be called after `pending_irq_number()`.\n+    ///\n+    /// # Safety\n+    ///\n+    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n+    ///   of `&mut self`.\n+    #[allow(clippy::trivially_copy_pass_by_ref)]\n+    pub fn mark_comleted<'irq_context>(\n+        &self,\n+        irq_number: u32,\n+        _ic: &exception::asynchronous::IRQContext<'irq_context>,\n+    ) {\n+        self.registers.EOIR.write(EOIR::EOIINTID.val(irq_number));\n+    }\n+}\n\ndiff -uNr 12_integrated_testing/kernel/src/bsp/device_driver/arm/gicv2/gicd.rs 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/arm/gicv2/gicd.rs\n--- 12_integrated_testing/kernel/src/bsp/device_driver/arm/gicv2/gicd.rs\n+++ 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/arm/gicv2/gicd.rs\n@@ -0,0 +1,199 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! GICD Driver - GIC Distributor.\n+//!\n+//! # Glossary\n+//!   - SPI - Shared Peripheral Interrupt.\n+\n+use crate::{\n+    bsp::device_driver::common::MMIODerefWrapper, state, synchronization,\n+    synchronization::IRQSafeNullLock,\n+};\n+use tock_registers::{\n+    interfaces::{Readable, Writeable},\n+    register_bitfields, register_structs,\n+    registers::{ReadOnly, ReadWrite},\n+};\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+register_bitfields! {\n+    u32,\n+\n+    /// Distributor Control Register\n+    CTLR [\n+        Enable OFFSET(0) NUMBITS(1) []\n+    ],\n+\n+    /// Interrupt Controller Type Register\n+    TYPER [\n+        ITLinesNumber OFFSET(0)  NUMBITS(5) []\n+    ],\n+\n+    /// Interrupt Processor Targets Registers\n+    ITARGETSR [\n+        Offset3 OFFSET(24) NUMBITS(8) [],\n+        Offset2 OFFSET(16) NUMBITS(8) [],\n+        Offset1 OFFSET(8)  NUMBITS(8) [],\n+        Offset0 OFFSET(0)  NUMBITS(8) []\n+    ]\n+}\n+\n+register_structs! {\n+    #[allow(non_snake_case)]\n+    SharedRegisterBlock {\n+        (0x000 => CTLR: ReadWrite<u32, CTLR::Register>),\n+        (0x004 => TYPER: ReadOnly<u32, TYPER::Register>),\n+        (0x008 => _reserved1),\n+        (0x104 => ISENABLER: [ReadWrite<u32>; 31]),\n+        (0x180 => _reserved2),\n+        (0x820 => ITARGETSR: [ReadWrite<u32, ITARGETSR::Register>; 248]),\n+        (0xC00 => @END),\n+    }\n+}\n+\n+register_structs! {\n+    #[allow(non_snake_case)]\n+    BankedRegisterBlock {\n+        (0x000 => _reserved1),\n+        (0x100 => ISENABLER: ReadWrite<u32>),\n+        (0x104 => _reserved2),\n+        (0x800 => ITARGETSR: [ReadOnly<u32, ITARGETSR::Register>; 8]),\n+        (0x820 => @END),\n+    }\n+}\n+\n+/// Abstraction for the non-banked parts of the associated MMIO registers.\n+type SharedRegisters = MMIODerefWrapper<SharedRegisterBlock>;\n+\n+/// Abstraction for the banked parts of the associated MMIO registers.\n+type BankedRegisters = MMIODerefWrapper<BankedRegisterBlock>;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Representation of the GIC Distributor.\n+pub struct GICD {\n+    /// Access to shared registers is guarded with a lock.\n+    shared_registers: IRQSafeNullLock<SharedRegisters>,\n+\n+    /// Access to banked registers is unguarded.\n+    banked_registers: BankedRegisters,\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Code\n+//--------------------------------------------------------------------------------------------------\n+\n+impl SharedRegisters {\n+    /// Return the number of IRQs that this HW implements.\n+    #[inline(always)]\n+    fn num_irqs(&mut self) -> usize {\n+        // Query number of implemented IRQs.\n+        //\n+        // Refer to GICv2 Architecture Specification, Section 4.3.2.\n+        ((self.TYPER.read(TYPER::ITLinesNumber) as usize) + 1) * 32\n+    }\n+\n+    /// Return a slice of the implemented ITARGETSR.\n+    #[inline(always)]\n+    fn implemented_itargets_slice(&mut self) -> &[ReadWrite<u32, ITARGETSR::Register>] {\n+        assert!(self.num_irqs() >= 36);\n+\n+        // Calculate the max index of the shared ITARGETSR array.\n+        //\n+        // The first 32 IRQs are private, so not included in `shared_registers`. Each ITARGETS\n+        // register has four entries, so shift right by two. Subtract one because we start\n+        // counting at zero.\n+        let spi_itargetsr_max_index = ((self.num_irqs() - 32) >> 2) - 1;\n+\n+        // Rust automatically inserts slice range sanity check, i.e. max >= min.\n+        &self.ITARGETSR[0..spi_itargetsr_max_index]\n+    }\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+use synchronization::interface::Mutex;\n+\n+impl GICD {\n+    /// Create an instance.\n+    ///\n+    /// # Safety\n+    ///\n+    /// - The user must ensure to provide a correct MMIO start address.\n+    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n+        Self {\n+            shared_registers: IRQSafeNullLock::new(SharedRegisters::new(mmio_start_addr)),\n+            banked_registers: BankedRegisters::new(mmio_start_addr),\n+        }\n+    }\n+\n+    /// Use a banked ITARGETSR to retrieve the executing core's GIC target mask.\n+    ///\n+    /// Quoting the GICv2 Architecture Specification:\n+    ///\n+    ///   \"GICD_ITARGETSR0 to GICD_ITARGETSR7 are read-only, and each field returns a value that\n+    ///    corresponds only to the processor reading the register.\"\n+    fn local_gic_target_mask(&self) -> u32 {\n+        self.banked_registers.ITARGETSR[0].read(ITARGETSR::Offset0)\n+    }\n+\n+    /// Route all SPIs to the boot core and enable the distributor.\n+    pub fn boot_core_init(&self) {\n+        assert!(\n+            state::state_manager().is_init(),\n+            \"Only allowed during kernel init phase\"\n+        );\n+\n+        // Target all SPIs to the boot core only.\n+        let mask = self.local_gic_target_mask();\n+\n+        self.shared_registers.lock(|regs| {\n+            for i in regs.implemented_itargets_slice().iter() {\n+                i.write(\n+                    ITARGETSR::Offset3.val(mask)\n+                        + ITARGETSR::Offset2.val(mask)\n+                        + ITARGETSR::Offset1.val(mask)\n+                        + ITARGETSR::Offset0.val(mask),\n+                );\n+            }\n+\n+            regs.CTLR.write(CTLR::Enable::SET);\n+        });\n+    }\n+\n+    /// Enable an interrupt.\n+    pub fn enable(&self, irq_num: &super::IRQNumber) {\n+        let irq_num = irq_num.get();\n+\n+        // Each bit in the u32 enable register corresponds to one IRQ number. Shift right by 5\n+        // (division by 32) and arrive at the index for the respective ISENABLER[i].\n+        let enable_reg_index = irq_num >> 5;\n+        let enable_bit: u32 = 1u32 << (irq_num modulo 32);\n+\n+        // Check if we are handling a private or shared IRQ.\n+        match irq_num {\n+            // Private.\n+            0..=31 => {\n+                let enable_reg = &self.banked_registers.ISENABLER;\n+                enable_reg.set(enable_reg.get() | enable_bit);\n+            }\n+            // Shared.\n+            _ => {\n+                let enable_reg_index_shared = enable_reg_index - 1;\n+\n+                self.shared_registers.lock(|regs| {\n+                    let enable_reg = &regs.ISENABLER[enable_reg_index_shared];\n+                    enable_reg.set(enable_reg.get() | enable_bit);\n+                });\n+            }\n+        }\n+    }\n+}\n\ndiff -uNr 12_integrated_testing/kernel/src/bsp/device_driver/arm/gicv2.rs 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/arm/gicv2.rs\n--- 12_integrated_testing/kernel/src/bsp/device_driver/arm/gicv2.rs\n+++ 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/arm/gicv2.rs\n@@ -0,0 +1,226 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! GICv2 Driver - ARM Generic Interrupt Controller v2.\n+//!\n+//! The following is a collection of excerpts with useful information from\n+//!   - `Programmer's Guide for ARMv8-A`\n+//!   - `ARM Generic Interrupt Controller Architecture Specification`\n+//!\n+//! # Programmer's Guide - 10.6.1 Configuration\n+//!\n+//! The GIC is accessed as a memory-mapped peripheral.\n+//!\n+//! All cores can access the common Distributor, but the CPU interface is banked, that is, each core\n+//! uses the same address to access its own private CPU interface.\n+//!\n+//! It is not possible for a core to access the CPU interface of another core.\n+//!\n+//! # Architecture Specification - 10.6.2 Initialization\n+//!\n+//! Both the Distributor and the CPU interfaces are disabled at reset. The GIC must be initialized\n+//! after reset before it can deliver interrupts to the core.\n+//!\n+//! In the Distributor, software must configure the priority, target, security and enable individual\n+//! interrupts. The Distributor must subsequently be enabled through its control register\n+//! (GICD_CTLR). For each CPU interface, software must program the priority mask and preemption\n+//! settings.\n+//!\n+//! Each CPU interface block itself must be enabled through its control register (GICD_CTLR). This\n+//! prepares the GIC to deliver interrupts to the core.\n+//!\n+//! Before interrupts are expected in the core, software prepares the core to take interrupts by\n+//! setting a valid interrupt vector in the vector table, and clearing interrupt mask bits in\n+//! PSTATE, and setting the routing controls.\n+//!\n+//! The entire interrupt mechanism in the system can be disabled by disabling the Distributor.\n+//! Interrupt delivery to an individual core can be disabled by disabling its CPU interface.\n+//! Individual interrupts can also be disabled (or enabled) in the distributor.\n+//!\n+//! For an interrupt to reach the core, the individual interrupt, Distributor and CPU interface must\n+//! all be enabled. The interrupt also needs to be of sufficient priority, that is, higher than the\n+//! core's priority mask.\n+//!\n+//! # Architecture Specification - 1.4.2 Interrupt types\n+//!\n+//! - Peripheral interrupt\n+//!     - Private Peripheral Interrupt (PPI)\n+//!         - This is a peripheral interrupt that is specific to a single processor.\n+//!     - Shared Peripheral Interrupt (SPI)\n+//!         - This is a peripheral interrupt that the Distributor can route to any of a specified\n+//!           combination of processors.\n+//!\n+//! - Software-generated interrupt (SGI)\n+//!     - This is an interrupt generated by software writing to a GICD_SGIR register in the GIC. The\n+//!       system uses SGIs for interprocessor communication.\n+//!     - An SGI has edge-triggered properties. The software triggering of the interrupt is\n+//!       equivalent to the edge transition of the interrupt request signal.\n+//!     - When an SGI occurs in a multiprocessor implementation, the CPUID field in the Interrupt\n+//!       Acknowledge Register, GICC_IAR, or the Aliased Interrupt Acknowledge Register, GICC_AIAR,\n+//!       identifies the processor that requested the interrupt.\n+//!\n+//! # Architecture Specification - 2.2.1 Interrupt IDs\n+//!\n+//! Interrupts from sources are identified using ID numbers. Each CPU interface can see up to 1020\n+//! interrupts. The banking of SPIs and PPIs increases the total number of interrupts supported by\n+//! the Distributor.\n+//!\n+//! The GIC assigns interrupt ID numbers ID0-ID1019 as follows:\n+//!   - Interrupt numbers 32..1019 are used for SPIs.\n+//!   - Interrupt numbers 0..31 are used for interrupts that are private to a CPU interface. These\n+//!     interrupts are banked in the Distributor.\n+//!       - A banked interrupt is one where the Distributor can have multiple interrupts with the\n+//!         same ID. A banked interrupt is identified uniquely by its ID number and its associated\n+//!         CPU interface number. Of the banked interrupt IDs:\n+//!           - 00..15 SGIs\n+//!           - 16..31 PPIs\n+\n+mod gicc;\n+mod gicd;\n+\n+use crate::{\n+    bsp::{self, device_driver::common::BoundedUsize},\n+    cpu, driver, exception, synchronization,\n+    synchronization::InitStateLock,\n+};\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+type HandlerTable = [Option<exception::asynchronous::IRQHandlerDescriptor<IRQNumber>>;\n+    IRQNumber::MAX_INCLUSIVE + 1];\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Used for the associated type of trait [`exception::asynchronous::interface::IRQManager`].\n+pub type IRQNumber = BoundedUsize<{ GICv2::MAX_IRQ_NUMBER }>;\n+\n+/// Representation of the GIC.\n+pub struct GICv2 {\n+    /// The Distributor.\n+    gicd: gicd::GICD,\n+\n+    /// The CPU Interface.\n+    gicc: gicc::GICC,\n+\n+    /// Stores registered IRQ handlers. Writable only during kernel init. RO afterwards.\n+    handler_table: InitStateLock<HandlerTable>,\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+impl GICv2 {\n+    const MAX_IRQ_NUMBER: usize = 300; // Normally 1019, but keep it lower to save some space.\n+\n+    pub const COMPATIBLE: &'static str = \"GICv2 (ARM Generic Interrupt Controller v2)\";\n+\n+    /// Create an instance.\n+    ///\n+    /// # Safety\n+    ///\n+    /// - The user must ensure to provide a correct MMIO start address.\n+    pub const unsafe fn new(gicd_mmio_start_addr: usize, gicc_mmio_start_addr: usize) -> Self {\n+        Self {\n+            gicd: gicd::GICD::new(gicd_mmio_start_addr),\n+            gicc: gicc::GICC::new(gicc_mmio_start_addr),\n+            handler_table: InitStateLock::new([None; IRQNumber::MAX_INCLUSIVE + 1]),\n+        }\n+    }\n+}\n+\n+//------------------------------------------------------------------------------\n+// OS Interface Code\n+//------------------------------------------------------------------------------\n+use synchronization::interface::ReadWriteEx;\n+\n+impl driver::interface::DeviceDriver for GICv2 {\n+    type IRQNumberType = IRQNumber;\n+\n+    fn compatible(&self) -> &'static str {\n+        Self::COMPATIBLE\n+    }\n+\n+    unsafe fn init(&self) -> Result<(), &'static str> {\n+        if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() {\n+            self.gicd.boot_core_init();\n+        }\n+\n+        self.gicc.priority_accept_all();\n+        self.gicc.enable();\n+\n+        Ok(())\n+    }\n+}\n+\n+impl exception::asynchronous::interface::IRQManager for GICv2 {\n+    type IRQNumberType = IRQNumber;\n+\n+    fn register_handler(\n+        &self,\n+        irq_handler_descriptor: exception::asynchronous::IRQHandlerDescriptor<Self::IRQNumberType>,\n+    ) -> Result<(), &'static str> {\n+        self.handler_table.write(|table| {\n+            let irq_number = irq_handler_descriptor.number().get();\n+\n+            if table[irq_number].is_some() {\n+                return Err(\"IRQ handler already registered\");\n+            }\n+\n+            table[irq_number] = Some(irq_handler_descriptor);\n+\n+            Ok(())\n+        })\n+    }\n+\n+    fn enable(&self, irq_number: &Self::IRQNumberType) {\n+        self.gicd.enable(irq_number);\n+    }\n+\n+    fn handle_pending_irqs<'irq_context>(\n+        &'irq_context self,\n+        ic: &exception::asynchronous::IRQContext<'irq_context>,\n+    ) {\n+        // Extract the highest priority pending IRQ number from the Interrupt Acknowledge Register\n+        // (IAR).\n+        let irq_number = self.gicc.pending_irq_number(ic);\n+\n+        // Guard against spurious interrupts.\n+        if irq_number > GICv2::MAX_IRQ_NUMBER {\n+            return;\n+        }\n+\n+        // Call the IRQ handler. Panic if there is none.\n+        self.handler_table.read(|table| {\n+            match table[irq_number] {\n+                None => panic!(\"No handler registered for IRQ {}\", irq_number),\n+                Some(descriptor) => {\n+                    // Call the IRQ handler. Panics on failure.\n+                    descriptor.handler().handle().expect(\"Error handling IRQ\");\n+                }\n+            }\n+        });\n+\n+        // Signal completion of handling.\n+        self.gicc.mark_comleted(irq_number as u32, ic);\n+    }\n+\n+    fn print_handler(&self) {\n+        use crate::info;\n+\n+        info!(\"      Peripheral handler:\");\n+\n+        self.handler_table.read(|table| {\n+            for (i, opt) in table.iter().skip(32).enumerate() {\n+                if let Some(handler) = opt {\n+                    info!(\"            {: >3}. {}\", i + 32, handler.name());\n+                }\n+            }\n+        });\n+    }\n+}\n\ndiff -uNr 12_integrated_testing/kernel/src/bsp/device_driver/arm.rs 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/arm.rs\n--- 12_integrated_testing/kernel/src/bsp/device_driver/arm.rs\n+++ 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/arm.rs\n@@ -0,0 +1,9 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! ARM driver top level.\n+\n+pub mod gicv2;\n+\n+pub use gicv2::*;\n\ndiff -uNr 12_integrated_testing/kernel/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs\n--- 12_integrated_testing/kernel/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs\n+++ 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs\n@@ -5,8 +5,8 @@\n //! GPIO Driver.\n\n use crate::{\n-    bsp::device_driver::common::MMIODerefWrapper, driver, synchronization,\n-    synchronization::NullLock,\n+    bsp::device_driver::common::MMIODerefWrapper, driver, exception::asynchronous::IRQNumber,\n+    synchronization, synchronization::IRQSafeNullLock,\n };\n use tock_registers::{\n     interfaces::{ReadWriteable, Writeable},\n@@ -118,7 +118,7 @@\n\n /// Representation of the GPIO HW.\n pub struct GPIO {\n-    inner: NullLock<GPIOInner>,\n+    inner: IRQSafeNullLock<GPIOInner>,\n }\n\n //--------------------------------------------------------------------------------------------------\n@@ -200,7 +200,7 @@\n     /// - The user must ensure to provide a correct MMIO start address.\n     pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n         Self {\n-            inner: NullLock::new(GPIOInner::new(mmio_start_addr)),\n+            inner: IRQSafeNullLock::new(GPIOInner::new(mmio_start_addr)),\n         }\n     }\n\n@@ -216,6 +216,8 @@\n use synchronization::interface::Mutex;\n\n impl driver::interface::DeviceDriver for GPIO {\n+    type IRQNumberType = IRQNumber;\n+\n     fn compatible(&self) -> &'static str {\n         Self::COMPATIBLE\n     }\n\ndiff -uNr 12_integrated_testing/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs\n--- 12_integrated_testing/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs\n+++ 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs\n@@ -0,0 +1,170 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Peripheral Interrupt Controller Driver.\n+//!\n+//! # Resources\n+//!\n+//! - <https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf>\n+\n+use super::{PendingIRQs, PeripheralIRQ};\n+use crate::{\n+    bsp::device_driver::common::MMIODerefWrapper,\n+    exception, synchronization,\n+    synchronization::{IRQSafeNullLock, InitStateLock},\n+};\n+use tock_registers::{\n+    interfaces::{Readable, Writeable},\n+    register_structs,\n+    registers::{ReadOnly, WriteOnly},\n+};\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+register_structs! {\n+    #[allow(non_snake_case)]\n+    WORegisterBlock {\n+        (0x00 => _reserved1),\n+        (0x10 => ENABLE_1: WriteOnly<u32>),\n+        (0x14 => ENABLE_2: WriteOnly<u32>),\n+        (0x18 => @END),\n+    }\n+}\n+\n+register_structs! {\n+    #[allow(non_snake_case)]\n+    RORegisterBlock {\n+        (0x00 => _reserved1),\n+        (0x04 => PENDING_1: ReadOnly<u32>),\n+        (0x08 => PENDING_2: ReadOnly<u32>),\n+        (0x0c => @END),\n+    }\n+}\n+\n+/// Abstraction for the WriteOnly parts of the associated MMIO registers.\n+type WriteOnlyRegisters = MMIODerefWrapper<WORegisterBlock>;\n+\n+/// Abstraction for the ReadOnly parts of the associated MMIO registers.\n+type ReadOnlyRegisters = MMIODerefWrapper<RORegisterBlock>;\n+\n+type HandlerTable = [Option<exception::asynchronous::IRQHandlerDescriptor<PeripheralIRQ>>;\n+    PeripheralIRQ::MAX_INCLUSIVE + 1];\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Representation of the peripheral interrupt controller.\n+pub struct PeripheralIC {\n+    /// Access to write registers is guarded with a lock.\n+    wo_registers: IRQSafeNullLock<WriteOnlyRegisters>,\n+\n+    /// Register read access is unguarded.\n+    ro_registers: ReadOnlyRegisters,\n+\n+    /// Stores registered IRQ handlers. Writable only during kernel init. RO afterwards.\n+    handler_table: InitStateLock<HandlerTable>,\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+impl PeripheralIC {\n+    /// Create an instance.\n+    ///\n+    /// # Safety\n+    ///\n+    /// - The user must ensure to provide a correct MMIO start address.\n+    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n+        Self {\n+            wo_registers: IRQSafeNullLock::new(WriteOnlyRegisters::new(mmio_start_addr)),\n+            ro_registers: ReadOnlyRegisters::new(mmio_start_addr),\n+            handler_table: InitStateLock::new([None; PeripheralIRQ::MAX_INCLUSIVE + 1]),\n+        }\n+    }\n+\n+    /// Query the list of pending IRQs.\n+    fn pending_irqs(&self) -> PendingIRQs {\n+        let pending_mask: u64 = (u64::from(self.ro_registers.PENDING_2.get()) << 32)\n+            | u64::from(self.ro_registers.PENDING_1.get());\n+\n+        PendingIRQs::new(pending_mask)\n+    }\n+}\n+\n+//------------------------------------------------------------------------------\n+// OS Interface Code\n+//------------------------------------------------------------------------------\n+use synchronization::interface::{Mutex, ReadWriteEx};\n+\n+impl exception::asynchronous::interface::IRQManager for PeripheralIC {\n+    type IRQNumberType = PeripheralIRQ;\n+\n+    fn register_handler(\n+        &self,\n+        irq_handler_descriptor: exception::asynchronous::IRQHandlerDescriptor<Self::IRQNumberType>,\n+    ) -> Result<(), &'static str> {\n+        self.handler_table.write(|table| {\n+            let irq_number = irq_handler_descriptor.number().get();\n+\n+            if table[irq_number].is_some() {\n+                return Err(\"IRQ handler already registered\");\n+            }\n+\n+            table[irq_number] = Some(irq_handler_descriptor);\n+\n+            Ok(())\n+        })\n+    }\n+\n+    fn enable(&self, irq: &Self::IRQNumberType) {\n+        self.wo_registers.lock(|regs| {\n+            let enable_reg = if irq.get() <= 31 {\n+                &regs.ENABLE_1\n+            } else {\n+                &regs.ENABLE_2\n+            };\n+\n+            let enable_bit: u32 = 1 << (irq.get() modulo 32);\n+\n+            // Writing a 1 to a bit will set the corresponding IRQ enable bit. All other IRQ enable\n+            // bits are unaffected. So we don't need read and OR'ing here.\n+            enable_reg.set(enable_bit);\n+        });\n+    }\n+\n+    fn handle_pending_irqs<'irq_context>(\n+        &'irq_context self,\n+        _ic: &exception::asynchronous::IRQContext<'irq_context>,\n+    ) {\n+        self.handler_table.read(|table| {\n+            for irq_number in self.pending_irqs() {\n+                match table[irq_number] {\n+                    None => panic!(\"No handler registered for IRQ {}\", irq_number),\n+                    Some(descriptor) => {\n+                        // Call the IRQ handler. Panics on failure.\n+                        descriptor.handler().handle().expect(\"Error handling IRQ\");\n+                    }\n+                }\n+            }\n+        })\n+    }\n+\n+    fn print_handler(&self) {\n+        use crate::info;\n+\n+        info!(\"      Peripheral handler:\");\n+\n+        self.handler_table.read(|table| {\n+            for (i, opt) in table.iter().enumerate() {\n+                if let Some(handler) = opt {\n+                    info!(\"            {: >3}. {}\", i, handler.name());\n+                }\n+            }\n+        });\n+    }\n+}\n\ndiff -uNr 12_integrated_testing/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs\n--- 12_integrated_testing/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs\n+++ 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs\n@@ -0,0 +1,152 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Interrupt Controller Driver.\n+\n+mod peripheral_ic;\n+\n+use crate::{\n+    bsp::device_driver::common::BoundedUsize,\n+    driver,\n+    exception::{self, asynchronous::IRQHandlerDescriptor},\n+};\n+use core::fmt;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Wrapper struct for a bitmask indicating pending IRQ numbers.\n+struct PendingIRQs {\n+    bitmask: u64,\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+pub type LocalIRQ = BoundedUsize<{ InterruptController::MAX_LOCAL_IRQ_NUMBER }>;\n+pub type PeripheralIRQ = BoundedUsize<{ InterruptController::MAX_PERIPHERAL_IRQ_NUMBER }>;\n+\n+/// Used for the associated type of trait [`exception::asynchronous::interface::IRQManager`].\n+#[derive(Copy, Clone)]\n+#[allow(missing_docs)]\n+pub enum IRQNumber {\n+    Local(LocalIRQ),\n+    Peripheral(PeripheralIRQ),\n+}\n+\n+/// Representation of the Interrupt Controller.\n+pub struct InterruptController {\n+    periph: peripheral_ic::PeripheralIC,\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Code\n+//--------------------------------------------------------------------------------------------------\n+\n+impl PendingIRQs {\n+    pub fn new(bitmask: u64) -> Self {\n+        Self { bitmask }\n+    }\n+}\n+\n+impl Iterator for PendingIRQs {\n+    type Item = usize;\n+\n+    fn next(&mut self) -> Option<Self::Item> {\n+        if self.bitmask == 0 {\n+            return None;\n+        }\n+\n+        let next = self.bitmask.trailing_zeros() as usize;\n+        self.bitmask &= self.bitmask.wrapping_sub(1);\n+        Some(next)\n+    }\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+impl fmt::Display for IRQNumber {\n+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n+        match self {\n+            Self::Local(number) => write!(f, \"Local({})\", number),\n+            Self::Peripheral(number) => write!(f, \"Peripheral({})\", number),\n+        }\n+    }\n+}\n+\n+impl InterruptController {\n+    // Restrict to 3 for now. This makes future code for local_ic.rs more straight forward.\n+    const MAX_LOCAL_IRQ_NUMBER: usize = 3;\n+    const MAX_PERIPHERAL_IRQ_NUMBER: usize = 63;\n+\n+    pub const COMPATIBLE: &'static str = \"BCM Interrupt Controller\";\n+\n+    /// Create an instance.\n+    ///\n+    /// # Safety\n+    ///\n+    /// - The user must ensure to provide a correct MMIO start address.\n+    pub const unsafe fn new(periph_mmio_start_addr: usize) -> Self {\n+        Self {\n+            periph: peripheral_ic::PeripheralIC::new(periph_mmio_start_addr),\n+        }\n+    }\n+}\n+\n+//------------------------------------------------------------------------------\n+// OS Interface Code\n+//------------------------------------------------------------------------------\n+\n+impl driver::interface::DeviceDriver for InterruptController {\n+    type IRQNumberType = IRQNumber;\n+\n+    fn compatible(&self) -> &'static str {\n+        Self::COMPATIBLE\n+    }\n+}\n+\n+impl exception::asynchronous::interface::IRQManager for InterruptController {\n+    type IRQNumberType = IRQNumber;\n+\n+    fn register_handler(\n+        &self,\n+        irq_handler_descriptor: exception::asynchronous::IRQHandlerDescriptor<Self::IRQNumberType>,\n+    ) -> Result<(), &'static str> {\n+        match irq_handler_descriptor.number() {\n+            IRQNumber::Local(_) => unimplemented!(\"Local IRQ controller not implemented.\"),\n+            IRQNumber::Peripheral(pirq) => {\n+                let periph_descriptor = IRQHandlerDescriptor::new(\n+                    pirq,\n+                    irq_handler_descriptor.name(),\n+                    irq_handler_descriptor.handler(),\n+                );\n+\n+                self.periph.register_handler(periph_descriptor)\n+            }\n+        }\n+    }\n+\n+    fn enable(&self, irq: &Self::IRQNumberType) {\n+        match irq {\n+            IRQNumber::Local(_) => unimplemented!(\"Local IRQ controller not implemented.\"),\n+            IRQNumber::Peripheral(pirq) => self.periph.enable(pirq),\n+        }\n+    }\n+\n+    fn handle_pending_irqs<'irq_context>(\n+        &'irq_context self,\n+        ic: &exception::asynchronous::IRQContext<'irq_context>,\n+    ) {\n+        // It can only be a peripheral IRQ pending because enable() does not support local IRQs yet.\n+        self.periph.handle_pending_irqs(ic)\n+    }\n+\n+    fn print_handler(&self) {\n+        self.periph.print_handler();\n+    }\n+}\n\ndiff -uNr 12_integrated_testing/kernel/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs\n--- 12_integrated_testing/kernel/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs\n+++ 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs\n@@ -10,8 +10,11 @@\n //! - <https://developer.arm.com/documentation/ddi0183/latest>\n\n use crate::{\n-    bsp::device_driver::common::MMIODerefWrapper, console, cpu, driver, synchronization,\n-    synchronization::NullLock,\n+    bsp::device_driver::common::MMIODerefWrapper,\n+    console, cpu, driver,\n+    exception::{self, asynchronous::IRQNumber},\n+    synchronization,\n+    synchronization::IRQSafeNullLock,\n };\n use core::fmt;\n use tock_registers::{\n@@ -134,6 +137,52 @@\n         ]\n     ],\n\n+    /// Interrupt FIFO Level Select Register.\n+    IFLS [\n+        /// Receive interrupt FIFO level select. The trigger points for the receive interrupt are as\n+        /// follows.\n+        RXIFLSEL OFFSET(3) NUMBITS(5) [\n+            OneEigth = 0b000,\n+            OneQuarter = 0b001,\n+            OneHalf = 0b010,\n+            ThreeQuarters = 0b011,\n+            SevenEights = 0b100\n+        ]\n+    ],\n+\n+    /// Interrupt Mask Set/Clear Register.\n+    IMSC [\n+        /// Receive timeout interrupt mask. A read returns the current mask for the UARTRTINTR\n+        /// interrupt.\n+        ///\n+        /// - On a write of 1, the mask of the UARTRTINTR interrupt is set.\n+        /// - A write of 0 clears the mask.\n+        RTIM OFFSET(6) NUMBITS(1) [\n+            Disabled = 0,\n+            Enabled = 1\n+        ],\n+\n+        /// Receive interrupt mask. A read returns the current mask for the UARTRXINTR interrupt.\n+        ///\n+        /// - On a write of 1, the mask of the UARTRXINTR interrupt is set.\n+        /// - A write of 0 clears the mask.\n+        RXIM OFFSET(4) NUMBITS(1) [\n+            Disabled = 0,\n+            Enabled = 1\n+        ]\n+    ],\n+\n+    /// Masked Interrupt Status Register.\n+    MIS [\n+        /// Receive timeout masked interrupt status. Returns the masked interrupt state of the\n+        /// UARTRTINTR interrupt.\n+        RTMIS OFFSET(6) NUMBITS(1) [],\n+\n+        /// Receive masked interrupt status. Returns the masked interrupt state of the UARTRXINTR\n+        /// interrupt.\n+        RXMIS OFFSET(4) NUMBITS(1) []\n+    ],\n+\n     /// Interrupt Clear Register.\n     ICR [\n         /// Meta field for all pending interrupts.\n@@ -152,7 +201,10 @@\n         (0x28 => FBRD: WriteOnly<u32, FBRD::Register>),\n         (0x2c => LCR_H: WriteOnly<u32, LCR_H::Register>),\n         (0x30 => CR: WriteOnly<u32, CR::Register>),\n-        (0x34 => _reserved3),\n+        (0x34 => IFLS: ReadWrite<u32, IFLS::Register>),\n+        (0x38 => IMSC: ReadWrite<u32, IMSC::Register>),\n+        (0x3C => _reserved3),\n+        (0x40 => MIS: ReadOnly<u32, MIS::Register>),\n         (0x44 => ICR: WriteOnly<u32, ICR::Register>),\n         (0x48 => @END),\n     }\n@@ -179,7 +231,7 @@\n\n /// Representation of the UART.\n pub struct PL011Uart {\n-    inner: NullLock<PL011UartInner>,\n+    inner: IRQSafeNullLock<PL011UartInner>,\n }\n\n //--------------------------------------------------------------------------------------------------\n@@ -247,6 +299,14 @@\n             .LCR_H\n             .write(LCR_H::WLEN::EightBit + LCR_H::FEN::FifosEnabled);\n\n+        // Set RX FIFO fill level at 1/8.\n+        self.registers.IFLS.write(IFLS::RXIFLSEL::OneEigth);\n+\n+        // Enable RX IRQ + RX timeout IRQ.\n+        self.registers\n+            .IMSC\n+            .write(IMSC::RXIM::Enabled + IMSC::RTIM::Enabled);\n+\n         // Turn the UART on.\n         self.registers\n             .CR\n@@ -337,7 +397,7 @@\n     /// - The user must ensure to provide a correct MMIO start address.\n     pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n         Self {\n-            inner: NullLock::new(PL011UartInner::new(mmio_start_addr)),\n+            inner: IRQSafeNullLock::new(PL011UartInner::new(mmio_start_addr)),\n         }\n     }\n }\n@@ -348,6 +408,8 @@\n use synchronization::interface::Mutex;\n\n impl driver::interface::DeviceDriver for PL011Uart {\n+    type IRQNumberType = IRQNumber;\n+\n     fn compatible(&self) -> &'static str {\n         Self::COMPATIBLE\n     }\n@@ -357,6 +419,20 @@\n\n         Ok(())\n     }\n+\n+    fn register_and_enable_irq_handler(\n+        &'static self,\n+        irq_number: &Self::IRQNumberType,\n+    ) -> Result<(), &'static str> {\n+        use exception::asynchronous::{irq_manager, IRQHandlerDescriptor};\n+\n+        let descriptor = IRQHandlerDescriptor::new(*irq_number, Self::COMPATIBLE, self);\n+\n+        irq_manager().register_handler(descriptor)?;\n+        irq_manager().enable(irq_number);\n+\n+        Ok(())\n+    }\n }\n\n impl console::interface::Write for PL011Uart {\n@@ -405,3 +481,24 @@\n }\n\n impl console::interface::All for PL011Uart {}\n+\n+impl exception::asynchronous::interface::IRQHandler for PL011Uart {\n+    fn handle(&self) -> Result<(), &'static str> {\n+        self.inner.lock(|inner| {\n+            let pending = inner.registers.MIS.extract();\n+\n+            // Clear all pending IRQs.\n+            inner.registers.ICR.write(ICR::ALL::CLEAR);\n+\n+            // Check for any kind of RX interrupt.\n+            if pending.matches_any(MIS::RXMIS::SET + MIS::RTMIS::SET) {\n+                // Echo any received characters.\n+                while let Some(c) = inner.read_char_converting(BlockingMode::NonBlocking) {\n+                    inner.write_char(c)\n+                }\n+            }\n+        });\n+\n+        Ok(())\n+    }\n+}\n\ndiff -uNr 12_integrated_testing/kernel/src/bsp/device_driver/bcm.rs 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/bcm.rs\n--- 12_integrated_testing/kernel/src/bsp/device_driver/bcm.rs\n+++ 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/bcm.rs\n@@ -5,7 +5,11 @@\n //! BCM driver top level.\n\n mod bcm2xxx_gpio;\n+#[cfg(feature = \"bsp_rpi3\")]\n+mod bcm2xxx_interrupt_controller;\n mod bcm2xxx_pl011_uart;\n\n pub use bcm2xxx_gpio::*;\n+#[cfg(feature = \"bsp_rpi3\")]\n+pub use bcm2xxx_interrupt_controller::*;\n pub use bcm2xxx_pl011_uart::*;\n\ndiff -uNr 12_integrated_testing/kernel/src/bsp/device_driver/common.rs 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/common.rs\n--- 12_integrated_testing/kernel/src/bsp/device_driver/common.rs\n+++ 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/common.rs\n@@ -4,7 +4,7 @@\n\n //! Common device driver code.\n\n-use core::{marker::PhantomData, ops};\n+use core::{fmt, marker::PhantomData, ops};\n\n //--------------------------------------------------------------------------------------------------\n // Public Definitions\n@@ -15,6 +15,10 @@\n     phantom: PhantomData<fn() -> T>,\n }\n\n+/// A wrapper type for usize with integrated range bound check.\n+#[derive(Copy, Clone)]\n+pub struct BoundedUsize<const MAX_INCLUSIVE: usize>(usize);\n+\n //--------------------------------------------------------------------------------------------------\n // Public Code\n //--------------------------------------------------------------------------------------------------\n@@ -36,3 +40,25 @@\n         unsafe { &*(self.start_addr as *const _) }\n     }\n }\n+\n+impl<const MAX_INCLUSIVE: usize> BoundedUsize<{ MAX_INCLUSIVE }> {\n+    pub const MAX_INCLUSIVE: usize = MAX_INCLUSIVE;\n+\n+    /// Creates a new instance if number <= MAX_INCLUSIVE.\n+    pub const fn new(number: usize) -> Self {\n+        assert!(number <= MAX_INCLUSIVE);\n+\n+        Self(number)\n+    }\n+\n+    /// Return the wrapped number.\n+    pub const fn get(self) -> usize {\n+        self.0\n+    }\n+}\n+\n+impl<const MAX_INCLUSIVE: usize> fmt::Display for BoundedUsize<{ MAX_INCLUSIVE }> {\n+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n+        write!(f, \"{}\", self.0)\n+    }\n+}\n\ndiff -uNr 12_integrated_testing/kernel/src/bsp/device_driver.rs 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver.rs\n--- 12_integrated_testing/kernel/src/bsp/device_driver.rs\n+++ 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver.rs\n@@ -4,9 +4,13 @@\n\n //! Device driver.\n\n+#[cfg(feature = \"bsp_rpi4\")]\n+mod arm;\n #[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\n mod bcm;\n mod common;\n\n+#[cfg(feature = \"bsp_rpi4\")]\n+pub use arm::*;\n #[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\n pub use bcm::*;\n\ndiff -uNr 12_integrated_testing/kernel/src/bsp/raspberrypi/driver.rs 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/raspberrypi/driver.rs\n--- 12_integrated_testing/kernel/src/bsp/raspberrypi/driver.rs\n+++ 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/raspberrypi/driver.rs\n@@ -4,8 +4,12 @@\n\n //! BSP driver support.\n\n-use super::memory::map::mmio;\n-use crate::{bsp::device_driver, console, driver as generic_driver};\n+use super::{exception, memory::map::mmio};\n+use crate::{\n+    bsp::device_driver,\n+    console, driver as generic_driver,\n+    exception::{self as generic_exception},\n+};\n use core::sync::atomic::{AtomicBool, Ordering};\n\n //--------------------------------------------------------------------------------------------------\n@@ -16,6 +20,14 @@\n     unsafe { device_driver::PL011Uart::new(mmio::PL011_UART_START) };\n static GPIO: device_driver::GPIO = unsafe { device_driver::GPIO::new(mmio::GPIO_START) };\n\n+#[cfg(feature = \"bsp_rpi3\")]\n+static INTERRUPT_CONTROLLER: device_driver::InterruptController =\n+    unsafe { device_driver::InterruptController::new(mmio::PERIPHERAL_IC_START) };\n+\n+#[cfg(feature = \"bsp_rpi4\")]\n+static INTERRUPT_CONTROLLER: device_driver::GICv2 =\n+    unsafe { device_driver::GICv2::new(mmio::GICD_START, mmio::GICC_START) };\n+\n //--------------------------------------------------------------------------------------------------\n // Private Code\n //--------------------------------------------------------------------------------------------------\n@@ -33,21 +45,43 @@\n     Ok(())\n }\n\n+/// This must be called only after successful init of the interrupt controller driver.\n+fn post_init_interrupt_controller() -> Result<(), &'static str> {\n+    generic_exception::asynchronous::register_irq_manager(&INTERRUPT_CONTROLLER);\n+\n+    Ok(())\n+}\n+\n fn driver_uart() -> Result<(), &'static str> {\n-    let uart_descriptor =\n-        generic_driver::DeviceDriverDescriptor::new(&PL011_UART, Some(post_init_uart));\n+    let uart_descriptor = generic_driver::DeviceDriverDescriptor::new(\n+        &PL011_UART,\n+        Some(post_init_uart),\n+        Some(exception::asynchronous::irq_map::PL011_UART),\n+    );\n     generic_driver::driver_manager().register_driver(uart_descriptor);\n\n     Ok(())\n }\n\n fn driver_gpio() -> Result<(), &'static str> {\n-    let gpio_descriptor = generic_driver::DeviceDriverDescriptor::new(&GPIO, Some(post_init_gpio));\n+    let gpio_descriptor =\n+        generic_driver::DeviceDriverDescriptor::new(&GPIO, Some(post_init_gpio), None);\n     generic_driver::driver_manager().register_driver(gpio_descriptor);\n\n     Ok(())\n }\n\n+fn driver_interrupt_controller() -> Result<(), &'static str> {\n+    let interrupt_controller_descriptor = generic_driver::DeviceDriverDescriptor::new(\n+        &INTERRUPT_CONTROLLER,\n+        Some(post_init_interrupt_controller),\n+        None,\n+    );\n+    generic_driver::driver_manager().register_driver(interrupt_controller_descriptor);\n+\n+    Ok(())\n+}\n+\n //--------------------------------------------------------------------------------------------------\n // Public Code\n //--------------------------------------------------------------------------------------------------\n@@ -65,6 +99,7 @@\n\n     driver_uart()?;\n     driver_gpio()?;\n+    driver_interrupt_controller()?;\n\n     INIT_DONE.store(true, Ordering::Relaxed);\n     Ok(())\n\ndiff -uNr 12_integrated_testing/kernel/src/bsp/raspberrypi/exception/asynchronous.rs 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/raspberrypi/exception/asynchronous.rs\n--- 12_integrated_testing/kernel/src/bsp/raspberrypi/exception/asynchronous.rs\n+++ 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/raspberrypi/exception/asynchronous.rs\n@@ -0,0 +1,28 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! BSP asynchronous exception handling.\n+\n+use crate::bsp;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Export for reuse in generic asynchronous.rs.\n+pub use bsp::device_driver::IRQNumber;\n+\n+#[cfg(feature = \"bsp_rpi3\")]\n+pub(in crate::bsp) mod irq_map {\n+    use super::bsp::device_driver::{IRQNumber, PeripheralIRQ};\n+\n+    pub const PL011_UART: IRQNumber = IRQNumber::Peripheral(PeripheralIRQ::new(57));\n+}\n+\n+#[cfg(feature = \"bsp_rpi4\")]\n+pub(in crate::bsp) mod irq_map {\n+    use super::bsp::device_driver::IRQNumber;\n+\n+    pub const PL011_UART: IRQNumber = IRQNumber::new(153);\n+}\n\ndiff -uNr 12_integrated_testing/kernel/src/bsp/raspberrypi/exception.rs 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/raspberrypi/exception.rs\n--- 12_integrated_testing/kernel/src/bsp/raspberrypi/exception.rs\n+++ 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/raspberrypi/exception.rs\n@@ -0,0 +1,7 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! BSP synchronous and asynchronous exception handling.\n+\n+pub mod asynchronous;\n\ndiff -uNr 12_integrated_testing/kernel/src/bsp/raspberrypi/memory.rs 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/raspberrypi/memory.rs\n--- 12_integrated_testing/kernel/src/bsp/raspberrypi/memory.rs\n+++ 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/raspberrypi/memory.rs\n@@ -73,10 +73,11 @@\n     pub mod mmio {\n         use super::*;\n\n-        pub const START:            usize =         0x3F00_0000;\n-        pub const GPIO_START:       usize = START + GPIO_OFFSET;\n-        pub const PL011_UART_START: usize = START + UART_OFFSET;\n-        pub const END_INCLUSIVE:    usize =         0x4000_FFFF;\n+        pub const START:               usize =         0x3F00_0000;\n+        pub const PERIPHERAL_IC_START: usize = START + 0x0000_B200;\n+        pub const GPIO_START:          usize = START + GPIO_OFFSET;\n+        pub const PL011_UART_START:    usize = START + UART_OFFSET;\n+        pub const END_INCLUSIVE:       usize =         0x4000_FFFF;\n     }\n\n     /// Physical devices.\n@@ -87,6 +88,8 @@\n         pub const START:            usize =         0xFE00_0000;\n         pub const GPIO_START:       usize = START + GPIO_OFFSET;\n         pub const PL011_UART_START: usize = START + UART_OFFSET;\n+        pub const GICD_START:       usize =         0xFF84_1000;\n+        pub const GICC_START:       usize =         0xFF84_2000;\n         pub const END_INCLUSIVE:    usize =         0xFF84_FFFF;\n     }\n }\n\ndiff -uNr 12_integrated_testing/kernel/src/bsp/raspberrypi.rs 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/raspberrypi.rs\n--- 12_integrated_testing/kernel/src/bsp/raspberrypi.rs\n+++ 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/raspberrypi.rs\n@@ -6,6 +6,7 @@\n\n pub mod cpu;\n pub mod driver;\n+pub mod exception;\n pub mod memory;\n\n //--------------------------------------------------------------------------------------------------\n\ndiff -uNr 12_integrated_testing/kernel/src/console.rs 13_exceptions_part2_peripheral_IRQs/kernel/src/console.rs\n--- 12_integrated_testing/kernel/src/console.rs\n+++ 13_exceptions_part2_peripheral_IRQs/kernel/src/console.rs\n@@ -6,7 +6,7 @@\n\n mod null_console;\n\n-use crate::synchronization::{self, NullLock};\n+use crate::synchronization;\n\n //--------------------------------------------------------------------------------------------------\n // Public Definitions\n@@ -60,22 +60,22 @@\n // Global instances\n //--------------------------------------------------------------------------------------------------\n\n-static CUR_CONSOLE: NullLock<&'static (dyn interface::All + Sync)> =\n-    NullLock::new(&null_console::NULL_CONSOLE);\n+static CUR_CONSOLE: InitStateLock<&'static (dyn interface::All + Sync)> =\n+    InitStateLock::new(&null_console::NULL_CONSOLE);\n\n //--------------------------------------------------------------------------------------------------\n // Public Code\n //--------------------------------------------------------------------------------------------------\n-use synchronization::interface::Mutex;\n+use synchronization::{interface::ReadWriteEx, InitStateLock};\n\n /// Register a new console.\n pub fn register_console(new_console: &'static (dyn interface::All + Sync)) {\n-    CUR_CONSOLE.lock(|con| *con = new_console);\n+    CUR_CONSOLE.write(|con| *con = new_console);\n }\n\n /// Return a reference to the currently registered console.\n ///\n /// This is the global console used by all printing macros.\n pub fn console() -> &'static dyn interface::All {\n-    CUR_CONSOLE.lock(|con| *con)\n+    CUR_CONSOLE.read(|con| *con)\n }\n\ndiff -uNr 12_integrated_testing/kernel/src/cpu/smp.rs 13_exceptions_part2_peripheral_IRQs/kernel/src/cpu/smp.rs\n--- 12_integrated_testing/kernel/src/cpu/smp.rs\n+++ 13_exceptions_part2_peripheral_IRQs/kernel/src/cpu/smp.rs\n@@ -0,0 +1,14 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Symmetric multiprocessing.\n+\n+#[cfg(target_arch = \"aarch64\")]\n+#[path = \"../_arch/aarch64/cpu/smp.rs\"]\n+mod arch_smp;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Architectural Public Reexports\n+//--------------------------------------------------------------------------------------------------\n+pub use arch_smp::core_id;\n\ndiff -uNr 12_integrated_testing/kernel/src/cpu.rs 13_exceptions_part2_peripheral_IRQs/kernel/src/cpu.rs\n--- 12_integrated_testing/kernel/src/cpu.rs\n+++ 13_exceptions_part2_peripheral_IRQs/kernel/src/cpu.rs\n@@ -10,6 +10,8 @@\n\n mod boot;\n\n+pub mod smp;\n+\n //--------------------------------------------------------------------------------------------------\n // Architectural Public Reexports\n //--------------------------------------------------------------------------------------------------\n\ndiff -uNr 12_integrated_testing/kernel/src/driver.rs 13_exceptions_part2_peripheral_IRQs/kernel/src/driver.rs\n--- 12_integrated_testing/kernel/src/driver.rs\n+++ 13_exceptions_part2_peripheral_IRQs/kernel/src/driver.rs\n@@ -5,9 +5,10 @@\n //! Driver support.\n\n use crate::{\n-    info,\n-    synchronization::{interface::Mutex, NullLock},\n+    exception, info,\n+    synchronization::{interface::ReadWriteEx, InitStateLock},\n };\n+use core::fmt;\n\n //--------------------------------------------------------------------------------------------------\n // Private Definitions\n@@ -15,9 +16,12 @@\n\n const NUM_DRIVERS: usize = 5;\n\n-struct DriverManagerInner {\n+struct DriverManagerInner<T>\n+where\n+    T: 'static,\n+{\n     next_index: usize,\n-    descriptors: [Option<DeviceDriverDescriptor>; NUM_DRIVERS],\n+    descriptors: [Option<DeviceDriverDescriptor<T>>; NUM_DRIVERS],\n }\n\n //--------------------------------------------------------------------------------------------------\n@@ -28,6 +32,9 @@\n pub mod interface {\n     /// Device Driver functions.\n     pub trait DeviceDriver {\n+        /// Different interrupt controllers might use different types for IRQ number.\n+        type IRQNumberType: super::fmt::Display;\n+\n         /// Return a compatibility string for identifying the driver.\n         fn compatible(&self) -> &'static str;\n\n@@ -39,6 +46,21 @@\n         unsafe fn init(&self) -> Result<(), &'static str> {\n             Ok(())\n         }\n+\n+        /// Called by the kernel to register and enable the device's IRQ handler.\n+        ///\n+        /// Rust's type system will prevent a call to this function unless the calling instance\n+        /// itself has static lifetime.\n+        fn register_and_enable_irq_handler(\n+            &'static self,\n+            irq_number: &Self::IRQNumberType,\n+        ) -> Result<(), &'static str> {\n+            panic!(\n+                \"Attempt to enable IRQ {} for device {}, but driver does not support this\",\n+                irq_number,\n+                self.compatible()\n+            )\n+        }\n     }\n }\n\n@@ -47,27 +69,37 @@\n\n /// A descriptor for device drivers.\n #[derive(Copy, Clone)]\n-pub struct DeviceDriverDescriptor {\n-    device_driver: &'static (dyn interface::DeviceDriver + Sync),\n+pub struct DeviceDriverDescriptor<T>\n+where\n+    T: 'static,\n+{\n+    device_driver: &'static (dyn interface::DeviceDriver<IRQNumberType = T> + Sync),\n     post_init_callback: Option<DeviceDriverPostInitCallback>,\n+    irq_number: Option<T>,\n }\n\n /// Provides device driver management functions.\n-pub struct DriverManager {\n-    inner: NullLock<DriverManagerInner>,\n+pub struct DriverManager<T>\n+where\n+    T: 'static,\n+{\n+    inner: InitStateLock<DriverManagerInner<T>>,\n }\n\n //--------------------------------------------------------------------------------------------------\n // Global instances\n //--------------------------------------------------------------------------------------------------\n\n-static DRIVER_MANAGER: DriverManager = DriverManager::new();\n+static DRIVER_MANAGER: DriverManager<exception::asynchronous::IRQNumber> = DriverManager::new();\n\n //--------------------------------------------------------------------------------------------------\n // Private Code\n //--------------------------------------------------------------------------------------------------\n\n-impl DriverManagerInner {\n+impl<T> DriverManagerInner<T>\n+where\n+    T: 'static + Copy,\n+{\n     /// Create an instance.\n     pub const fn new() -> Self {\n         Self {\n@@ -81,43 +113,48 @@\n // Public Code\n //--------------------------------------------------------------------------------------------------\n\n-impl DeviceDriverDescriptor {\n+impl<T> DeviceDriverDescriptor<T> {\n     /// Create an instance.\n     pub fn new(\n-        device_driver: &'static (dyn interface::DeviceDriver + Sync),\n+        device_driver: &'static (dyn interface::DeviceDriver<IRQNumberType = T> + Sync),\n         post_init_callback: Option<DeviceDriverPostInitCallback>,\n+        irq_number: Option<T>,\n     ) -> Self {\n         Self {\n             device_driver,\n             post_init_callback,\n+            irq_number,\n         }\n     }\n }\n\n /// Return a reference to the global DriverManager.\n-pub fn driver_manager() -> &'static DriverManager {\n+pub fn driver_manager() -> &'static DriverManager<exception::asynchronous::IRQNumber> {\n     &DRIVER_MANAGER\n }\n\n-impl DriverManager {\n+impl<T> DriverManager<T>\n+where\n+    T: fmt::Display + Copy,\n+{\n     /// Create an instance.\n     pub const fn new() -> Self {\n         Self {\n-            inner: NullLock::new(DriverManagerInner::new()),\n+            inner: InitStateLock::new(DriverManagerInner::new()),\n         }\n     }\n\n     /// Register a device driver with the kernel.\n-    pub fn register_driver(&self, descriptor: DeviceDriverDescriptor) {\n-        self.inner.lock(|inner| {\n+    pub fn register_driver(&self, descriptor: DeviceDriverDescriptor<T>) {\n+        self.inner.write(|inner| {\n             inner.descriptors[inner.next_index] = Some(descriptor);\n             inner.next_index += 1;\n         })\n     }\n\n     /// Helper for iterating over registered drivers.\n-    fn for_each_descriptor<'a>(&'a self, f: impl FnMut(&'a DeviceDriverDescriptor)) {\n-        self.inner.lock(|inner| {\n+    fn for_each_descriptor<'a>(&'a self, f: impl FnMut(&'a DeviceDriverDescriptor<T>)) {\n+        self.inner.read(|inner| {\n             inner\n                 .descriptors\n                 .iter()\n@@ -126,12 +163,12 @@\n         })\n     }\n\n-    /// Fully initialize all drivers.\n+    /// Fully initialize all drivers and their interrupts handlers.\n     ///\n     /// # Safety\n     ///\n     /// - During init, drivers might do stuff with system-wide impact.\n-    pub unsafe fn init_drivers(&self) {\n+    pub unsafe fn init_drivers_and_irqs(&self) {\n         self.for_each_descriptor(|descriptor| {\n             // 1. Initialize driver.\n             if let Err(x) = descriptor.device_driver.init() {\n@@ -150,6 +187,23 @@\n                         descriptor.device_driver.compatible(),\n                         x\n                     );\n+                }\n+            }\n+        });\n+\n+        // 3. After all post-init callbacks were done, the interrupt controller should be\n+        //    registered and functional. So let drivers register with it now.\n+        self.for_each_descriptor(|descriptor| {\n+            if let Some(irq_number) = &descriptor.irq_number {\n+                if let Err(x) = descriptor\n+                    .device_driver\n+                    .register_and_enable_irq_handler(irq_number)\n+                {\n+                    panic!(\n+                        \"Error during driver interrupt handler registration: {}: {}\",\n+                        descriptor.device_driver.compatible(),\n+                        x\n+                    );\n                 }\n             }\n         });\n\ndiff -uNr 12_integrated_testing/kernel/src/exception/asynchronous/null_irq_manager.rs 13_exceptions_part2_peripheral_IRQs/kernel/src/exception/asynchronous/null_irq_manager.rs\n--- 12_integrated_testing/kernel/src/exception/asynchronous/null_irq_manager.rs\n+++ 13_exceptions_part2_peripheral_IRQs/kernel/src/exception/asynchronous/null_irq_manager.rs\n@@ -0,0 +1,42 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Null IRQ Manager.\n+\n+use super::{interface, IRQContext, IRQHandlerDescriptor};\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+pub struct NullIRQManager;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Global instances\n+//--------------------------------------------------------------------------------------------------\n+\n+pub static NULL_IRQ_MANAGER: NullIRQManager = NullIRQManager {};\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+impl interface::IRQManager for NullIRQManager {\n+    type IRQNumberType = super::IRQNumber;\n+\n+    fn register_handler(\n+        &self,\n+        _descriptor: IRQHandlerDescriptor<Self::IRQNumberType>,\n+    ) -> Result<(), &'static str> {\n+        panic!(\"No IRQ Manager registered yet\");\n+    }\n+\n+    fn enable(&self, _irq_number: &Self::IRQNumberType) {\n+        panic!(\"No IRQ Manager registered yet\");\n+    }\n+\n+    fn handle_pending_irqs<'irq_context>(&'irq_context self, _ic: &IRQContext<'irq_context>) {\n+        panic!(\"No IRQ Manager registered yet\");\n+    }\n+}\n\ndiff -uNr 12_integrated_testing/kernel/src/exception/asynchronous.rs 13_exceptions_part2_peripheral_IRQs/kernel/src/exception/asynchronous.rs\n--- 12_integrated_testing/kernel/src/exception/asynchronous.rs\n+++ 13_exceptions_part2_peripheral_IRQs/kernel/src/exception/asynchronous.rs\n@@ -7,8 +7,184 @@\n #[cfg(target_arch = \"aarch64\")]\n #[path = \"../_arch/aarch64/exception/asynchronous.rs\"]\n mod arch_asynchronous;\n+mod null_irq_manager;\n+\n+use crate::{bsp, synchronization};\n+use core::marker::PhantomData;\n\n //--------------------------------------------------------------------------------------------------\n // Architectural Public Reexports\n //--------------------------------------------------------------------------------------------------\n-pub use arch_asynchronous::print_state;\n+pub use arch_asynchronous::{\n+    is_local_irq_masked, local_irq_mask, local_irq_mask_save, local_irq_restore, local_irq_unmask,\n+    print_state,\n+};\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Interrupt number as defined by the BSP.\n+pub type IRQNumber = bsp::exception::asynchronous::IRQNumber;\n+\n+/// Interrupt descriptor.\n+#[derive(Copy, Clone)]\n+pub struct IRQHandlerDescriptor<T>\n+where\n+    T: Copy,\n+{\n+    /// The IRQ number.\n+    number: T,\n+\n+    /// Descriptive name.\n+    name: &'static str,\n+\n+    /// Reference to handler trait object.\n+    handler: &'static (dyn interface::IRQHandler + Sync),\n+}\n+\n+/// IRQContext token.\n+///\n+/// An instance of this type indicates that the local core is currently executing in IRQ\n+/// context, aka executing an interrupt vector or subcalls of it.\n+///\n+/// Concept and implementation derived from the `CriticalSection` introduced in\n+/// <https://github.com/rust-embedded/bare-metal>\n+#[derive(Clone, Copy)]\n+pub struct IRQContext<'irq_context> {\n+    _0: PhantomData<&'irq_context ()>,\n+}\n+\n+/// Asynchronous exception handling interfaces.\n+pub mod interface {\n+\n+    /// Implemented by types that handle IRQs.\n+    pub trait IRQHandler {\n+        /// Called when the corresponding interrupt is asserted.\n+        fn handle(&self) -> Result<(), &'static str>;\n+    }\n+\n+    /// IRQ management functions.\n+    ///\n+    /// The `BSP` is supposed to supply one global instance. Typically implemented by the\n+    /// platform's interrupt controller.\n+    pub trait IRQManager {\n+        /// The IRQ number type depends on the implementation.\n+        type IRQNumberType: Copy;\n+\n+        /// Register a handler.\n+        fn register_handler(\n+            &self,\n+            irq_handler_descriptor: super::IRQHandlerDescriptor<Self::IRQNumberType>,\n+        ) -> Result<(), &'static str>;\n+\n+        /// Enable an interrupt in the controller.\n+        fn enable(&self, irq_number: &Self::IRQNumberType);\n+\n+        /// Handle pending interrupts.\n+        ///\n+        /// This function is called directly from the CPU's IRQ exception vector. On AArch64,\n+        /// this means that the respective CPU core has disabled exception handling.\n+        /// This function can therefore not be preempted and runs start to finish.\n+        ///\n+        /// Takes an IRQContext token to ensure it can only be called from IRQ context.\n+        #[allow(clippy::trivially_copy_pass_by_ref)]\n+        fn handle_pending_irqs<'irq_context>(\n+            &'irq_context self,\n+            ic: &super::IRQContext<'irq_context>,\n+        );\n+\n+        /// Print list of registered handlers.\n+        fn print_handler(&self) {}\n+    }\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Global instances\n+//--------------------------------------------------------------------------------------------------\n+\n+static CUR_IRQ_MANAGER: InitStateLock<\n+    &'static (dyn interface::IRQManager<IRQNumberType = IRQNumber> + Sync),\n+> = InitStateLock::new(&null_irq_manager::NULL_IRQ_MANAGER);\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+use synchronization::{interface::ReadWriteEx, InitStateLock};\n+\n+impl<T> IRQHandlerDescriptor<T>\n+where\n+    T: Copy,\n+{\n+    /// Create an instance.\n+    pub const fn new(\n+        number: T,\n+        name: &'static str,\n+        handler: &'static (dyn interface::IRQHandler + Sync),\n+    ) -> Self {\n+        Self {\n+            number,\n+            name,\n+            handler,\n+        }\n+    }\n+\n+    /// Return the number.\n+    pub const fn number(&self) -> T {\n+        self.number\n+    }\n+\n+    /// Return the name.\n+    pub const fn name(&self) -> &'static str {\n+        self.name\n+    }\n+\n+    /// Return the handler.\n+    pub const fn handler(&self) -> &'static (dyn interface::IRQHandler + Sync) {\n+        self.handler\n+    }\n+}\n+\n+impl<'irq_context> IRQContext<'irq_context> {\n+    /// Creates an IRQContext token.\n+    ///\n+    /// # Safety\n+    ///\n+    /// - This must only be called when the current core is in an interrupt context and will not\n+    ///   live beyond the end of it. That is, creation is allowed in interrupt vector functions. For\n+    ///   example, in the ARMv8-A case, in `extern \"C\" fn current_elx_irq()`.\n+    /// - Note that the lifetime `'irq_context` of the returned instance is unconstrained. User code\n+    ///   must not be able to influence the lifetime picked for this type, since that might cause it\n+    ///   to be inferred to `'static`.\n+    #[inline(always)]\n+    pub unsafe fn new() -> Self {\n+        IRQContext { _0: PhantomData }\n+    }\n+}\n+\n+/// Executes the provided closure while IRQs are masked on the executing core.\n+///\n+/// While the function temporarily changes the HW state of the executing core, it restores it to the\n+/// previous state before returning, so this is deemed safe.\n+#[inline(always)]\n+pub fn exec_with_irq_masked<T>(f: impl FnOnce() -> T) -> T {\n+    let saved = local_irq_mask_save();\n+    let ret = f();\n+    local_irq_restore(saved);\n+\n+    ret\n+}\n+\n+/// Register a new IRQ manager.\n+pub fn register_irq_manager(\n+    new_manager: &'static (dyn interface::IRQManager<IRQNumberType = IRQNumber> + Sync),\n+) {\n+    CUR_IRQ_MANAGER.write(|manager| *manager = new_manager);\n+}\n+\n+/// Return a reference to the currently registered IRQ manager.\n+///\n+/// This is the IRQ manager used by the architectural interrupt handling code.\n+pub fn irq_manager() -> &'static dyn interface::IRQManager<IRQNumberType = IRQNumber> {\n+    CUR_IRQ_MANAGER.read(|manager| *manager)\n+}\n\ndiff -uNr 12_integrated_testing/kernel/src/lib.rs 13_exceptions_part2_peripheral_IRQs/kernel/src/lib.rs\n--- 12_integrated_testing/kernel/src/lib.rs\n+++ 13_exceptions_part2_peripheral_IRQs/kernel/src/lib.rs\n@@ -138,6 +138,7 @@\n pub mod exception;\n pub mod memory;\n pub mod print;\n+pub mod state;\n pub mod time;\n\n //--------------------------------------------------------------------------------------------------\n\ndiff -uNr 12_integrated_testing/kernel/src/main.rs 13_exceptions_part2_peripheral_IRQs/kernel/src/main.rs\n--- 12_integrated_testing/kernel/src/main.rs\n+++ 13_exceptions_part2_peripheral_IRQs/kernel/src/main.rs\n@@ -13,7 +13,7 @@\n #![no_main]\n #![no_std]\n\n-use libkernel::{bsp, console, driver, exception, info, memory, time};\n+use libkernel::{bsp, cpu, driver, exception, info, memory, state, time};\n\n /// Early init code.\n ///\n@@ -23,7 +23,7 @@\n /// - The init calls in this function must appear in the correct order:\n ///     - MMU + Data caching must be activated at the earliest. Without it, any atomic operations,\n ///       e.g. the yet-to-be-introduced spinlocks in the device drivers (which currently employ\n-///       NullLocks instead of spinlocks), will fail to work (properly) on the RPi SoCs.\n+///       IRQSafeNullLocks instead of spinlocks), will fail to work (properly) on the RPi SoCs.\n #[no_mangle]\n unsafe fn kernel_init() -> ! {\n     use memory::mmu::interface::MMU;\n@@ -40,8 +40,13 @@\n     }\n\n     // Initialize all device drivers.\n-    driver::driver_manager().init_drivers();\n-    // println! is usable from here on.\n+    driver::driver_manager().init_drivers_and_irqs();\n+\n+    // Unmask interrupts on the boot CPU core.\n+    exception::asynchronous::local_irq_unmask();\n+\n+    // Announce conclusion of the kernel_init() phase.\n+    state::state_manager().transition_to_single_core_main();\n\n     // Transition from unsafe to safe.\n     kernel_main()\n@@ -49,8 +54,6 @@\n\n /// The main function running after the early init.\n fn kernel_main() -> ! {\n-    use console::console;\n-\n     info!(\"{}\", libkernel::version());\n     info!(\"Booting on: {}\", bsp::board_name());\n\n@@ -71,12 +74,9 @@\n     info!(\"Drivers loaded:\");\n     driver::driver_manager().enumerate();\n\n-    info!(\"Echoing input now\");\n+    info!(\"Registered IRQ handlers:\");\n+    exception::asynchronous::irq_manager().print_handler();\n\n-    // Discard any spurious received characters before going into echo mode.\n-    console().clear_rx();\n-    loop {\n-        let c = console().read_char();\n-        console().write_char(c);\n-    }\n+    info!(\"Echoing input now\");\n+    cpu::wait_forever();\n }\n\ndiff -uNr 12_integrated_testing/kernel/src/panic_wait.rs 13_exceptions_part2_peripheral_IRQs/kernel/src/panic_wait.rs\n--- 12_integrated_testing/kernel/src/panic_wait.rs\n+++ 13_exceptions_part2_peripheral_IRQs/kernel/src/panic_wait.rs\n@@ -4,7 +4,7 @@\n\n //! A panic handler that infinitely waits.\n\n-use crate::{cpu, println};\n+use crate::{cpu, exception, println};\n use core::panic::PanicInfo;\n\n //--------------------------------------------------------------------------------------------------\n@@ -59,6 +59,8 @@\n\n #[panic_handler]\n fn panic(info: &PanicInfo) -> ! {\n+    exception::asynchronous::local_irq_mask();\n+\n     // Protect against panic infinite loops if any of the following code panics itself.\n     panic_prevent_reenter();\n\n\ndiff -uNr 12_integrated_testing/kernel/src/state.rs 13_exceptions_part2_peripheral_IRQs/kernel/src/state.rs\n--- 12_integrated_testing/kernel/src/state.rs\n+++ 13_exceptions_part2_peripheral_IRQs/kernel/src/state.rs\n@@ -0,0 +1,92 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! State information about the kernel itself.\n+\n+use core::sync::atomic::{AtomicU8, Ordering};\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Different stages in the kernel execution.\n+#[derive(Copy, Clone, Eq, PartialEq)]\n+enum State {\n+    /// The kernel starts booting in this state.\n+    Init,\n+\n+    /// The kernel transitions to this state when jumping to `kernel_main()` (at the end of\n+    /// `kernel_init()`, after all init calls are done).\n+    SingleCoreMain,\n+\n+    /// The kernel transitions to this state when it boots the secondary cores, aka switches\n+    /// exectution mode to symmetric multiprocessing (SMP).\n+    MultiCoreMain,\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Maintains the kernel state and state transitions.\n+pub struct StateManager(AtomicU8);\n+\n+//--------------------------------------------------------------------------------------------------\n+// Global instances\n+//--------------------------------------------------------------------------------------------------\n+\n+static STATE_MANAGER: StateManager = StateManager::new();\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Return a reference to the global StateManager.\n+pub fn state_manager() -> &'static StateManager {\n+    &STATE_MANAGER\n+}\n+\n+impl StateManager {\n+    const INIT: u8 = 0;\n+    const SINGLE_CORE_MAIN: u8 = 1;\n+    const MULTI_CORE_MAIN: u8 = 2;\n+\n+    /// Create a new instance.\n+    pub const fn new() -> Self {\n+        Self(AtomicU8::new(Self::INIT))\n+    }\n+\n+    /// Return the current state.\n+    fn state(&self) -> State {\n+        let state = self.0.load(Ordering::Acquire);\n+\n+        match state {\n+            Self::INIT => State::Init,\n+            Self::SINGLE_CORE_MAIN => State::SingleCoreMain,\n+            Self::MULTI_CORE_MAIN => State::MultiCoreMain,\n+            _ => panic!(\"Invalid KERNEL_STATE\"),\n+        }\n+    }\n+\n+    /// Return if the kernel is init state.\n+    pub fn is_init(&self) -> bool {\n+        self.state() == State::Init\n+    }\n+\n+    /// Transition from Init to SingleCoreMain.\n+    pub fn transition_to_single_core_main(&self) {\n+        if self\n+            .0\n+            .compare_exchange(\n+                Self::INIT,\n+                Self::SINGLE_CORE_MAIN,\n+                Ordering::Acquire,\n+                Ordering::Relaxed,\n+            )\n+            .is_err()\n+        {\n+            panic!(\"transition_to_single_core_main() called while state != Init\");\n+        }\n+    }\n+}\n\ndiff -uNr 12_integrated_testing/kernel/src/synchronization.rs 13_exceptions_part2_peripheral_IRQs/kernel/src/synchronization.rs\n--- 12_integrated_testing/kernel/src/synchronization.rs\n+++ 13_exceptions_part2_peripheral_IRQs/kernel/src/synchronization.rs\n@@ -28,6 +28,21 @@\n         /// Locks the mutex and grants the closure temporary mutable access to the wrapped data.\n         fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R;\n     }\n+\n+    /// A reader-writer exclusion type.\n+    ///\n+    /// The implementing object allows either a number of readers or at most one writer at any point\n+    /// in time.\n+    pub trait ReadWriteEx {\n+        /// The type of encapsulated data.\n+        type Data;\n+\n+        /// Grants temporary mutable access to the encapsulated data.\n+        fn write<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R;\n+\n+        /// Grants temporary immutable access to the encapsulated data.\n+        fn read<'a, R>(&'a self, f: impl FnOnce(&'a Self::Data) -> R) -> R;\n+    }\n }\n\n /// A pseudo-lock for teaching purposes.\n@@ -36,8 +51,18 @@\n /// other cores to the contained data. This part is preserved for later lessons.\n ///\n /// The lock will only be used as long as it is safe to do so, i.e. as long as the kernel is\n-/// executing single-threaded, aka only running on a single core with interrupts disabled.\n-pub struct NullLock<T>\n+/// executing on a single core.\n+pub struct IRQSafeNullLock<T>\n+where\n+    T: ?Sized,\n+{\n+    data: UnsafeCell<T>,\n+}\n+\n+/// A pseudo-lock that is RW during the single-core kernel init phase and RO afterwards.\n+///\n+/// Intended to encapsulate data that is populated during kernel init when no concurrency exists.\n+pub struct InitStateLock<T>\n where\n     T: ?Sized,\n {\n@@ -48,10 +73,22 @@\n // Public Code\n //--------------------------------------------------------------------------------------------------\n\n-unsafe impl<T> Send for NullLock<T> where T: ?Sized + Send {}\n-unsafe impl<T> Sync for NullLock<T> where T: ?Sized + Send {}\n+unsafe impl<T> Send for IRQSafeNullLock<T> where T: ?Sized + Send {}\n+unsafe impl<T> Sync for IRQSafeNullLock<T> where T: ?Sized + Send {}\n+\n+impl<T> IRQSafeNullLock<T> {\n+    /// Create an instance.\n+    pub const fn new(data: T) -> Self {\n+        Self {\n+            data: UnsafeCell::new(data),\n+        }\n+    }\n+}\n+\n+unsafe impl<T> Send for InitStateLock<T> where T: ?Sized + Send {}\n+unsafe impl<T> Sync for InitStateLock<T> where T: ?Sized + Send {}\n\n-impl<T> NullLock<T> {\n+impl<T> InitStateLock<T> {\n     /// Create an instance.\n     pub const fn new(data: T) -> Self {\n         Self {\n@@ -63,8 +100,9 @@\n //------------------------------------------------------------------------------\n // OS Interface Code\n //------------------------------------------------------------------------------\n+use crate::{exception, state};\n\n-impl<T> interface::Mutex for NullLock<T> {\n+impl<T> interface::Mutex for IRQSafeNullLock<T> {\n     type Data = T;\n\n     fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R {\n@@ -72,6 +110,50 @@\n         // mutable reference will ever only be given out once at a time.\n         let data = unsafe { &mut *self.data.get() };\n\n+        // Execute the closure while IRQs are masked.\n+        exception::asynchronous::exec_with_irq_masked(|| f(data))\n+    }\n+}\n+\n+impl<T> interface::ReadWriteEx for InitStateLock<T> {\n+    type Data = T;\n+\n+    fn write<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R {\n+        assert!(\n+            state::state_manager().is_init(),\n+            \"InitStateLock::write called after kernel init phase\"\n+        );\n+        assert!(\n+            !exception::asynchronous::is_local_irq_masked(),\n+            \"InitStateLock::write called with IRQs unmasked\"\n+        );\n+\n+        let data = unsafe { &mut *self.data.get() };\n+\n         f(data)\n     }\n+\n+    fn read<'a, R>(&'a self, f: impl FnOnce(&'a Self::Data) -> R) -> R {\n+        let data = unsafe { &*self.data.get() };\n+\n+        f(data)\n+    }\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Testing\n+//--------------------------------------------------------------------------------------------------\n+\n+#[cfg(test)]\n+mod tests {\n+    use super::*;\n+    use test_macros::kernel_test;\n+\n+    /// InitStateLock must be transparent.\n+    #[kernel_test]\n+    fn init_state_lock_is_transparent() {\n+        use core::mem::size_of;\n+\n+        assert_eq!(size_of::<InitStateLock<u64>>(), size_of::<u64>());\n+    }\n }\n\ndiff -uNr 12_integrated_testing/kernel/tests/04_exception_irq_sanity.rs 13_exceptions_part2_peripheral_IRQs/kernel/tests/04_exception_irq_sanity.rs\n--- 12_integrated_testing/kernel/tests/04_exception_irq_sanity.rs\n+++ 13_exceptions_part2_peripheral_IRQs/kernel/tests/04_exception_irq_sanity.rs\n@@ -0,0 +1,66 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! IRQ handling sanity tests.\n+\n+#![feature(custom_test_frameworks)]\n+#![no_main]\n+#![no_std]\n+#![reexport_test_harness_main = \"test_main\"]\n+#![test_runner(libkernel::test_runner)]\n+\n+use libkernel::{bsp, cpu, exception};\n+use test_macros::kernel_test;\n+\n+#[no_mangle]\n+unsafe fn kernel_init() -> ! {\n+    bsp::driver::qemu_bring_up_console();\n+\n+    exception::handling_init();\n+    exception::asynchronous::local_irq_unmask();\n+\n+    test_main();\n+\n+    cpu::qemu_exit_success()\n+}\n+\n+/// Check that IRQ masking works.\n+#[kernel_test]\n+fn local_irq_mask_works() {\n+    // Precondition: IRQs are unmasked.\n+    assert!(exception::asynchronous::is_local_irq_masked());\n+\n+    exception::asynchronous::local_irq_mask();\n+    assert!(!exception::asynchronous::is_local_irq_masked());\n+\n+    // Restore earlier state.\n+    exception::asynchronous::local_irq_unmask();\n+}\n+\n+/// Check that IRQ unmasking works.\n+#[kernel_test]\n+fn local_irq_unmask_works() {\n+    // Precondition: IRQs are masked.\n+    exception::asynchronous::local_irq_mask();\n+    assert!(!exception::asynchronous::is_local_irq_masked());\n+\n+    exception::asynchronous::local_irq_unmask();\n+    assert!(exception::asynchronous::is_local_irq_masked());\n+}\n+\n+/// Check that IRQ mask save is saving \"something\".\n+#[kernel_test]\n+fn local_irq_mask_save_works() {\n+    // Precondition: IRQs are unmasked.\n+    assert!(exception::asynchronous::is_local_irq_masked());\n+\n+    let first = exception::asynchronous::local_irq_mask_save();\n+    assert!(!exception::asynchronous::is_local_irq_masked());\n+\n+    let second = exception::asynchronous::local_irq_mask_save();\n+    assert_ne!(first, second);\n+\n+    exception::asynchronous::local_irq_restore(first);\n+    assert!(exception::asynchronous::is_local_irq_masked());\n+}\n\n```\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/Cargo.toml",
    "content": "[package]\nname = \"mingo\"\nversion = \"0.13.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n\n[features]\ndefault = []\nbsp_rpi3 = [\"tock-registers\"]\nbsp_rpi4 = [\"tock-registers\"]\ntest_build = [\"qemu-exit\"]\n\n##--------------------------------------------------------------------------------------------------\n## Dependencies\n##--------------------------------------------------------------------------------------------------\n\n[dependencies]\ntest-types = { path = \"../libraries/test-types\" }\n\n# Optional dependencies\ntock-registers = { version = \"0.8.x\", default-features = false, features = [\"register_types\"], optional = true }\nqemu-exit = { version = \"3.x.x\", optional = true }\n\n# Platform specific dependencies\n[target.'cfg(target_arch = \"aarch64\")'.dependencies]\naarch64-cpu = { version = \"9.x.x\" }\n\n##--------------------------------------------------------------------------------------------------\n## Testing\n##--------------------------------------------------------------------------------------------------\n\n[dev-dependencies]\ntest-macros = { path = \"../libraries/test-macros\" }\n\n# Unit tests are done in the library part of the kernel.\n[lib]\nname = \"libkernel\"\ntest = true\n\n# Disable unit tests for the kernel binary.\n[[bin]]\nname = \"kernel\"\npath = \"src/main.rs\"\ntest = false\n\n# List of tests without harness.\n[[test]]\nname = \"00_console_sanity\"\nharness = false\n\n[[test]]\nname = \"02_exception_sync_page_fault\"\nharness = false\n\n[[test]]\nname = \"03_exception_restore_sanity\"\nharness = false\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/build.rs",
    "content": "use std::{env, fs, process};\n\nfn main() {\n    let ld_script_path = match env::var(\"LD_SCRIPT_PATH\") {\n        Ok(var) => var,\n        _ => process::exit(0),\n    };\n\n    let files = fs::read_dir(ld_script_path).unwrap();\n    files\n        .filter_map(Result::ok)\n        .filter(|d| {\n            if let Some(e) = d.path().extension() {\n                e == \"ld\"\n            } else {\n                false\n            }\n        })\n        .for_each(|f| println!(\"cargo:rerun-if-changed={}\", f.path().display()));\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/_arch/aarch64/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural boot code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::boot::arch_boot\n\nuse aarch64_cpu::{asm, registers::*};\nuse core::arch::global_asm;\nuse tock_registers::interfaces::Writeable;\n\n// Assembly counterpart to this file.\nglobal_asm!(\n    include_str!(\"boot.s\"),\n    CONST_CURRENTEL_EL2 = const 0x8,\n    CONST_CORE_ID_MASK = const 0b11\n);\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Prepares the transition from EL2 to EL1.\n///\n/// # Safety\n///\n/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.\n/// - The HW state of EL1 must be prepared in a sound way.\n#[inline(always)]\nunsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: u64) {\n    // Enable timer counter registers for EL1.\n    CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET);\n\n    // No offset for reading the counters.\n    CNTVOFF_EL2.set(0);\n\n    // Set EL1 execution state to AArch64.\n    HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64);\n\n    // Set up a simulated exception return.\n    //\n    // First, fake a saved program status where all interrupts were masked and SP_EL1 was used as a\n    // stack pointer.\n    SPSR_EL2.write(\n        SPSR_EL2::D::Masked\n            + SPSR_EL2::A::Masked\n            + SPSR_EL2::I::Masked\n            + SPSR_EL2::F::Masked\n            + SPSR_EL2::M::EL1h,\n    );\n\n    // Second, let the link register point to kernel_init().\n    ELR_EL2.set(crate::kernel_init as *const () as u64);\n\n    // Set up SP_EL1 (stack pointer), which will be used by EL1 once we \"return\" to it. Since there\n    // are no plans to ever return to EL2, just re-use the same stack.\n    SP_EL1.set(phys_boot_core_stack_end_exclusive_addr);\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The Rust entry of the `kernel` binary.\n///\n/// The function is called from the assembly `_start` function.\n///\n/// # Safety\n///\n/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`.\n#[no_mangle]\npub unsafe extern \"C\" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! {\n    prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr);\n\n    // Use `eret` to \"return\" to EL1. This results in execution of kernel_init() in EL1.\n    asm::eret()\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/_arch/aarch64/cpu/boot.s",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2022 Andre Richter <andre.o.richter@gmail.com>\n\n//--------------------------------------------------------------------------------------------------\n// Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Load the address of a symbol into a register, PC-relative.\n//\n// The symbol must lie within +/- 4 GiB of the Program Counter.\n//\n// # Resources\n//\n// - https://sourceware.org/binutils/docs-2.36/as/AArch64_002dRelocations.html\n.macro ADR_REL register, symbol\n\tadrp\t\\register, \\symbol\n\tadd\t\\register, \\register, #:lo12:\\symbol\n.endm\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n.section .text._start\n\n//------------------------------------------------------------------------------\n// fn _start()\n//------------------------------------------------------------------------------\n_start:\n\t// Only proceed if the core executes in EL2. Park it otherwise.\n\tmrs\tx0, CurrentEL\n\tcmp\tx0, {CONST_CURRENTEL_EL2}\n\tb.ne\t.L_parking_loop\n\n\t// Only proceed on the boot core. Park it otherwise.\n\tmrs\tx1, MPIDR_EL1\n\tand\tx1, x1, {CONST_CORE_ID_MASK}\n\tldr\tx2, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs\n\tcmp\tx1, x2\n\tb.ne\t.L_parking_loop\n\n\t// If execution reaches here, it is the boot core.\n\n\t// Initialize DRAM.\n\tADR_REL\tx0, __bss_start\n\tADR_REL x1, __bss_end_exclusive\n\n.L_bss_init_loop:\n\tcmp\tx0, x1\n\tb.eq\t.L_prepare_rust\n\tstp\txzr, xzr, [x0], #16\n\tb\t.L_bss_init_loop\n\n\t// Prepare the jump to Rust code.\n.L_prepare_rust:\n\t// Set the stack pointer. This ensures that any code in EL2 that needs the stack will work.\n\tADR_REL\tx0, __boot_core_stack_end_exclusive\n\tmov\tsp, x0\n\n\t// Read the CPU's timer counter frequency and store it in ARCH_TIMER_COUNTER_FREQUENCY.\n\t// Abort if the frequency read back as 0.\n\tADR_REL\tx1, ARCH_TIMER_COUNTER_FREQUENCY // provided by aarch64/time.rs\n\tmrs\tx2, CNTFRQ_EL0\n\tcmp\tx2, xzr\n\tb.eq\t.L_parking_loop\n\tstr\tw2, [x1]\n\n\t// Jump to Rust code. x0 holds the function argument provided to _start_rust().\n\tb\t_start_rust\n\n\t// Infinitely wait for events (aka \"park the core\").\n.L_parking_loop:\n\twfe\n\tb\t.L_parking_loop\n\n.size\t_start, . - _start\n.type\t_start, function\n.global\t_start\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/_arch/aarch64/cpu/smp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural symmetric multiprocessing.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::smp::arch_smp\n\nuse aarch64_cpu::registers::*;\nuse tock_registers::interfaces::Readable;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return the executing core's id.\n#[inline(always)]\npub fn core_id<T>() -> T\nwhere\n    T: From<u8>,\n{\n    const CORE_MASK: u64 = 0b11;\n\n    T::from((MPIDR_EL1.get() & CORE_MASK) as u8)\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/_arch/aarch64/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural processor code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::arch_cpu\n\nuse aarch64_cpu::asm;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\npub use asm::nop;\n\n/// Pause execution on the core.\n#[inline(always)]\npub fn wait_forever() -> ! {\n    loop {\n        asm::wfe()\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n#[cfg(feature = \"test_build\")]\nuse qemu_exit::QEMUExit;\n\n#[cfg(feature = \"test_build\")]\nconst QEMU_EXIT_HANDLE: qemu_exit::AArch64 = qemu_exit::AArch64::new();\n\n/// Make the host QEMU binary execute `exit(1)`.\n#[cfg(feature = \"test_build\")]\npub fn qemu_exit_failure() -> ! {\n    QEMU_EXIT_HANDLE.exit_failure()\n}\n\n/// Make the host QEMU binary execute `exit(0)`.\n#[cfg(feature = \"test_build\")]\npub fn qemu_exit_success() -> ! {\n    QEMU_EXIT_HANDLE.exit_success()\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/_arch/aarch64/exception/asynchronous.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural asynchronous exception handling.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::exception::asynchronous::arch_asynchronous\n\nuse aarch64_cpu::registers::*;\nuse core::arch::asm;\nuse tock_registers::interfaces::{Readable, Writeable};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nmod daif_bits {\n    pub const IRQ: u8 = 0b0010;\n}\n\ntrait DaifField {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register>;\n}\n\nstruct Debug;\nstruct SError;\nstruct IRQ;\nstruct FIQ;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl DaifField for Debug {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::D\n    }\n}\n\nimpl DaifField for SError {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::A\n    }\n}\n\nimpl DaifField for IRQ {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::I\n    }\n}\n\nimpl DaifField for FIQ {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::F\n    }\n}\n\nfn is_masked<T>() -> bool\nwhere\n    T: DaifField,\n{\n    DAIF.is_set(T::daif_field())\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Returns whether IRQs are masked on the executing core.\npub fn is_local_irq_masked() -> bool {\n    !is_masked::<IRQ>()\n}\n\n/// Unmask IRQs on the executing core.\n///\n/// It is not needed to place an explicit instruction synchronization barrier after the `msr`.\n/// Quoting the Architecture Reference Manual for ARMv8-A, section C5.1.3:\n///\n/// \"Writes to PSTATE.{PAN, D, A, I, F} occur in program order without the need for additional\n/// synchronization.\"\n#[inline(always)]\npub fn local_irq_unmask() {\n    unsafe {\n        asm!(\n            \"msr DAIFClr, {arg}\",\n            arg = const daif_bits::IRQ,\n            options(nomem, nostack, preserves_flags)\n        );\n    }\n}\n\n/// Mask IRQs on the executing core.\n#[inline(always)]\npub fn local_irq_mask() {\n    unsafe {\n        asm!(\n            \"msr DAIFSet, {arg}\",\n            arg = const daif_bits::IRQ,\n            options(nomem, nostack, preserves_flags)\n        );\n    }\n}\n\n/// Mask IRQs on the executing core and return the previously saved interrupt mask bits (DAIF).\n#[inline(always)]\npub fn local_irq_mask_save() -> u64 {\n    let saved = DAIF.get();\n    local_irq_mask();\n\n    saved\n}\n\n/// Restore the interrupt mask bits (DAIF) using the callee's argument.\n///\n/// # Invariant\n///\n/// - No sanity checks on the input.\n#[inline(always)]\npub fn local_irq_restore(saved: u64) {\n    DAIF.set(saved);\n}\n\n/// Print the AArch64 exceptions status.\n#[rustfmt::skip]\npub fn print_state() {\n    use crate::info;\n\n    let to_mask_str = |x| -> _ {\n        if x { \"Masked\" } else { \"Unmasked\" }\n    };\n\n    info!(\"      Debug:  {}\", to_mask_str(is_masked::<Debug>()));\n    info!(\"      SError: {}\", to_mask_str(is_masked::<SError>()));\n    info!(\"      IRQ:    {}\", to_mask_str(is_masked::<IRQ>()));\n    info!(\"      FIQ:    {}\", to_mask_str(is_masked::<FIQ>()));\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/_arch/aarch64/exception.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural synchronous and asynchronous exception handling.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::exception::arch_exception\n\nuse crate::exception;\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::{arch::global_asm, cell::UnsafeCell, fmt};\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    registers::InMemoryRegister,\n};\n\n// Assembly counterpart to this file.\nglobal_asm!(include_str!(\"exception.s\"));\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Wrapper structs for memory copies of registers.\n#[repr(transparent)]\nstruct SpsrEL1(InMemoryRegister<u64, SPSR_EL1::Register>);\nstruct EsrEL1(InMemoryRegister<u64, ESR_EL1::Register>);\n\n/// The exception context as it is stored on the stack on exception entry.\n#[repr(C)]\nstruct ExceptionContext {\n    /// General Purpose Registers.\n    gpr: [u64; 30],\n\n    /// The link register, aka x30.\n    lr: u64,\n\n    /// Exception link register. The program counter at the time the exception happened.\n    elr_el1: u64,\n\n    /// Saved program status.\n    spsr_el1: SpsrEL1,\n\n    /// Exception syndrome register.\n    esr_el1: EsrEL1,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Prints verbose information about the exception and then panics.\nfn default_exception_handler(exc: &ExceptionContext) {\n    panic!(\n        \"CPU Exception!\\n\\n\\\n        {}\",\n        exc\n    );\n}\n\n//------------------------------------------------------------------------------\n// Current, EL0\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn current_el0_synchronous(_e: &mut ExceptionContext) {\n    panic!(\"Should not be here. Use of SP_EL0 in EL1 is not supported.\")\n}\n\n#[no_mangle]\nextern \"C\" fn current_el0_irq(_e: &mut ExceptionContext) {\n    panic!(\"Should not be here. Use of SP_EL0 in EL1 is not supported.\")\n}\n\n#[no_mangle]\nextern \"C\" fn current_el0_serror(_e: &mut ExceptionContext) {\n    panic!(\"Should not be here. Use of SP_EL0 in EL1 is not supported.\")\n}\n\n//------------------------------------------------------------------------------\n// Current, ELx\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn current_elx_synchronous(e: &mut ExceptionContext) {\n    #[cfg(feature = \"test_build\")]\n    {\n        const TEST_SVC_ID: u64 = 0x1337;\n\n        if let Some(ESR_EL1::EC::Value::SVC64) = e.esr_el1.exception_class() {\n            if e.esr_el1.iss() == TEST_SVC_ID {\n                return;\n            }\n        }\n    }\n\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn current_elx_irq(_e: &mut ExceptionContext) {\n    let token = unsafe { &exception::asynchronous::IRQContext::new() };\n    exception::asynchronous::irq_manager().handle_pending_irqs(token);\n}\n\n#[no_mangle]\nextern \"C\" fn current_elx_serror(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n//------------------------------------------------------------------------------\n// Lower, AArch64\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn lower_aarch64_synchronous(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch64_irq(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch64_serror(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n//------------------------------------------------------------------------------\n// Lower, AArch32\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn lower_aarch32_synchronous(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch32_irq(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch32_serror(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n//------------------------------------------------------------------------------\n// Misc\n//------------------------------------------------------------------------------\n\n/// Human readable SPSR_EL1.\n#[rustfmt::skip]\nimpl fmt::Display for SpsrEL1 {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        // Raw value.\n        writeln!(f, \"SPSR_EL1: {:#010x}\", self.0.get())?;\n\n        let to_flag_str = |x| -> _ {\n            if x { \"Set\" } else { \"Not set\" }\n         };\n\n        writeln!(f, \"      Flags:\")?;\n        writeln!(f, \"            Negative (N): {}\", to_flag_str(self.0.is_set(SPSR_EL1::N)))?;\n        writeln!(f, \"            Zero     (Z): {}\", to_flag_str(self.0.is_set(SPSR_EL1::Z)))?;\n        writeln!(f, \"            Carry    (C): {}\", to_flag_str(self.0.is_set(SPSR_EL1::C)))?;\n        writeln!(f, \"            Overflow (V): {}\", to_flag_str(self.0.is_set(SPSR_EL1::V)))?;\n\n        let to_mask_str = |x| -> _ {\n            if x { \"Masked\" } else { \"Unmasked\" }\n        };\n\n        writeln!(f, \"      Exception handling state:\")?;\n        writeln!(f, \"            Debug  (D): {}\", to_mask_str(self.0.is_set(SPSR_EL1::D)))?;\n        writeln!(f, \"            SError (A): {}\", to_mask_str(self.0.is_set(SPSR_EL1::A)))?;\n        writeln!(f, \"            IRQ    (I): {}\", to_mask_str(self.0.is_set(SPSR_EL1::I)))?;\n        writeln!(f, \"            FIQ    (F): {}\", to_mask_str(self.0.is_set(SPSR_EL1::F)))?;\n\n        write!(f, \"      Illegal Execution State (IL): {}\",\n            to_flag_str(self.0.is_set(SPSR_EL1::IL))\n        )\n    }\n}\n\nimpl EsrEL1 {\n    #[inline(always)]\n    fn exception_class(&self) -> Option<ESR_EL1::EC::Value> {\n        self.0.read_as_enum(ESR_EL1::EC)\n    }\n\n    #[cfg(feature = \"test_build\")]\n    #[inline(always)]\n    fn iss(&self) -> u64 {\n        self.0.read(ESR_EL1::ISS)\n    }\n}\n\n/// Human readable ESR_EL1.\n#[rustfmt::skip]\nimpl fmt::Display for EsrEL1 {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        // Raw print of whole register.\n        writeln!(f, \"ESR_EL1: {:#010x}\", self.0.get())?;\n\n        // Raw print of exception class.\n        write!(f, \"      Exception Class         (EC) : {:#x}\", self.0.read(ESR_EL1::EC))?;\n\n        // Exception class.\n        let ec_translation = match self.exception_class() {\n            Some(ESR_EL1::EC::Value::DataAbortCurrentEL) => \"Data Abort, current EL\",\n            _ => \"N/A\",\n        };\n        writeln!(f, \" - {}\", ec_translation)?;\n\n        // Raw print of instruction specific syndrome.\n        write!(f, \"      Instr Specific Syndrome (ISS): {:#x}\", self.0.read(ESR_EL1::ISS))\n    }\n}\n\nimpl ExceptionContext {\n    #[inline(always)]\n    fn exception_class(&self) -> Option<ESR_EL1::EC::Value> {\n        self.esr_el1.exception_class()\n    }\n\n    #[inline(always)]\n    fn fault_address_valid(&self) -> bool {\n        use ESR_EL1::EC::Value::*;\n\n        match self.exception_class() {\n            None => false,\n            Some(ec) => matches!(\n                ec,\n                InstrAbortLowerEL\n                    | InstrAbortCurrentEL\n                    | PCAlignmentFault\n                    | DataAbortLowerEL\n                    | DataAbortCurrentEL\n                    | WatchpointLowerEL\n                    | WatchpointCurrentEL\n            ),\n        }\n    }\n}\n\n/// Human readable print of the exception context.\nimpl fmt::Display for ExceptionContext {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        writeln!(f, \"{}\", self.esr_el1)?;\n\n        if self.fault_address_valid() {\n            writeln!(f, \"FAR_EL1: {:#018x}\", FAR_EL1.get() as usize)?;\n        }\n\n        writeln!(f, \"{}\", self.spsr_el1)?;\n        writeln!(f, \"ELR_EL1: {:#018x}\", self.elr_el1)?;\n        writeln!(f)?;\n        writeln!(f, \"General purpose register:\")?;\n\n        #[rustfmt::skip]\n        let alternating = |x| -> _ {\n            if x % 2 == 0 { \"   \" } else { \"\\n\" }\n        };\n\n        // Print two registers per line.\n        for (i, reg) in self.gpr.iter().enumerate() {\n            write!(f, \"      x{: <2}: {: >#018x}{}\", i, reg, alternating(i))?;\n        }\n        write!(f, \"      lr : {:#018x}\", self.lr)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse crate::exception::PrivilegeLevel;\n\n/// The processing element's current privilege level.\npub fn current_privilege_level() -> (PrivilegeLevel, &'static str) {\n    let el = CurrentEL.read_as_enum(CurrentEL::EL);\n    match el {\n        Some(CurrentEL::EL::Value::EL2) => (PrivilegeLevel::Hypervisor, \"EL2\"),\n        Some(CurrentEL::EL::Value::EL1) => (PrivilegeLevel::Kernel, \"EL1\"),\n        Some(CurrentEL::EL::Value::EL0) => (PrivilegeLevel::User, \"EL0\"),\n        _ => (PrivilegeLevel::Unknown, \"Unknown\"),\n    }\n}\n\n/// Init exception handling by setting the exception vector base address register.\n///\n/// # Safety\n///\n/// - Changes the HW state of the executing core.\n/// - The vector table and the symbol `__exception_vector_table_start` from the linker script must\n///   adhere to the alignment and size constraints demanded by the ARMv8-A Architecture Reference\n///   Manual.\npub unsafe fn handling_init() {\n    // Provided by exception.S.\n    extern \"Rust\" {\n        static __exception_vector_start: UnsafeCell<()>;\n    }\n\n    VBAR_EL1.set(__exception_vector_start.get() as u64);\n\n    // Force VBAR update to complete before next instruction.\n    barrier::isb(barrier::SY);\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/_arch/aarch64/exception.s",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2022 Andre Richter <andre.o.richter@gmail.com>\n\n//--------------------------------------------------------------------------------------------------\n// Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Call the function provided by parameter `\\handler` after saving the exception context. Provide\n/// the context as the first parameter to '\\handler'.\n.macro CALL_WITH_CONTEXT handler\n__vector_\\handler:\n\t// Make room on the stack for the exception context.\n\tsub\tsp,  sp,  #16 * 17\n\n\t// Store all general purpose registers on the stack.\n\tstp\tx0,  x1,  [sp, #16 * 0]\n\tstp\tx2,  x3,  [sp, #16 * 1]\n\tstp\tx4,  x5,  [sp, #16 * 2]\n\tstp\tx6,  x7,  [sp, #16 * 3]\n\tstp\tx8,  x9,  [sp, #16 * 4]\n\tstp\tx10, x11, [sp, #16 * 5]\n\tstp\tx12, x13, [sp, #16 * 6]\n\tstp\tx14, x15, [sp, #16 * 7]\n\tstp\tx16, x17, [sp, #16 * 8]\n\tstp\tx18, x19, [sp, #16 * 9]\n\tstp\tx20, x21, [sp, #16 * 10]\n\tstp\tx22, x23, [sp, #16 * 11]\n\tstp\tx24, x25, [sp, #16 * 12]\n\tstp\tx26, x27, [sp, #16 * 13]\n\tstp\tx28, x29, [sp, #16 * 14]\n\n\t// Add the exception link register (ELR_EL1), saved program status (SPSR_EL1) and exception\n\t// syndrome register (ESR_EL1).\n\tmrs\tx1,  ELR_EL1\n\tmrs\tx2,  SPSR_EL1\n\tmrs\tx3,  ESR_EL1\n\n\tstp\tlr,  x1,  [sp, #16 * 15]\n\tstp\tx2,  x3,  [sp, #16 * 16]\n\n\t// x0 is the first argument for the function called through `\\handler`.\n\tmov\tx0,  sp\n\n\t// Call `\\handler`.\n\tbl\t\\handler\n\n\t// After returning from exception handling code, replay the saved context and return via\n\t// `eret`.\n\tb\t__exception_restore_context\n\n.size\t__vector_\\handler, . - __vector_\\handler\n.type\t__vector_\\handler, function\n.endm\n\n.macro FIQ_SUSPEND\n1:\twfe\n\tb\t1b\n.endm\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n.section .text\n\n//------------------------------------------------------------------------------\n// The exception vector table.\n//------------------------------------------------------------------------------\n\n// Align by 2^11 bytes, as demanded by ARMv8-A. Same as ALIGN(2048) in an ld script.\n.align 11\n\n// Export a symbol for the Rust code to use.\n__exception_vector_start:\n\n// Current exception level with SP_EL0.\n//\n// .org sets the offset relative to section start.\n//\n// # Safety\n//\n// - It must be ensured that `CALL_WITH_CONTEXT` <= 0x80 bytes.\n.org 0x000\n\tCALL_WITH_CONTEXT current_el0_synchronous\n.org 0x080\n\tCALL_WITH_CONTEXT current_el0_irq\n.org 0x100\n\tFIQ_SUSPEND\n.org 0x180\n\tCALL_WITH_CONTEXT current_el0_serror\n\n// Current exception level with SP_ELx, x > 0.\n.org 0x200\n\tCALL_WITH_CONTEXT current_elx_synchronous\n.org 0x280\n\tCALL_WITH_CONTEXT current_elx_irq\n.org 0x300\n\tFIQ_SUSPEND\n.org 0x380\n\tCALL_WITH_CONTEXT current_elx_serror\n\n// Lower exception level, AArch64\n.org 0x400\n\tCALL_WITH_CONTEXT lower_aarch64_synchronous\n.org 0x480\n\tCALL_WITH_CONTEXT lower_aarch64_irq\n.org 0x500\n\tFIQ_SUSPEND\n.org 0x580\n\tCALL_WITH_CONTEXT lower_aarch64_serror\n\n// Lower exception level, AArch32\n.org 0x600\n\tCALL_WITH_CONTEXT lower_aarch32_synchronous\n.org 0x680\n\tCALL_WITH_CONTEXT lower_aarch32_irq\n.org 0x700\n\tFIQ_SUSPEND\n.org 0x780\n\tCALL_WITH_CONTEXT lower_aarch32_serror\n.org 0x800\n\n//------------------------------------------------------------------------------\n// fn __exception_restore_context()\n//------------------------------------------------------------------------------\n__exception_restore_context:\n\tldr\tw19,      [sp, #16 * 16]\n\tldp\tlr,  x20, [sp, #16 * 15]\n\n\tmsr\tSPSR_EL1, x19\n\tmsr\tELR_EL1,  x20\n\n\tldp\tx0,  x1,  [sp, #16 * 0]\n\tldp\tx2,  x3,  [sp, #16 * 1]\n\tldp\tx4,  x5,  [sp, #16 * 2]\n\tldp\tx6,  x7,  [sp, #16 * 3]\n\tldp\tx8,  x9,  [sp, #16 * 4]\n\tldp\tx10, x11, [sp, #16 * 5]\n\tldp\tx12, x13, [sp, #16 * 6]\n\tldp\tx14, x15, [sp, #16 * 7]\n\tldp\tx16, x17, [sp, #16 * 8]\n\tldp\tx18, x19, [sp, #16 * 9]\n\tldp\tx20, x21, [sp, #16 * 10]\n\tldp\tx22, x23, [sp, #16 * 11]\n\tldp\tx24, x25, [sp, #16 * 12]\n\tldp\tx26, x27, [sp, #16 * 13]\n\tldp\tx28, x29, [sp, #16 * 14]\n\n\tadd\tsp,  sp,  #16 * 17\n\n\teret\n\n.size\t__exception_restore_context, . - __exception_restore_context\n.type\t__exception_restore_context, function\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/_arch/aarch64/memory/mmu/translation_table.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural translation table.\n//!\n//! Only 64 KiB granule is supported.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::memory::mmu::translation_table::arch_translation_table\n\nuse crate::{\n    bsp, memory,\n    memory::mmu::{\n        arch_mmu::{Granule512MiB, Granule64KiB},\n        AccessPermissions, AttributeFields, MemAttributes,\n    },\n};\nuse core::convert;\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields,\n    registers::InMemoryRegister,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// A table descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-15.\nregister_bitfields! {u64,\n    STAGE1_TABLE_DESCRIPTOR [\n        /// Physical address of the next descriptor.\n        NEXT_LEVEL_TABLE_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16]\n\n        TYPE  OFFSET(1) NUMBITS(1) [\n            Block = 0,\n            Table = 1\n        ],\n\n        VALID OFFSET(0) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ]\n    ]\n}\n\n// A level 3 page descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-17.\nregister_bitfields! {u64,\n    STAGE1_PAGE_DESCRIPTOR [\n        /// Unprivileged execute-never.\n        UXN      OFFSET(54) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ],\n\n        /// Privileged execute-never.\n        PXN      OFFSET(53) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ],\n\n        /// Physical address of the next table descriptor (lvl2) or the page descriptor (lvl3).\n        OUTPUT_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16]\n\n        /// Access flag.\n        AF       OFFSET(10) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ],\n\n        /// Shareability field.\n        SH       OFFSET(8) NUMBITS(2) [\n            OuterShareable = 0b10,\n            InnerShareable = 0b11\n        ],\n\n        /// Access Permissions.\n        AP       OFFSET(6) NUMBITS(2) [\n            RW_EL1 = 0b00,\n            RW_EL1_EL0 = 0b01,\n            RO_EL1 = 0b10,\n            RO_EL1_EL0 = 0b11\n        ],\n\n        /// Memory attributes index into the MAIR_EL1 register.\n        AttrIndx OFFSET(2) NUMBITS(3) [],\n\n        TYPE     OFFSET(1) NUMBITS(1) [\n            Reserved_Invalid = 0,\n            Page = 1\n        ],\n\n        VALID    OFFSET(0) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ]\n    ]\n}\n\n/// A table descriptor for 64 KiB aperture.\n///\n/// The output points to the next table.\n#[derive(Copy, Clone)]\n#[repr(C)]\nstruct TableDescriptor {\n    value: u64,\n}\n\n/// A page descriptor with 64 KiB aperture.\n///\n/// The output points to physical memory.\n#[derive(Copy, Clone)]\n#[repr(C)]\nstruct PageDescriptor {\n    value: u64,\n}\n\ntrait StartAddr {\n    fn phys_start_addr_u64(&self) -> u64;\n    fn phys_start_addr_usize(&self) -> usize;\n}\n\nconst NUM_LVL2_TABLES: usize = bsp::memory::mmu::KernelAddrSpace::SIZE >> Granule512MiB::SHIFT;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB\n/// aligned, so the lvl3 is put first.\n#[repr(C)]\n#[repr(align(65536))]\npub struct FixedSizeTranslationTable<const NUM_TABLES: usize> {\n    /// Page descriptors, covering 64 KiB windows per entry.\n    lvl3: [[PageDescriptor; 8192]; NUM_TABLES],\n\n    /// Table descriptors, covering 512 MiB windows.\n    lvl2: [TableDescriptor; NUM_TABLES],\n}\n\n/// A translation table type for the kernel space.\npub type KernelTranslationTable = FixedSizeTranslationTable<NUM_LVL2_TABLES>;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n// The binary is still identity mapped, so we don't need to convert here.\nimpl<T, const N: usize> StartAddr for [T; N] {\n    fn phys_start_addr_u64(&self) -> u64 {\n        self as *const T as u64\n    }\n\n    fn phys_start_addr_usize(&self) -> usize {\n        self as *const _ as usize\n    }\n}\n\nimpl TableDescriptor {\n    /// Create an instance.\n    ///\n    /// Descriptor is invalid by default.\n    pub const fn new_zeroed() -> Self {\n        Self { value: 0 }\n    }\n\n    /// Create an instance pointing to the supplied address.\n    pub fn from_next_lvl_table_addr(phys_next_lvl_table_addr: usize) -> Self {\n        let val = InMemoryRegister::<u64, STAGE1_TABLE_DESCRIPTOR::Register>::new(0);\n\n        let shifted = phys_next_lvl_table_addr >> Granule64KiB::SHIFT;\n        val.write(\n            STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64)\n                + STAGE1_TABLE_DESCRIPTOR::TYPE::Table\n                + STAGE1_TABLE_DESCRIPTOR::VALID::True,\n        );\n\n        TableDescriptor { value: val.get() }\n    }\n}\n\n/// Convert the kernel's generic memory attributes to HW-specific attributes of the MMU.\nimpl convert::From<AttributeFields>\n    for tock_registers::fields::FieldValue<u64, STAGE1_PAGE_DESCRIPTOR::Register>\n{\n    fn from(attribute_fields: AttributeFields) -> Self {\n        // Memory attributes.\n        let mut desc = match attribute_fields.mem_attributes {\n            MemAttributes::CacheableDRAM => {\n                STAGE1_PAGE_DESCRIPTOR::SH::InnerShareable\n                    + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::NORMAL)\n            }\n            MemAttributes::Device => {\n                STAGE1_PAGE_DESCRIPTOR::SH::OuterShareable\n                    + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::DEVICE)\n            }\n        };\n\n        // Access Permissions.\n        desc += match attribute_fields.acc_perms {\n            AccessPermissions::ReadOnly => STAGE1_PAGE_DESCRIPTOR::AP::RO_EL1,\n            AccessPermissions::ReadWrite => STAGE1_PAGE_DESCRIPTOR::AP::RW_EL1,\n        };\n\n        // The execute-never attribute is mapped to PXN in AArch64.\n        desc += if attribute_fields.execute_never {\n            STAGE1_PAGE_DESCRIPTOR::PXN::True\n        } else {\n            STAGE1_PAGE_DESCRIPTOR::PXN::False\n        };\n\n        // Always set unprivileged exectue-never as long as userspace is not implemented yet.\n        desc += STAGE1_PAGE_DESCRIPTOR::UXN::True;\n\n        desc\n    }\n}\n\nimpl PageDescriptor {\n    /// Create an instance.\n    ///\n    /// Descriptor is invalid by default.\n    pub const fn new_zeroed() -> Self {\n        Self { value: 0 }\n    }\n\n    /// Create an instance.\n    pub fn from_output_addr(phys_output_addr: usize, attribute_fields: &AttributeFields) -> Self {\n        let val = InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(0);\n\n        let shifted = phys_output_addr as u64 >> Granule64KiB::SHIFT;\n        val.write(\n            STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted)\n                + STAGE1_PAGE_DESCRIPTOR::AF::True\n                + STAGE1_PAGE_DESCRIPTOR::TYPE::Page\n                + STAGE1_PAGE_DESCRIPTOR::VALID::True\n                + (*attribute_fields).into(),\n        );\n\n        Self { value: val.get() }\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<const NUM_TABLES: usize> FixedSizeTranslationTable<NUM_TABLES> {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        // Can't have a zero-sized address space.\n        assert!(NUM_TABLES > 0);\n\n        Self {\n            lvl3: [[PageDescriptor::new_zeroed(); 8192]; NUM_TABLES],\n            lvl2: [TableDescriptor::new_zeroed(); NUM_TABLES],\n        }\n    }\n\n    /// Iterates over all static translation table entries and fills them at once.\n    ///\n    /// # Safety\n    ///\n    /// - Modifies a `static mut`. Ensure it only happens from here.\n    pub unsafe fn populate_tt_entries(&mut self) -> Result<(), &'static str> {\n        for (l2_nr, l2_entry) in self.lvl2.iter_mut().enumerate() {\n            *l2_entry =\n                TableDescriptor::from_next_lvl_table_addr(self.lvl3[l2_nr].phys_start_addr_usize());\n\n            for (l3_nr, l3_entry) in self.lvl3[l2_nr].iter_mut().enumerate() {\n                let virt_addr = (l2_nr << Granule512MiB::SHIFT) + (l3_nr << Granule64KiB::SHIFT);\n\n                let (phys_output_addr, attribute_fields) =\n                    bsp::memory::mmu::virt_mem_layout().virt_addr_properties(virt_addr)?;\n\n                *l3_entry = PageDescriptor::from_output_addr(phys_output_addr, &attribute_fields);\n            }\n        }\n\n        Ok(())\n    }\n\n    /// The translation table's base address to be used for programming the MMU.\n    pub fn phys_base_address(&self) -> u64 {\n        self.lvl2.phys_start_addr_u64()\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// Check if the size of `struct TableDescriptor` is as expected.\n    #[kernel_test]\n    fn size_of_tabledescriptor_equals_64_bit() {\n        assert_eq!(\n            core::mem::size_of::<TableDescriptor>(),\n            core::mem::size_of::<u64>()\n        );\n    }\n\n    /// Check if the size of `struct PageDescriptor` is as expected.\n    #[kernel_test]\n    fn size_of_pagedescriptor_equals_64_bit() {\n        assert_eq!(\n            core::mem::size_of::<PageDescriptor>(),\n            core::mem::size_of::<u64>()\n        );\n    }\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/_arch/aarch64/memory/mmu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management Unit Driver.\n//!\n//! Only 64 KiB granule is supported.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::memory::mmu::arch_mmu\n\nuse crate::{\n    bsp, memory,\n    memory::mmu::{translation_table::KernelTranslationTable, TranslationGranule},\n};\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::intrinsics::unlikely;\nuse tock_registers::interfaces::{ReadWriteable, Readable, Writeable};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Memory Management Unit type.\nstruct MemoryManagementUnit;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub type Granule512MiB = TranslationGranule<{ 512 * 1024 * 1024 }>;\npub type Granule64KiB = TranslationGranule<{ 64 * 1024 }>;\n\n/// Constants for indexing the MAIR_EL1.\n#[allow(dead_code)]\npub mod mair {\n    pub const DEVICE: u64 = 0;\n    pub const NORMAL: u64 = 1;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n/// The kernel translation tables.\n///\n/// # Safety\n///\n/// - Supposed to land in `.bss`. Therefore, ensure that all initial member values boil down to \"0\".\nstatic mut KERNEL_TABLES: KernelTranslationTable = KernelTranslationTable::new();\n\nstatic MMU: MemoryManagementUnit = MemoryManagementUnit;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<const AS_SIZE: usize> memory::mmu::AddressSpace<AS_SIZE> {\n    /// Checks for architectural restrictions.\n    pub const fn arch_address_space_size_sanity_checks() {\n        // Size must be at least one full 512 MiB table.\n        assert!((AS_SIZE % Granule512MiB::SIZE) == 0);\n\n        // Check for 48 bit virtual address size as maximum, which is supported by any ARMv8\n        // version.\n        assert!(AS_SIZE <= (1 << 48));\n    }\n}\n\nimpl MemoryManagementUnit {\n    /// Setup function for the MAIR_EL1 register.\n    fn set_up_mair(&self) {\n        // Define the memory types being mapped.\n        MAIR_EL1.write(\n            // Attribute 1 - Cacheable normal DRAM.\n            MAIR_EL1::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc +\n        MAIR_EL1::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc +\n\n        // Attribute 0 - Device.\n        MAIR_EL1::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck,\n        );\n    }\n\n    /// Configure various settings of stage 1 of the EL1 translation regime.\n    fn configure_translation_control(&self) {\n        let t0sz = (64 - bsp::memory::mmu::KernelAddrSpace::SIZE_SHIFT) as u64;\n\n        TCR_EL1.write(\n            TCR_EL1::TBI0::Used\n                + TCR_EL1::IPS::Bits_40\n                + TCR_EL1::TG0::KiB_64\n                + TCR_EL1::SH0::Inner\n                + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable\n                + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable\n                + TCR_EL1::EPD0::EnableTTBR0Walks\n                + TCR_EL1::A1::TTBR0\n                + TCR_EL1::T0SZ.val(t0sz)\n                + TCR_EL1::EPD1::DisableTTBR1Walks,\n        );\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the MMU instance.\npub fn mmu() -> &'static impl memory::mmu::interface::MMU {\n    &MMU\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse memory::mmu::MMUEnableError;\n\nimpl memory::mmu::interface::MMU for MemoryManagementUnit {\n    unsafe fn enable_mmu_and_caching(&self) -> Result<(), MMUEnableError> {\n        if unlikely(self.is_enabled()) {\n            return Err(MMUEnableError::AlreadyEnabled);\n        }\n\n        // Fail early if translation granule is not supported.\n        if unlikely(!ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran64::Supported)) {\n            return Err(MMUEnableError::Other(\n                \"Translation granule not supported in HW\",\n            ));\n        }\n\n        // Prepare the memory attribute indirection register.\n        self.set_up_mair();\n\n        // Populate translation tables.\n        KERNEL_TABLES\n            .populate_tt_entries()\n            .map_err(MMUEnableError::Other)?;\n\n        // Set the \"Translation Table Base Register\".\n        TTBR0_EL1.set_baddr(KERNEL_TABLES.phys_base_address());\n\n        self.configure_translation_control();\n\n        // Switch the MMU on.\n        //\n        // First, force all previous changes to be seen before the MMU is enabled.\n        barrier::isb(barrier::SY);\n\n        // Enable the MMU and turn on data and instruction caching.\n        SCTLR_EL1.modify(SCTLR_EL1::M::Enable + SCTLR_EL1::C::Cacheable + SCTLR_EL1::I::Cacheable);\n\n        // Force MMU init to complete before next instruction.\n        barrier::isb(barrier::SY);\n\n        Ok(())\n    }\n\n    #[inline(always)]\n    fn is_enabled(&self) -> bool {\n        SCTLR_EL1.matches_all(SCTLR_EL1::M::Enable)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use core::{cell::UnsafeCell, ops::Range};\n    use test_macros::kernel_test;\n\n    /// Check if KERNEL_TABLES is in .bss.\n    #[kernel_test]\n    fn kernel_tables_in_bss() {\n        extern \"Rust\" {\n            static __bss_start: UnsafeCell<u64>;\n            static __bss_end_exclusive: UnsafeCell<u64>;\n        }\n\n        let bss_range = unsafe {\n            Range {\n                start: __bss_start.get(),\n                end: __bss_end_exclusive.get(),\n            }\n        };\n        let kernel_tables_addr = unsafe { &KERNEL_TABLES as *const _ as usize as *mut u64 };\n\n        assert!(bss_range.contains(&kernel_tables_addr));\n    }\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/_arch/aarch64/time.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural timer primitives.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::time::arch_time\n\nuse crate::warn;\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::{\n    num::{NonZeroU128, NonZeroU32, NonZeroU64},\n    ops::{Add, Div},\n    time::Duration,\n};\nuse tock_registers::interfaces::Readable;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nconst NANOSEC_PER_SEC: NonZeroU64 = NonZeroU64::new(1_000_000_000).unwrap();\n\n#[derive(Copy, Clone, PartialOrd, PartialEq)]\nstruct GenericTimerCounterValue(u64);\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n/// Boot assembly code overwrites this value with the value of CNTFRQ_EL0 before any Rust code is\n/// executed. This given value here is just a (safe) dummy.\n#[no_mangle]\nstatic ARCH_TIMER_COUNTER_FREQUENCY: NonZeroU32 = NonZeroU32::MIN;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nfn arch_timer_counter_frequency() -> NonZeroU32 {\n    // Read volatile is needed here to prevent the compiler from optimizing\n    // ARCH_TIMER_COUNTER_FREQUENCY away.\n    //\n    // This is safe, because all the safety requirements as stated in read_volatile()'s\n    // documentation are fulfilled.\n    unsafe { core::ptr::read_volatile(&ARCH_TIMER_COUNTER_FREQUENCY) }\n}\n\nimpl GenericTimerCounterValue {\n    pub const MAX: Self = GenericTimerCounterValue(u64::MAX);\n}\n\nimpl Add for GenericTimerCounterValue {\n    type Output = Self;\n\n    fn add(self, other: Self) -> Self {\n        GenericTimerCounterValue(self.0.wrapping_add(other.0))\n    }\n}\n\nimpl From<GenericTimerCounterValue> for Duration {\n    fn from(counter_value: GenericTimerCounterValue) -> Self {\n        if counter_value.0 == 0 {\n            return Duration::ZERO;\n        }\n\n        let frequency: NonZeroU64 = arch_timer_counter_frequency().into();\n\n        // Div<NonZeroU64> implementation for u64 cannot panic.\n        let secs = counter_value.0.div(frequency);\n\n        // This is safe, because frequency can never be greater than u32::MAX, which means the\n        // largest theoretical value for sub_second_counter_value is (u32::MAX - 1). Therefore,\n        // (sub_second_counter_value * NANOSEC_PER_SEC) cannot overflow an u64.\n        //\n        // The subsequent division ensures the result fits into u32, since the max result is smaller\n        // than NANOSEC_PER_SEC. Therefore, just cast it to u32 using `as`.\n        let sub_second_counter_value = counter_value.0 % frequency;\n        let nanos = unsafe { sub_second_counter_value.unchecked_mul(u64::from(NANOSEC_PER_SEC)) }\n            .div(frequency) as u32;\n\n        Duration::new(secs, nanos)\n    }\n}\n\nfn max_duration() -> Duration {\n    Duration::from(GenericTimerCounterValue::MAX)\n}\n\nimpl TryFrom<Duration> for GenericTimerCounterValue {\n    type Error = &'static str;\n\n    fn try_from(duration: Duration) -> Result<Self, Self::Error> {\n        if duration < resolution() {\n            return Ok(GenericTimerCounterValue(0));\n        }\n\n        if duration > max_duration() {\n            return Err(\"Conversion error. Duration too big\");\n        }\n\n        let frequency: u128 = u32::from(arch_timer_counter_frequency()) as u128;\n        let duration: u128 = duration.as_nanos();\n\n        // This is safe, because frequency can never be greater than u32::MAX, and\n        // (Duration::MAX.as_nanos() * u32::MAX) < u128::MAX.\n        let counter_value =\n            unsafe { duration.unchecked_mul(frequency) }.div(NonZeroU128::from(NANOSEC_PER_SEC));\n\n        // Since we checked above that we are <= max_duration(), just cast to u64.\n        Ok(GenericTimerCounterValue(counter_value as u64))\n    }\n}\n\n#[inline(always)]\nfn read_cntpct() -> GenericTimerCounterValue {\n    // Prevent that the counter is read ahead of time due to out-of-order execution.\n    barrier::isb(barrier::SY);\n    let cnt = CNTPCT_EL0.get();\n\n    GenericTimerCounterValue(cnt)\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The timer's resolution.\npub fn resolution() -> Duration {\n    Duration::from(GenericTimerCounterValue(1))\n}\n\n/// The uptime since power-on of the device.\n///\n/// This includes time consumed by firmware and bootloaders.\npub fn uptime() -> Duration {\n    read_cntpct().into()\n}\n\n/// Spin for a given duration.\npub fn spin_for(duration: Duration) {\n    let curr_counter_value = read_cntpct();\n\n    let counter_value_delta: GenericTimerCounterValue = match duration.try_into() {\n        Err(msg) => {\n            warn!(\"spin_for: {}. Skipping\", msg);\n            return;\n        }\n        Ok(val) => val,\n    };\n    let counter_value_target = curr_counter_value + counter_value_delta;\n\n    // Busy wait.\n    //\n    // Read CNTPCT_EL0 directly to avoid the ISB that is part of [`read_cntpct`].\n    while GenericTimerCounterValue(CNTPCT_EL0.get()) < counter_value_target {}\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/arm/gicv2/gicc.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GICC Driver - GIC CPU interface.\n\nuse crate::{bsp::device_driver::common::MMIODerefWrapper, exception};\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields, register_structs,\n    registers::ReadWrite,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nregister_bitfields! {\n    u32,\n\n    /// CPU Interface Control Register\n    CTLR [\n        Enable OFFSET(0) NUMBITS(1) []\n    ],\n\n    /// Interrupt Priority Mask Register\n    PMR [\n        Priority OFFSET(0) NUMBITS(8) []\n    ],\n\n    /// Interrupt Acknowledge Register\n    IAR [\n        InterruptID OFFSET(0) NUMBITS(10) []\n    ],\n\n    /// End of Interrupt Register\n    EOIR [\n        EOIINTID OFFSET(0) NUMBITS(10) []\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    pub RegisterBlock {\n        (0x000 => CTLR: ReadWrite<u32, CTLR::Register>),\n        (0x004 => PMR: ReadWrite<u32, PMR::Register>),\n        (0x008 => _reserved1),\n        (0x00C => IAR: ReadWrite<u32, IAR::Register>),\n        (0x010 => EOIR: ReadWrite<u32, EOIR::Register>),\n        (0x014  => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the GIC CPU interface.\npub struct GICC {\n    registers: Registers,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GICC {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n        }\n    }\n\n    /// Accept interrupts of any priority.\n    ///\n    /// Quoting the GICv2 Architecture Specification:\n    ///\n    ///   \"Writing 255 to the GICC_PMR always sets it to the largest supported priority field\n    ///    value.\"\n    ///\n    /// # Safety\n    ///\n    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n    ///   of `&mut self`.\n    pub fn priority_accept_all(&self) {\n        self.registers.PMR.write(PMR::Priority.val(255)); // Comment in arch spec.\n    }\n\n    /// Enable the interface - start accepting IRQs.\n    ///\n    /// # Safety\n    ///\n    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n    ///   of `&mut self`.\n    pub fn enable(&self) {\n        self.registers.CTLR.write(CTLR::Enable::SET);\n    }\n\n    /// Extract the number of the highest-priority pending IRQ.\n    ///\n    /// Can only be called from IRQ context, which is ensured by taking an `IRQContext` token.\n    ///\n    /// # Safety\n    ///\n    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n    ///   of `&mut self`.\n    #[allow(clippy::trivially_copy_pass_by_ref)]\n    pub fn pending_irq_number<'irq_context>(\n        &self,\n        _ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) -> usize {\n        self.registers.IAR.read(IAR::InterruptID) as usize\n    }\n\n    /// Complete handling of the currently active IRQ.\n    ///\n    /// Can only be called from IRQ context, which is ensured by taking an `IRQContext` token.\n    ///\n    /// To be called after `pending_irq_number()`.\n    ///\n    /// # Safety\n    ///\n    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n    ///   of `&mut self`.\n    #[allow(clippy::trivially_copy_pass_by_ref)]\n    pub fn mark_comleted<'irq_context>(\n        &self,\n        irq_number: u32,\n        _ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) {\n        self.registers.EOIR.write(EOIR::EOIINTID.val(irq_number));\n    }\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/arm/gicv2/gicd.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GICD Driver - GIC Distributor.\n//!\n//! # Glossary\n//!   - SPI - Shared Peripheral Interrupt.\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper, state, synchronization,\n    synchronization::IRQSafeNullLock,\n};\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields, register_structs,\n    registers::{ReadOnly, ReadWrite},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nregister_bitfields! {\n    u32,\n\n    /// Distributor Control Register\n    CTLR [\n        Enable OFFSET(0) NUMBITS(1) []\n    ],\n\n    /// Interrupt Controller Type Register\n    TYPER [\n        ITLinesNumber OFFSET(0)  NUMBITS(5) []\n    ],\n\n    /// Interrupt Processor Targets Registers\n    ITARGETSR [\n        Offset3 OFFSET(24) NUMBITS(8) [],\n        Offset2 OFFSET(16) NUMBITS(8) [],\n        Offset1 OFFSET(8)  NUMBITS(8) [],\n        Offset0 OFFSET(0)  NUMBITS(8) []\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    SharedRegisterBlock {\n        (0x000 => CTLR: ReadWrite<u32, CTLR::Register>),\n        (0x004 => TYPER: ReadOnly<u32, TYPER::Register>),\n        (0x008 => _reserved1),\n        (0x104 => ISENABLER: [ReadWrite<u32>; 31]),\n        (0x180 => _reserved2),\n        (0x820 => ITARGETSR: [ReadWrite<u32, ITARGETSR::Register>; 248]),\n        (0xC00 => @END),\n    }\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    BankedRegisterBlock {\n        (0x000 => _reserved1),\n        (0x100 => ISENABLER: ReadWrite<u32>),\n        (0x104 => _reserved2),\n        (0x800 => ITARGETSR: [ReadOnly<u32, ITARGETSR::Register>; 8]),\n        (0x820 => @END),\n    }\n}\n\n/// Abstraction for the non-banked parts of the associated MMIO registers.\ntype SharedRegisters = MMIODerefWrapper<SharedRegisterBlock>;\n\n/// Abstraction for the banked parts of the associated MMIO registers.\ntype BankedRegisters = MMIODerefWrapper<BankedRegisterBlock>;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the GIC Distributor.\npub struct GICD {\n    /// Access to shared registers is guarded with a lock.\n    shared_registers: IRQSafeNullLock<SharedRegisters>,\n\n    /// Access to banked registers is unguarded.\n    banked_registers: BankedRegisters,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl SharedRegisters {\n    /// Return the number of IRQs that this HW implements.\n    #[inline(always)]\n    fn num_irqs(&mut self) -> usize {\n        // Query number of implemented IRQs.\n        //\n        // Refer to GICv2 Architecture Specification, Section 4.3.2.\n        ((self.TYPER.read(TYPER::ITLinesNumber) as usize) + 1) * 32\n    }\n\n    /// Return a slice of the implemented ITARGETSR.\n    #[inline(always)]\n    fn implemented_itargets_slice(&mut self) -> &[ReadWrite<u32, ITARGETSR::Register>] {\n        assert!(self.num_irqs() >= 36);\n\n        // Calculate the max index of the shared ITARGETSR array.\n        //\n        // The first 32 IRQs are private, so not included in `shared_registers`. Each ITARGETS\n        // register has four entries, so shift right by two. Subtract one because we start\n        // counting at zero.\n        let spi_itargetsr_max_index = ((self.num_irqs() - 32) >> 2) - 1;\n\n        // Rust automatically inserts slice range sanity check, i.e. max >= min.\n        &self.ITARGETSR[0..spi_itargetsr_max_index]\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl GICD {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            shared_registers: IRQSafeNullLock::new(SharedRegisters::new(mmio_start_addr)),\n            banked_registers: BankedRegisters::new(mmio_start_addr),\n        }\n    }\n\n    /// Use a banked ITARGETSR to retrieve the executing core's GIC target mask.\n    ///\n    /// Quoting the GICv2 Architecture Specification:\n    ///\n    ///   \"GICD_ITARGETSR0 to GICD_ITARGETSR7 are read-only, and each field returns a value that\n    ///    corresponds only to the processor reading the register.\"\n    fn local_gic_target_mask(&self) -> u32 {\n        self.banked_registers.ITARGETSR[0].read(ITARGETSR::Offset0)\n    }\n\n    /// Route all SPIs to the boot core and enable the distributor.\n    pub fn boot_core_init(&self) {\n        assert!(\n            state::state_manager().is_init(),\n            \"Only allowed during kernel init phase\"\n        );\n\n        // Target all SPIs to the boot core only.\n        let mask = self.local_gic_target_mask();\n\n        self.shared_registers.lock(|regs| {\n            for i in regs.implemented_itargets_slice().iter() {\n                i.write(\n                    ITARGETSR::Offset3.val(mask)\n                        + ITARGETSR::Offset2.val(mask)\n                        + ITARGETSR::Offset1.val(mask)\n                        + ITARGETSR::Offset0.val(mask),\n                );\n            }\n\n            regs.CTLR.write(CTLR::Enable::SET);\n        });\n    }\n\n    /// Enable an interrupt.\n    pub fn enable(&self, irq_num: &super::IRQNumber) {\n        let irq_num = irq_num.get();\n\n        // Each bit in the u32 enable register corresponds to one IRQ number. Shift right by 5\n        // (division by 32) and arrive at the index for the respective ISENABLER[i].\n        let enable_reg_index = irq_num >> 5;\n        let enable_bit: u32 = 1u32 << (irq_num % 32);\n\n        // Check if we are handling a private or shared IRQ.\n        match irq_num {\n            // Private.\n            0..=31 => {\n                let enable_reg = &self.banked_registers.ISENABLER;\n                enable_reg.set(enable_reg.get() | enable_bit);\n            }\n            // Shared.\n            _ => {\n                let enable_reg_index_shared = enable_reg_index - 1;\n\n                self.shared_registers.lock(|regs| {\n                    let enable_reg = &regs.ISENABLER[enable_reg_index_shared];\n                    enable_reg.set(enable_reg.get() | enable_bit);\n                });\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/arm/gicv2.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GICv2 Driver - ARM Generic Interrupt Controller v2.\n//!\n//! The following is a collection of excerpts with useful information from\n//!   - `Programmer's Guide for ARMv8-A`\n//!   - `ARM Generic Interrupt Controller Architecture Specification`\n//!\n//! # Programmer's Guide - 10.6.1 Configuration\n//!\n//! The GIC is accessed as a memory-mapped peripheral.\n//!\n//! All cores can access the common Distributor, but the CPU interface is banked, that is, each core\n//! uses the same address to access its own private CPU interface.\n//!\n//! It is not possible for a core to access the CPU interface of another core.\n//!\n//! # Architecture Specification - 10.6.2 Initialization\n//!\n//! Both the Distributor and the CPU interfaces are disabled at reset. The GIC must be initialized\n//! after reset before it can deliver interrupts to the core.\n//!\n//! In the Distributor, software must configure the priority, target, security and enable individual\n//! interrupts. The Distributor must subsequently be enabled through its control register\n//! (GICD_CTLR). For each CPU interface, software must program the priority mask and preemption\n//! settings.\n//!\n//! Each CPU interface block itself must be enabled through its control register (GICD_CTLR). This\n//! prepares the GIC to deliver interrupts to the core.\n//!\n//! Before interrupts are expected in the core, software prepares the core to take interrupts by\n//! setting a valid interrupt vector in the vector table, and clearing interrupt mask bits in\n//! PSTATE, and setting the routing controls.\n//!\n//! The entire interrupt mechanism in the system can be disabled by disabling the Distributor.\n//! Interrupt delivery to an individual core can be disabled by disabling its CPU interface.\n//! Individual interrupts can also be disabled (or enabled) in the distributor.\n//!\n//! For an interrupt to reach the core, the individual interrupt, Distributor and CPU interface must\n//! all be enabled. The interrupt also needs to be of sufficient priority, that is, higher than the\n//! core's priority mask.\n//!\n//! # Architecture Specification - 1.4.2 Interrupt types\n//!\n//! - Peripheral interrupt\n//!     - Private Peripheral Interrupt (PPI)\n//!         - This is a peripheral interrupt that is specific to a single processor.\n//!     - Shared Peripheral Interrupt (SPI)\n//!         - This is a peripheral interrupt that the Distributor can route to any of a specified\n//!           combination of processors.\n//!\n//! - Software-generated interrupt (SGI)\n//!     - This is an interrupt generated by software writing to a GICD_SGIR register in the GIC. The\n//!       system uses SGIs for interprocessor communication.\n//!     - An SGI has edge-triggered properties. The software triggering of the interrupt is\n//!       equivalent to the edge transition of the interrupt request signal.\n//!     - When an SGI occurs in a multiprocessor implementation, the CPUID field in the Interrupt\n//!       Acknowledge Register, GICC_IAR, or the Aliased Interrupt Acknowledge Register, GICC_AIAR,\n//!       identifies the processor that requested the interrupt.\n//!\n//! # Architecture Specification - 2.2.1 Interrupt IDs\n//!\n//! Interrupts from sources are identified using ID numbers. Each CPU interface can see up to 1020\n//! interrupts. The banking of SPIs and PPIs increases the total number of interrupts supported by\n//! the Distributor.\n//!\n//! The GIC assigns interrupt ID numbers ID0-ID1019 as follows:\n//!   - Interrupt numbers 32..1019 are used for SPIs.\n//!   - Interrupt numbers 0..31 are used for interrupts that are private to a CPU interface. These\n//!     interrupts are banked in the Distributor.\n//!       - A banked interrupt is one where the Distributor can have multiple interrupts with the\n//!         same ID. A banked interrupt is identified uniquely by its ID number and its associated\n//!         CPU interface number. Of the banked interrupt IDs:\n//!           - 00..15 SGIs\n//!           - 16..31 PPIs\n\nmod gicc;\nmod gicd;\n\nuse crate::{\n    bsp::{self, device_driver::common::BoundedUsize},\n    cpu, driver, exception, synchronization,\n    synchronization::InitStateLock,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\ntype HandlerTable = [Option<exception::asynchronous::IRQHandlerDescriptor<IRQNumber>>;\n    IRQNumber::MAX_INCLUSIVE + 1];\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Used for the associated type of trait [`exception::asynchronous::interface::IRQManager`].\npub type IRQNumber = BoundedUsize<{ GICv2::MAX_IRQ_NUMBER }>;\n\n/// Representation of the GIC.\npub struct GICv2 {\n    /// The Distributor.\n    gicd: gicd::GICD,\n\n    /// The CPU Interface.\n    gicc: gicc::GICC,\n\n    /// Stores registered IRQ handlers. Writable only during kernel init. RO afterwards.\n    handler_table: InitStateLock<HandlerTable>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GICv2 {\n    const MAX_IRQ_NUMBER: usize = 300; // Normally 1019, but keep it lower to save some space.\n\n    pub const COMPATIBLE: &'static str = \"GICv2 (ARM Generic Interrupt Controller v2)\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(gicd_mmio_start_addr: usize, gicc_mmio_start_addr: usize) -> Self {\n        Self {\n            gicd: gicd::GICD::new(gicd_mmio_start_addr),\n            gicc: gicc::GICC::new(gicc_mmio_start_addr),\n            handler_table: InitStateLock::new([None; IRQNumber::MAX_INCLUSIVE + 1]),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::ReadWriteEx;\n\nimpl driver::interface::DeviceDriver for GICv2 {\n    type IRQNumberType = IRQNumber;\n\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n\n    unsafe fn init(&self) -> Result<(), &'static str> {\n        if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() {\n            self.gicd.boot_core_init();\n        }\n\n        self.gicc.priority_accept_all();\n        self.gicc.enable();\n\n        Ok(())\n    }\n}\n\nimpl exception::asynchronous::interface::IRQManager for GICv2 {\n    type IRQNumberType = IRQNumber;\n\n    fn register_handler(\n        &self,\n        irq_handler_descriptor: exception::asynchronous::IRQHandlerDescriptor<Self::IRQNumberType>,\n    ) -> Result<(), &'static str> {\n        self.handler_table.write(|table| {\n            let irq_number = irq_handler_descriptor.number().get();\n\n            if table[irq_number].is_some() {\n                return Err(\"IRQ handler already registered\");\n            }\n\n            table[irq_number] = Some(irq_handler_descriptor);\n\n            Ok(())\n        })\n    }\n\n    fn enable(&self, irq_number: &Self::IRQNumberType) {\n        self.gicd.enable(irq_number);\n    }\n\n    fn handle_pending_irqs<'irq_context>(\n        &'irq_context self,\n        ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) {\n        // Extract the highest priority pending IRQ number from the Interrupt Acknowledge Register\n        // (IAR).\n        let irq_number = self.gicc.pending_irq_number(ic);\n\n        // Guard against spurious interrupts.\n        if irq_number > GICv2::MAX_IRQ_NUMBER {\n            return;\n        }\n\n        // Call the IRQ handler. Panic if there is none.\n        self.handler_table.read(|table| {\n            match table[irq_number] {\n                None => panic!(\"No handler registered for IRQ {}\", irq_number),\n                Some(descriptor) => {\n                    // Call the IRQ handler. Panics on failure.\n                    descriptor.handler().handle().expect(\"Error handling IRQ\");\n                }\n            }\n        });\n\n        // Signal completion of handling.\n        self.gicc.mark_comleted(irq_number as u32, ic);\n    }\n\n    fn print_handler(&self) {\n        use crate::info;\n\n        info!(\"      Peripheral handler:\");\n\n        self.handler_table.read(|table| {\n            for (i, opt) in table.iter().skip(32).enumerate() {\n                if let Some(handler) = opt {\n                    info!(\"            {: >3}. {}\", i + 32, handler.name());\n                }\n            }\n        });\n    }\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/arm.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! ARM driver top level.\n\npub mod gicv2;\n\npub use gicv2::*;\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GPIO Driver.\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper, driver, exception::asynchronous::IRQNumber,\n    synchronization, synchronization::IRQSafeNullLock,\n};\nuse tock_registers::{\n    interfaces::{ReadWriteable, Writeable},\n    register_bitfields, register_structs,\n    registers::ReadWrite,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// GPIO registers.\n//\n// Descriptions taken from\n// - https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf\n// - https://datasheets.raspberrypi.org/bcm2711/bcm2711-peripherals.pdf\nregister_bitfields! {\n    u32,\n\n    /// GPIO Function Select 1\n    GPFSEL1 [\n        /// Pin 15\n        FSEL15 OFFSET(15) NUMBITS(3) [\n            Input = 0b000,\n            Output = 0b001,\n            AltFunc0 = 0b100  // PL011 UART RX\n\n        ],\n\n        /// Pin 14\n        FSEL14 OFFSET(12) NUMBITS(3) [\n            Input = 0b000,\n            Output = 0b001,\n            AltFunc0 = 0b100  // PL011 UART TX\n        ]\n    ],\n\n    /// GPIO Pull-up/down Register\n    ///\n    /// BCM2837 only.\n    GPPUD [\n        /// Controls the actuation of the internal pull-up/down control line to ALL the GPIO pins.\n        PUD OFFSET(0) NUMBITS(2) [\n            Off = 0b00,\n            PullDown = 0b01,\n            PullUp = 0b10\n        ]\n    ],\n\n    /// GPIO Pull-up/down Clock Register 0\n    ///\n    /// BCM2837 only.\n    GPPUDCLK0 [\n        /// Pin 15\n        PUDCLK15 OFFSET(15) NUMBITS(1) [\n            NoEffect = 0,\n            AssertClock = 1\n        ],\n\n        /// Pin 14\n        PUDCLK14 OFFSET(14) NUMBITS(1) [\n            NoEffect = 0,\n            AssertClock = 1\n        ]\n    ],\n\n    /// GPIO Pull-up / Pull-down Register 0\n    ///\n    /// BCM2711 only.\n    GPIO_PUP_PDN_CNTRL_REG0 [\n        /// Pin 15\n        GPIO_PUP_PDN_CNTRL15 OFFSET(30) NUMBITS(2) [\n            NoResistor = 0b00,\n            PullUp = 0b01\n        ],\n\n        /// Pin 14\n        GPIO_PUP_PDN_CNTRL14 OFFSET(28) NUMBITS(2) [\n            NoResistor = 0b00,\n            PullUp = 0b01\n        ]\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    RegisterBlock {\n        (0x00 => _reserved1),\n        (0x04 => GPFSEL1: ReadWrite<u32, GPFSEL1::Register>),\n        (0x08 => _reserved2),\n        (0x94 => GPPUD: ReadWrite<u32, GPPUD::Register>),\n        (0x98 => GPPUDCLK0: ReadWrite<u32, GPPUDCLK0::Register>),\n        (0x9C => _reserved3),\n        (0xE4 => GPIO_PUP_PDN_CNTRL_REG0: ReadWrite<u32, GPIO_PUP_PDN_CNTRL_REG0::Register>),\n        (0xE8 => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\nstruct GPIOInner {\n    registers: Registers,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the GPIO HW.\npub struct GPIO {\n    inner: IRQSafeNullLock<GPIOInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GPIOInner {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n        }\n    }\n\n    /// Disable pull-up/down on pins 14 and 15.\n    #[cfg(feature = \"bsp_rpi3\")]\n    fn disable_pud_14_15_bcm2837(&mut self) {\n        use crate::time;\n        use core::time::Duration;\n\n        // The Linux 2837 GPIO driver waits 1 µs between the steps.\n        const DELAY: Duration = Duration::from_micros(1);\n\n        self.registers.GPPUD.write(GPPUD::PUD::Off);\n        time::time_manager().spin_for(DELAY);\n\n        self.registers\n            .GPPUDCLK0\n            .write(GPPUDCLK0::PUDCLK15::AssertClock + GPPUDCLK0::PUDCLK14::AssertClock);\n        time::time_manager().spin_for(DELAY);\n\n        self.registers.GPPUD.write(GPPUD::PUD::Off);\n        self.registers.GPPUDCLK0.set(0);\n    }\n\n    /// Disable pull-up/down on pins 14 and 15.\n    #[cfg(feature = \"bsp_rpi4\")]\n    fn disable_pud_14_15_bcm2711(&mut self) {\n        self.registers.GPIO_PUP_PDN_CNTRL_REG0.write(\n            GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL15::PullUp\n                + GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL14::PullUp,\n        );\n    }\n\n    /// Map PL011 UART as standard output.\n    ///\n    /// TX to pin 14\n    /// RX to pin 15\n    pub fn map_pl011_uart(&mut self) {\n        // Select the UART on pins 14 and 15.\n        self.registers\n            .GPFSEL1\n            .modify(GPFSEL1::FSEL15::AltFunc0 + GPFSEL1::FSEL14::AltFunc0);\n\n        // Disable pull-up/down on pins 14 and 15.\n        #[cfg(feature = \"bsp_rpi3\")]\n        self.disable_pud_14_15_bcm2837();\n\n        #[cfg(feature = \"bsp_rpi4\")]\n        self.disable_pud_14_15_bcm2711();\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GPIO {\n    pub const COMPATIBLE: &'static str = \"BCM GPIO\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            inner: IRQSafeNullLock::new(GPIOInner::new(mmio_start_addr)),\n        }\n    }\n\n    /// Concurrency safe version of `GPIOInner.map_pl011_uart()`\n    pub fn map_pl011_uart(&self) {\n        self.inner.lock(|inner| inner.map_pl011_uart())\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl driver::interface::DeviceDriver for GPIO {\n    type IRQNumberType = IRQNumber;\n\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Peripheral Interrupt Controller Driver.\n//!\n//! # Resources\n//!\n//! - <https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf>\n\nuse super::{PendingIRQs, PeripheralIRQ};\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    exception, synchronization,\n    synchronization::{IRQSafeNullLock, InitStateLock},\n};\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_structs,\n    registers::{ReadOnly, WriteOnly},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    WORegisterBlock {\n        (0x00 => _reserved1),\n        (0x10 => ENABLE_1: WriteOnly<u32>),\n        (0x14 => ENABLE_2: WriteOnly<u32>),\n        (0x18 => @END),\n    }\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    RORegisterBlock {\n        (0x00 => _reserved1),\n        (0x04 => PENDING_1: ReadOnly<u32>),\n        (0x08 => PENDING_2: ReadOnly<u32>),\n        (0x0c => @END),\n    }\n}\n\n/// Abstraction for the WriteOnly parts of the associated MMIO registers.\ntype WriteOnlyRegisters = MMIODerefWrapper<WORegisterBlock>;\n\n/// Abstraction for the ReadOnly parts of the associated MMIO registers.\ntype ReadOnlyRegisters = MMIODerefWrapper<RORegisterBlock>;\n\ntype HandlerTable = [Option<exception::asynchronous::IRQHandlerDescriptor<PeripheralIRQ>>;\n    PeripheralIRQ::MAX_INCLUSIVE + 1];\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the peripheral interrupt controller.\npub struct PeripheralIC {\n    /// Access to write registers is guarded with a lock.\n    wo_registers: IRQSafeNullLock<WriteOnlyRegisters>,\n\n    /// Register read access is unguarded.\n    ro_registers: ReadOnlyRegisters,\n\n    /// Stores registered IRQ handlers. Writable only during kernel init. RO afterwards.\n    handler_table: InitStateLock<HandlerTable>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PeripheralIC {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            wo_registers: IRQSafeNullLock::new(WriteOnlyRegisters::new(mmio_start_addr)),\n            ro_registers: ReadOnlyRegisters::new(mmio_start_addr),\n            handler_table: InitStateLock::new([None; PeripheralIRQ::MAX_INCLUSIVE + 1]),\n        }\n    }\n\n    /// Query the list of pending IRQs.\n    fn pending_irqs(&self) -> PendingIRQs {\n        let pending_mask: u64 = (u64::from(self.ro_registers.PENDING_2.get()) << 32)\n            | u64::from(self.ro_registers.PENDING_1.get());\n\n        PendingIRQs::new(pending_mask)\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::{Mutex, ReadWriteEx};\n\nimpl exception::asynchronous::interface::IRQManager for PeripheralIC {\n    type IRQNumberType = PeripheralIRQ;\n\n    fn register_handler(\n        &self,\n        irq_handler_descriptor: exception::asynchronous::IRQHandlerDescriptor<Self::IRQNumberType>,\n    ) -> Result<(), &'static str> {\n        self.handler_table.write(|table| {\n            let irq_number = irq_handler_descriptor.number().get();\n\n            if table[irq_number].is_some() {\n                return Err(\"IRQ handler already registered\");\n            }\n\n            table[irq_number] = Some(irq_handler_descriptor);\n\n            Ok(())\n        })\n    }\n\n    fn enable(&self, irq: &Self::IRQNumberType) {\n        self.wo_registers.lock(|regs| {\n            let enable_reg = if irq.get() <= 31 {\n                &regs.ENABLE_1\n            } else {\n                &regs.ENABLE_2\n            };\n\n            let enable_bit: u32 = 1 << (irq.get() % 32);\n\n            // Writing a 1 to a bit will set the corresponding IRQ enable bit. All other IRQ enable\n            // bits are unaffected. So we don't need read and OR'ing here.\n            enable_reg.set(enable_bit);\n        });\n    }\n\n    fn handle_pending_irqs<'irq_context>(\n        &'irq_context self,\n        _ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) {\n        self.handler_table.read(|table| {\n            for irq_number in self.pending_irqs() {\n                match table[irq_number] {\n                    None => panic!(\"No handler registered for IRQ {}\", irq_number),\n                    Some(descriptor) => {\n                        // Call the IRQ handler. Panics on failure.\n                        descriptor.handler().handle().expect(\"Error handling IRQ\");\n                    }\n                }\n            }\n        })\n    }\n\n    fn print_handler(&self) {\n        use crate::info;\n\n        info!(\"      Peripheral handler:\");\n\n        self.handler_table.read(|table| {\n            for (i, opt) in table.iter().enumerate() {\n                if let Some(handler) = opt {\n                    info!(\"            {: >3}. {}\", i, handler.name());\n                }\n            }\n        });\n    }\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Interrupt Controller Driver.\n\nmod peripheral_ic;\n\nuse crate::{\n    bsp::device_driver::common::BoundedUsize,\n    driver,\n    exception::{self, asynchronous::IRQHandlerDescriptor},\n};\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Wrapper struct for a bitmask indicating pending IRQ numbers.\nstruct PendingIRQs {\n    bitmask: u64,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub type LocalIRQ = BoundedUsize<{ InterruptController::MAX_LOCAL_IRQ_NUMBER }>;\npub type PeripheralIRQ = BoundedUsize<{ InterruptController::MAX_PERIPHERAL_IRQ_NUMBER }>;\n\n/// Used for the associated type of trait [`exception::asynchronous::interface::IRQManager`].\n#[derive(Copy, Clone)]\n#[allow(missing_docs)]\npub enum IRQNumber {\n    Local(LocalIRQ),\n    Peripheral(PeripheralIRQ),\n}\n\n/// Representation of the Interrupt Controller.\npub struct InterruptController {\n    periph: peripheral_ic::PeripheralIC,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PendingIRQs {\n    pub fn new(bitmask: u64) -> Self {\n        Self { bitmask }\n    }\n}\n\nimpl Iterator for PendingIRQs {\n    type Item = usize;\n\n    fn next(&mut self) -> Option<Self::Item> {\n        if self.bitmask == 0 {\n            return None;\n        }\n\n        let next = self.bitmask.trailing_zeros() as usize;\n        self.bitmask &= self.bitmask.wrapping_sub(1);\n        Some(next)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl fmt::Display for IRQNumber {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        match self {\n            Self::Local(number) => write!(f, \"Local({})\", number),\n            Self::Peripheral(number) => write!(f, \"Peripheral({})\", number),\n        }\n    }\n}\n\nimpl InterruptController {\n    // Restrict to 3 for now. This makes future code for local_ic.rs more straight forward.\n    const MAX_LOCAL_IRQ_NUMBER: usize = 3;\n    const MAX_PERIPHERAL_IRQ_NUMBER: usize = 63;\n\n    pub const COMPATIBLE: &'static str = \"BCM Interrupt Controller\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(periph_mmio_start_addr: usize) -> Self {\n        Self {\n            periph: peripheral_ic::PeripheralIC::new(periph_mmio_start_addr),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\n\nimpl driver::interface::DeviceDriver for InterruptController {\n    type IRQNumberType = IRQNumber;\n\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n}\n\nimpl exception::asynchronous::interface::IRQManager for InterruptController {\n    type IRQNumberType = IRQNumber;\n\n    fn register_handler(\n        &self,\n        irq_handler_descriptor: exception::asynchronous::IRQHandlerDescriptor<Self::IRQNumberType>,\n    ) -> Result<(), &'static str> {\n        match irq_handler_descriptor.number() {\n            IRQNumber::Local(_) => unimplemented!(\"Local IRQ controller not implemented.\"),\n            IRQNumber::Peripheral(pirq) => {\n                let periph_descriptor = IRQHandlerDescriptor::new(\n                    pirq,\n                    irq_handler_descriptor.name(),\n                    irq_handler_descriptor.handler(),\n                );\n\n                self.periph.register_handler(periph_descriptor)\n            }\n        }\n    }\n\n    fn enable(&self, irq: &Self::IRQNumberType) {\n        match irq {\n            IRQNumber::Local(_) => unimplemented!(\"Local IRQ controller not implemented.\"),\n            IRQNumber::Peripheral(pirq) => self.periph.enable(pirq),\n        }\n    }\n\n    fn handle_pending_irqs<'irq_context>(\n        &'irq_context self,\n        ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) {\n        // It can only be a peripheral IRQ pending because enable() does not support local IRQs yet.\n        self.periph.handle_pending_irqs(ic)\n    }\n\n    fn print_handler(&self) {\n        self.periph.print_handler();\n    }\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! PL011 UART driver.\n//!\n//! # Resources\n//!\n//! - <https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf>\n//! - <https://developer.arm.com/documentation/ddi0183/latest>\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    console, cpu, driver,\n    exception::{self, asynchronous::IRQNumber},\n    synchronization,\n    synchronization::IRQSafeNullLock,\n};\nuse core::fmt;\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields, register_structs,\n    registers::{ReadOnly, ReadWrite, WriteOnly},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// PL011 UART registers.\n//\n// Descriptions taken from \"PrimeCell UART (PL011) Technical Reference Manual\" r1p5.\nregister_bitfields! {\n    u32,\n\n    /// Flag Register.\n    FR [\n        /// Transmit FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n        /// Line Control Register, LCR_H.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the transmit holding register is empty.\n        /// - If the FIFO is enabled, the TXFE bit is set when the transmit FIFO is empty.\n        /// - This bit does not indicate if there is data in the transmit shift register.\n        TXFE OFFSET(7) NUMBITS(1) [],\n\n        /// Transmit FIFO full. The meaning of this bit depends on the state of the FEN bit in the\n        /// LCR_H Register.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the transmit holding register is full.\n        /// - If the FIFO is enabled, the TXFF bit is set when the transmit FIFO is full.\n        TXFF OFFSET(5) NUMBITS(1) [],\n\n        /// Receive FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n        /// LCR_H Register.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the receive holding register is empty.\n        /// - If the FIFO is enabled, the RXFE bit is set when the receive FIFO is empty.\n        RXFE OFFSET(4) NUMBITS(1) [],\n\n        /// UART busy. If this bit is set to 1, the UART is busy transmitting data. This bit remains\n        /// set until the complete byte, including all the stop bits, has been sent from the shift\n        /// register.\n        ///\n        /// This bit is set as soon as the transmit FIFO becomes non-empty, regardless of whether\n        /// the UART is enabled or not.\n        BUSY OFFSET(3) NUMBITS(1) []\n    ],\n\n    /// Integer Baud Rate Divisor.\n    IBRD [\n        /// The integer baud rate divisor.\n        BAUD_DIVINT OFFSET(0) NUMBITS(16) []\n    ],\n\n    /// Fractional Baud Rate Divisor.\n    FBRD [\n        ///  The fractional baud rate divisor.\n        BAUD_DIVFRAC OFFSET(0) NUMBITS(6) []\n    ],\n\n    /// Line Control Register.\n    LCR_H [\n        /// Word length. These bits indicate the number of data bits transmitted or received in a\n        /// frame.\n        #[allow(clippy::enum_variant_names)]\n        WLEN OFFSET(5) NUMBITS(2) [\n            FiveBit = 0b00,\n            SixBit = 0b01,\n            SevenBit = 0b10,\n            EightBit = 0b11\n        ],\n\n        /// Enable FIFOs:\n        ///\n        /// 0 = FIFOs are disabled (character mode) that is, the FIFOs become 1-byte-deep holding\n        /// registers.\n        ///\n        /// 1 = Transmit and receive FIFO buffers are enabled (FIFO mode).\n        FEN  OFFSET(4) NUMBITS(1) [\n            FifosDisabled = 0,\n            FifosEnabled = 1\n        ]\n    ],\n\n    /// Control Register.\n    CR [\n        /// Receive enable. If this bit is set to 1, the receive section of the UART is enabled.\n        /// Data reception occurs for either UART signals or SIR signals depending on the setting of\n        /// the SIREN bit. When the UART is disabled in the middle of reception, it completes the\n        /// current character before stopping.\n        RXE OFFSET(9) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// Transmit enable. If this bit is set to 1, the transmit section of the UART is enabled.\n        /// Data transmission occurs for either UART signals, or SIR signals depending on the\n        /// setting of the SIREN bit. When the UART is disabled in the middle of transmission, it\n        /// completes the current character before stopping.\n        TXE OFFSET(8) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// UART enable:\n        ///\n        /// 0 = UART is disabled. If the UART is disabled in the middle of transmission or\n        /// reception, it completes the current character before stopping.\n        ///\n        /// 1 = The UART is enabled. Data transmission and reception occurs for either UART signals\n        /// or SIR signals depending on the setting of the SIREN bit\n        UARTEN OFFSET(0) NUMBITS(1) [\n            /// If the UART is disabled in the middle of transmission or reception, it completes the\n            /// current character before stopping.\n            Disabled = 0,\n            Enabled = 1\n        ]\n    ],\n\n    /// Interrupt FIFO Level Select Register.\n    IFLS [\n        /// Receive interrupt FIFO level select. The trigger points for the receive interrupt are as\n        /// follows.\n        RXIFLSEL OFFSET(3) NUMBITS(5) [\n            OneEigth = 0b000,\n            OneQuarter = 0b001,\n            OneHalf = 0b010,\n            ThreeQuarters = 0b011,\n            SevenEights = 0b100\n        ]\n    ],\n\n    /// Interrupt Mask Set/Clear Register.\n    IMSC [\n        /// Receive timeout interrupt mask. A read returns the current mask for the UARTRTINTR\n        /// interrupt.\n        ///\n        /// - On a write of 1, the mask of the UARTRTINTR interrupt is set.\n        /// - A write of 0 clears the mask.\n        RTIM OFFSET(6) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// Receive interrupt mask. A read returns the current mask for the UARTRXINTR interrupt.\n        ///\n        /// - On a write of 1, the mask of the UARTRXINTR interrupt is set.\n        /// - A write of 0 clears the mask.\n        RXIM OFFSET(4) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ]\n    ],\n\n    /// Masked Interrupt Status Register.\n    MIS [\n        /// Receive timeout masked interrupt status. Returns the masked interrupt state of the\n        /// UARTRTINTR interrupt.\n        RTMIS OFFSET(6) NUMBITS(1) [],\n\n        /// Receive masked interrupt status. Returns the masked interrupt state of the UARTRXINTR\n        /// interrupt.\n        RXMIS OFFSET(4) NUMBITS(1) []\n    ],\n\n    /// Interrupt Clear Register.\n    ICR [\n        /// Meta field for all pending interrupts.\n        ALL OFFSET(0) NUMBITS(11) []\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    pub RegisterBlock {\n        (0x00 => DR: ReadWrite<u32>),\n        (0x04 => _reserved1),\n        (0x18 => FR: ReadOnly<u32, FR::Register>),\n        (0x1c => _reserved2),\n        (0x24 => IBRD: WriteOnly<u32, IBRD::Register>),\n        (0x28 => FBRD: WriteOnly<u32, FBRD::Register>),\n        (0x2c => LCR_H: WriteOnly<u32, LCR_H::Register>),\n        (0x30 => CR: WriteOnly<u32, CR::Register>),\n        (0x34 => IFLS: ReadWrite<u32, IFLS::Register>),\n        (0x38 => IMSC: ReadWrite<u32, IMSC::Register>),\n        (0x3C => _reserved3),\n        (0x40 => MIS: ReadOnly<u32, MIS::Register>),\n        (0x44 => ICR: WriteOnly<u32, ICR::Register>),\n        (0x48 => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\n#[derive(PartialEq)]\nenum BlockingMode {\n    Blocking,\n    NonBlocking,\n}\n\nstruct PL011UartInner {\n    registers: Registers,\n    chars_written: usize,\n    chars_read: usize,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the UART.\npub struct PL011Uart {\n    inner: IRQSafeNullLock<PL011UartInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PL011UartInner {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n            chars_written: 0,\n            chars_read: 0,\n        }\n    }\n\n    /// Set up baud rate and characteristics.\n    ///\n    /// This results in 8N1 and 921_600 baud.\n    ///\n    /// The calculation for the BRD is (we set the clock to 48 MHz in config.txt):\n    /// `(48_000_000 / 16) / 921_600 = 3.2552083`.\n    ///\n    /// This means the integer part is `3` and goes into the `IBRD`.\n    /// The fractional part is `0.2552083`.\n    ///\n    /// `FBRD` calculation according to the PL011 Technical Reference Manual:\n    /// `INTEGER((0.2552083 * 64) + 0.5) = 16`.\n    ///\n    /// Therefore, the generated baud rate divider is: `3 + 16/64 = 3.25`. Which results in a\n    /// genrated baud rate of `48_000_000 / (16 * 3.25) = 923_077`.\n    ///\n    /// Error = `((923_077 - 921_600) / 921_600) * 100 = 0.16%`.\n    pub fn init(&mut self) {\n        // Execution can arrive here while there are still characters queued in the TX FIFO and\n        // actively being sent out by the UART hardware. If the UART is turned off in this case,\n        // those queued characters would be lost.\n        //\n        // For example, this can happen during runtime on a call to panic!(), because panic!()\n        // initializes its own UART instance and calls init().\n        //\n        // Hence, flush first to ensure all pending characters are transmitted.\n        self.flush();\n\n        // Turn the UART off temporarily.\n        self.registers.CR.set(0);\n\n        // Clear all pending interrupts.\n        self.registers.ICR.write(ICR::ALL::CLEAR);\n\n        // From the PL011 Technical Reference Manual:\n        //\n        // The LCR_H, IBRD, and FBRD registers form the single 30-bit wide LCR Register that is\n        // updated on a single write strobe generated by a LCR_H write. So, to internally update the\n        // contents of IBRD or FBRD, a LCR_H write must always be performed at the end.\n        //\n        // Set the baud rate, 8N1 and FIFO enabled.\n        self.registers.IBRD.write(IBRD::BAUD_DIVINT.val(3));\n        self.registers.FBRD.write(FBRD::BAUD_DIVFRAC.val(16));\n        self.registers\n            .LCR_H\n            .write(LCR_H::WLEN::EightBit + LCR_H::FEN::FifosEnabled);\n\n        // Set RX FIFO fill level at 1/8.\n        self.registers.IFLS.write(IFLS::RXIFLSEL::OneEigth);\n\n        // Enable RX IRQ + RX timeout IRQ.\n        self.registers\n            .IMSC\n            .write(IMSC::RXIM::Enabled + IMSC::RTIM::Enabled);\n\n        // Turn the UART on.\n        self.registers\n            .CR\n            .write(CR::UARTEN::Enabled + CR::TXE::Enabled + CR::RXE::Enabled);\n    }\n\n    /// Send a character.\n    fn write_char(&mut self, c: char) {\n        // Spin while TX FIFO full is set, waiting for an empty slot.\n        while self.registers.FR.matches_all(FR::TXFF::SET) {\n            cpu::nop();\n        }\n\n        // Write the character to the buffer.\n        self.registers.DR.set(c as u32);\n\n        self.chars_written += 1;\n    }\n\n    /// Block execution until the last buffered character has been physically put on the TX wire.\n    fn flush(&self) {\n        // Spin until the busy bit is cleared.\n        while self.registers.FR.matches_all(FR::BUSY::SET) {\n            cpu::nop();\n        }\n    }\n\n    /// Retrieve a character.\n    fn read_char_converting(&mut self, blocking_mode: BlockingMode) -> Option<char> {\n        // If RX FIFO is empty,\n        if self.registers.FR.matches_all(FR::RXFE::SET) {\n            // immediately return in non-blocking mode.\n            if blocking_mode == BlockingMode::NonBlocking {\n                return None;\n            }\n\n            // Otherwise, wait until a char was received.\n            while self.registers.FR.matches_all(FR::RXFE::SET) {\n                cpu::nop();\n            }\n        }\n\n        // Read one character.\n        let mut ret = self.registers.DR.get() as u8 as char;\n\n        // Convert carrige return to newline.\n        if ret == '\\r' {\n            ret = '\\n'\n        }\n\n        // Update statistics.\n        self.chars_read += 1;\n\n        Some(ret)\n    }\n}\n\n/// Implementing `core::fmt::Write` enables usage of the `format_args!` macros, which in turn are\n/// used to implement the `kernel`'s `print!` and `println!` macros. By implementing `write_str()`,\n/// we get `write_fmt()` automatically.\n///\n/// The function takes an `&mut self`, so it must be implemented for the inner struct.\n///\n/// See [`src/print.rs`].\n///\n/// [`src/print.rs`]: ../../print/index.html\nimpl fmt::Write for PL011UartInner {\n    fn write_str(&mut self, s: &str) -> fmt::Result {\n        for c in s.chars() {\n            self.write_char(c);\n        }\n\n        Ok(())\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PL011Uart {\n    pub const COMPATIBLE: &'static str = \"BCM PL011 UART\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            inner: IRQSafeNullLock::new(PL011UartInner::new(mmio_start_addr)),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl driver::interface::DeviceDriver for PL011Uart {\n    type IRQNumberType = IRQNumber;\n\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n\n    unsafe fn init(&self) -> Result<(), &'static str> {\n        self.inner.lock(|inner| inner.init());\n\n        Ok(())\n    }\n\n    fn register_and_enable_irq_handler(\n        &'static self,\n        irq_number: &Self::IRQNumberType,\n    ) -> Result<(), &'static str> {\n        use exception::asynchronous::{irq_manager, IRQHandlerDescriptor};\n\n        let descriptor = IRQHandlerDescriptor::new(*irq_number, Self::COMPATIBLE, self);\n\n        irq_manager().register_handler(descriptor)?;\n        irq_manager().enable(irq_number);\n\n        Ok(())\n    }\n}\n\nimpl console::interface::Write for PL011Uart {\n    /// Passthrough of `args` to the `core::fmt::Write` implementation, but guarded by a Mutex to\n    /// serialize access.\n    fn write_char(&self, c: char) {\n        self.inner.lock(|inner| inner.write_char(c));\n    }\n\n    fn write_fmt(&self, args: core::fmt::Arguments) -> fmt::Result {\n        // Fully qualified syntax for the call to `core::fmt::Write::write_fmt()` to increase\n        // readability.\n        self.inner.lock(|inner| fmt::Write::write_fmt(inner, args))\n    }\n\n    fn flush(&self) {\n        // Spin until TX FIFO empty is set.\n        self.inner.lock(|inner| inner.flush());\n    }\n}\n\nimpl console::interface::Read for PL011Uart {\n    fn read_char(&self) -> char {\n        self.inner\n            .lock(|inner| inner.read_char_converting(BlockingMode::Blocking).unwrap())\n    }\n\n    fn clear_rx(&self) {\n        // Read from the RX FIFO until it is indicating empty.\n        while self\n            .inner\n            .lock(|inner| inner.read_char_converting(BlockingMode::NonBlocking))\n            .is_some()\n        {}\n    }\n}\n\nimpl console::interface::Statistics for PL011Uart {\n    fn chars_written(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_written)\n    }\n\n    fn chars_read(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_read)\n    }\n}\n\nimpl console::interface::All for PL011Uart {}\n\nimpl exception::asynchronous::interface::IRQHandler for PL011Uart {\n    fn handle(&self) -> Result<(), &'static str> {\n        self.inner.lock(|inner| {\n            let pending = inner.registers.MIS.extract();\n\n            // Clear all pending IRQs.\n            inner.registers.ICR.write(ICR::ALL::CLEAR);\n\n            // Check for any kind of RX interrupt.\n            if pending.matches_any(MIS::RXMIS::SET + MIS::RTMIS::SET) {\n                // Echo any received characters.\n                while let Some(c) = inner.read_char_converting(BlockingMode::NonBlocking) {\n                    inner.write_char(c)\n                }\n            }\n        });\n\n        Ok(())\n    }\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/bcm.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BCM driver top level.\n\nmod bcm2xxx_gpio;\n#[cfg(feature = \"bsp_rpi3\")]\nmod bcm2xxx_interrupt_controller;\nmod bcm2xxx_pl011_uart;\n\npub use bcm2xxx_gpio::*;\n#[cfg(feature = \"bsp_rpi3\")]\npub use bcm2xxx_interrupt_controller::*;\npub use bcm2xxx_pl011_uart::*;\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/common.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Common device driver code.\n\nuse core::{fmt, marker::PhantomData, ops};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct MMIODerefWrapper<T> {\n    start_addr: usize,\n    phantom: PhantomData<fn() -> T>,\n}\n\n/// A wrapper type for usize with integrated range bound check.\n#[derive(Copy, Clone)]\npub struct BoundedUsize<const MAX_INCLUSIVE: usize>(usize);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T> MMIODerefWrapper<T> {\n    /// Create an instance.\n    pub const unsafe fn new(start_addr: usize) -> Self {\n        Self {\n            start_addr,\n            phantom: PhantomData,\n        }\n    }\n}\n\nimpl<T> ops::Deref for MMIODerefWrapper<T> {\n    type Target = T;\n\n    fn deref(&self) -> &Self::Target {\n        unsafe { &*(self.start_addr as *const _) }\n    }\n}\n\nimpl<const MAX_INCLUSIVE: usize> BoundedUsize<{ MAX_INCLUSIVE }> {\n    pub const MAX_INCLUSIVE: usize = MAX_INCLUSIVE;\n\n    /// Creates a new instance if number <= MAX_INCLUSIVE.\n    pub const fn new(number: usize) -> Self {\n        assert!(number <= MAX_INCLUSIVE);\n\n        Self(number)\n    }\n\n    /// Return the wrapped number.\n    pub const fn get(self) -> usize {\n        self.0\n    }\n}\n\nimpl<const MAX_INCLUSIVE: usize> fmt::Display for BoundedUsize<{ MAX_INCLUSIVE }> {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        write!(f, \"{}\", self.0)\n    }\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Device driver.\n\n#[cfg(feature = \"bsp_rpi4\")]\nmod arm;\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod bcm;\nmod common;\n\n#[cfg(feature = \"bsp_rpi4\")]\npub use arm::*;\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use bcm::*;\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/raspberrypi/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Processor code.\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Used by `arch` code to find the early boot core.\n#[no_mangle]\n#[link_section = \".text._start_arguments\"]\npub static BOOT_CORE_ID: u64 = 0;\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/raspberrypi/driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP driver support.\n\nuse super::{exception, memory::map::mmio};\nuse crate::{\n    bsp::device_driver,\n    console, driver as generic_driver,\n    exception::{self as generic_exception},\n};\nuse core::sync::atomic::{AtomicBool, Ordering};\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic PL011_UART: device_driver::PL011Uart =\n    unsafe { device_driver::PL011Uart::new(mmio::PL011_UART_START) };\nstatic GPIO: device_driver::GPIO = unsafe { device_driver::GPIO::new(mmio::GPIO_START) };\n\n#[cfg(feature = \"bsp_rpi3\")]\nstatic INTERRUPT_CONTROLLER: device_driver::InterruptController =\n    unsafe { device_driver::InterruptController::new(mmio::PERIPHERAL_IC_START) };\n\n#[cfg(feature = \"bsp_rpi4\")]\nstatic INTERRUPT_CONTROLLER: device_driver::GICv2 =\n    unsafe { device_driver::GICv2::new(mmio::GICD_START, mmio::GICC_START) };\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// This must be called only after successful init of the UART driver.\nfn post_init_uart() -> Result<(), &'static str> {\n    console::register_console(&PL011_UART);\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the GPIO driver.\nfn post_init_gpio() -> Result<(), &'static str> {\n    GPIO.map_pl011_uart();\n    Ok(())\n}\n\n/// This must be called only after successful init of the interrupt controller driver.\nfn post_init_interrupt_controller() -> Result<(), &'static str> {\n    generic_exception::asynchronous::register_irq_manager(&INTERRUPT_CONTROLLER);\n\n    Ok(())\n}\n\nfn driver_uart() -> Result<(), &'static str> {\n    let uart_descriptor = generic_driver::DeviceDriverDescriptor::new(\n        &PL011_UART,\n        Some(post_init_uart),\n        Some(exception::asynchronous::irq_map::PL011_UART),\n    );\n    generic_driver::driver_manager().register_driver(uart_descriptor);\n\n    Ok(())\n}\n\nfn driver_gpio() -> Result<(), &'static str> {\n    let gpio_descriptor =\n        generic_driver::DeviceDriverDescriptor::new(&GPIO, Some(post_init_gpio), None);\n    generic_driver::driver_manager().register_driver(gpio_descriptor);\n\n    Ok(())\n}\n\nfn driver_interrupt_controller() -> Result<(), &'static str> {\n    let interrupt_controller_descriptor = generic_driver::DeviceDriverDescriptor::new(\n        &INTERRUPT_CONTROLLER,\n        Some(post_init_interrupt_controller),\n        None,\n    );\n    generic_driver::driver_manager().register_driver(interrupt_controller_descriptor);\n\n    Ok(())\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Initialize the driver subsystem.\n///\n/// # Safety\n///\n/// See child function calls.\npub unsafe fn init() -> Result<(), &'static str> {\n    static INIT_DONE: AtomicBool = AtomicBool::new(false);\n    if INIT_DONE.load(Ordering::Relaxed) {\n        return Err(\"Init already done\");\n    }\n\n    driver_uart()?;\n    driver_gpio()?;\n    driver_interrupt_controller()?;\n\n    INIT_DONE.store(true, Ordering::Relaxed);\n    Ok(())\n}\n\n/// Minimal code needed to bring up the console in QEMU (for testing only). This is often less steps\n/// than on real hardware due to QEMU's abstractions.\n#[cfg(feature = \"test_build\")]\npub fn qemu_bring_up_console() {\n    console::register_console(&PL011_UART);\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/raspberrypi/exception/asynchronous.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP asynchronous exception handling.\n\nuse crate::bsp;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Export for reuse in generic asynchronous.rs.\npub use bsp::device_driver::IRQNumber;\n\n#[cfg(feature = \"bsp_rpi3\")]\npub(in crate::bsp) mod irq_map {\n    use super::bsp::device_driver::{IRQNumber, PeripheralIRQ};\n\n    pub const PL011_UART: IRQNumber = IRQNumber::Peripheral(PeripheralIRQ::new(57));\n}\n\n#[cfg(feature = \"bsp_rpi4\")]\npub(in crate::bsp) mod irq_map {\n    use super::bsp::device_driver::IRQNumber;\n\n    pub const PL011_UART: IRQNumber = IRQNumber::new(153);\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/raspberrypi/exception.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP synchronous and asynchronous exception handling.\n\npub mod asynchronous;\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/raspberrypi/kernel.ld",
    "content": "/* SPDX-License-Identifier: MIT OR Apache-2.0\n *\n * Copyright (c) 2018-2022 Andre Richter <andre.o.richter@gmail.com>\n */\n\nPAGE_SIZE = 64K;\nPAGE_MASK = PAGE_SIZE - 1;\n\n__rpi_phys_dram_start_addr = 0;\n\n/* The physical address at which the the kernel binary will be loaded by the Raspberry's firmware */\n__rpi_phys_binary_load_addr = 0x80000;\n\n\nENTRY(__rpi_phys_binary_load_addr)\n\n/* Flags:\n *     4 == R\n *     5 == RX\n *     6 == RW\n *\n * Segments are marked PT_LOAD below so that the ELF file provides virtual and physical addresses.\n * It doesn't mean all of them need actually be loaded.\n */\nPHDRS\n{\n    segment_boot_core_stack PT_LOAD FLAGS(6);\n    segment_code            PT_LOAD FLAGS(5);\n    segment_data            PT_LOAD FLAGS(6);\n}\n\nSECTIONS\n{\n    . =  __rpi_phys_dram_start_addr;\n\n    /***********************************************************************************************\n    * Boot Core Stack\n    ***********************************************************************************************/\n    .boot_core_stack (NOLOAD) :\n    {\n                                             /*   ^             */\n                                             /*   | stack       */\n        . += __rpi_phys_binary_load_addr;    /*   | growth      */\n                                             /*   | direction   */\n        __boot_core_stack_end_exclusive = .; /*   |             */\n    } :segment_boot_core_stack\n\n    ASSERT((. & PAGE_MASK) == 0, \"End of boot core stack is not page aligned\")\n\n    /***********************************************************************************************\n    * Code + RO Data + Global Offset Table\n    ***********************************************************************************************/\n    __code_start = .;\n    .text :\n    {\n        KEEP(*(.text._start))\n        *(.text._start_arguments) /* Constants (or statics in Rust speak) read by _start(). */\n        *(.text._start_rust)      /* The Rust entry point */\n        *(.text*)                 /* Everything else */\n    } :segment_code\n\n    .rodata : ALIGN(8) { *(.rodata*) } :segment_code\n\n    . = ALIGN(PAGE_SIZE);\n    __code_end_exclusive = .;\n\n    /***********************************************************************************************\n    * Data + BSS\n    ***********************************************************************************************/\n    .data : { *(.data*) } :segment_data\n\n    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */\n    .bss (NOLOAD) : ALIGN(16)\n    {\n        __bss_start = .;\n        *(.bss*);\n        . = ALIGN(16);\n        __bss_end_exclusive = .;\n    } :segment_data\n\n    /***********************************************************************************************\n    * Misc\n    ***********************************************************************************************/\n    .got : { *(.got*) }\n    ASSERT(SIZEOF(.got) == 0, \"Relocation support not expected\")\n\n    /DISCARD/ : { *(.comment*) }\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/raspberrypi/memory/mmu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Memory Management Unit.\n\nuse super::map as memory_map;\nuse crate::memory::mmu::*;\nuse core::ops::RangeInclusive;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// The kernel's address space defined by this BSP.\npub type KernelAddrSpace = AddressSpace<{ memory_map::END_INCLUSIVE + 1 }>;\n\nconst NUM_MEM_RANGES: usize = 2;\n\n/// The virtual memory layout.\n///\n/// The layout must contain only special ranges, aka anything that is _not_ normal cacheable DRAM.\n/// It is agnostic of the paging granularity that the architecture's MMU will use.\npub static LAYOUT: KernelVirtualLayout<NUM_MEM_RANGES> = KernelVirtualLayout::new(\n    memory_map::END_INCLUSIVE,\n    [\n        TranslationDescriptor {\n            name: \"Kernel code and RO data\",\n            virtual_range: code_range_inclusive,\n            physical_range_translation: Translation::Identity,\n            attribute_fields: AttributeFields {\n                mem_attributes: MemAttributes::CacheableDRAM,\n                acc_perms: AccessPermissions::ReadOnly,\n                execute_never: false,\n            },\n        },\n        TranslationDescriptor {\n            name: \"Device MMIO\",\n            virtual_range: mmio_range_inclusive,\n            physical_range_translation: Translation::Identity,\n            attribute_fields: AttributeFields {\n                mem_attributes: MemAttributes::Device,\n                acc_perms: AccessPermissions::ReadWrite,\n                execute_never: true,\n            },\n        },\n    ],\n);\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nfn code_range_inclusive() -> RangeInclusive<usize> {\n    // Notice the subtraction to turn the exclusive end into an inclusive end.\n    #[allow(clippy::range_minus_one)]\n    RangeInclusive::new(super::code_start(), super::code_end_exclusive() - 1)\n}\n\nfn mmio_range_inclusive() -> RangeInclusive<usize> {\n    RangeInclusive::new(memory_map::mmio::START, memory_map::mmio::END_INCLUSIVE)\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the virtual memory layout.\npub fn virt_mem_layout() -> &'static KernelVirtualLayout<NUM_MEM_RANGES> {\n    &LAYOUT\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// Check alignment of the kernel's virtual memory layout sections.\n    #[kernel_test]\n    fn virt_mem_layout_sections_are_64KiB_aligned() {\n        const SIXTYFOUR_KIB: usize = 65536;\n\n        for i in LAYOUT.inner().iter() {\n            let start: usize = *(i.virtual_range)().start();\n            let end: usize = *(i.virtual_range)().end() + 1;\n\n            assert_eq!(start % SIXTYFOUR_KIB, 0);\n            assert_eq!(end % SIXTYFOUR_KIB, 0);\n            assert!(end >= start);\n        }\n    }\n\n    /// Ensure the kernel's virtual memory layout is free of overlaps.\n    #[kernel_test]\n    fn virt_mem_layout_has_no_overlaps() {\n        let layout = virt_mem_layout().inner();\n\n        for (i, first) in layout.iter().enumerate() {\n            for second in layout.iter().skip(i + 1) {\n                let first_range = first.virtual_range;\n                let second_range = second.virtual_range;\n\n                assert!(!first_range().contains(second_range().start()));\n                assert!(!first_range().contains(second_range().end()));\n                assert!(!second_range().contains(first_range().start()));\n                assert!(!second_range().contains(first_range().end()));\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/raspberrypi/memory.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Memory Management.\n//!\n//! The physical memory layout.\n//!\n//! The Raspberry's firmware copies the kernel binary to 0x8_0000. The preceding region will be used\n//! as the boot core's stack.\n//!\n//! +---------------------------------------+\n//! |                                       | 0x0\n//! |                                       |                                ^\n//! | Boot-core Stack                       |                                | stack\n//! |                                       |                                | growth\n//! |                                       |                                | direction\n//! +---------------------------------------+\n//! |                                       | code_start @ 0x8_0000\n//! | .text                                 |\n//! | .rodata                               |\n//! | .got                                  |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       | code_end_exclusive\n//! | .data                                 |\n//! | .bss                                  |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       |\n//! |                                       |\npub mod mmu;\n\nuse core::cell::UnsafeCell;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Symbols from the linker script.\nextern \"Rust\" {\n    static __code_start: UnsafeCell<()>;\n    static __code_end_exclusive: UnsafeCell<()>;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// The board's physical memory map.\n#[rustfmt::skip]\npub(super) mod map {\n    /// The inclusive end address of the memory map.\n    ///\n    /// End address + 1 must be power of two.\n    ///\n    /// # Note\n    ///\n    /// RPi3 and RPi4 boards can have different amounts of RAM. To make our code lean for\n    /// educational purposes, we set the max size of the address space to 4 GiB regardless of board.\n    /// This way, we can map the entire range that we need (end of MMIO for RPi4) in one take.\n    ///\n    /// However, making this trade-off has the downside of making it possible for the CPU to assert a\n    /// physical address that is not backed by any DRAM (e.g. accessing an address close to 4 GiB on\n    /// an RPi3 that comes with 1 GiB of RAM). This would result in a crash or other kind of error.\n    pub const END_INCLUSIVE:       usize = 0xFFFF_FFFF;\n\n    pub const GPIO_OFFSET:         usize = 0x0020_0000;\n    pub const UART_OFFSET:         usize = 0x0020_1000;\n\n    /// Physical devices.\n    #[cfg(feature = \"bsp_rpi3\")]\n    pub mod mmio {\n        use super::*;\n\n        pub const START:               usize =         0x3F00_0000;\n        pub const PERIPHERAL_IC_START: usize = START + 0x0000_B200;\n        pub const GPIO_START:          usize = START + GPIO_OFFSET;\n        pub const PL011_UART_START:    usize = START + UART_OFFSET;\n        pub const END_INCLUSIVE:       usize =         0x4000_FFFF;\n    }\n\n    /// Physical devices.\n    #[cfg(feature = \"bsp_rpi4\")]\n    pub mod mmio {\n        use super::*;\n\n        pub const START:            usize =         0xFE00_0000;\n        pub const GPIO_START:       usize = START + GPIO_OFFSET;\n        pub const PL011_UART_START: usize = START + UART_OFFSET;\n        pub const GICD_START:       usize =         0xFF84_1000;\n        pub const GICC_START:       usize =         0xFF84_2000;\n        pub const END_INCLUSIVE:    usize =         0xFF84_FFFF;\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Start page address of the code segment.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn code_start() -> usize {\n    unsafe { __code_start.get() as usize }\n}\n\n/// Exclusive end page address of the code segment.\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn code_end_exclusive() -> usize {\n    unsafe { __code_end_exclusive.get() as usize }\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/raspberrypi.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Top-level BSP file for the Raspberry Pi 3 and 4.\n\npub mod cpu;\npub mod driver;\npub mod exception;\npub mod memory;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Board identification.\npub fn board_name() -> &'static str {\n    #[cfg(feature = \"bsp_rpi3\")]\n    {\n        \"Raspberry Pi 3\"\n    }\n\n    #[cfg(feature = \"bsp_rpi4\")]\n    {\n        \"Raspberry Pi 4\"\n    }\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/bsp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Conditional reexporting of Board Support Packages.\n\nmod device_driver;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod raspberrypi;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use raspberrypi::*;\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/common.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! General purpose code.\n\n/// Convert a size into human readable format.\npub const fn size_human_readable_ceil(size: usize) -> (usize, &'static str) {\n    const KIB: usize = 1024;\n    const MIB: usize = 1024 * 1024;\n    const GIB: usize = 1024 * 1024 * 1024;\n\n    if (size / GIB) > 0 {\n        (size.div_ceil(GIB), \"GiB\")\n    } else if (size / MIB) > 0 {\n        (size.div_ceil(MIB), \"MiB\")\n    } else if (size / KIB) > 0 {\n        (size.div_ceil(KIB), \"KiB\")\n    } else {\n        (size, \"Byte\")\n    }\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/console/null_console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Null console.\n\nuse super::interface;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct NullConsole;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\npub static NULL_CONSOLE: NullConsole = NullConsole {};\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl interface::Write for NullConsole {\n    fn write_char(&self, _c: char) {}\n\n    fn write_fmt(&self, _args: fmt::Arguments) -> fmt::Result {\n        fmt::Result::Ok(())\n    }\n\n    fn flush(&self) {}\n}\n\nimpl interface::Read for NullConsole {\n    fn clear_rx(&self) {}\n}\n\nimpl interface::Statistics for NullConsole {}\nimpl interface::All for NullConsole {}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! System console.\n\nmod null_console;\n\nuse crate::synchronization;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Console interfaces.\npub mod interface {\n    use core::fmt;\n\n    /// Console write functions.\n    pub trait Write {\n        /// Write a single character.\n        fn write_char(&self, c: char);\n\n        /// Write a Rust format string.\n        fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result;\n\n        /// Block until the last buffered character has been physically put on the TX wire.\n        fn flush(&self);\n    }\n\n    /// Console read functions.\n    pub trait Read {\n        /// Read a single character.\n        fn read_char(&self) -> char {\n            ' '\n        }\n\n        /// Clear RX buffers, if any.\n        fn clear_rx(&self);\n    }\n\n    /// Console statistics.\n    pub trait Statistics {\n        /// Return the number of characters written.\n        fn chars_written(&self) -> usize {\n            0\n        }\n\n        /// Return the number of characters read.\n        fn chars_read(&self) -> usize {\n            0\n        }\n    }\n\n    /// Trait alias for a full-fledged console.\n    pub trait All: Write + Read + Statistics {}\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic CUR_CONSOLE: InitStateLock<&'static (dyn interface::All + Sync)> =\n    InitStateLock::new(&null_console::NULL_CONSOLE);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::{interface::ReadWriteEx, InitStateLock};\n\n/// Register a new console.\npub fn register_console(new_console: &'static (dyn interface::All + Sync)) {\n    CUR_CONSOLE.write(|con| *con = new_console);\n}\n\n/// Return a reference to the currently registered console.\n///\n/// This is the global console used by all printing macros.\npub fn console() -> &'static dyn interface::All {\n    CUR_CONSOLE.read(|con| *con)\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Boot code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/cpu/boot.rs\"]\nmod arch_boot;\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/cpu/smp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Symmetric multiprocessing.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/cpu/smp.rs\"]\nmod arch_smp;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_smp::core_id;\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Processor code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/cpu.rs\"]\nmod arch_cpu;\n\nmod boot;\n\npub mod smp;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_cpu::{nop, wait_forever};\n\n#[cfg(feature = \"test_build\")]\npub use arch_cpu::{qemu_exit_failure, qemu_exit_success};\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Driver support.\n\nuse crate::{\n    exception, info,\n    synchronization::{interface::ReadWriteEx, InitStateLock},\n};\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nconst NUM_DRIVERS: usize = 5;\n\nstruct DriverManagerInner<T>\nwhere\n    T: 'static,\n{\n    next_index: usize,\n    descriptors: [Option<DeviceDriverDescriptor<T>>; NUM_DRIVERS],\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Driver interfaces.\npub mod interface {\n    /// Device Driver functions.\n    pub trait DeviceDriver {\n        /// Different interrupt controllers might use different types for IRQ number.\n        type IRQNumberType: super::fmt::Display;\n\n        /// Return a compatibility string for identifying the driver.\n        fn compatible(&self) -> &'static str;\n\n        /// Called by the kernel to bring up the device.\n        ///\n        /// # Safety\n        ///\n        /// - During init, drivers might do stuff with system-wide impact.\n        unsafe fn init(&self) -> Result<(), &'static str> {\n            Ok(())\n        }\n\n        /// Called by the kernel to register and enable the device's IRQ handler.\n        ///\n        /// Rust's type system will prevent a call to this function unless the calling instance\n        /// itself has static lifetime.\n        fn register_and_enable_irq_handler(\n            &'static self,\n            irq_number: &Self::IRQNumberType,\n        ) -> Result<(), &'static str> {\n            panic!(\n                \"Attempt to enable IRQ {} for device {}, but driver does not support this\",\n                irq_number,\n                self.compatible()\n            )\n        }\n    }\n}\n\n/// Tpye to be used as an optional callback after a driver's init() has run.\npub type DeviceDriverPostInitCallback = unsafe fn() -> Result<(), &'static str>;\n\n/// A descriptor for device drivers.\n#[derive(Copy, Clone)]\npub struct DeviceDriverDescriptor<T>\nwhere\n    T: 'static,\n{\n    device_driver: &'static (dyn interface::DeviceDriver<IRQNumberType = T> + Sync),\n    post_init_callback: Option<DeviceDriverPostInitCallback>,\n    irq_number: Option<T>,\n}\n\n/// Provides device driver management functions.\npub struct DriverManager<T>\nwhere\n    T: 'static,\n{\n    inner: InitStateLock<DriverManagerInner<T>>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic DRIVER_MANAGER: DriverManager<exception::asynchronous::IRQNumber> = DriverManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T> DriverManagerInner<T>\nwhere\n    T: 'static + Copy,\n{\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            next_index: 0,\n            descriptors: [None; NUM_DRIVERS],\n        }\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T> DeviceDriverDescriptor<T> {\n    /// Create an instance.\n    pub fn new(\n        device_driver: &'static (dyn interface::DeviceDriver<IRQNumberType = T> + Sync),\n        post_init_callback: Option<DeviceDriverPostInitCallback>,\n        irq_number: Option<T>,\n    ) -> Self {\n        Self {\n            device_driver,\n            post_init_callback,\n            irq_number,\n        }\n    }\n}\n\n/// Return a reference to the global DriverManager.\npub fn driver_manager() -> &'static DriverManager<exception::asynchronous::IRQNumber> {\n    &DRIVER_MANAGER\n}\n\nimpl<T> DriverManager<T>\nwhere\n    T: fmt::Display + Copy,\n{\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            inner: InitStateLock::new(DriverManagerInner::new()),\n        }\n    }\n\n    /// Register a device driver with the kernel.\n    pub fn register_driver(&self, descriptor: DeviceDriverDescriptor<T>) {\n        self.inner.write(|inner| {\n            inner.descriptors[inner.next_index] = Some(descriptor);\n            inner.next_index += 1;\n        })\n    }\n\n    /// Helper for iterating over registered drivers.\n    fn for_each_descriptor<'a>(&'a self, f: impl FnMut(&'a DeviceDriverDescriptor<T>)) {\n        self.inner.read(|inner| {\n            inner\n                .descriptors\n                .iter()\n                .filter_map(|x| x.as_ref())\n                .for_each(f)\n        })\n    }\n\n    /// Fully initialize all drivers and their interrupts handlers.\n    ///\n    /// # Safety\n    ///\n    /// - During init, drivers might do stuff with system-wide impact.\n    pub unsafe fn init_drivers_and_irqs(&self) {\n        self.for_each_descriptor(|descriptor| {\n            // 1. Initialize driver.\n            if let Err(x) = descriptor.device_driver.init() {\n                panic!(\n                    \"Error initializing driver: {}: {}\",\n                    descriptor.device_driver.compatible(),\n                    x\n                );\n            }\n\n            // 2. Call corresponding post init callback.\n            if let Some(callback) = &descriptor.post_init_callback {\n                if let Err(x) = callback() {\n                    panic!(\n                        \"Error during driver post-init callback: {}: {}\",\n                        descriptor.device_driver.compatible(),\n                        x\n                    );\n                }\n            }\n        });\n\n        // 3. After all post-init callbacks were done, the interrupt controller should be\n        //    registered and functional. So let drivers register with it now.\n        self.for_each_descriptor(|descriptor| {\n            if let Some(irq_number) = &descriptor.irq_number {\n                if let Err(x) = descriptor\n                    .device_driver\n                    .register_and_enable_irq_handler(irq_number)\n                {\n                    panic!(\n                        \"Error during driver interrupt handler registration: {}: {}\",\n                        descriptor.device_driver.compatible(),\n                        x\n                    );\n                }\n            }\n        });\n    }\n\n    /// Enumerate all registered device drivers.\n    pub fn enumerate(&self) {\n        let mut i: usize = 1;\n        self.for_each_descriptor(|descriptor| {\n            info!(\"      {}. {}\", i, descriptor.device_driver.compatible());\n\n            i += 1;\n        });\n    }\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/exception/asynchronous/null_irq_manager.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Null IRQ Manager.\n\nuse super::{interface, IRQContext, IRQHandlerDescriptor};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct NullIRQManager;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\npub static NULL_IRQ_MANAGER: NullIRQManager = NullIRQManager {};\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl interface::IRQManager for NullIRQManager {\n    type IRQNumberType = super::IRQNumber;\n\n    fn register_handler(\n        &self,\n        _descriptor: IRQHandlerDescriptor<Self::IRQNumberType>,\n    ) -> Result<(), &'static str> {\n        panic!(\"No IRQ Manager registered yet\");\n    }\n\n    fn enable(&self, _irq_number: &Self::IRQNumberType) {\n        panic!(\"No IRQ Manager registered yet\");\n    }\n\n    fn handle_pending_irqs<'irq_context>(&'irq_context self, _ic: &IRQContext<'irq_context>) {\n        panic!(\"No IRQ Manager registered yet\");\n    }\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/exception/asynchronous.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Asynchronous exception handling.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/exception/asynchronous.rs\"]\nmod arch_asynchronous;\nmod null_irq_manager;\n\nuse crate::{bsp, synchronization};\nuse core::marker::PhantomData;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_asynchronous::{\n    is_local_irq_masked, local_irq_mask, local_irq_mask_save, local_irq_restore, local_irq_unmask,\n    print_state,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Interrupt number as defined by the BSP.\npub type IRQNumber = bsp::exception::asynchronous::IRQNumber;\n\n/// Interrupt descriptor.\n#[derive(Copy, Clone)]\npub struct IRQHandlerDescriptor<T>\nwhere\n    T: Copy,\n{\n    /// The IRQ number.\n    number: T,\n\n    /// Descriptive name.\n    name: &'static str,\n\n    /// Reference to handler trait object.\n    handler: &'static (dyn interface::IRQHandler + Sync),\n}\n\n/// IRQContext token.\n///\n/// An instance of this type indicates that the local core is currently executing in IRQ\n/// context, aka executing an interrupt vector or subcalls of it.\n///\n/// Concept and implementation derived from the `CriticalSection` introduced in\n/// <https://github.com/rust-embedded/bare-metal>\n#[derive(Clone, Copy)]\npub struct IRQContext<'irq_context> {\n    _0: PhantomData<&'irq_context ()>,\n}\n\n/// Asynchronous exception handling interfaces.\npub mod interface {\n\n    /// Implemented by types that handle IRQs.\n    pub trait IRQHandler {\n        /// Called when the corresponding interrupt is asserted.\n        fn handle(&self) -> Result<(), &'static str>;\n    }\n\n    /// IRQ management functions.\n    ///\n    /// The `BSP` is supposed to supply one global instance. Typically implemented by the\n    /// platform's interrupt controller.\n    pub trait IRQManager {\n        /// The IRQ number type depends on the implementation.\n        type IRQNumberType: Copy;\n\n        /// Register a handler.\n        fn register_handler(\n            &self,\n            irq_handler_descriptor: super::IRQHandlerDescriptor<Self::IRQNumberType>,\n        ) -> Result<(), &'static str>;\n\n        /// Enable an interrupt in the controller.\n        fn enable(&self, irq_number: &Self::IRQNumberType);\n\n        /// Handle pending interrupts.\n        ///\n        /// This function is called directly from the CPU's IRQ exception vector. On AArch64,\n        /// this means that the respective CPU core has disabled exception handling.\n        /// This function can therefore not be preempted and runs start to finish.\n        ///\n        /// Takes an IRQContext token to ensure it can only be called from IRQ context.\n        #[allow(clippy::trivially_copy_pass_by_ref)]\n        fn handle_pending_irqs<'irq_context>(\n            &'irq_context self,\n            ic: &super::IRQContext<'irq_context>,\n        );\n\n        /// Print list of registered handlers.\n        fn print_handler(&self) {}\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic CUR_IRQ_MANAGER: InitStateLock<\n    &'static (dyn interface::IRQManager<IRQNumberType = IRQNumber> + Sync),\n> = InitStateLock::new(&null_irq_manager::NULL_IRQ_MANAGER);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::{interface::ReadWriteEx, InitStateLock};\n\nimpl<T> IRQHandlerDescriptor<T>\nwhere\n    T: Copy,\n{\n    /// Create an instance.\n    pub const fn new(\n        number: T,\n        name: &'static str,\n        handler: &'static (dyn interface::IRQHandler + Sync),\n    ) -> Self {\n        Self {\n            number,\n            name,\n            handler,\n        }\n    }\n\n    /// Return the number.\n    pub const fn number(&self) -> T {\n        self.number\n    }\n\n    /// Return the name.\n    pub const fn name(&self) -> &'static str {\n        self.name\n    }\n\n    /// Return the handler.\n    pub const fn handler(&self) -> &'static (dyn interface::IRQHandler + Sync) {\n        self.handler\n    }\n}\n\nimpl<'irq_context> IRQContext<'irq_context> {\n    /// Creates an IRQContext token.\n    ///\n    /// # Safety\n    ///\n    /// - This must only be called when the current core is in an interrupt context and will not\n    ///   live beyond the end of it. That is, creation is allowed in interrupt vector functions. For\n    ///   example, in the ARMv8-A case, in `extern \"C\" fn current_elx_irq()`.\n    /// - Note that the lifetime `'irq_context` of the returned instance is unconstrained. User code\n    ///   must not be able to influence the lifetime picked for this type, since that might cause it\n    ///   to be inferred to `'static`.\n    #[inline(always)]\n    pub unsafe fn new() -> Self {\n        IRQContext { _0: PhantomData }\n    }\n}\n\n/// Executes the provided closure while IRQs are masked on the executing core.\n///\n/// While the function temporarily changes the HW state of the executing core, it restores it to the\n/// previous state before returning, so this is deemed safe.\n#[inline(always)]\npub fn exec_with_irq_masked<T>(f: impl FnOnce() -> T) -> T {\n    let saved = local_irq_mask_save();\n    let ret = f();\n    local_irq_restore(saved);\n\n    ret\n}\n\n/// Register a new IRQ manager.\npub fn register_irq_manager(\n    new_manager: &'static (dyn interface::IRQManager<IRQNumberType = IRQNumber> + Sync),\n) {\n    CUR_IRQ_MANAGER.write(|manager| *manager = new_manager);\n}\n\n/// Return a reference to the currently registered IRQ manager.\n///\n/// This is the IRQ manager used by the architectural interrupt handling code.\npub fn irq_manager() -> &'static dyn interface::IRQManager<IRQNumberType = IRQNumber> {\n    CUR_IRQ_MANAGER.read(|manager| *manager)\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/exception.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Synchronous and asynchronous exception handling.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/exception.rs\"]\nmod arch_exception;\n\npub mod asynchronous;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_exception::{current_privilege_level, handling_init};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Kernel privilege levels.\n#[allow(missing_docs)]\n#[derive(Eq, PartialEq)]\npub enum PrivilegeLevel {\n    User,\n    Kernel,\n    Hypervisor,\n    Unknown,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// Libkernel unit tests must execute in kernel mode.\n    #[kernel_test]\n    fn test_runner_executes_in_kernel_mode() {\n        let (level, _) = current_privilege_level();\n\n        assert!(level == PrivilegeLevel::Kernel)\n    }\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/lib.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n// Rust embedded logo for `make doc`.\n#![doc(\n    html_logo_url = \"https://raw.githubusercontent.com/rust-embedded/wg/master/assets/logo/ewg-logo-blue-white-on-transparent.png\"\n)]\n\n//! The `kernel` library.\n//!\n//! Used to compose the final kernel binary.\n//!\n//! # Code organization and architecture\n//!\n//! The code is divided into different *modules*, each representing a typical **subsystem** of the\n//! `kernel`. Top-level module files of subsystems reside directly in the `src` folder. For example,\n//! `src/memory.rs` contains code that is concerned with all things memory management.\n//!\n//! ## Visibility of processor architecture code\n//!\n//! Some of the `kernel`'s subsystems depend on low-level code that is specific to the target\n//! processor architecture. For each supported processor architecture, there exists a subfolder in\n//! `src/_arch`, for example, `src/_arch/aarch64`.\n//!\n//! The architecture folders mirror the subsystem modules laid out in `src`. For example,\n//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go\n//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in\n//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic\n//! module's name prefixed with `arch_`.\n//!\n//! For example, this is the top of `src/memory/mmu.rs`:\n//!\n//! ```\n//! #[cfg(target_arch = \"aarch64\")]\n//! #[path = \"../_arch/aarch64/memory/mmu.rs\"]\n//! mod arch_mmu;\n//! ```\n//!\n//! Often times, items from the `arch_ module` will be publicly reexported by the parent module.\n//! This way, each architecture specific module can provide its implementation of an item, while the\n//! caller must not be concerned which architecture has been conditionally compiled.\n//!\n//! ## BSP code\n//!\n//! `BSP` stands for Board Support Package. `BSP` code is organized under `src/bsp.rs` and contains\n//! target board specific definitions and functions. These are things such as the board's memory map\n//! or instances of drivers for devices that are featured on the respective board.\n//!\n//! Just like processor architecture code, the `BSP` code's module structure tries to mirror the\n//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is\n//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`.\n//!\n//! ## Kernel interfaces\n//!\n//! Both `arch` and `bsp` contain code that is conditionally compiled depending on the actual target\n//! and board for which the kernel is compiled. For example, the `interrupt controller` hardware of\n//! the `Raspberry Pi 3` and the `Raspberry Pi 4` is different, but we want the rest of the `kernel`\n//! code to play nicely with any of the two without much hassle.\n//!\n//! In order to provide a clean abstraction between `arch`, `bsp` and `generic kernel code`,\n//! `interface` traits are provided *whenever possible* and *where it makes sense*. They are defined\n//! in the respective subsystem module and help to enforce the idiom of *program to an interface,\n//! not an implementation*. For example, there will be a common IRQ handling interface which the two\n//! different interrupt controller `drivers` of both Raspberrys will implement, and only export the\n//! interface to the rest of the `kernel`.\n//!\n//! ```\n//!         +-------------------+\n//!         | Interface (Trait) |\n//!         |                   |\n//!         +--+-------------+--+\n//!            ^             ^\n//!            |             |\n//!            |             |\n//! +----------+--+       +--+----------+\n//! | kernel code |       |  bsp code   |\n//! |             |       |  arch code  |\n//! +-------------+       +-------------+\n//! ```\n//!\n//! # Summary\n//!\n//! For a logical `kernel` subsystem, corresponding code can be distributed over several physical\n//! locations. Here is an example for the **memory** subsystem:\n//!\n//! - `src/memory.rs` and `src/memory/**/*`\n//!   - Common code that is agnostic of target processor architecture and `BSP` characteristics.\n//!     - Example: A function to zero a chunk of memory.\n//!   - Interfaces for the memory subsystem that are implemented by `arch` or `BSP` code.\n//!     - Example: An `MMU` interface that defines `MMU` function prototypes.\n//! - `src/bsp/__board_name__/memory.rs` and `src/bsp/__board_name__/memory/**/*`\n//!   - `BSP` specific code.\n//!   - Example: The board's memory map (physical addresses of DRAM and MMIO devices).\n//! - `src/_arch/__arch_name__/memory.rs` and `src/_arch/__arch_name__/memory/**/*`\n//!   - Processor architecture specific code.\n//!   - Example: Implementation of the `MMU` interface for the `__arch_name__` processor\n//!     architecture.\n//!\n//! From a namespace perspective, **memory** subsystem code lives in:\n//!\n//! - `crate::memory::*`\n//! - `crate::bsp::memory::*`\n//!\n//! # Boot flow\n//!\n//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.\n//!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.\n//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.\n\n#![allow(clippy::upper_case_acronyms)]\n#![allow(incomplete_features)]\n#![feature(asm_const)]\n#![feature(const_option)]\n#![feature(core_intrinsics)]\n#![feature(format_args_nl)]\n#![feature(int_roundings)]\n#![feature(linkage)]\n#![feature(nonzero_min_max)]\n#![feature(panic_info_message)]\n#![feature(trait_alias)]\n#![feature(unchecked_math)]\n#![no_std]\n// Testing\n#![cfg_attr(test, no_main)]\n#![feature(custom_test_frameworks)]\n#![reexport_test_harness_main = \"test_main\"]\n#![test_runner(crate::test_runner)]\n\nmod panic_wait;\nmod synchronization;\n\npub mod bsp;\npub mod common;\npub mod console;\npub mod cpu;\npub mod driver;\npub mod exception;\npub mod memory;\npub mod print;\npub mod state;\npub mod time;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Version string.\npub fn version() -> &'static str {\n    concat!(\n        env!(\"CARGO_PKG_NAME\"),\n        \" version \",\n        env!(\"CARGO_PKG_VERSION\")\n    )\n}\n\n#[cfg(not(test))]\nextern \"Rust\" {\n    fn kernel_init() -> !;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n/// The default runner for unit tests.\npub fn test_runner(tests: &[&test_types::UnitTest]) {\n    // This line will be printed as the test header.\n    println!(\"Running {} tests\", tests.len());\n\n    for (i, test) in tests.iter().enumerate() {\n        print!(\"{:>3}. {:.<58}\", i + 1, test.name);\n\n        // Run the actual test.\n        (test.test_func)();\n\n        // Failed tests call panic!(). Execution reaches here only if the test has passed.\n        println!(\"[ok]\")\n    }\n}\n\n/// The `kernel_init()` for unit tests.\n#[cfg(test)]\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    bsp::driver::qemu_bring_up_console();\n\n    test_main();\n\n    cpu::qemu_exit_success()\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/main.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n// Rust embedded logo for `make doc`.\n#![doc(\n    html_logo_url = \"https://raw.githubusercontent.com/rust-embedded/wg/master/assets/logo/ewg-logo-blue-white-on-transparent.png\"\n)]\n\n//! The `kernel` binary.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\nuse libkernel::{bsp, cpu, driver, exception, info, memory, state, time};\n\n/// Early init code.\n///\n/// # Safety\n///\n/// - Only a single core must be active and running this function.\n/// - The init calls in this function must appear in the correct order:\n///     - MMU + Data caching must be activated at the earliest. Without it, any atomic operations,\n///       e.g. the yet-to-be-introduced spinlocks in the device drivers (which currently employ\n///       IRQSafeNullLocks instead of spinlocks), will fail to work (properly) on the RPi SoCs.\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    use memory::mmu::interface::MMU;\n\n    exception::handling_init();\n\n    if let Err(string) = memory::mmu::mmu().enable_mmu_and_caching() {\n        panic!(\"MMU: {}\", string);\n    }\n\n    // Initialize the BSP driver subsystem.\n    if let Err(x) = bsp::driver::init() {\n        panic!(\"Error initializing BSP driver subsystem: {}\", x);\n    }\n\n    // Initialize all device drivers.\n    driver::driver_manager().init_drivers_and_irqs();\n\n    // Unmask interrupts on the boot CPU core.\n    exception::asynchronous::local_irq_unmask();\n\n    // Announce conclusion of the kernel_init() phase.\n    state::state_manager().transition_to_single_core_main();\n\n    // Transition from unsafe to safe.\n    kernel_main()\n}\n\n/// The main function running after the early init.\nfn kernel_main() -> ! {\n    info!(\"{}\", libkernel::version());\n    info!(\"Booting on: {}\", bsp::board_name());\n\n    info!(\"MMU online. Special regions:\");\n    bsp::memory::mmu::virt_mem_layout().print_layout();\n\n    let (_, privilege_level) = exception::current_privilege_level();\n    info!(\"Current privilege level: {}\", privilege_level);\n\n    info!(\"Exception handling state:\");\n    exception::asynchronous::print_state();\n\n    info!(\n        \"Architectural timer resolution: {} ns\",\n        time::time_manager().resolution().as_nanos()\n    );\n\n    info!(\"Drivers loaded:\");\n    driver::driver_manager().enumerate();\n\n    info!(\"Registered IRQ handlers:\");\n    exception::asynchronous::irq_manager().print_handler();\n\n    info!(\"Echoing input now\");\n    cpu::wait_forever();\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/memory/mmu/translation_table.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Translation table.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../../_arch/aarch64/memory/mmu/translation_table.rs\"]\nmod arch_translation_table;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_translation_table::KernelTranslationTable;\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/memory/mmu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management Unit.\n//!\n//! In order to decouple `BSP` and `arch` parts of the MMU code (to keep them pluggable), this file\n//! provides types for composing an architecture-agnostic description of the kernel's virtual memory\n//! layout.\n//!\n//! The `BSP` provides such a description through the `bsp::memory::mmu::virt_mem_layout()`\n//! function.\n//!\n//! The `MMU` driver of the `arch` code uses `bsp::memory::mmu::virt_mem_layout()` to compile and\n//! install respective translation tables.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/memory/mmu.rs\"]\nmod arch_mmu;\n\nmod translation_table;\n\nuse crate::common;\nuse core::{fmt, ops::RangeInclusive};\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_mmu::mmu;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// MMU enable errors variants.\n#[allow(missing_docs)]\n#[derive(Debug)]\npub enum MMUEnableError {\n    AlreadyEnabled,\n    Other(&'static str),\n}\n\n/// Memory Management interfaces.\npub mod interface {\n    use super::*;\n\n    /// MMU functions.\n    pub trait MMU {\n        /// Called by the kernel during early init. Supposed to take the translation tables from the\n        /// `BSP`-supplied `virt_mem_layout()` and install/activate them for the respective MMU.\n        ///\n        /// # Safety\n        ///\n        /// - Changes the HW's global state.\n        unsafe fn enable_mmu_and_caching(&self) -> Result<(), MMUEnableError>;\n\n        /// Returns true if the MMU is enabled, false otherwise.\n        fn is_enabled(&self) -> bool;\n    }\n}\n\n/// Describes the characteristics of a translation granule.\npub struct TranslationGranule<const GRANULE_SIZE: usize>;\n\n/// Describes properties of an address space.\npub struct AddressSpace<const AS_SIZE: usize>;\n\n/// Architecture agnostic translation types.\n#[allow(missing_docs)]\n#[derive(Copy, Clone)]\npub enum Translation {\n    Identity,\n    Offset(usize),\n}\n\n/// Architecture agnostic memory attributes.\n#[allow(missing_docs)]\n#[derive(Copy, Clone)]\npub enum MemAttributes {\n    CacheableDRAM,\n    Device,\n}\n\n/// Architecture agnostic access permissions.\n#[allow(missing_docs)]\n#[derive(Copy, Clone)]\npub enum AccessPermissions {\n    ReadOnly,\n    ReadWrite,\n}\n\n/// Collection of memory attributes.\n#[allow(missing_docs)]\n#[derive(Copy, Clone)]\npub struct AttributeFields {\n    pub mem_attributes: MemAttributes,\n    pub acc_perms: AccessPermissions,\n    pub execute_never: bool,\n}\n\n/// Architecture agnostic descriptor for a memory range.\n#[allow(missing_docs)]\npub struct TranslationDescriptor {\n    pub name: &'static str,\n    pub virtual_range: fn() -> RangeInclusive<usize>,\n    pub physical_range_translation: Translation,\n    pub attribute_fields: AttributeFields,\n}\n\n/// Type for expressing the kernel's virtual memory layout.\npub struct KernelVirtualLayout<const NUM_SPECIAL_RANGES: usize> {\n    /// The last (inclusive) address of the address space.\n    max_virt_addr_inclusive: usize,\n\n    /// Array of descriptors for non-standard (normal cacheable DRAM) memory regions.\n    inner: [TranslationDescriptor; NUM_SPECIAL_RANGES],\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl fmt::Display for MMUEnableError {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        match self {\n            MMUEnableError::AlreadyEnabled => write!(f, \"MMU is already enabled\"),\n            MMUEnableError::Other(x) => write!(f, \"{}\", x),\n        }\n    }\n}\n\nimpl<const GRANULE_SIZE: usize> TranslationGranule<GRANULE_SIZE> {\n    /// The granule's size.\n    pub const SIZE: usize = Self::size_checked();\n\n    /// The granule's shift, aka log2(size).\n    pub const SHIFT: usize = Self::SIZE.trailing_zeros() as usize;\n\n    const fn size_checked() -> usize {\n        assert!(GRANULE_SIZE.is_power_of_two());\n\n        GRANULE_SIZE\n    }\n}\n\nimpl<const AS_SIZE: usize> AddressSpace<AS_SIZE> {\n    /// The address space size.\n    pub const SIZE: usize = Self::size_checked();\n\n    /// The address space shift, aka log2(size).\n    pub const SIZE_SHIFT: usize = Self::SIZE.trailing_zeros() as usize;\n\n    const fn size_checked() -> usize {\n        assert!(AS_SIZE.is_power_of_two());\n\n        // Check for architectural restrictions as well.\n        Self::arch_address_space_size_sanity_checks();\n\n        AS_SIZE\n    }\n}\n\nimpl Default for AttributeFields {\n    fn default() -> AttributeFields {\n        AttributeFields {\n            mem_attributes: MemAttributes::CacheableDRAM,\n            acc_perms: AccessPermissions::ReadWrite,\n            execute_never: true,\n        }\n    }\n}\n\n/// Human-readable output of a TranslationDescriptor.\nimpl fmt::Display for TranslationDescriptor {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        // Call the function to which self.range points, and dereference the result, which causes\n        // Rust to copy the value.\n        let start = *(self.virtual_range)().start();\n        let end = *(self.virtual_range)().end();\n        let size = end - start + 1;\n\n        let (size, unit) = common::size_human_readable_ceil(size);\n\n        let attr = match self.attribute_fields.mem_attributes {\n            MemAttributes::CacheableDRAM => \"C\",\n            MemAttributes::Device => \"Dev\",\n        };\n\n        let acc_p = match self.attribute_fields.acc_perms {\n            AccessPermissions::ReadOnly => \"RO\",\n            AccessPermissions::ReadWrite => \"RW\",\n        };\n\n        let xn = if self.attribute_fields.execute_never {\n            \"PXN\"\n        } else {\n            \"PX\"\n        };\n\n        write!(\n            f,\n            \"      {:#010x} - {:#010x} | {: >3} {} | {: <3} {} {: <3} | {}\",\n            start, end, size, unit, attr, acc_p, xn, self.name\n        )\n    }\n}\n\nimpl<const NUM_SPECIAL_RANGES: usize> KernelVirtualLayout<{ NUM_SPECIAL_RANGES }> {\n    /// Create a new instance.\n    pub const fn new(max: usize, layout: [TranslationDescriptor; NUM_SPECIAL_RANGES]) -> Self {\n        Self {\n            max_virt_addr_inclusive: max,\n            inner: layout,\n        }\n    }\n\n    /// For a virtual address, find and return the physical output address and corresponding\n    /// attributes.\n    ///\n    /// If the address is not found in `inner`, return an identity mapped default with normal\n    /// cacheable DRAM attributes.\n    pub fn virt_addr_properties(\n        &self,\n        virt_addr: usize,\n    ) -> Result<(usize, AttributeFields), &'static str> {\n        if virt_addr > self.max_virt_addr_inclusive {\n            return Err(\"Address out of range\");\n        }\n\n        for i in self.inner.iter() {\n            if (i.virtual_range)().contains(&virt_addr) {\n                let output_addr = match i.physical_range_translation {\n                    Translation::Identity => virt_addr,\n                    Translation::Offset(a) => a + (virt_addr - (i.virtual_range)().start()),\n                };\n\n                return Ok((output_addr, i.attribute_fields));\n            }\n        }\n\n        Ok((virt_addr, AttributeFields::default()))\n    }\n\n    /// Print the memory layout.\n    pub fn print_layout(&self) {\n        use crate::info;\n\n        for i in self.inner.iter() {\n            info!(\"{}\", i);\n        }\n    }\n\n    #[cfg(test)]\n    pub fn inner(&self) -> &[TranslationDescriptor; NUM_SPECIAL_RANGES] {\n        &self.inner\n    }\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/memory.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management.\n\npub mod mmu;\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/panic_wait.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A panic handler that infinitely waits.\n\nuse crate::{cpu, exception, println};\nuse core::panic::PanicInfo;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// The point of exit for `libkernel`.\n///\n/// It is linked weakly, so that the integration tests can overload its standard behavior.\n#[linkage = \"weak\"]\n#[no_mangle]\nfn _panic_exit() -> ! {\n    #[cfg(not(feature = \"test_build\"))]\n    {\n        cpu::wait_forever()\n    }\n\n    #[cfg(feature = \"test_build\")]\n    {\n        cpu::qemu_exit_failure()\n    }\n}\n\n/// Stop immediately if called a second time.\n///\n/// # Note\n///\n/// Using atomics here relieves us from needing to use `unsafe` for the static variable.\n///\n/// On `AArch64`, which is the only implemented architecture at the time of writing this,\n/// [`AtomicBool::load`] and [`AtomicBool::store`] are lowered to ordinary load and store\n/// instructions. They are therefore safe to use even with MMU + caching deactivated.\n///\n/// [`AtomicBool::load`]: core::sync::atomic::AtomicBool::load\n/// [`AtomicBool::store`]: core::sync::atomic::AtomicBool::store\nfn panic_prevent_reenter() {\n    use core::sync::atomic::{AtomicBool, Ordering};\n\n    #[cfg(not(target_arch = \"aarch64\"))]\n    compile_error!(\"Add the target_arch to above's check if the following code is safe to use\");\n\n    static PANIC_IN_PROGRESS: AtomicBool = AtomicBool::new(false);\n\n    if !PANIC_IN_PROGRESS.load(Ordering::Relaxed) {\n        PANIC_IN_PROGRESS.store(true, Ordering::Relaxed);\n\n        return;\n    }\n\n    _panic_exit()\n}\n\n#[panic_handler]\nfn panic(info: &PanicInfo) -> ! {\n    exception::asynchronous::local_irq_mask();\n\n    // Protect against panic infinite loops if any of the following code panics itself.\n    panic_prevent_reenter();\n\n    let timestamp = crate::time::time_manager().uptime();\n    let (location, line, column) = match info.location() {\n        Some(loc) => (loc.file(), loc.line(), loc.column()),\n        _ => (\"???\", 0, 0),\n    };\n\n    println!(\n        \"[  {:>3}.{:06}] Kernel panic!\\n\\n\\\n        Panic location:\\n      File '{}', line {}, column {}\\n\\n\\\n        {}\",\n        timestamp.as_secs(),\n        timestamp.subsec_micros(),\n        location,\n        line,\n        column,\n        info.message().unwrap_or(&format_args!(\"\")),\n    );\n\n    _panic_exit()\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/print.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Printing.\n\nuse crate::console;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n#[doc(hidden)]\npub fn _print(args: fmt::Arguments) {\n    console::console().write_fmt(args).unwrap();\n}\n\n/// Prints without a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! print {\n    ($($arg:tt)*) => ($crate::print::_print(format_args!($($arg)*)));\n}\n\n/// Prints with a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! println {\n    () => ($crate::print!(\"\\n\"));\n    ($($arg:tt)*) => ({\n        $crate::print::_print(format_args_nl!($($arg)*));\n    })\n}\n\n/// Prints an info, with a newline.\n#[macro_export]\nmacro_rules! info {\n    ($string:expr) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[  {:>3}.{:06}] \", $string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n        ));\n    });\n    ($format_string:expr, $($arg:tt)*) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[  {:>3}.{:06}] \", $format_string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n            $($arg)*\n        ));\n    })\n}\n\n/// Prints a warning, with a newline.\n#[macro_export]\nmacro_rules! warn {\n    ($string:expr) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[W {:>3}.{:06}] \", $string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n        ));\n    });\n    ($format_string:expr, $($arg:tt)*) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[W {:>3}.{:06}] \", $format_string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n            $($arg)*\n        ));\n    })\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/state.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! State information about the kernel itself.\n\nuse core::sync::atomic::{AtomicU8, Ordering};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Different stages in the kernel execution.\n#[derive(Copy, Clone, Eq, PartialEq)]\nenum State {\n    /// The kernel starts booting in this state.\n    Init,\n\n    /// The kernel transitions to this state when jumping to `kernel_main()` (at the end of\n    /// `kernel_init()`, after all init calls are done).\n    SingleCoreMain,\n\n    /// The kernel transitions to this state when it boots the secondary cores, aka switches\n    /// exectution mode to symmetric multiprocessing (SMP).\n    MultiCoreMain,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Maintains the kernel state and state transitions.\npub struct StateManager(AtomicU8);\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic STATE_MANAGER: StateManager = StateManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the global StateManager.\npub fn state_manager() -> &'static StateManager {\n    &STATE_MANAGER\n}\n\nimpl StateManager {\n    const INIT: u8 = 0;\n    const SINGLE_CORE_MAIN: u8 = 1;\n    const MULTI_CORE_MAIN: u8 = 2;\n\n    /// Create a new instance.\n    pub const fn new() -> Self {\n        Self(AtomicU8::new(Self::INIT))\n    }\n\n    /// Return the current state.\n    fn state(&self) -> State {\n        let state = self.0.load(Ordering::Acquire);\n\n        match state {\n            Self::INIT => State::Init,\n            Self::SINGLE_CORE_MAIN => State::SingleCoreMain,\n            Self::MULTI_CORE_MAIN => State::MultiCoreMain,\n            _ => panic!(\"Invalid KERNEL_STATE\"),\n        }\n    }\n\n    /// Return if the kernel is init state.\n    pub fn is_init(&self) -> bool {\n        self.state() == State::Init\n    }\n\n    /// Transition from Init to SingleCoreMain.\n    pub fn transition_to_single_core_main(&self) {\n        if self\n            .0\n            .compare_exchange(\n                Self::INIT,\n                Self::SINGLE_CORE_MAIN,\n                Ordering::Acquire,\n                Ordering::Relaxed,\n            )\n            .is_err()\n        {\n            panic!(\"transition_to_single_core_main() called while state != Init\");\n        }\n    }\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/synchronization.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Synchronization primitives.\n//!\n//! # Resources\n//!\n//!   - <https://doc.rust-lang.org/book/ch16-04-extensible-concurrency-sync-and-send.html>\n//!   - <https://stackoverflow.com/questions/59428096/understanding-the-send-trait>\n//!   - <https://doc.rust-lang.org/std/cell/index.html>\n\nuse core::cell::UnsafeCell;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Synchronization interfaces.\npub mod interface {\n\n    /// Any object implementing this trait guarantees exclusive access to the data wrapped within\n    /// the Mutex for the duration of the provided closure.\n    pub trait Mutex {\n        /// The type of the data that is wrapped by this mutex.\n        type Data;\n\n        /// Locks the mutex and grants the closure temporary mutable access to the wrapped data.\n        fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R;\n    }\n\n    /// A reader-writer exclusion type.\n    ///\n    /// The implementing object allows either a number of readers or at most one writer at any point\n    /// in time.\n    pub trait ReadWriteEx {\n        /// The type of encapsulated data.\n        type Data;\n\n        /// Grants temporary mutable access to the encapsulated data.\n        fn write<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R;\n\n        /// Grants temporary immutable access to the encapsulated data.\n        fn read<'a, R>(&'a self, f: impl FnOnce(&'a Self::Data) -> R) -> R;\n    }\n}\n\n/// A pseudo-lock for teaching purposes.\n///\n/// In contrast to a real Mutex implementation, does not protect against concurrent access from\n/// other cores to the contained data. This part is preserved for later lessons.\n///\n/// The lock will only be used as long as it is safe to do so, i.e. as long as the kernel is\n/// executing on a single core.\npub struct IRQSafeNullLock<T>\nwhere\n    T: ?Sized,\n{\n    data: UnsafeCell<T>,\n}\n\n/// A pseudo-lock that is RW during the single-core kernel init phase and RO afterwards.\n///\n/// Intended to encapsulate data that is populated during kernel init when no concurrency exists.\npub struct InitStateLock<T>\nwhere\n    T: ?Sized,\n{\n    data: UnsafeCell<T>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nunsafe impl<T> Send for IRQSafeNullLock<T> where T: ?Sized + Send {}\nunsafe impl<T> Sync for IRQSafeNullLock<T> where T: ?Sized + Send {}\n\nimpl<T> IRQSafeNullLock<T> {\n    /// Create an instance.\n    pub const fn new(data: T) -> Self {\n        Self {\n            data: UnsafeCell::new(data),\n        }\n    }\n}\n\nunsafe impl<T> Send for InitStateLock<T> where T: ?Sized + Send {}\nunsafe impl<T> Sync for InitStateLock<T> where T: ?Sized + Send {}\n\nimpl<T> InitStateLock<T> {\n    /// Create an instance.\n    pub const fn new(data: T) -> Self {\n        Self {\n            data: UnsafeCell::new(data),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse crate::{exception, state};\n\nimpl<T> interface::Mutex for IRQSafeNullLock<T> {\n    type Data = T;\n\n    fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R {\n        // In a real lock, there would be code encapsulating this line that ensures that this\n        // mutable reference will ever only be given out once at a time.\n        let data = unsafe { &mut *self.data.get() };\n\n        // Execute the closure while IRQs are masked.\n        exception::asynchronous::exec_with_irq_masked(|| f(data))\n    }\n}\n\nimpl<T> interface::ReadWriteEx for InitStateLock<T> {\n    type Data = T;\n\n    fn write<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R {\n        assert!(\n            state::state_manager().is_init(),\n            \"InitStateLock::write called after kernel init phase\"\n        );\n        assert!(\n            !exception::asynchronous::is_local_irq_masked(),\n            \"InitStateLock::write called with IRQs unmasked\"\n        );\n\n        let data = unsafe { &mut *self.data.get() };\n\n        f(data)\n    }\n\n    fn read<'a, R>(&'a self, f: impl FnOnce(&'a Self::Data) -> R) -> R {\n        let data = unsafe { &*self.data.get() };\n\n        f(data)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// InitStateLock must be transparent.\n    #[kernel_test]\n    fn init_state_lock_is_transparent() {\n        use core::mem::size_of;\n\n        assert_eq!(size_of::<InitStateLock<u64>>(), size_of::<u64>());\n    }\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/src/time.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Timer primitives.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/time.rs\"]\nmod arch_time;\n\nuse core::time::Duration;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Provides time management functions.\npub struct TimeManager;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic TIME_MANAGER: TimeManager = TimeManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the global TimeManager.\npub fn time_manager() -> &'static TimeManager {\n    &TIME_MANAGER\n}\n\nimpl TimeManager {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self\n    }\n\n    /// The timer's resolution.\n    pub fn resolution(&self) -> Duration {\n        arch_time::resolution()\n    }\n\n    /// The uptime since power-on of the device.\n    ///\n    /// This includes time consumed by firmware and bootloaders.\n    pub fn uptime(&self) -> Duration {\n        arch_time::uptime()\n    }\n\n    /// Spin for a given duration.\n    pub fn spin_for(&self, duration: Duration) {\n        arch_time::spin_for(duration)\n    }\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/tests/00_console_sanity.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'console_io_test'\n\n# Verify sending and receiving works as expected.\nclass TxRxHandshakeTest < SubtestBase\n    def name\n        'Transmit and Receive handshake'\n    end\n\n    def run(qemu_out, qemu_in)\n        qemu_in.write_nonblock('ABC')\n        expect_or_raise(qemu_out, 'OK1234')\n    end\nend\n\n# Check for correct TX statistics implementation. Depends on test 1 being run first.\nclass TxStatisticsTest < SubtestBase\n    def name\n        'Transmit statistics'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, '6')\n    end\nend\n\n# Check for correct RX statistics implementation. Depends on test 1 being run first.\nclass RxStatisticsTest < SubtestBase\n    def name\n        'Receive statistics'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, '3')\n    end\nend\n\n## -------------------------------------------------------------------------------------------------\n## Test registration\n## -------------------------------------------------------------------------------------------------\ndef subtest_collection\n    [TxRxHandshakeTest.new, TxStatisticsTest.new, RxStatisticsTest.new]\nend\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/tests/00_console_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Console sanity tests - RX, TX and statistics.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Console tests should time out on the I/O harness in case of panic.\nmod panic_wait_forever;\n\nuse libkernel::{bsp, console, cpu, exception, print};\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    use console::console;\n\n    exception::handling_init();\n    bsp::driver::qemu_bring_up_console();\n\n    // Handshake\n    assert_eq!(console().read_char(), 'A');\n    assert_eq!(console().read_char(), 'B');\n    assert_eq!(console().read_char(), 'C');\n    print!(\"OK1234\");\n\n    // 6\n    print!(\"{}\", console().chars_written());\n\n    // 3\n    print!(\"{}\", console().chars_read());\n\n    // The QEMU process running this test will be closed by the I/O test harness.\n    cpu::wait_forever();\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/tests/01_timer_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Timer sanity tests.\n\n#![feature(custom_test_frameworks)]\n#![no_main]\n#![no_std]\n#![reexport_test_harness_main = \"test_main\"]\n#![test_runner(libkernel::test_runner)]\n\nuse core::time::Duration;\nuse libkernel::{bsp, cpu, exception, time};\nuse test_macros::kernel_test;\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    bsp::driver::qemu_bring_up_console();\n\n    // Depending on CPU arch, some timer bring-up code could go here. Not needed for the RPi.\n\n    test_main();\n\n    cpu::qemu_exit_success()\n}\n\n/// Simple check that the timer is running.\n#[kernel_test]\nfn timer_is_counting() {\n    assert!(time::time_manager().uptime().as_nanos() > 0)\n}\n\n/// Timer resolution must be sufficient.\n#[kernel_test]\nfn timer_resolution_is_sufficient() {\n    assert!(time::time_manager().resolution().as_nanos() > 0);\n    assert!(time::time_manager().resolution().as_nanos() < 100)\n}\n\n/// Sanity check spin_for() implementation.\n#[kernel_test]\nfn spin_accuracy_check_1_second() {\n    let t1 = time::time_manager().uptime();\n    time::time_manager().spin_for(Duration::from_secs(1));\n    let t2 = time::time_manager().uptime();\n\n    assert_eq!((t2 - t1).as_secs(), 1)\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/tests/02_exception_sync_page_fault.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Page faults must result in synchronous exceptions.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Overwrites libkernel's `panic_wait::_panic_exit()` so that it returns a \"success\" code.\n///\n/// In this test, reaching the panic is a success, because it is called from the synchronous\n/// exception handler, which is what this test wants to achieve.\n///\n/// It also means that this integration test can not use any other code that calls panic!() directly\n/// or indirectly.\nmod panic_exit_success;\n\nuse libkernel::{bsp, cpu, exception, info, memory, println};\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    use memory::mmu::interface::MMU;\n\n    exception::handling_init();\n    bsp::driver::qemu_bring_up_console();\n\n    // This line will be printed as the test header.\n    println!(\"Testing synchronous exception handling by causing a page fault\");\n\n    if let Err(string) = memory::mmu::mmu().enable_mmu_and_caching() {\n        info!(\"MMU: {}\", string);\n        cpu::qemu_exit_failure()\n    }\n\n    info!(\"Writing beyond mapped area to address 9 GiB...\");\n    let big_addr: u64 = 9 * 1024 * 1024 * 1024;\n    core::ptr::read_volatile(big_addr as *mut u64);\n\n    // If execution reaches here, the memory access above did not cause a page fault exception.\n    cpu::qemu_exit_failure()\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/tests/03_exception_restore_sanity.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'console_io_test'\n\n# Verify that exception restore works.\nclass ExceptionRestoreTest < SubtestBase\n    def name\n        'Exception restore'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, 'Back from system call!')\n    end\nend\n\n## -------------------------------------------------------------------------------------------------\n## Test registration\n## -------------------------------------------------------------------------------------------------\ndef subtest_collection\n    [ExceptionRestoreTest.new]\nend\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/tests/03_exception_restore_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A simple sanity test to see if exception restore code works.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Console tests should time out on the I/O harness in case of panic.\nmod panic_wait_forever;\n\nuse core::arch::asm;\nuse libkernel::{bsp, cpu, exception, info, memory, println};\n\n#[inline(never)]\nfn nested_system_call() {\n    #[cfg(target_arch = \"aarch64\")]\n    unsafe {\n        asm!(\"svc #0x1337\", options(nomem, nostack, preserves_flags));\n    }\n\n    #[cfg(not(target_arch = \"aarch64\"))]\n    {\n        info!(\"Not supported yet\");\n        cpu::wait_forever();\n    }\n}\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    use memory::mmu::interface::MMU;\n\n    exception::handling_init();\n    bsp::driver::qemu_bring_up_console();\n\n    // This line will be printed as the test header.\n    println!(\"Testing exception restore\");\n\n    if let Err(string) = memory::mmu::mmu().enable_mmu_and_caching() {\n        info!(\"MMU: {}\", string);\n        cpu::qemu_exit_failure()\n    }\n\n    info!(\"Making a dummy system call\");\n\n    // Calling this inside a function indirectly tests if the link register is restored properly.\n    nested_system_call();\n\n    info!(\"Back from system call!\");\n\n    // The QEMU process running this test will be closed by the I/O test harness.\n    cpu::wait_forever();\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/tests/04_exception_irq_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! IRQ handling sanity tests.\n\n#![feature(custom_test_frameworks)]\n#![no_main]\n#![no_std]\n#![reexport_test_harness_main = \"test_main\"]\n#![test_runner(libkernel::test_runner)]\n\nuse libkernel::{bsp, cpu, exception};\nuse test_macros::kernel_test;\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    bsp::driver::qemu_bring_up_console();\n\n    exception::handling_init();\n    exception::asynchronous::local_irq_unmask();\n\n    test_main();\n\n    cpu::qemu_exit_success()\n}\n\n/// Check that IRQ masking works.\n#[kernel_test]\nfn local_irq_mask_works() {\n    // Precondition: IRQs are unmasked.\n    assert!(exception::asynchronous::is_local_irq_masked());\n\n    exception::asynchronous::local_irq_mask();\n    assert!(!exception::asynchronous::is_local_irq_masked());\n\n    // Restore earlier state.\n    exception::asynchronous::local_irq_unmask();\n}\n\n/// Check that IRQ unmasking works.\n#[kernel_test]\nfn local_irq_unmask_works() {\n    // Precondition: IRQs are masked.\n    exception::asynchronous::local_irq_mask();\n    assert!(!exception::asynchronous::is_local_irq_masked());\n\n    exception::asynchronous::local_irq_unmask();\n    assert!(exception::asynchronous::is_local_irq_masked());\n}\n\n/// Check that IRQ mask save is saving \"something\".\n#[kernel_test]\nfn local_irq_mask_save_works() {\n    // Precondition: IRQs are unmasked.\n    assert!(exception::asynchronous::is_local_irq_masked());\n\n    let first = exception::asynchronous::local_irq_mask_save();\n    assert!(!exception::asynchronous::is_local_irq_masked());\n\n    let second = exception::asynchronous::local_irq_mask_save();\n    assert_ne!(first, second);\n\n    exception::asynchronous::local_irq_restore(first);\n    assert!(exception::asynchronous::is_local_irq_masked());\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/tests/boot_test_string.rb",
    "content": "# frozen_string_literal: true\n\nEXPECTED_PRINT = 'Echoing input now'\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/tests/panic_exit_success/mod.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n/// Overwrites libkernel's `panic_wait::_panic_exit()` with the QEMU-exit version.\n#[no_mangle]\nfn _panic_exit() -> ! {\n    libkernel::cpu::qemu_exit_success()\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/kernel/tests/panic_wait_forever/mod.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n/// Overwrites libkernel's `panic_wait::_panic_exit()` with wait_forever.\n#[no_mangle]\nfn _panic_exit() -> ! {\n    libkernel::cpu::wait_forever()\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/libraries/test-macros/Cargo.toml",
    "content": "[package]\nname = \"test-macros\"\nversion = \"0.1.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n\n[lib]\nproc-macro = true\n\n[dependencies]\nproc-macro2 = \"1.x\"\nquote = \"1.x\"\nsyn = { version = \"1.x\", features = [\"full\"] }\ntest-types = { path = \"../test-types\" }\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/libraries/test-macros/src/lib.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\nuse proc_macro::TokenStream;\nuse proc_macro2::Span;\nuse quote::quote;\nuse syn::{parse_macro_input, Ident, ItemFn};\n\n#[proc_macro_attribute]\npub fn kernel_test(_attr: TokenStream, input: TokenStream) -> TokenStream {\n    let f = parse_macro_input!(input as ItemFn);\n\n    let test_name = &format!(\"{}\", f.sig.ident);\n    let test_ident = Ident::new(\n        &format!(\"{}_TEST_CONTAINER\", f.sig.ident.to_string().to_uppercase()),\n        Span::call_site(),\n    );\n    let test_code_block = f.block;\n\n    quote!(\n        #[test_case]\n        const #test_ident: test_types::UnitTest = test_types::UnitTest {\n            name: #test_name,\n            test_func: || #test_code_block,\n        };\n    )\n    .into()\n}\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/libraries/test-types/Cargo.toml",
    "content": "[package]\nname = \"test-types\"\nversion = \"0.1.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n"
  },
  {
    "path": "13_exceptions_part2_peripheral_IRQs/libraries/test-types/src/lib.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Types for the `custom_test_frameworks` implementation.\n\n#![no_std]\n\n/// Unit test container.\npub struct UnitTest {\n    /// Name of the test.\n    pub name: &'static str,\n\n    /// Function pointer to the test.\n    pub test_func: fn(),\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/.cargo/config.toml",
    "content": "[target.'cfg(target_os = \"none\")']\nrunner = \"target/kernel_test_runner.sh\"\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/.vscode/settings.json",
    "content": "{\n    \"editor.formatOnSave\": true,\n    \"editor.rulers\": [100],\n    \"rust-analyzer.cargo.target\": \"aarch64-unknown-none-softfloat\",\n    \"rust-analyzer.cargo.features\": [\"bsp_rpi3\"],\n    \"rust-analyzer.checkOnSave.allTargets\": false,\n    \"rust-analyzer.checkOnSave.extraArgs\": [\"--lib\", \"--bins\"],\n    \"rust-analyzer.lens.debug\": false,\n    \"rust-analyzer.lens.run\": false\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/Cargo.toml",
    "content": "[workspace]\n\nmembers = [\n        \"libraries/*\",\n        \"kernel\"\n]\n\n[profile.release]\nlto = true\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/Makefile",
    "content": "## SPDX-License-Identifier: MIT OR Apache-2.0\n##\n## Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\ninclude ../common/docker.mk\ninclude ../common/format.mk\ninclude ../common/operating_system.mk\n\n##--------------------------------------------------------------------------------------------------\n## Optional, user-provided configuration values\n##--------------------------------------------------------------------------------------------------\n\n# Default to the RPi3.\nBSP ?= rpi3\n\n# Default to a serial device name that is common in Linux.\nDEV_SERIAL ?= /dev/ttyUSB0\n\n# Optional integration test name.\nifdef TEST\n    TEST_ARG = --test $(TEST)\nelse\n    TEST_ARG = --test '*'\nendif\n\n\n\n##--------------------------------------------------------------------------------------------------\n## BSP-specific configuration values\n##--------------------------------------------------------------------------------------------------\nQEMU_MISSING_STRING = \"This board is not yet supported for QEMU.\"\n\nifeq ($(BSP),rpi3)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE = raspi3\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    QEMU_TEST_ARGS    = $(QEMU_RELEASE_ARGS) -semihosting\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    OPENOCD_ARG       = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi3.cfg\n    JTAG_BOOT_IMAGE   = ../X1_JTAG_boot/jtag_boot_rpi3.img\n    LD_SCRIPT_PATH    = $(shell pwd)/kernel/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a53\nelse ifeq ($(BSP),rpi4)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE =\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    QEMU_TEST_ARGS    = $(QEMU_RELEASE_ARGS) -semihosting\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    OPENOCD_ARG       = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi4.cfg\n    JTAG_BOOT_IMAGE   = ../X1_JTAG_boot/jtag_boot_rpi4.img\n    LD_SCRIPT_PATH    = $(shell pwd)/kernel/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a72\nendif\n\n# Export for build.rs.\nexport LD_SCRIPT_PATH\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets and Prerequisites\n##--------------------------------------------------------------------------------------------------\nKERNEL_MANIFEST      = kernel/Cargo.toml\nKERNEL_LINKER_SCRIPT = kernel.ld\nLAST_BUILD_CONFIG    = target/$(BSP).build_config\n\nKERNEL_ELF      = target/$(TARGET)/release/kernel\n# This parses cargo's dep-info file.\n# https://doc.rust-lang.org/cargo/guide/build-cache.html#dep-info-files\nKERNEL_ELF_DEPS = $(filter-out %: ,$(file < $(KERNEL_ELF).d)) $(KERNEL_MANIFEST) $(LAST_BUILD_CONFIG)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Command building blocks\n##--------------------------------------------------------------------------------------------------\nRUSTFLAGS = $(RUSTC_MISC_ARGS)                   \\\n    -C link-arg=--library-path=$(LD_SCRIPT_PATH) \\\n    -C link-arg=--script=$(KERNEL_LINKER_SCRIPT)\n\nRUSTFLAGS_PEDANTIC = $(RUSTFLAGS) \\\n    -D warnings                   \\\n    -D missing_docs\n\nFEATURES      = --features bsp_$(BSP)\nCOMPILER_ARGS = --target=$(TARGET) \\\n    $(FEATURES)                    \\\n    --release\n\nRUSTC_CMD   = cargo rustc $(COMPILER_ARGS) --manifest-path $(KERNEL_MANIFEST)\nDOC_CMD     = cargo doc $(COMPILER_ARGS)\nCLIPPY_CMD  = cargo clippy $(COMPILER_ARGS)\nTEST_CMD    = cargo test $(COMPILER_ARGS) --manifest-path $(KERNEL_MANIFEST)\nOBJCOPY_CMD = rust-objcopy \\\n    --strip-all            \\\n    -O binary\n\nEXEC_QEMU          = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE)\nEXEC_TEST_DISPATCH = ruby ../common/tests/dispatch.rb\nEXEC_MINIPUSH      = ruby ../common/serial/minipush.rb\n\n##------------------------------------------------------------------------------\n## Dockerization\n##------------------------------------------------------------------------------\nDOCKER_CMD            = docker run -t --rm -v $(shell pwd):/work/tutorial -w /work/tutorial\nDOCKER_CMD_INTERACT   = $(DOCKER_CMD) -i\nDOCKER_ARG_DIR_COMMON = -v $(shell pwd)/../common:/work/common\nDOCKER_ARG_DIR_JTAG   = -v $(shell pwd)/../X1_JTAG_boot:/work/X1_JTAG_boot\nDOCKER_ARG_DEV        = --privileged -v /dev:/dev\nDOCKER_ARG_NET        = --network host\n\n# DOCKER_IMAGE defined in include file (see top of this file).\nDOCKER_QEMU  = $(DOCKER_CMD_INTERACT) $(DOCKER_IMAGE)\nDOCKER_TOOLS = $(DOCKER_CMD) $(DOCKER_IMAGE)\nDOCKER_TEST  = $(DOCKER_CMD) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\nDOCKER_GDB   = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_NET) $(DOCKER_IMAGE)\n\n# Dockerize commands, which require USB device passthrough, only on Linux.\nifeq ($(shell uname -s),Linux)\n    DOCKER_CMD_DEV = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_DEV)\n\n    DOCKER_CHAINBOOT = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\n    DOCKER_JTAGBOOT  = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_ARG_DIR_JTAG) $(DOCKER_IMAGE)\n    DOCKER_OPENOCD   = $(DOCKER_CMD_DEV) $(DOCKER_ARG_NET) $(DOCKER_IMAGE)\nelse\n    DOCKER_OPENOCD   = echo \"Not yet supported on non-Linux systems.\"; \\#\nendif\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: all doc qemu chainboot clippy clean readelf objdump nm check\n\nall: $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Save the configuration as a file, so make understands if it changed.\n##------------------------------------------------------------------------------\n$(LAST_BUILD_CONFIG):\n\t@rm -f target/*.build_config\n\t@mkdir -p target\n\t@touch $(LAST_BUILD_CONFIG)\n\n##------------------------------------------------------------------------------\n## Compile the kernel ELF\n##------------------------------------------------------------------------------\n$(KERNEL_ELF): $(KERNEL_ELF_DEPS)\n\t$(call color_header, \"Compiling kernel ELF - $(BSP)\")\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(RUSTC_CMD)\n\n##------------------------------------------------------------------------------\n## Generate the stripped kernel binary\n##------------------------------------------------------------------------------\n$(KERNEL_BIN): $(KERNEL_ELF)\n\t$(call color_header, \"Generating stripped binary\")\n\t@$(OBJCOPY_CMD) $(KERNEL_ELF) $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Name\")\n\t@echo $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Size\")\n\t$(call disk_usage_KiB, $(KERNEL_BIN))\n\n##------------------------------------------------------------------------------\n## Generate the documentation\n##------------------------------------------------------------------------------\ndoc:\n\t$(call color_header, \"Generating docs\")\n\t@$(DOC_CMD) --document-private-items --open\n\n##------------------------------------------------------------------------------\n## Run the kernel in QEMU\n##------------------------------------------------------------------------------\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\nqemu:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\nqemu: $(KERNEL_BIN)\n\t$(call color_header, \"Launching QEMU\")\n\t@$(DOCKER_QEMU) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\nendif\n\n##------------------------------------------------------------------------------\n## Push the kernel to the real HW target\n##------------------------------------------------------------------------------\nchainboot: $(KERNEL_BIN)\n\t@$(DOCKER_CHAINBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run clippy\n##------------------------------------------------------------------------------\nclippy:\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(CLIPPY_CMD)\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(CLIPPY_CMD) --features test_build --tests \\\n                --manifest-path $(KERNEL_MANIFEST)\n\n##------------------------------------------------------------------------------\n## Clean\n##------------------------------------------------------------------------------\nclean:\n\trm -rf target $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run readelf\n##------------------------------------------------------------------------------\nreadelf: $(KERNEL_ELF)\n\t$(call color_header, \"Launching readelf\")\n\t@$(DOCKER_TOOLS) $(READELF_BINARY) --headers $(KERNEL_ELF)\n\n##------------------------------------------------------------------------------\n## Run objdump\n##------------------------------------------------------------------------------\nobjdump: $(KERNEL_ELF)\n\t$(call color_header, \"Launching objdump\")\n\t@$(DOCKER_TOOLS) $(OBJDUMP_BINARY) --disassemble --demangle \\\n                --section .text   \\\n                --section .rodata \\\n                $(KERNEL_ELF) | rustfilt\n\n##------------------------------------------------------------------------------\n## Run nm\n##------------------------------------------------------------------------------\nnm: $(KERNEL_ELF)\n\t$(call color_header, \"Launching nm\")\n\t@$(DOCKER_TOOLS) $(NM_BINARY) --demangle --print-size $(KERNEL_ELF) | sort | rustfilt\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Debugging targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: jtagboot openocd gdb gdb-opt0\n\n##------------------------------------------------------------------------------\n## Push the JTAG boot image to the real HW target\n##------------------------------------------------------------------------------\njtagboot:\n\t@$(DOCKER_JTAGBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(JTAG_BOOT_IMAGE)\n\n##------------------------------------------------------------------------------\n## Start OpenOCD session\n##------------------------------------------------------------------------------\nopenocd:\n\t$(call color_header, \"Launching OpenOCD\")\n\t@$(DOCKER_OPENOCD) openocd $(OPENOCD_ARG)\n\n##------------------------------------------------------------------------------\n## Start GDB session\n##------------------------------------------------------------------------------\ngdb: RUSTC_MISC_ARGS += -C debuginfo=2\ngdb-opt0: RUSTC_MISC_ARGS += -C debuginfo=2 -C opt-level=0\ngdb gdb-opt0: $(KERNEL_ELF)\n\t$(call color_header, \"Launching GDB\")\n\t@$(DOCKER_GDB) gdb-multiarch -q $(KERNEL_ELF)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Testing targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: test test_boot test_unit test_integration\n\ntest_unit test_integration: FEATURES += --features test_build\n\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\ntest_boot test_unit test_integration test:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\n##------------------------------------------------------------------------------\n## Run boot test\n##------------------------------------------------------------------------------\ntest_boot: $(KERNEL_BIN)\n\t$(call color_header, \"Boot test - $(BSP)\")\n\t@$(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Helpers for unit and integration test targets\n##------------------------------------------------------------------------------\ndefine KERNEL_TEST_RUNNER\n#!/usr/bin/env bash\n\n    # The cargo test runner seems to change into the crate under test's directory. Therefore, ensure\n    # this script executes from the root.\n    cd $(shell pwd)\n\n    TEST_ELF=$$(echo $$1 | sed -e 's/.*target/target/g')\n    TEST_BINARY=$$(echo $$1.img | sed -e 's/.*target/target/g')\n\n    $(OBJCOPY_CMD) $$TEST_ELF $$TEST_BINARY\n    $(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_TEST_ARGS) -kernel $$TEST_BINARY\nendef\n\nexport KERNEL_TEST_RUNNER\n\ndefine test_prepare\n    @mkdir -p target\n    @echo \"$$KERNEL_TEST_RUNNER\" > target/kernel_test_runner.sh\n    @chmod +x target/kernel_test_runner.sh\nendef\n\n##------------------------------------------------------------------------------\n## Run unit test(s)\n##------------------------------------------------------------------------------\ntest_unit:\n\t$(call color_header, \"Compiling unit test(s) - $(BSP)\")\n\t$(call test_prepare)\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(TEST_CMD) --lib\n\n##------------------------------------------------------------------------------\n## Run integration test(s)\n##------------------------------------------------------------------------------\ntest_integration:\n\t$(call color_header, \"Compiling integration test(s) - $(BSP)\")\n\t$(call test_prepare)\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(TEST_CMD) $(TEST_ARG)\n\ntest: test_boot test_unit test_integration\n\nendif\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/README.md",
    "content": "# Tutorial 14 - Virtual Memory Part 2: MMIO Remap\n\n## tl;dr\n\n- We introduce a first set of changes which is eventually needed for separating `kernel` and `user`\n  address spaces.\n- The memory mapping strategy gets more sophisticated as we do away with `identity mapping` the\n  whole of the board's address space.\n- Instead, only ranges that are actually needed are mapped:\n    - The `kernel binary` stays `identity mapped` for now.\n    - Device `MMIO regions` are remapped lazily (to a special reserved virtual address region).\n\n## Table of Contents\n\n- [Introduction](#introduction)\n- [Implementation](#implementation)\n  - [A New Mapping API in `src/memory/mmu.rs`](#a-new-mapping-api-in-srcmemorymmutranslationtablers)\n  - [The new APIs in action](#the-new-apis-in-action)\n  - [MMIO Virtual Address Allocation](#mmio-virtual-address-allocation)\n  - [Supporting Changes](#supporting-changes)\n- [Test it](#test-it)\n- [Diff to previous](#diff-to-previous)\n\n## Introduction\n\nThis tutorial is a first step of many needed for enabling `userspace applications` (which we\nhopefully will have some day in the very distant future).\n\nFor this, one of the features we want is a clean separation of `kernel` and `user` address spaces.\nFortunately, `ARMv8` has convenient architecture support to realize this. The following text and\npictue gives some more motivation and technical information. It is quoted from the _[ARM Cortex-A\nSeries Programmer’s Guide for ARMv8-A], Chapter 12.2, Separation of kernel and application Virtual\nAddress spaces_:\n\n> Operating systems typically have a number of applications or tasks running concurrently. Each of\n> these has its own unique set of translation tables and the kernel switches from one to another as\n> part of the process of switching context between one task and another. However, much of the memory\n> system is used only by the kernel and has fixed virtual to Physical Address mappings where the\n> translation table entries rarely change. The ARMv8 architecture provides a number of features to\n> efficiently handle this requirement.\n>\n> The table base addresses are specified in the Translation Table Base Registers `TTBR0_EL1` and\n> `TTBR1_EL1`. The translation table pointed to by `TTBR0` is selected when the upper bits of the VA\n> are all 0. `TTBR1` is selected when the upper bits of the VA are all set to 1. [...]\n>\n> Figure 12-4 shows how the kernel space can be mapped to the most significant area of memory and\n> the Virtual Address space associated with each application mapped to the least significant area of\n> memory. However, both of these are mapped to a much smaller Physical Address space.\n\n<p align=\"center\">\n    <img src=\"../doc/15_kernel_user_address_space_partitioning.png\" height=\"500\" align=\"center\">\n</p>\n\nThis approach is also sometimes called a \"[higher half kernel]\". To eventually achieve this\nseparation, this tutorial makes a start by changing the following things:\n\n1. Instead of bulk-`identity mapping` the whole of the board's address space, only the particular\n   parts that are needed will be mapped.\n1. For now, the `kernel binary` stays identity mapped. This will be changed in the coming tutorials\n   as it is a quite difficult and peculiar exercise to remap the kernel.\n1. Device `MMIO regions` are lazily remapped during device driver bringup (using the new\n   `DriverManage` function `instantiate_drivers()`).\n   1. A dedicated region of virtual addresses that we reserve using `BSP` code and the `linker\n      script` is used for this.\n1. We keep using `TTBR0` for the kernel translation tables for now. This will be changed when we\n   remap the `kernel binary` in the coming tutorials.\n\n[ARM Cortex-A Series Programmer’s Guide for ARMv8-A]: https://developer.arm.com/documentation/den0024/latest/\n[higher half kernel]: https://wiki.osdev.org/Higher_Half_Kernel\n\n## Implementation\n\nUntil now, the whole address space of the board was identity mapped at once. The **architecture**\n(`src/_arch/_/memory/**`) and **bsp** (`src/bsp/_/memory/**`) parts of the kernel worked\ntogether directly while setting up the translation tables, without any indirection through **generic\nkernel code** (`src/memory/**`).\n\nThe way it worked was that the `architectural MMU code` would query the `bsp code` about the start\nand end of the physical address space, and any special regions in this space that need a mapping\nthat _is not_ normal chacheable DRAM. It would then go ahead and map the whole address space at once\nand never touch the translation tables again during runtime.\n\nChanging in this tutorial, **architecture** and **bsp** code will no longer autonomously create the\nvirtual memory mappings. Instead, this is now orchestrated by the kernel's **generic MMU subsystem\ncode**.\n\n### A New Mapping API in `src/memory/mmu/translation_table.rs`\n\nFirst, we define an interface for operating on `translation tables`:\n\n```rust\n/// Translation table operations.\npub trait TranslationTable {\n    /// Anything that needs to run before any of the other provided functions can be used.\n    ///\n    /// # Safety\n    ///\n    /// - Implementor must ensure that this function can run only once or is harmless if invoked\n    ///   multiple times.\n    fn init(&mut self);\n\n    /// The translation table's base address to be used for programming the MMU.\n    fn phys_base_address(&self) -> Address<Physical>;\n\n    /// Map the given virtual memory region to the given physical memory region.\n    unsafe fn map_at(\n        &mut self,\n        virt_region: &MemoryRegion<Virtual>,\n        phys_region: &MemoryRegion<Physical>,\n        attr: &AttributeFields,\n    ) -> Result<(), &'static str>;\n}\n```\n\nIn order to enable the generic kernel code to manipulate the kernel's translation tables, they must\nfirst be made accessible. Until now, they were just a \"hidden\" struct in the `architectural` MMU\ndriver (`src/arch/.../memory/mmu.rs`). This made sense because the MMU driver code was the only code\nthat needed to be concerned with the table data structure, so having it accessible locally\nsimplified things.\n\nSince the tables need to be exposed to the rest of the kernel code now, it makes sense to move them\nto `BSP` code. Because ultimately, it is the `BSP` that is defining the translation table's\nproperties, such as the size of the virtual address space that the tables need to cover.\n\nThey are now defined in the global instances region of `src/bsp/.../memory/mmu.rs`. To control\naccess, they are  guarded by an `InitStateLock`.\n\n```rust\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n/// The kernel translation tables.\nstatic KERNEL_TABLES: InitStateLock<KernelTranslationTable> =\n    InitStateLock::new(KernelTranslationTable::new());\n```\n\nThe struct `KernelTranslationTable` is a type alias defined in the same file, which in turn gets its\ndefinition from an associated type of type `KernelVirtAddrSpace`, which itself is a type alias of\n`memory::mmu::AddressSpace`. I know this sounds horribly complicated, but in the end this is just\nsome layers of `const generics` whose implementation is scattered between `generic` and `arch` code.\nThis is done to (1) ensure a sane compile-time definition of the translation table struct (by doing\nvarious bounds checks), and (2) to separate concerns between generic `MMU` code and specializations\nthat come from the `architectural` part.\n\nIn the end, these tables can be accessed by calling `bsp::memory::mmu::kernel_translation_tables()`:\n\n```rust\n/// Return a reference to the kernel's translation tables.\npub fn kernel_translation_tables() -> &'static InitStateLock<KernelTranslationTable> {\n    &KERNEL_TABLES\n}\n```\n\nFinally, the generic kernel code (`src/memory/mmu.rs`) now provides a couple of memory mapping\nfunctions that access and manipulate this instance. They  are exported for the rest of the kernel to\nuse:\n\n```rust\n/// Raw mapping of a virtual to physical region in the kernel translation tables.\n///\n/// Prevents mapping into the MMIO range of the tables.\npub unsafe fn kernel_map_at(\n    name: &'static str,\n    virt_region: &MemoryRegion<Virtual>,\n    phys_region: &MemoryRegion<Physical>,\n    attr: &AttributeFields,\n) -> Result<(), &'static str>;\n\n/// MMIO remapping in the kernel translation tables.\n///\n/// Typically used by device drivers.\npub unsafe fn kernel_map_mmio(\n    name: &'static str,\n    mmio_descriptor: &MMIODescriptor,\n) -> Result<Address<Virtual>, &'static str>;\n\n/// Map the kernel's binary. Returns the translation table's base address.\npub unsafe fn kernel_map_binary() -> Result<Address<Physical>, &'static str>;\n\n/// Enable the MMU and data + instruction caching.\npub unsafe fn enable_mmu_and_caching(\n    phys_tables_base_addr: Address<Physical>,\n) -> Result<(), MMUEnableError>;\n```\n\n### The new APIs in action\n\n`kernel_map_binary()` and `enable_mmu_and_caching()` are used early in `kernel_init()` to set up\nvirtual memory:\n\n```rust\nlet phys_kernel_tables_base_addr = match memory::mmu::kernel_map_binary() {\n    Err(string) => panic!(\"Error mapping kernel binary: {}\", string),\n    Ok(addr) => addr,\n};\n\nif let Err(e) = memory::mmu::enable_mmu_and_caching(phys_kernel_tables_base_addr) {\n    panic!(\"Enabling MMU failed: {}\", e);\n}\n```\n\nBoth functions internally use `bsp` and `arch` specific code to achieve their goals. For example,\n`memory::mmu::kernel_map_binary()` itself wraps around a `bsp` function of the same name\n(`bsp::memory::mmu::kernel_map_binary()`):\n\n```rust\n/// Map the kernel binary.\npub unsafe fn kernel_map_binary() -> Result<(), &'static str> {\n    generic_mmu::kernel_map_at(\n        \"Kernel boot-core stack\",\n        // omitted for brevity.\n    )?;\n\n    generic_mmu::kernel_map_at(\n        \"Kernel code and RO data\",\n        &virt_code_region(),\n        &kernel_virt_to_phys_region(virt_code_region()),\n        &AttributeFields {\n            mem_attributes: MemAttributes::CacheableDRAM,\n            acc_perms: AccessPermissions::ReadOnly,\n            execute_never: false,\n        },\n    )?;\n\n    generic_mmu::kernel_map_at(\n        \"Kernel data and bss\",\n        // omitted for brevity.\n    )?;\n\n    Ok(())\n}\n```\n\nAnother user of the new APIs is the **driver subsystem**. As has been said in the introduction, the\ngoal is to remap the `MMIO` regions of the drivers. To achieve this in a seamless way, some changes\nto the architecture of the driver subsystem were needed.\n\nUntil now, the drivers were `static instances` which had their `MMIO addresses` statically set in\nthe constructor. This was fine, because even if virtual memory was activated, only `identity\nmapping` was used, so the hardcoded addresses would be valid with and without the MMU being active.\n\nWith `remapped MMIO addresses`, this is not possible anymore, since the remapping will only happen\nat runtime. Therefore, the new approach is to defer the whole instantiation of the drivers until the\nremapped addresses are known. To achieve this, in `src/bsp/raspberrypi/drivers.rs`, the static\ndriver instances are now wrapped into a `MaybeUninit` (and are also `mut` now):\n\n```rust\nstatic mut PL011_UART: MaybeUninit<device_driver::PL011Uart> = MaybeUninit::uninit();\nstatic mut GPIO: MaybeUninit<device_driver::GPIO> = MaybeUninit::uninit();\n\n#[cfg(feature = \"bsp_rpi3\")]\nstatic mut INTERRUPT_CONTROLLER: MaybeUninit<device_driver::InterruptController> =\n    MaybeUninit::uninit();\n\n#[cfg(feature = \"bsp_rpi4\")]\nstatic mut INTERRUPT_CONTROLLER: MaybeUninit<device_driver::GICv2> = MaybeUninit::uninit();\n```\n\nAccordingly, new dedicated `instantiate_xyz()` functions have been added, which will be called by\nthe corresponding `driver_xyz()` functions. Here is an example for the `UART`:\n\n```rust\n/// This must be called only after successful init of the memory subsystem.\nunsafe fn instantiate_uart() -> Result<(), &'static str> {\n    let mmio_descriptor = MMIODescriptor::new(mmio::PL011_UART_START, mmio::PL011_UART_SIZE);\n    let virt_addr =\n        memory::mmu::kernel_map_mmio(device_driver::PL011Uart::COMPATIBLE, &mmio_descriptor)?;\n\n    PL011_UART.write(device_driver::PL011Uart::new(virt_addr));\n\n    Ok(())\n}\n```\n\n```rust\n/// Function needs to ensure that driver registration happens only after correct instantiation.\nunsafe fn driver_uart() -> Result<(), &'static str> {\n    instantiate_uart()?;\n\n    let uart_descriptor = generic_driver::DeviceDriverDescriptor::new(\n        PL011_UART.assume_init_ref(),\n        Some(post_init_uart),\n        Some(exception::asynchronous::irq_map::PL011_UART),\n    );\n    generic_driver::driver_manager().register_driver(uart_descriptor);\n\n    Ok(())\n}\n```\n\nThe code shows that an `MMIODescriptor` is created first, and then used to remap the MMIO region\nusing `memory::mmu::kernel_map_mmio()`. This function will be discussed in detail in the next\nchapter. What's important for now is that it returns the new `Virtual Address` of the remapped MMIO\nregion. The constructor of the `UART` driver now also expects a virtual address.\n\nNext, a new instance of the `PL011Uart` driver is created, and written into the `PL011_UART` global\nvariable (remember, it is defined as `MaybeUninit<device_driver::PL011Uart> =\nMaybeUninit::uninit()`). Meaning, after this line of code, `PL011_UART` is properly initialized.\nOnly then, the driver is registered with the kernel and thus becomes accessible for the first time.\nThis ensures that nobody can use the UART before its memory has been initialized properly.\n\n### MMIO Virtual Address Allocation\n\nGetting back to the remapping part, let's peek inside `memory::mmu::kernel_map_mmio()`. We can see\nthat a `virtual address region` is obtained from an `allocator` before remapping:\n\n```rust\npub unsafe fn kernel_map_mmio(\n    name: &'static str,\n    mmio_descriptor: &MMIODescriptor,\n) -> Result<Address<Virtual>, &'static str> {\n\n    // omitted\n\n        let virt_region =\n            page_alloc::kernel_mmio_va_allocator().lock(|allocator| allocator.alloc(num_pages))?;\n\n        kernel_map_at_unchecked(\n            name,\n            &virt_region,\n            &phys_region,\n            &AttributeFields {\n                mem_attributes: MemAttributes::Device,\n                acc_perms: AccessPermissions::ReadWrite,\n                execute_never: true,\n            },\n        )?;\n\n    // omitted\n}\n```\n\nThis allocator is defined and implemented in the added file `src/memory/mmu/page_alloc.rs`. Like\nother parts of the mapping code, its implementation makes use of the newly introduced\n`PageAddress<ATYPE>` and `MemoryRegion<ATYPE>` types (in\n[`src/memory/mmu/types.rs`](kernel/src/memory/mmu/types.rs)), but apart from that is rather straight\nforward. Therefore, it won't be covered in details here.\n\nThe more interesting question is: How does the allocator get to learn which VAs it can use?\n\nThis is happening in the following function, which gets called as part of\n`memory::mmu::post_enable_init()`, which in turn gets called in `kernel_init()` after the MMU has\nbeen turned on.\n\n```rust\n/// Query the BSP for the reserved virtual addresses for MMIO remapping and initialize the kernel's\n/// MMIO VA allocator with it.\nfn kernel_init_mmio_va_allocator() {\n    let region = bsp::memory::mmu::virt_mmio_remap_region();\n\n    page_alloc::kernel_mmio_va_allocator().lock(|allocator| allocator.init(region));\n}\n```\n\nAgain, it is the `BSP` that provides the information. The `BSP` itself indirectly gets it from the\nlinker script. In it, we have defined an `8 MiB` region right after the `.data` segment:\n\n```ld.s\n__data_end_exclusive = .;\n\n/***********************************************************************************************\n* MMIO Remap Reserved\n***********************************************************************************************/\n__mmio_remap_start = .;\n. += 8 * 1024 * 1024;\n__mmio_remap_end_exclusive = .;\n\nASSERT((. & PAGE_MASK) == 0, \"MMIO remap reservation is not page aligned\")\n```\n\nThe two symbols `__mmio_remap_start` and `__mmio_remap_end_exclusive` are used by the `BSP` to learn\nthe VA range.\n\n### Supporting Changes\n\nThere's a couple of changes more not covered in this tutorial text, but the reader should ideally\nskim through them:\n\n- [`src/memory.rs`](kernel/src/memory.rs) and\n  [`src/memory/mmu/types.rs`](kernel/src/memory/mmu/types.rs) introduce supporting types,\n  like`Address<ATYPE>`, `PageAddress<ATYPE>` and `MemoryRegion<ATYPE>`. It is worth reading their\n  implementations.\n- [`src/memory/mmu/mapping_record.rs`](kernel/src/memory/mmu/mapping_record.rs) provides the generic\n  kernel code's way of tracking previous memory mappings for use cases such as reusing existing\n  mappings (in case of drivers that have their MMIO ranges in the same `64 KiB` page) or printing\n  mappings statistics.\n\n## Test it\n\nWhen you load the kernel, you can now see that the driver's MMIO virtual addresses start right after\nthe `.data` section:\n\nRaspberry Pi 3:\n\n```console\n$ make chainboot\n[...]\nMinipush 1.0\n\n[MP] ⏳ Waiting for /dev/ttyUSB0\n[MP] ✅ Serial connected\n[MP] 🔌 Please power the target now\n\n __  __ _      _ _                 _\n|  \\/  (_)_ _ (_) |   ___  __ _ __| |\n| |\\/| | | ' \\| | |__/ _ \\/ _` / _` |\n|_|  |_|_|_||_|_|____\\___/\\__,_\\__,_|\n\n           Raspberry Pi 3\n\n[ML] Requesting binary\n[MP] ⏩ Pushing 65 KiB =========================================🦀 100% 0 KiB/s Time: 00:00:00\n[ML] Loaded! Executing the payload now\n\n[    0.740694] mingo version 0.14.0\n[    0.740902] Booting on: Raspberry Pi 3\n[    0.741357] MMU online:\n[    0.741649]       -------------------------------------------------------------------------------------------------------------------------------------------\n[    0.743393]                         Virtual                                   Physical               Size       Attr                    Entity\n[    0.745138]       -------------------------------------------------------------------------------------------------------------------------------------------\n[    0.746883]       0x0000_0000_0000_0000..0x0000_0000_0007_ffff --> 0x00_0000_0000..0x00_0007_ffff | 512 KiB | C   RW XN | Kernel boot-core stack\n[    0.748486]       0x0000_0000_0008_0000..0x0000_0000_0008_ffff --> 0x00_0008_0000..0x00_0008_ffff |  64 KiB | C   RO X  | Kernel code and RO data\n[    0.750099]       0x0000_0000_0009_0000..0x0000_0000_000e_ffff --> 0x00_0009_0000..0x00_000e_ffff | 384 KiB | C   RW XN | Kernel data and bss\n[    0.751670]       0x0000_0000_000f_0000..0x0000_0000_000f_ffff --> 0x00_3f20_0000..0x00_3f20_ffff |  64 KiB | Dev RW XN | BCM PL011 UART\n[    0.753187]                                                                                                             | BCM GPIO\n[    0.754638]       0x0000_0000_0010_0000..0x0000_0000_0010_ffff --> 0x00_3f00_0000..0x00_3f00_ffff |  64 KiB | Dev RW XN | BCM Interrupt Controller\n[    0.756264]       -------------------------------------------------------------------------------------------------------------------------------------------\n```\n\nRaspberry Pi 4:\n\n```console\n$ BSP=rpi4 make chainboot\n[...]\nMinipush 1.0\n\n[MP] ⏳ Waiting for /dev/ttyUSB0\n[MP] ✅ Serial connected\n[MP] 🔌 Please power the target now\n\n __  __ _      _ _                 _\n|  \\/  (_)_ _ (_) |   ___  __ _ __| |\n| |\\/| | | ' \\| | |__/ _ \\/ _` / _` |\n|_|  |_|_|_||_|_|____\\___/\\__,_\\__,_|\n\n           Raspberry Pi 4\n\n[ML] Requesting binary\n[MP] ⏩ Pushing 65 KiB =========================================🦀 100% 0 KiB/s Time: 00:00:00\n[ML] Loaded! Executing the payload now\n\n[    0.736136] mingo version 0.14.0\n[    0.736170] Booting on: Raspberry Pi 4\n[    0.736625] MMU online:\n[    0.736918]       -------------------------------------------------------------------------------------------------------------------------------------------\n[    0.738662]                         Virtual                                   Physical               Size       Attr                    Entity\n[    0.740406]       -------------------------------------------------------------------------------------------------------------------------------------------\n[    0.742151]       0x0000_0000_0000_0000..0x0000_0000_0007_ffff --> 0x00_0000_0000..0x00_0007_ffff | 512 KiB | C   RW XN | Kernel boot-core stack\n[    0.743754]       0x0000_0000_0008_0000..0x0000_0000_0008_ffff --> 0x00_0008_0000..0x00_0008_ffff |  64 KiB | C   RO X  | Kernel code and RO data\n[    0.745368]       0x0000_0000_0009_0000..0x0000_0000_000d_ffff --> 0x00_0009_0000..0x00_000d_ffff | 320 KiB | C   RW XN | Kernel data and bss\n[    0.746938]       0x0000_0000_000e_0000..0x0000_0000_000e_ffff --> 0x00_fe20_0000..0x00_fe20_ffff |  64 KiB | Dev RW XN | BCM PL011 UART\n[    0.748455]                                                                                                             | BCM GPIO\n[    0.749907]       0x0000_0000_000f_0000..0x0000_0000_000f_ffff --> 0x00_ff84_0000..0x00_ff84_ffff |  64 KiB | Dev RW XN | GICv2 GICD\n[    0.751380]                                                                                                             | GICV2 GICC\n[    0.752853]       -------------------------------------------------------------------------------------------------------------------------------------------\n```\n\n## Diff to previous\n```diff\n\ndiff -uNr 13_exceptions_part2_peripheral_IRQs/kernel/Cargo.toml 14_virtual_mem_part2_mmio_remap/kernel/Cargo.toml\n--- 13_exceptions_part2_peripheral_IRQs/kernel/Cargo.toml\n+++ 14_virtual_mem_part2_mmio_remap/kernel/Cargo.toml\n@@ -1,6 +1,6 @@\n [package]\n name = \"mingo\"\n-version = \"0.13.0\"\n+version = \"0.14.0\"\n authors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\n edition = \"2021\"\n\n\ndiff -uNr 13_exceptions_part2_peripheral_IRQs/kernel/src/_arch/aarch64/memory/mmu/translation_table.rs 14_virtual_mem_part2_mmio_remap/kernel/src/_arch/aarch64/memory/mmu/translation_table.rs\n--- 13_exceptions_part2_peripheral_IRQs/kernel/src/_arch/aarch64/memory/mmu/translation_table.rs\n+++ 14_virtual_mem_part2_mmio_remap/kernel/src/_arch/aarch64/memory/mmu/translation_table.rs\n@@ -14,10 +14,14 @@\n //! crate::memory::mmu::translation_table::arch_translation_table\n\n use crate::{\n-    bsp, memory,\n-    memory::mmu::{\n-        arch_mmu::{Granule512MiB, Granule64KiB},\n-        AccessPermissions, AttributeFields, MemAttributes,\n+    bsp,\n+    memory::{\n+        self,\n+        mmu::{\n+            arch_mmu::{Granule512MiB, Granule64KiB},\n+            AccessPermissions, AttributeFields, MemAttributes, MemoryRegion, PageAddress,\n+        },\n+        Address, Physical, Virtual,\n     },\n };\n use core::convert;\n@@ -121,12 +125,9 @@\n }\n\n trait StartAddr {\n-    fn phys_start_addr_u64(&self) -> u64;\n-    fn phys_start_addr_usize(&self) -> usize;\n+    fn phys_start_addr(&self) -> Address<Physical>;\n }\n\n-const NUM_LVL2_TABLES: usize = bsp::memory::mmu::KernelAddrSpace::SIZE >> Granule512MiB::SHIFT;\n-\n //--------------------------------------------------------------------------------------------------\n // Public Definitions\n //--------------------------------------------------------------------------------------------------\n@@ -141,10 +142,10 @@\n\n     /// Table descriptors, covering 512 MiB windows.\n     lvl2: [TableDescriptor; NUM_TABLES],\n-}\n\n-/// A translation table type for the kernel space.\n-pub type KernelTranslationTable = FixedSizeTranslationTable<NUM_LVL2_TABLES>;\n+    /// Have the tables been initialized?\n+    initialized: bool,\n+}\n\n //--------------------------------------------------------------------------------------------------\n // Private Code\n@@ -152,12 +153,8 @@\n\n // The binary is still identity mapped, so we don't need to convert here.\n impl<T, const N: usize> StartAddr for [T; N] {\n-    fn phys_start_addr_u64(&self) -> u64 {\n-        self as *const T as u64\n-    }\n-\n-    fn phys_start_addr_usize(&self) -> usize {\n-        self as *const _ as usize\n+    fn phys_start_addr(&self) -> Address<Physical> {\n+        Address::new(self as *const _ as usize)\n     }\n }\n\n@@ -170,10 +167,10 @@\n     }\n\n     /// Create an instance pointing to the supplied address.\n-    pub fn from_next_lvl_table_addr(phys_next_lvl_table_addr: usize) -> Self {\n+    pub fn from_next_lvl_table_addr(phys_next_lvl_table_addr: Address<Physical>) -> Self {\n         let val = InMemoryRegister::<u64, STAGE1_TABLE_DESCRIPTOR::Register>::new(0);\n\n-        let shifted = phys_next_lvl_table_addr >> Granule64KiB::SHIFT;\n+        let shifted = phys_next_lvl_table_addr.as_usize() >> Granule64KiB::SHIFT;\n         val.write(\n             STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64)\n                 + STAGE1_TABLE_DESCRIPTOR::TYPE::Table\n@@ -230,12 +227,15 @@\n     }\n\n     /// Create an instance.\n-    pub fn from_output_addr(phys_output_addr: usize, attribute_fields: &AttributeFields) -> Self {\n+    pub fn from_output_page_addr(\n+        phys_output_page_addr: PageAddress<Physical>,\n+        attribute_fields: &AttributeFields,\n+    ) -> Self {\n         let val = InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(0);\n\n-        let shifted = phys_output_addr as u64 >> Granule64KiB::SHIFT;\n+        let shifted = phys_output_page_addr.into_inner().as_usize() >> Granule64KiB::SHIFT;\n         val.write(\n-            STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted)\n+            STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted as u64)\n                 + STAGE1_PAGE_DESCRIPTOR::AF::True\n                 + STAGE1_PAGE_DESCRIPTOR::TYPE::Page\n                 + STAGE1_PAGE_DESCRIPTOR::VALID::True\n@@ -244,50 +244,133 @@\n\n         Self { value: val.get() }\n     }\n+\n+    /// Returns the valid bit.\n+    fn is_valid(&self) -> bool {\n+        InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(self.value)\n+            .is_set(STAGE1_PAGE_DESCRIPTOR::VALID)\n+    }\n }\n\n //--------------------------------------------------------------------------------------------------\n // Public Code\n //--------------------------------------------------------------------------------------------------\n\n+impl<const AS_SIZE: usize> memory::mmu::AssociatedTranslationTable\n+    for memory::mmu::AddressSpace<AS_SIZE>\n+where\n+    [u8; Self::SIZE >> Granule512MiB::SHIFT]: Sized,\n+{\n+    type TableStartFromBottom = FixedSizeTranslationTable<{ Self::SIZE >> Granule512MiB::SHIFT }>;\n+}\n+\n impl<const NUM_TABLES: usize> FixedSizeTranslationTable<NUM_TABLES> {\n     /// Create an instance.\n+    #[allow(clippy::assertions_on_constants)]\n     pub const fn new() -> Self {\n+        assert!(bsp::memory::mmu::KernelGranule::SIZE == Granule64KiB::SIZE);\n+\n         // Can't have a zero-sized address space.\n         assert!(NUM_TABLES > 0);\n\n         Self {\n             lvl3: [[PageDescriptor::new_zeroed(); 8192]; NUM_TABLES],\n             lvl2: [TableDescriptor::new_zeroed(); NUM_TABLES],\n+            initialized: false,\n         }\n     }\n\n-    /// Iterates over all static translation table entries and fills them at once.\n-    ///\n-    /// # Safety\n-    ///\n-    /// - Modifies a `static mut`. Ensure it only happens from here.\n-    pub unsafe fn populate_tt_entries(&mut self) -> Result<(), &'static str> {\n-        for (l2_nr, l2_entry) in self.lvl2.iter_mut().enumerate() {\n-            *l2_entry =\n-                TableDescriptor::from_next_lvl_table_addr(self.lvl3[l2_nr].phys_start_addr_usize());\n+    /// Helper to calculate the lvl2 and lvl3 indices from an address.\n+    #[inline(always)]\n+    fn lvl2_lvl3_index_from_page_addr(\n+        &self,\n+        virt_page_addr: PageAddress<Virtual>,\n+    ) -> Result<(usize, usize), &'static str> {\n+        let addr = virt_page_addr.into_inner().as_usize();\n+        let lvl2_index = addr >> Granule512MiB::SHIFT;\n+        let lvl3_index = (addr & Granule512MiB::MASK) >> Granule64KiB::SHIFT;\n\n-            for (l3_nr, l3_entry) in self.lvl3[l2_nr].iter_mut().enumerate() {\n-                let virt_addr = (l2_nr << Granule512MiB::SHIFT) + (l3_nr << Granule64KiB::SHIFT);\n+        if lvl2_index > (NUM_TABLES - 1) {\n+            return Err(\"Virtual page is out of bounds of translation table\");\n+        }\n\n-                let (phys_output_addr, attribute_fields) =\n-                    bsp::memory::mmu::virt_mem_layout().virt_addr_properties(virt_addr)?;\n+        Ok((lvl2_index, lvl3_index))\n+    }\n\n-                *l3_entry = PageDescriptor::from_output_addr(phys_output_addr, &attribute_fields);\n-            }\n+    /// Sets the PageDescriptor corresponding to the supplied page address.\n+    ///\n+    /// Doesn't allow overriding an already valid page.\n+    #[inline(always)]\n+    fn set_page_descriptor_from_page_addr(\n+        &mut self,\n+        virt_page_addr: PageAddress<Virtual>,\n+        new_desc: &PageDescriptor,\n+    ) -> Result<(), &'static str> {\n+        let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from_page_addr(virt_page_addr)?;\n+        let desc = &mut self.lvl3[lvl2_index][lvl3_index];\n+\n+        if desc.is_valid() {\n+            return Err(\"Virtual page is already mapped\");\n         }\n\n+        *desc = *new_desc;\n         Ok(())\n     }\n+}\n\n-    /// The translation table's base address to be used for programming the MMU.\n-    pub fn phys_base_address(&self) -> u64 {\n-        self.lvl2.phys_start_addr_u64()\n+//------------------------------------------------------------------------------\n+// OS Interface Code\n+//------------------------------------------------------------------------------\n+\n+impl<const NUM_TABLES: usize> memory::mmu::translation_table::interface::TranslationTable\n+    for FixedSizeTranslationTable<NUM_TABLES>\n+{\n+    fn init(&mut self) {\n+        if self.initialized {\n+            return;\n+        }\n+\n+        // Populate the l2 entries.\n+        for (lvl2_nr, lvl2_entry) in self.lvl2.iter_mut().enumerate() {\n+            let phys_table_addr = self.lvl3[lvl2_nr].phys_start_addr();\n+\n+            let new_desc = TableDescriptor::from_next_lvl_table_addr(phys_table_addr);\n+            *lvl2_entry = new_desc;\n+        }\n+\n+        self.initialized = true;\n+    }\n+\n+    fn phys_base_address(&self) -> Address<Physical> {\n+        self.lvl2.phys_start_addr()\n+    }\n+\n+    unsafe fn map_at(\n+        &mut self,\n+        virt_region: &MemoryRegion<Virtual>,\n+        phys_region: &MemoryRegion<Physical>,\n+        attr: &AttributeFields,\n+    ) -> Result<(), &'static str> {\n+        assert!(self.initialized, \"Translation tables not initialized\");\n+\n+        if virt_region.size() != phys_region.size() {\n+            return Err(\"Tried to map memory regions with unequal sizes\");\n+        }\n+\n+        if phys_region.end_exclusive_page_addr() > bsp::memory::phys_addr_space_end_exclusive_addr()\n+        {\n+            return Err(\"Tried to map outside of physical address space\");\n+        }\n+\n+        let iter = phys_region.into_iter().zip(virt_region.into_iter());\n+        for (phys_page_addr, virt_page_addr) in iter {\n+            let new_desc = PageDescriptor::from_output_page_addr(phys_page_addr, attr);\n+            let virt_page = virt_page_addr;\n+\n+            self.set_page_descriptor_from_page_addr(virt_page, &new_desc)?;\n+        }\n+\n+        Ok(())\n     }\n }\n\n@@ -296,6 +379,9 @@\n //--------------------------------------------------------------------------------------------------\n\n #[cfg(test)]\n+pub type MinSizeTranslationTable = FixedSizeTranslationTable<1>;\n+\n+#[cfg(test)]\n mod tests {\n     use super::*;\n     use test_macros::kernel_test;\n\ndiff -uNr 13_exceptions_part2_peripheral_IRQs/kernel/src/_arch/aarch64/memory/mmu.rs 14_virtual_mem_part2_mmio_remap/kernel/src/_arch/aarch64/memory/mmu.rs\n--- 13_exceptions_part2_peripheral_IRQs/kernel/src/_arch/aarch64/memory/mmu.rs\n+++ 14_virtual_mem_part2_mmio_remap/kernel/src/_arch/aarch64/memory/mmu.rs\n@@ -15,7 +15,7 @@\n\n use crate::{\n     bsp, memory,\n-    memory::mmu::{translation_table::KernelTranslationTable, TranslationGranule},\n+    memory::{mmu::TranslationGranule, Address, Physical},\n };\n use aarch64_cpu::{asm::barrier, registers::*};\n use core::intrinsics::unlikely;\n@@ -46,13 +46,6 @@\n // Global instances\n //--------------------------------------------------------------------------------------------------\n\n-/// The kernel translation tables.\n-///\n-/// # Safety\n-///\n-/// - Supposed to land in `.bss`. Therefore, ensure that all initial member values boil down to \"0\".\n-static mut KERNEL_TABLES: KernelTranslationTable = KernelTranslationTable::new();\n-\n static MMU: MemoryManagementUnit = MemoryManagementUnit;\n\n //--------------------------------------------------------------------------------------------------\n@@ -87,7 +80,7 @@\n\n     /// Configure various settings of stage 1 of the EL1 translation regime.\n     fn configure_translation_control(&self) {\n-        let t0sz = (64 - bsp::memory::mmu::KernelAddrSpace::SIZE_SHIFT) as u64;\n+        let t0sz = (64 - bsp::memory::mmu::KernelVirtAddrSpace::SIZE_SHIFT) as u64;\n\n         TCR_EL1.write(\n             TCR_EL1::TBI0::Used\n@@ -119,7 +112,10 @@\n use memory::mmu::MMUEnableError;\n\n impl memory::mmu::interface::MMU for MemoryManagementUnit {\n-    unsafe fn enable_mmu_and_caching(&self) -> Result<(), MMUEnableError> {\n+    unsafe fn enable_mmu_and_caching(\n+        &self,\n+        phys_tables_base_addr: Address<Physical>,\n+    ) -> Result<(), MMUEnableError> {\n         if unlikely(self.is_enabled()) {\n             return Err(MMUEnableError::AlreadyEnabled);\n         }\n@@ -134,13 +130,8 @@\n         // Prepare the memory attribute indirection register.\n         self.set_up_mair();\n\n-        // Populate translation tables.\n-        KERNEL_TABLES\n-            .populate_tt_entries()\n-            .map_err(MMUEnableError::Other)?;\n-\n         // Set the \"Translation Table Base Register\".\n-        TTBR0_EL1.set_baddr(KERNEL_TABLES.phys_base_address());\n+        TTBR0_EL1.set_baddr(phys_tables_base_addr.as_usize() as u64);\n\n         self.configure_translation_control();\n\n@@ -163,33 +154,3 @@\n         SCTLR_EL1.matches_all(SCTLR_EL1::M::Enable)\n     }\n }\n-\n-//--------------------------------------------------------------------------------------------------\n-// Testing\n-//--------------------------------------------------------------------------------------------------\n-\n-#[cfg(test)]\n-mod tests {\n-    use super::*;\n-    use core::{cell::UnsafeCell, ops::Range};\n-    use test_macros::kernel_test;\n-\n-    /// Check if KERNEL_TABLES is in .bss.\n-    #[kernel_test]\n-    fn kernel_tables_in_bss() {\n-        extern \"Rust\" {\n-            static __bss_start: UnsafeCell<u64>;\n-            static __bss_end_exclusive: UnsafeCell<u64>;\n-        }\n-\n-        let bss_range = unsafe {\n-            Range {\n-                start: __bss_start.get(),\n-                end: __bss_end_exclusive.get(),\n-            }\n-        };\n-        let kernel_tables_addr = unsafe { &KERNEL_TABLES as *const _ as usize as *mut u64 };\n-\n-        assert!(bss_range.contains(&kernel_tables_addr));\n-    }\n-}\n\ndiff -uNr 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/arm/gicv2/gicc.rs 14_virtual_mem_part2_mmio_remap/kernel/src/bsp/device_driver/arm/gicv2/gicc.rs\n--- 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/arm/gicv2/gicc.rs\n+++ 14_virtual_mem_part2_mmio_remap/kernel/src/bsp/device_driver/arm/gicv2/gicc.rs\n@@ -4,7 +4,11 @@\n\n //! GICC Driver - GIC CPU interface.\n\n-use crate::{bsp::device_driver::common::MMIODerefWrapper, exception};\n+use crate::{\n+    bsp::device_driver::common::MMIODerefWrapper,\n+    exception,\n+    memory::{Address, Virtual},\n+};\n use tock_registers::{\n     interfaces::{Readable, Writeable},\n     register_bitfields, register_structs,\n@@ -73,7 +77,7 @@\n     /// # Safety\n     ///\n     /// - The user must ensure to provide a correct MMIO start address.\n-    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n+    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n         Self {\n             registers: Registers::new(mmio_start_addr),\n         }\n\ndiff -uNr 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/arm/gicv2/gicd.rs 14_virtual_mem_part2_mmio_remap/kernel/src/bsp/device_driver/arm/gicv2/gicd.rs\n--- 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/arm/gicv2/gicd.rs\n+++ 14_virtual_mem_part2_mmio_remap/kernel/src/bsp/device_driver/arm/gicv2/gicd.rs\n@@ -8,7 +8,9 @@\n //!   - SPI - Shared Peripheral Interrupt.\n\n use crate::{\n-    bsp::device_driver::common::MMIODerefWrapper, state, synchronization,\n+    bsp::device_driver::common::MMIODerefWrapper,\n+    memory::{Address, Virtual},\n+    state, synchronization,\n     synchronization::IRQSafeNullLock,\n };\n use tock_registers::{\n@@ -128,7 +130,7 @@\n     /// # Safety\n     ///\n     /// - The user must ensure to provide a correct MMIO start address.\n-    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n+    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n         Self {\n             shared_registers: IRQSafeNullLock::new(SharedRegisters::new(mmio_start_addr)),\n             banked_registers: BankedRegisters::new(mmio_start_addr),\n\ndiff -uNr 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/arm/gicv2.rs 14_virtual_mem_part2_mmio_remap/kernel/src/bsp/device_driver/arm/gicv2.rs\n--- 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/arm/gicv2.rs\n+++ 14_virtual_mem_part2_mmio_remap/kernel/src/bsp/device_driver/arm/gicv2.rs\n@@ -81,7 +81,9 @@\n\n use crate::{\n     bsp::{self, device_driver::common::BoundedUsize},\n-    cpu, driver, exception, synchronization,\n+    cpu, driver, exception,\n+    memory::{Address, Virtual},\n+    synchronization,\n     synchronization::InitStateLock,\n };\n\n@@ -125,7 +127,10 @@\n     /// # Safety\n     ///\n     /// - The user must ensure to provide a correct MMIO start address.\n-    pub const unsafe fn new(gicd_mmio_start_addr: usize, gicc_mmio_start_addr: usize) -> Self {\n+    pub const unsafe fn new(\n+        gicd_mmio_start_addr: Address<Virtual>,\n+        gicc_mmio_start_addr: Address<Virtual>,\n+    ) -> Self {\n         Self {\n             gicd: gicd::GICD::new(gicd_mmio_start_addr),\n             gicc: gicc::GICC::new(gicc_mmio_start_addr),\n\ndiff -uNr 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs 14_virtual_mem_part2_mmio_remap/kernel/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs\n--- 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs\n+++ 14_virtual_mem_part2_mmio_remap/kernel/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs\n@@ -5,8 +5,12 @@\n //! GPIO Driver.\n\n use crate::{\n-    bsp::device_driver::common::MMIODerefWrapper, driver, exception::asynchronous::IRQNumber,\n-    synchronization, synchronization::IRQSafeNullLock,\n+    bsp::device_driver::common::MMIODerefWrapper,\n+    driver,\n+    exception::asynchronous::IRQNumber,\n+    memory::{Address, Virtual},\n+    synchronization,\n+    synchronization::IRQSafeNullLock,\n };\n use tock_registers::{\n     interfaces::{ReadWriteable, Writeable},\n@@ -131,7 +135,7 @@\n     /// # Safety\n     ///\n     /// - The user must ensure to provide a correct MMIO start address.\n-    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n+    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n         Self {\n             registers: Registers::new(mmio_start_addr),\n         }\n@@ -198,7 +202,7 @@\n     /// # Safety\n     ///\n     /// - The user must ensure to provide a correct MMIO start address.\n-    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n+    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n         Self {\n             inner: IRQSafeNullLock::new(GPIOInner::new(mmio_start_addr)),\n         }\n\ndiff -uNr 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs 14_virtual_mem_part2_mmio_remap/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs\n--- 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs\n+++ 14_virtual_mem_part2_mmio_remap/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs\n@@ -11,7 +11,9 @@\n use super::{PendingIRQs, PeripheralIRQ};\n use crate::{\n     bsp::device_driver::common::MMIODerefWrapper,\n-    exception, synchronization,\n+    exception,\n+    memory::{Address, Virtual},\n+    synchronization,\n     synchronization::{IRQSafeNullLock, InitStateLock},\n };\n use tock_registers::{\n@@ -79,7 +81,7 @@\n     /// # Safety\n     ///\n     /// - The user must ensure to provide a correct MMIO start address.\n-    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n+    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n         Self {\n             wo_registers: IRQSafeNullLock::new(WriteOnlyRegisters::new(mmio_start_addr)),\n             ro_registers: ReadOnlyRegisters::new(mmio_start_addr),\n\ndiff -uNr 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs 14_virtual_mem_part2_mmio_remap/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs\n--- 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs\n+++ 14_virtual_mem_part2_mmio_remap/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs\n@@ -10,6 +10,7 @@\n     bsp::device_driver::common::BoundedUsize,\n     driver,\n     exception::{self, asynchronous::IRQHandlerDescriptor},\n+    memory::{Address, Virtual},\n };\n use core::fmt;\n\n@@ -91,7 +92,7 @@\n     /// # Safety\n     ///\n     /// - The user must ensure to provide a correct MMIO start address.\n-    pub const unsafe fn new(periph_mmio_start_addr: usize) -> Self {\n+    pub const unsafe fn new(periph_mmio_start_addr: Address<Virtual>) -> Self {\n         Self {\n             periph: peripheral_ic::PeripheralIC::new(periph_mmio_start_addr),\n         }\n\ndiff -uNr 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs 14_virtual_mem_part2_mmio_remap/kernel/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs\n--- 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs\n+++ 14_virtual_mem_part2_mmio_remap/kernel/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs\n@@ -13,6 +13,7 @@\n     bsp::device_driver::common::MMIODerefWrapper,\n     console, cpu, driver,\n     exception::{self, asynchronous::IRQNumber},\n+    memory::{Address, Virtual},\n     synchronization,\n     synchronization::IRQSafeNullLock,\n };\n@@ -244,7 +245,7 @@\n     /// # Safety\n     ///\n     /// - The user must ensure to provide a correct MMIO start address.\n-    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n+    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n         Self {\n             registers: Registers::new(mmio_start_addr),\n             chars_written: 0,\n@@ -395,7 +396,7 @@\n     /// # Safety\n     ///\n     /// - The user must ensure to provide a correct MMIO start address.\n-    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n+    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n         Self {\n             inner: IRQSafeNullLock::new(PL011UartInner::new(mmio_start_addr)),\n         }\n\ndiff -uNr 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/common.rs 14_virtual_mem_part2_mmio_remap/kernel/src/bsp/device_driver/common.rs\n--- 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/device_driver/common.rs\n+++ 14_virtual_mem_part2_mmio_remap/kernel/src/bsp/device_driver/common.rs\n@@ -4,6 +4,7 @@\n\n //! Common device driver code.\n\n+use crate::memory::{Address, Virtual};\n use core::{fmt, marker::PhantomData, ops};\n\n //--------------------------------------------------------------------------------------------------\n@@ -11,7 +12,7 @@\n //--------------------------------------------------------------------------------------------------\n\n pub struct MMIODerefWrapper<T> {\n-    start_addr: usize,\n+    start_addr: Address<Virtual>,\n     phantom: PhantomData<fn() -> T>,\n }\n\n@@ -25,7 +26,7 @@\n\n impl<T> MMIODerefWrapper<T> {\n     /// Create an instance.\n-    pub const unsafe fn new(start_addr: usize) -> Self {\n+    pub const unsafe fn new(start_addr: Address<Virtual>) -> Self {\n         Self {\n             start_addr,\n             phantom: PhantomData,\n@@ -37,7 +38,7 @@\n     type Target = T;\n\n     fn deref(&self) -> &Self::Target {\n-        unsafe { &*(self.start_addr as *const _) }\n+        unsafe { &*(self.start_addr.as_usize() as *const _) }\n     }\n }\n\n\ndiff -uNr 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/raspberrypi/driver.rs 14_virtual_mem_part2_mmio_remap/kernel/src/bsp/raspberrypi/driver.rs\n--- 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/raspberrypi/driver.rs\n+++ 14_virtual_mem_part2_mmio_remap/kernel/src/bsp/raspberrypi/driver.rs\n@@ -9,52 +9,109 @@\n     bsp::device_driver,\n     console, driver as generic_driver,\n     exception::{self as generic_exception},\n+    memory,\n+    memory::mmu::MMIODescriptor,\n+};\n+use core::{\n+    mem::MaybeUninit,\n+    sync::atomic::{AtomicBool, Ordering},\n };\n-use core::sync::atomic::{AtomicBool, Ordering};\n\n //--------------------------------------------------------------------------------------------------\n // Global instances\n //--------------------------------------------------------------------------------------------------\n\n-static PL011_UART: device_driver::PL011Uart =\n-    unsafe { device_driver::PL011Uart::new(mmio::PL011_UART_START) };\n-static GPIO: device_driver::GPIO = unsafe { device_driver::GPIO::new(mmio::GPIO_START) };\n+static mut PL011_UART: MaybeUninit<device_driver::PL011Uart> = MaybeUninit::uninit();\n+static mut GPIO: MaybeUninit<device_driver::GPIO> = MaybeUninit::uninit();\n\n #[cfg(feature = \"bsp_rpi3\")]\n-static INTERRUPT_CONTROLLER: device_driver::InterruptController =\n-    unsafe { device_driver::InterruptController::new(mmio::PERIPHERAL_IC_START) };\n+static mut INTERRUPT_CONTROLLER: MaybeUninit<device_driver::InterruptController> =\n+    MaybeUninit::uninit();\n\n #[cfg(feature = \"bsp_rpi4\")]\n-static INTERRUPT_CONTROLLER: device_driver::GICv2 =\n-    unsafe { device_driver::GICv2::new(mmio::GICD_START, mmio::GICC_START) };\n+static mut INTERRUPT_CONTROLLER: MaybeUninit<device_driver::GICv2> = MaybeUninit::uninit();\n\n //--------------------------------------------------------------------------------------------------\n // Private Code\n //--------------------------------------------------------------------------------------------------\n\n+/// This must be called only after successful init of the memory subsystem.\n+unsafe fn instantiate_uart() -> Result<(), &'static str> {\n+    let mmio_descriptor = MMIODescriptor::new(mmio::PL011_UART_START, mmio::PL011_UART_SIZE);\n+    let virt_addr =\n+        memory::mmu::kernel_map_mmio(device_driver::PL011Uart::COMPATIBLE, &mmio_descriptor)?;\n+\n+    PL011_UART.write(device_driver::PL011Uart::new(virt_addr));\n+\n+    Ok(())\n+}\n+\n /// This must be called only after successful init of the UART driver.\n-fn post_init_uart() -> Result<(), &'static str> {\n-    console::register_console(&PL011_UART);\n+unsafe fn post_init_uart() -> Result<(), &'static str> {\n+    console::register_console(PL011_UART.assume_init_ref());\n+\n+    Ok(())\n+}\n+\n+/// This must be called only after successful init of the memory subsystem.\n+unsafe fn instantiate_gpio() -> Result<(), &'static str> {\n+    let mmio_descriptor = MMIODescriptor::new(mmio::GPIO_START, mmio::GPIO_SIZE);\n+    let virt_addr =\n+        memory::mmu::kernel_map_mmio(device_driver::GPIO::COMPATIBLE, &mmio_descriptor)?;\n+\n+    GPIO.write(device_driver::GPIO::new(virt_addr));\n\n     Ok(())\n }\n\n /// This must be called only after successful init of the GPIO driver.\n-fn post_init_gpio() -> Result<(), &'static str> {\n-    GPIO.map_pl011_uart();\n+unsafe fn post_init_gpio() -> Result<(), &'static str> {\n+    GPIO.assume_init_ref().map_pl011_uart();\n+    Ok(())\n+}\n+\n+/// This must be called only after successful init of the memory subsystem.\n+#[cfg(feature = \"bsp_rpi3\")]\n+unsafe fn instantiate_interrupt_controller() -> Result<(), &'static str> {\n+    let periph_mmio_descriptor =\n+        MMIODescriptor::new(mmio::PERIPHERAL_IC_START, mmio::PERIPHERAL_IC_SIZE);\n+    let periph_virt_addr = memory::mmu::kernel_map_mmio(\n+        device_driver::InterruptController::COMPATIBLE,\n+        &periph_mmio_descriptor,\n+    )?;\n+\n+    INTERRUPT_CONTROLLER.write(device_driver::InterruptController::new(periph_virt_addr));\n+\n+    Ok(())\n+}\n+\n+/// This must be called only after successful init of the memory subsystem.\n+#[cfg(feature = \"bsp_rpi4\")]\n+unsafe fn instantiate_interrupt_controller() -> Result<(), &'static str> {\n+    let gicd_mmio_descriptor = MMIODescriptor::new(mmio::GICD_START, mmio::GICD_SIZE);\n+    let gicd_virt_addr = memory::mmu::kernel_map_mmio(\"GICv2 GICD\", &gicd_mmio_descriptor)?;\n+\n+    let gicc_mmio_descriptor = MMIODescriptor::new(mmio::GICC_START, mmio::GICC_SIZE);\n+    let gicc_virt_addr = memory::mmu::kernel_map_mmio(\"GICV2 GICC\", &gicc_mmio_descriptor)?;\n+\n+    INTERRUPT_CONTROLLER.write(device_driver::GICv2::new(gicd_virt_addr, gicc_virt_addr));\n+\n     Ok(())\n }\n\n /// This must be called only after successful init of the interrupt controller driver.\n-fn post_init_interrupt_controller() -> Result<(), &'static str> {\n-    generic_exception::asynchronous::register_irq_manager(&INTERRUPT_CONTROLLER);\n+unsafe fn post_init_interrupt_controller() -> Result<(), &'static str> {\n+    generic_exception::asynchronous::register_irq_manager(INTERRUPT_CONTROLLER.assume_init_ref());\n\n     Ok(())\n }\n\n-fn driver_uart() -> Result<(), &'static str> {\n+/// Function needs to ensure that driver registration happens only after correct instantiation.\n+unsafe fn driver_uart() -> Result<(), &'static str> {\n+    instantiate_uart()?;\n+\n     let uart_descriptor = generic_driver::DeviceDriverDescriptor::new(\n-        &PL011_UART,\n+        PL011_UART.assume_init_ref(),\n         Some(post_init_uart),\n         Some(exception::asynchronous::irq_map::PL011_UART),\n     );\n@@ -63,17 +120,26 @@\n     Ok(())\n }\n\n-fn driver_gpio() -> Result<(), &'static str> {\n-    let gpio_descriptor =\n-        generic_driver::DeviceDriverDescriptor::new(&GPIO, Some(post_init_gpio), None);\n+/// Function needs to ensure that driver registration happens only after correct instantiation.\n+unsafe fn driver_gpio() -> Result<(), &'static str> {\n+    instantiate_gpio()?;\n+\n+    let gpio_descriptor = generic_driver::DeviceDriverDescriptor::new(\n+        GPIO.assume_init_ref(),\n+        Some(post_init_gpio),\n+        None,\n+    );\n     generic_driver::driver_manager().register_driver(gpio_descriptor);\n\n     Ok(())\n }\n\n-fn driver_interrupt_controller() -> Result<(), &'static str> {\n+/// Function needs to ensure that driver registration happens only after correct instantiation.\n+unsafe fn driver_interrupt_controller() -> Result<(), &'static str> {\n+    instantiate_interrupt_controller()?;\n+\n     let interrupt_controller_descriptor = generic_driver::DeviceDriverDescriptor::new(\n-        &INTERRUPT_CONTROLLER,\n+        INTERRUPT_CONTROLLER.assume_init_ref(),\n         Some(post_init_interrupt_controller),\n         None,\n     );\n@@ -109,5 +175,10 @@\n /// than on real hardware due to QEMU's abstractions.\n #[cfg(feature = \"test_build\")]\n pub fn qemu_bring_up_console() {\n-    console::register_console(&PL011_UART);\n+    use crate::cpu;\n+\n+    unsafe {\n+        instantiate_uart().unwrap_or_else(|_| cpu::qemu_exit_failure());\n+        console::register_console(PL011_UART.assume_init_ref());\n+    };\n }\n\ndiff -uNr 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/raspberrypi/kernel.ld 14_virtual_mem_part2_mmio_remap/kernel/src/bsp/raspberrypi/kernel.ld\n--- 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/raspberrypi/kernel.ld\n+++ 14_virtual_mem_part2_mmio_remap/kernel/src/bsp/raspberrypi/kernel.ld\n@@ -38,7 +38,7 @@\n     ***********************************************************************************************/\n     .boot_core_stack (NOLOAD) :\n     {\n-                                             /*   ^             */\n+        __boot_core_stack_start = .;         /*   ^             */\n                                              /*   | stack       */\n         . += __rpi_phys_binary_load_addr;    /*   | growth      */\n                                              /*   | direction   */\n@@ -67,6 +67,7 @@\n     /***********************************************************************************************\n     * Data + BSS\n     ***********************************************************************************************/\n+    __data_start = .;\n     .data : { *(.data*) } :segment_data\n\n     /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */\n@@ -78,6 +79,18 @@\n         __bss_end_exclusive = .;\n     } :segment_data\n\n+    . = ALIGN(PAGE_SIZE);\n+    __data_end_exclusive = .;\n+\n+    /***********************************************************************************************\n+    * MMIO Remap Reserved\n+    ***********************************************************************************************/\n+    __mmio_remap_start = .;\n+    . += 8 * 1024 * 1024;\n+    __mmio_remap_end_exclusive = .;\n+\n+    ASSERT((. & PAGE_MASK) == 0, \"MMIO remap reservation is not page aligned\")\n+\n     /***********************************************************************************************\n     * Misc\n     ***********************************************************************************************/\n\ndiff -uNr 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/raspberrypi/memory/mmu.rs 14_virtual_mem_part2_mmio_remap/kernel/src/bsp/raspberrypi/memory/mmu.rs\n--- 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/raspberrypi/memory/mmu.rs\n+++ 14_virtual_mem_part2_mmio_remap/kernel/src/bsp/raspberrypi/memory/mmu.rs\n@@ -4,70 +4,163 @@\n\n //! BSP Memory Management Unit.\n\n-use super::map as memory_map;\n-use crate::memory::mmu::*;\n-use core::ops::RangeInclusive;\n+use crate::{\n+    memory::{\n+        mmu::{\n+            self as generic_mmu, AccessPermissions, AddressSpace, AssociatedTranslationTable,\n+            AttributeFields, MemAttributes, MemoryRegion, PageAddress, TranslationGranule,\n+        },\n+        Physical, Virtual,\n+    },\n+    synchronization::InitStateLock,\n+};\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+type KernelTranslationTable =\n+    <KernelVirtAddrSpace as AssociatedTranslationTable>::TableStartFromBottom;\n\n //--------------------------------------------------------------------------------------------------\n // Public Definitions\n //--------------------------------------------------------------------------------------------------\n\n-/// The kernel's address space defined by this BSP.\n-pub type KernelAddrSpace = AddressSpace<{ memory_map::END_INCLUSIVE + 1 }>;\n+/// The translation granule chosen by this BSP. This will be used everywhere else in the kernel to\n+/// derive respective data structures and their sizes. For example, the `crate::memory::mmu::Page`.\n+pub type KernelGranule = TranslationGranule<{ 64 * 1024 }>;\n+\n+/// The kernel's virtual address space defined by this BSP.\n+pub type KernelVirtAddrSpace = AddressSpace<{ 1024 * 1024 * 1024 }>;\n\n-const NUM_MEM_RANGES: usize = 2;\n+//--------------------------------------------------------------------------------------------------\n+// Global instances\n+//--------------------------------------------------------------------------------------------------\n\n-/// The virtual memory layout.\n+/// The kernel translation tables.\n ///\n-/// The layout must contain only special ranges, aka anything that is _not_ normal cacheable DRAM.\n-/// It is agnostic of the paging granularity that the architecture's MMU will use.\n-pub static LAYOUT: KernelVirtualLayout<NUM_MEM_RANGES> = KernelVirtualLayout::new(\n-    memory_map::END_INCLUSIVE,\n-    [\n-        TranslationDescriptor {\n-            name: \"Kernel code and RO data\",\n-            virtual_range: code_range_inclusive,\n-            physical_range_translation: Translation::Identity,\n-            attribute_fields: AttributeFields {\n-                mem_attributes: MemAttributes::CacheableDRAM,\n-                acc_perms: AccessPermissions::ReadOnly,\n-                execute_never: false,\n-            },\n-        },\n-        TranslationDescriptor {\n-            name: \"Device MMIO\",\n-            virtual_range: mmio_range_inclusive,\n-            physical_range_translation: Translation::Identity,\n-            attribute_fields: AttributeFields {\n-                mem_attributes: MemAttributes::Device,\n-                acc_perms: AccessPermissions::ReadWrite,\n-                execute_never: true,\n-            },\n-        },\n-    ],\n-);\n+/// It is mandatory that InitStateLock is transparent.\n+///\n+/// That is, `size_of(InitStateLock<KernelTranslationTable>) == size_of(KernelTranslationTable)`.\n+/// There is a unit tests that checks this porperty.\n+static KERNEL_TABLES: InitStateLock<KernelTranslationTable> =\n+    InitStateLock::new(KernelTranslationTable::new());\n\n //--------------------------------------------------------------------------------------------------\n // Private Code\n //--------------------------------------------------------------------------------------------------\n\n-fn code_range_inclusive() -> RangeInclusive<usize> {\n-    // Notice the subtraction to turn the exclusive end into an inclusive end.\n-    #[allow(clippy::range_minus_one)]\n-    RangeInclusive::new(super::code_start(), super::code_end_exclusive() - 1)\n+/// Helper function for calculating the number of pages the given parameter spans.\n+const fn size_to_num_pages(size: usize) -> usize {\n+    assert!(size > 0);\n+    assert!(size modulo KernelGranule::SIZE == 0);\n+\n+    size >> KernelGranule::SHIFT\n+}\n+\n+/// The code pages of the kernel binary.\n+fn virt_code_region() -> MemoryRegion<Virtual> {\n+    let num_pages = size_to_num_pages(super::code_size());\n+\n+    let start_page_addr = super::virt_code_start();\n+    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n+\n+    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n+}\n+\n+/// The data pages of the kernel binary.\n+fn virt_data_region() -> MemoryRegion<Virtual> {\n+    let num_pages = size_to_num_pages(super::data_size());\n+\n+    let start_page_addr = super::virt_data_start();\n+    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n+\n+    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n+}\n+\n+/// The boot core stack pages.\n+fn virt_boot_core_stack_region() -> MemoryRegion<Virtual> {\n+    let num_pages = size_to_num_pages(super::boot_core_stack_size());\n+\n+    let start_page_addr = super::virt_boot_core_stack_start();\n+    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n+\n+    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n }\n\n-fn mmio_range_inclusive() -> RangeInclusive<usize> {\n-    RangeInclusive::new(memory_map::mmio::START, memory_map::mmio::END_INCLUSIVE)\n+// The binary is still identity mapped, so use this trivial conversion function for mapping below.\n+\n+fn kernel_virt_to_phys_region(virt_region: MemoryRegion<Virtual>) -> MemoryRegion<Physical> {\n+    MemoryRegion::new(\n+        PageAddress::from(virt_region.start_page_addr().into_inner().as_usize()),\n+        PageAddress::from(\n+            virt_region\n+                .end_exclusive_page_addr()\n+                .into_inner()\n+                .as_usize(),\n+        ),\n+    )\n }\n\n //--------------------------------------------------------------------------------------------------\n // Public Code\n //--------------------------------------------------------------------------------------------------\n\n-/// Return a reference to the virtual memory layout.\n-pub fn virt_mem_layout() -> &'static KernelVirtualLayout<NUM_MEM_RANGES> {\n-    &LAYOUT\n+/// Return a reference to the kernel's translation tables.\n+pub fn kernel_translation_tables() -> &'static InitStateLock<KernelTranslationTable> {\n+    &KERNEL_TABLES\n+}\n+\n+/// The MMIO remap pages.\n+pub fn virt_mmio_remap_region() -> MemoryRegion<Virtual> {\n+    let num_pages = size_to_num_pages(super::mmio_remap_size());\n+\n+    let start_page_addr = super::virt_mmio_remap_start();\n+    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n+\n+    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n+}\n+\n+/// Map the kernel binary.\n+///\n+/// # Safety\n+///\n+/// - Any miscalculation or attribute error will likely be fatal. Needs careful manual checking.\n+pub unsafe fn kernel_map_binary() -> Result<(), &'static str> {\n+    generic_mmu::kernel_map_at(\n+        \"Kernel boot-core stack\",\n+        &virt_boot_core_stack_region(),\n+        &kernel_virt_to_phys_region(virt_boot_core_stack_region()),\n+        &AttributeFields {\n+            mem_attributes: MemAttributes::CacheableDRAM,\n+            acc_perms: AccessPermissions::ReadWrite,\n+            execute_never: true,\n+        },\n+    )?;\n+\n+    generic_mmu::kernel_map_at(\n+        \"Kernel code and RO data\",\n+        &virt_code_region(),\n+        &kernel_virt_to_phys_region(virt_code_region()),\n+        &AttributeFields {\n+            mem_attributes: MemAttributes::CacheableDRAM,\n+            acc_perms: AccessPermissions::ReadOnly,\n+            execute_never: false,\n+        },\n+    )?;\n+\n+    generic_mmu::kernel_map_at(\n+        \"Kernel data and bss\",\n+        &virt_data_region(),\n+        &kernel_virt_to_phys_region(virt_data_region()),\n+        &AttributeFields {\n+            mem_attributes: MemAttributes::CacheableDRAM,\n+            acc_perms: AccessPermissions::ReadWrite,\n+            execute_never: true,\n+        },\n+    )?;\n+\n+    Ok(())\n }\n\n //--------------------------------------------------------------------------------------------------\n@@ -77,38 +170,60 @@\n #[cfg(test)]\n mod tests {\n     use super::*;\n+    use core::{cell::UnsafeCell, ops::Range};\n     use test_macros::kernel_test;\n\n     /// Check alignment of the kernel's virtual memory layout sections.\n     #[kernel_test]\n     fn virt_mem_layout_sections_are_64KiB_aligned() {\n-        const SIXTYFOUR_KIB: usize = 65536;\n-\n-        for i in LAYOUT.inner().iter() {\n-            let start: usize = *(i.virtual_range)().start();\n-            let end: usize = *(i.virtual_range)().end() + 1;\n-\n-            assert_eq!(start modulo SIXTYFOUR_KIB, 0);\n-            assert_eq!(end modulo SIXTYFOUR_KIB, 0);\n-            assert!(end >= start);\n+        for i in [\n+            virt_boot_core_stack_region,\n+            virt_code_region,\n+            virt_data_region,\n+        ]\n+        .iter()\n+        {\n+            let start = i().start_page_addr().into_inner();\n+            let end_exclusive = i().end_exclusive_page_addr().into_inner();\n+\n+            assert!(start.is_page_aligned());\n+            assert!(end_exclusive.is_page_aligned());\n+            assert!(end_exclusive >= start);\n         }\n     }\n\n     /// Ensure the kernel's virtual memory layout is free of overlaps.\n     #[kernel_test]\n     fn virt_mem_layout_has_no_overlaps() {\n-        let layout = virt_mem_layout().inner();\n-\n-        for (i, first) in layout.iter().enumerate() {\n-            for second in layout.iter().skip(i + 1) {\n-                let first_range = first.virtual_range;\n-                let second_range = second.virtual_range;\n-\n-                assert!(!first_range().contains(second_range().start()));\n-                assert!(!first_range().contains(second_range().end()));\n-                assert!(!second_range().contains(first_range().start()));\n-                assert!(!second_range().contains(first_range().end()));\n+        let layout = [\n+            virt_boot_core_stack_region(),\n+            virt_code_region(),\n+            virt_data_region(),\n+        ];\n+\n+        for (i, first_range) in layout.iter().enumerate() {\n+            for second_range in layout.iter().skip(i + 1) {\n+                assert!(!first_range.overlaps(second_range))\n             }\n         }\n     }\n+\n+    /// Check if KERNEL_TABLES is in .bss.\n+    #[kernel_test]\n+    fn kernel_tables_in_bss() {\n+        extern \"Rust\" {\n+            static __bss_start: UnsafeCell<u64>;\n+            static __bss_end_exclusive: UnsafeCell<u64>;\n+        }\n+\n+        let bss_range = unsafe {\n+            Range {\n+                start: __bss_start.get(),\n+                end: __bss_end_exclusive.get(),\n+            }\n+        };\n+        let kernel_tables_addr = &KERNEL_TABLES as *const _ as usize as *mut u64;\n+\n+        assert!(bss_range.contains(&kernel_tables_addr));\n+    }\n }\n\ndiff -uNr 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/raspberrypi/memory.rs 14_virtual_mem_part2_mmio_remap/kernel/src/bsp/raspberrypi/memory.rs\n--- 13_exceptions_part2_peripheral_IRQs/kernel/src/bsp/raspberrypi/memory.rs\n+++ 14_virtual_mem_part2_mmio_remap/kernel/src/bsp/raspberrypi/memory.rs\n@@ -10,27 +10,59 @@\n //! as the boot core's stack.\n //!\n //! +---------------------------------------+\n-//! |                                       | 0x0\n+//! |                                       | boot_core_stack_start @ 0x0\n //! |                                       |                                ^\n //! | Boot-core Stack                       |                                | stack\n //! |                                       |                                | growth\n //! |                                       |                                | direction\n //! +---------------------------------------+\n-//! |                                       | code_start @ 0x8_0000\n+//! |                                       | code_start @ 0x8_0000 == boot_core_stack_end_exclusive\n //! | .text                                 |\n //! | .rodata                               |\n //! | .got                                  |\n //! |                                       |\n //! +---------------------------------------+\n-//! |                                       | code_end_exclusive\n+//! |                                       | data_start == code_end_exclusive\n //! | .data                                 |\n //! | .bss                                  |\n //! |                                       |\n //! +---------------------------------------+\n+//! |                                       | data_end_exclusive\n //! |                                       |\n+//!\n+//!\n+//!\n+//!\n+//!\n+//! The virtual memory layout is as follows:\n+//!\n+//! +---------------------------------------+\n+//! |                                       | boot_core_stack_start @ 0x0\n+//! |                                       |                                ^\n+//! | Boot-core Stack                       |                                | stack\n+//! |                                       |                                | growth\n+//! |                                       |                                | direction\n+//! +---------------------------------------+\n+//! |                                       | code_start @ 0x8_0000 == boot_core_stack_end_exclusive\n+//! | .text                                 |\n+//! | .rodata                               |\n+//! | .got                                  |\n+//! |                                       |\n+//! +---------------------------------------+\n+//! |                                       | data_start == code_end_exclusive\n+//! | .data                                 |\n+//! | .bss                                  |\n+//! |                                       |\n+//! +---------------------------------------+\n+//! |                                       |  mmio_remap_start == data_end_exclusive\n+//! | VA region for MMIO remapping          |\n+//! |                                       |\n+//! +---------------------------------------+\n+//! |                                       |  mmio_remap_end_exclusive\n //! |                                       |\n pub mod mmu;\n\n+use crate::memory::{mmu::PageAddress, Address, Physical, Virtual};\n use core::cell::UnsafeCell;\n\n //--------------------------------------------------------------------------------------------------\n@@ -41,6 +73,15 @@\n extern \"Rust\" {\n     static __code_start: UnsafeCell<()>;\n     static __code_end_exclusive: UnsafeCell<()>;\n+\n+    static __data_start: UnsafeCell<()>;\n+    static __data_end_exclusive: UnsafeCell<()>;\n+\n+    static __mmio_remap_start: UnsafeCell<()>;\n+    static __mmio_remap_end_exclusive: UnsafeCell<()>;\n+\n+    static __boot_core_stack_start: UnsafeCell<()>;\n+    static __boot_core_stack_end_exclusive: UnsafeCell<()>;\n }\n\n //--------------------------------------------------------------------------------------------------\n@@ -50,34 +91,23 @@\n /// The board's physical memory map.\n #[rustfmt::skip]\n pub(super) mod map {\n-    /// The inclusive end address of the memory map.\n-    ///\n-    /// End address + 1 must be power of two.\n-    ///\n-    /// # Note\n-    ///\n-    /// RPi3 and RPi4 boards can have different amounts of RAM. To make our code lean for\n-    /// educational purposes, we set the max size of the address space to 4 GiB regardless of board.\n-    /// This way, we can map the entire range that we need (end of MMIO for RPi4) in one take.\n-    ///\n-    /// However, making this trade-off has the downside of making it possible for the CPU to assert a\n-    /// physical address that is not backed by any DRAM (e.g. accessing an address close to 4 GiB on\n-    /// an RPi3 that comes with 1 GiB of RAM). This would result in a crash or other kind of error.\n-    pub const END_INCLUSIVE:       usize = 0xFFFF_FFFF;\n-\n-    pub const GPIO_OFFSET:         usize = 0x0020_0000;\n-    pub const UART_OFFSET:         usize = 0x0020_1000;\n+    use super::*;\n\n     /// Physical devices.\n     #[cfg(feature = \"bsp_rpi3\")]\n     pub mod mmio {\n         use super::*;\n\n-        pub const START:               usize =         0x3F00_0000;\n-        pub const PERIPHERAL_IC_START: usize = START + 0x0000_B200;\n-        pub const GPIO_START:          usize = START + GPIO_OFFSET;\n-        pub const PL011_UART_START:    usize = START + UART_OFFSET;\n-        pub const END_INCLUSIVE:       usize =         0x4000_FFFF;\n+        pub const PERIPHERAL_IC_START: Address<Physical> = Address::new(0x3F00_B200);\n+        pub const PERIPHERAL_IC_SIZE:  usize             =              0x24;\n+\n+        pub const GPIO_START:          Address<Physical> = Address::new(0x3F20_0000);\n+        pub const GPIO_SIZE:           usize             =              0xA0;\n+\n+        pub const PL011_UART_START:    Address<Physical> = Address::new(0x3F20_1000);\n+        pub const PL011_UART_SIZE:     usize             =              0x48;\n+\n+        pub const END:                 Address<Physical> = Address::new(0x4001_0000);\n     }\n\n     /// Physical devices.\n@@ -85,13 +115,22 @@\n     pub mod mmio {\n         use super::*;\n\n-        pub const START:            usize =         0xFE00_0000;\n-        pub const GPIO_START:       usize = START + GPIO_OFFSET;\n-        pub const PL011_UART_START: usize = START + UART_OFFSET;\n-        pub const GICD_START:       usize =         0xFF84_1000;\n-        pub const GICC_START:       usize =         0xFF84_2000;\n-        pub const END_INCLUSIVE:    usize =         0xFF84_FFFF;\n+        pub const GPIO_START:       Address<Physical> = Address::new(0xFE20_0000);\n+        pub const GPIO_SIZE:        usize             =              0xA0;\n+\n+        pub const PL011_UART_START: Address<Physical> = Address::new(0xFE20_1000);\n+        pub const PL011_UART_SIZE:  usize             =              0x48;\n+\n+        pub const GICD_START:       Address<Physical> = Address::new(0xFF84_1000);\n+        pub const GICD_SIZE:        usize             =              0x824;\n+\n+        pub const GICC_START:       Address<Physical> = Address::new(0xFF84_2000);\n+        pub const GICC_SIZE:        usize             =              0x14;\n+\n+        pub const END:              Address<Physical> = Address::new(0xFF85_0000);\n     }\n+\n+    pub const END: Address<Physical> = mmio::END;\n }\n\n //--------------------------------------------------------------------------------------------------\n@@ -104,15 +143,76 @@\n ///\n /// - Value is provided by the linker script and must be trusted as-is.\n #[inline(always)]\n-fn code_start() -> usize {\n-    unsafe { __code_start.get() as usize }\n+fn virt_code_start() -> PageAddress<Virtual> {\n+    PageAddress::from(unsafe { __code_start.get() as usize })\n }\n\n-/// Exclusive end page address of the code segment.\n+/// Size of the code segment.\n+///\n /// # Safety\n ///\n /// - Value is provided by the linker script and must be trusted as-is.\n #[inline(always)]\n-fn code_end_exclusive() -> usize {\n-    unsafe { __code_end_exclusive.get() as usize }\n+fn code_size() -> usize {\n+    unsafe { (__code_end_exclusive.get() as usize) - (__code_start.get() as usize) }\n+}\n+\n+/// Start page address of the data segment.\n+#[inline(always)]\n+fn virt_data_start() -> PageAddress<Virtual> {\n+    PageAddress::from(unsafe { __data_start.get() as usize })\n+}\n+\n+/// Size of the data segment.\n+///\n+/// # Safety\n+///\n+/// - Value is provided by the linker script and must be trusted as-is.\n+#[inline(always)]\n+fn data_size() -> usize {\n+    unsafe { (__data_end_exclusive.get() as usize) - (__data_start.get() as usize) }\n+}\n+\n+/// Start page address of the MMIO remap reservation.\n+///\n+/// # Safety\n+///\n+/// - Value is provided by the linker script and must be trusted as-is.\n+#[inline(always)]\n+fn virt_mmio_remap_start() -> PageAddress<Virtual> {\n+    PageAddress::from(unsafe { __mmio_remap_start.get() as usize })\n+}\n+\n+/// Size of the MMIO remap reservation.\n+///\n+/// # Safety\n+///\n+/// - Value is provided by the linker script and must be trusted as-is.\n+#[inline(always)]\n+fn mmio_remap_size() -> usize {\n+    unsafe { (__mmio_remap_end_exclusive.get() as usize) - (__mmio_remap_start.get() as usize) }\n+}\n+\n+/// Start page address of the boot core's stack.\n+#[inline(always)]\n+fn virt_boot_core_stack_start() -> PageAddress<Virtual> {\n+    PageAddress::from(unsafe { __boot_core_stack_start.get() as usize })\n+}\n+\n+/// Size of the boot core's stack.\n+#[inline(always)]\n+fn boot_core_stack_size() -> usize {\n+    unsafe {\n+        (__boot_core_stack_end_exclusive.get() as usize) - (__boot_core_stack_start.get() as usize)\n+    }\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Exclusive end address of the physical address space.\n+#[inline(always)]\n+pub fn phys_addr_space_end_exclusive_addr() -> PageAddress<Physical> {\n+    PageAddress::from(map::END)\n }\n\ndiff -uNr 13_exceptions_part2_peripheral_IRQs/kernel/src/common.rs 14_virtual_mem_part2_mmio_remap/kernel/src/common.rs\n--- 13_exceptions_part2_peripheral_IRQs/kernel/src/common.rs\n+++ 14_virtual_mem_part2_mmio_remap/kernel/src/common.rs\n@@ -4,6 +4,30 @@\n\n //! General purpose code.\n\n+/// Check if a value is aligned to a given size.\n+#[inline(always)]\n+pub const fn is_aligned(value: usize, alignment: usize) -> bool {\n+    assert!(alignment.is_power_of_two());\n+\n+    (value & (alignment - 1)) == 0\n+}\n+\n+/// Align down.\n+#[inline(always)]\n+pub const fn align_down(value: usize, alignment: usize) -> usize {\n+    assert!(alignment.is_power_of_two());\n+\n+    value & !(alignment - 1)\n+}\n+\n+/// Align up.\n+#[inline(always)]\n+pub const fn align_up(value: usize, alignment: usize) -> usize {\n+    assert!(alignment.is_power_of_two());\n+\n+    (value + alignment - 1) & !(alignment - 1)\n+}\n+\n /// Convert a size into human readable format.\n pub const fn size_human_readable_ceil(size: usize) -> (usize, &'static str) {\n     const KIB: usize = 1024;\n\ndiff -uNr 13_exceptions_part2_peripheral_IRQs/kernel/src/lib.rs 14_virtual_mem_part2_mmio_remap/kernel/src/lib.rs\n--- 13_exceptions_part2_peripheral_IRQs/kernel/src/lib.rs\n+++ 14_virtual_mem_part2_mmio_remap/kernel/src/lib.rs\n@@ -114,10 +114,13 @@\n #![feature(const_option)]\n #![feature(core_intrinsics)]\n #![feature(format_args_nl)]\n+#![feature(generic_const_exprs)]\n #![feature(int_roundings)]\n+#![feature(is_sorted)]\n #![feature(linkage)]\n #![feature(nonzero_min_max)]\n #![feature(panic_info_message)]\n+#![feature(step_trait)]\n #![feature(trait_alias)]\n #![feature(unchecked_math)]\n #![no_std]\n@@ -184,6 +187,17 @@\n #[no_mangle]\n unsafe fn kernel_init() -> ! {\n     exception::handling_init();\n+\n+    let phys_kernel_tables_base_addr = match memory::mmu::kernel_map_binary() {\n+        Err(string) => panic!(\"Error mapping kernel binary: {}\", string),\n+        Ok(addr) => addr,\n+    };\n+\n+    if let Err(e) = memory::mmu::enable_mmu_and_caching(phys_kernel_tables_base_addr) {\n+        panic!(\"Enabling MMU failed: {}\", e);\n+    }\n+\n+    memory::mmu::post_enable_init();\n     bsp::driver::qemu_bring_up_console();\n\n     test_main();\n\ndiff -uNr 13_exceptions_part2_peripheral_IRQs/kernel/src/main.rs 14_virtual_mem_part2_mmio_remap/kernel/src/main.rs\n--- 13_exceptions_part2_peripheral_IRQs/kernel/src/main.rs\n+++ 14_virtual_mem_part2_mmio_remap/kernel/src/main.rs\n@@ -26,14 +26,19 @@\n ///       IRQSafeNullLocks instead of spinlocks), will fail to work (properly) on the RPi SoCs.\n #[no_mangle]\n unsafe fn kernel_init() -> ! {\n-    use memory::mmu::interface::MMU;\n-\n     exception::handling_init();\n\n-    if let Err(string) = memory::mmu::mmu().enable_mmu_and_caching() {\n-        panic!(\"MMU: {}\", string);\n+    let phys_kernel_tables_base_addr = match memory::mmu::kernel_map_binary() {\n+        Err(string) => panic!(\"Error mapping kernel binary: {}\", string),\n+        Ok(addr) => addr,\n+    };\n+\n+    if let Err(e) = memory::mmu::enable_mmu_and_caching(phys_kernel_tables_base_addr) {\n+        panic!(\"Enabling MMU failed: {}\", e);\n     }\n\n+    memory::mmu::post_enable_init();\n+\n     // Initialize the BSP driver subsystem.\n     if let Err(x) = bsp::driver::init() {\n         panic!(\"Error initializing BSP driver subsystem: {}\", x);\n@@ -57,8 +62,8 @@\n     info!(\"{}\", libkernel::version());\n     info!(\"Booting on: {}\", bsp::board_name());\n\n-    info!(\"MMU online. Special regions:\");\n-    bsp::memory::mmu::virt_mem_layout().print_layout();\n+    info!(\"MMU online:\");\n+    memory::mmu::kernel_print_mappings();\n\n     let (_, privilege_level) = exception::current_privilege_level();\n     info!(\"Current privilege level: {}\", privilege_level);\n\ndiff -uNr 13_exceptions_part2_peripheral_IRQs/kernel/src/memory/mmu/mapping_record.rs 14_virtual_mem_part2_mmio_remap/kernel/src/memory/mmu/mapping_record.rs\n--- 13_exceptions_part2_peripheral_IRQs/kernel/src/memory/mmu/mapping_record.rs\n+++ 14_virtual_mem_part2_mmio_remap/kernel/src/memory/mmu/mapping_record.rs\n@@ -0,0 +1,238 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! A record of mapped pages.\n+\n+use super::{\n+    AccessPermissions, Address, AttributeFields, MMIODescriptor, MemAttributes, MemoryRegion,\n+    Physical, Virtual,\n+};\n+use crate::{bsp, common, info, synchronization, synchronization::InitStateLock, warn};\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Type describing a virtual memory mapping.\n+#[allow(missing_docs)]\n+#[derive(Copy, Clone)]\n+struct MappingRecordEntry {\n+    pub users: [Option<&'static str>; 5],\n+    pub phys_start_addr: Address<Physical>,\n+    pub virt_start_addr: Address<Virtual>,\n+    pub num_pages: usize,\n+    pub attribute_fields: AttributeFields,\n+}\n+\n+struct MappingRecord {\n+    inner: [Option<MappingRecordEntry>; 12],\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Global instances\n+//--------------------------------------------------------------------------------------------------\n+\n+static KERNEL_MAPPING_RECORD: InitStateLock<MappingRecord> =\n+    InitStateLock::new(MappingRecord::new());\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Code\n+//--------------------------------------------------------------------------------------------------\n+\n+impl MappingRecordEntry {\n+    pub fn new(\n+        name: &'static str,\n+        virt_region: &MemoryRegion<Virtual>,\n+        phys_region: &MemoryRegion<Physical>,\n+        attr: &AttributeFields,\n+    ) -> Self {\n+        Self {\n+            users: [Some(name), None, None, None, None],\n+            phys_start_addr: phys_region.start_addr(),\n+            virt_start_addr: virt_region.start_addr(),\n+            num_pages: phys_region.num_pages(),\n+            attribute_fields: *attr,\n+        }\n+    }\n+\n+    fn find_next_free_user(&mut self) -> Result<&mut Option<&'static str>, &'static str> {\n+        if let Some(x) = self.users.iter_mut().find(|x| x.is_none()) {\n+            return Ok(x);\n+        };\n+\n+        Err(\"Storage for user info exhausted\")\n+    }\n+\n+    pub fn add_user(&mut self, user: &'static str) -> Result<(), &'static str> {\n+        let x = self.find_next_free_user()?;\n+        *x = Some(user);\n+        Ok(())\n+    }\n+}\n+\n+impl MappingRecord {\n+    pub const fn new() -> Self {\n+        Self { inner: [None; 12] }\n+    }\n+\n+    fn size(&self) -> usize {\n+        self.inner.iter().filter(|x| x.is_some()).count()\n+    }\n+\n+    fn sort(&mut self) {\n+        let upper_bound_exclusive = self.size();\n+        let entries = &mut self.inner[0..upper_bound_exclusive];\n+\n+        if !entries.is_sorted_by_key(|item| item.unwrap().virt_start_addr) {\n+            entries.sort_unstable_by_key(|item| item.unwrap().virt_start_addr)\n+        }\n+    }\n+\n+    fn find_next_free(&mut self) -> Result<&mut Option<MappingRecordEntry>, &'static str> {\n+        if let Some(x) = self.inner.iter_mut().find(|x| x.is_none()) {\n+            return Ok(x);\n+        }\n+\n+        Err(\"Storage for mapping info exhausted\")\n+    }\n+\n+    fn find_duplicate(\n+        &mut self,\n+        phys_region: &MemoryRegion<Physical>,\n+    ) -> Option<&mut MappingRecordEntry> {\n+        self.inner\n+            .iter_mut()\n+            .filter_map(|x| x.as_mut())\n+            .filter(|x| x.attribute_fields.mem_attributes == MemAttributes::Device)\n+            .find(|x| {\n+                if x.phys_start_addr != phys_region.start_addr() {\n+                    return false;\n+                }\n+\n+                if x.num_pages != phys_region.num_pages() {\n+                    return false;\n+                }\n+\n+                true\n+            })\n+    }\n+\n+    pub fn add(\n+        &mut self,\n+        name: &'static str,\n+        virt_region: &MemoryRegion<Virtual>,\n+        phys_region: &MemoryRegion<Physical>,\n+        attr: &AttributeFields,\n+    ) -> Result<(), &'static str> {\n+        let x = self.find_next_free()?;\n+\n+        *x = Some(MappingRecordEntry::new(\n+            name,\n+            virt_region,\n+            phys_region,\n+            attr,\n+        ));\n+\n+        self.sort();\n+\n+        Ok(())\n+    }\n+\n+    pub fn print(&self) {\n+        info!(\"      -------------------------------------------------------------------------------------------------------------------------------------------\");\n+        info!(\n+            \"      {:^44}     {:^30}   {:^7}   {:^9}   {:^35}\",\n+            \"Virtual\", \"Physical\", \"Size\", \"Attr\", \"Entity\"\n+        );\n+        info!(\"      -------------------------------------------------------------------------------------------------------------------------------------------\");\n+\n+        for i in self.inner.iter().flatten() {\n+            let size = i.num_pages * bsp::memory::mmu::KernelGranule::SIZE;\n+            let virt_start = i.virt_start_addr;\n+            let virt_end_inclusive = virt_start + (size - 1);\n+            let phys_start = i.phys_start_addr;\n+            let phys_end_inclusive = phys_start + (size - 1);\n+\n+            let (size, unit) = common::size_human_readable_ceil(size);\n+\n+            let attr = match i.attribute_fields.mem_attributes {\n+                MemAttributes::CacheableDRAM => \"C\",\n+                MemAttributes::Device => \"Dev\",\n+            };\n+\n+            let acc_p = match i.attribute_fields.acc_perms {\n+                AccessPermissions::ReadOnly => \"RO\",\n+                AccessPermissions::ReadWrite => \"RW\",\n+            };\n+\n+            let xn = if i.attribute_fields.execute_never {\n+                \"XN\"\n+            } else {\n+                \"X\"\n+            };\n+\n+            info!(\n+                \"      {}..{} --> {}..{} | {:>3} {} | {:<3} {} {:<2} | {}\",\n+                virt_start,\n+                virt_end_inclusive,\n+                phys_start,\n+                phys_end_inclusive,\n+                size,\n+                unit,\n+                attr,\n+                acc_p,\n+                xn,\n+                i.users[0].unwrap()\n+            );\n+\n+            for k in i.users[1..].iter() {\n+                if let Some(additional_user) = *k {\n+                    info!(\n+                        \"                                                                                                            | {}\",\n+                        additional_user\n+                    );\n+                }\n+            }\n+        }\n+\n+        info!(\"      -------------------------------------------------------------------------------------------------------------------------------------------\");\n+    }\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+use synchronization::interface::ReadWriteEx;\n+\n+/// Add an entry to the mapping info record.\n+pub fn kernel_add(\n+    name: &'static str,\n+    virt_region: &MemoryRegion<Virtual>,\n+    phys_region: &MemoryRegion<Physical>,\n+    attr: &AttributeFields,\n+) -> Result<(), &'static str> {\n+    KERNEL_MAPPING_RECORD.write(|mr| mr.add(name, virt_region, phys_region, attr))\n+}\n+\n+pub fn kernel_find_and_insert_mmio_duplicate(\n+    mmio_descriptor: &MMIODescriptor,\n+    new_user: &'static str,\n+) -> Option<Address<Virtual>> {\n+    let phys_region: MemoryRegion<Physical> = (*mmio_descriptor).into();\n+\n+    KERNEL_MAPPING_RECORD.write(|mr| {\n+        let dup = mr.find_duplicate(&phys_region)?;\n+\n+        if let Err(x) = dup.add_user(new_user) {\n+            warn!(\"{}\", x);\n+        }\n+\n+        Some(dup.virt_start_addr)\n+    })\n+}\n+\n+/// Human-readable print of all recorded kernel mappings.\n+pub fn kernel_print() {\n+    KERNEL_MAPPING_RECORD.read(|mr| mr.print());\n+}\n\ndiff -uNr 13_exceptions_part2_peripheral_IRQs/kernel/src/memory/mmu/page_alloc.rs 14_virtual_mem_part2_mmio_remap/kernel/src/memory/mmu/page_alloc.rs\n--- 13_exceptions_part2_peripheral_IRQs/kernel/src/memory/mmu/page_alloc.rs\n+++ 14_virtual_mem_part2_mmio_remap/kernel/src/memory/mmu/page_alloc.rs\n@@ -0,0 +1,70 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Page allocation.\n+\n+use super::MemoryRegion;\n+use crate::{\n+    memory::{AddressType, Virtual},\n+    synchronization::IRQSafeNullLock,\n+    warn,\n+};\n+use core::num::NonZeroUsize;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// A page allocator that can be lazyily initialized.\n+pub struct PageAllocator<ATYPE: AddressType> {\n+    pool: Option<MemoryRegion<ATYPE>>,\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Global instances\n+//--------------------------------------------------------------------------------------------------\n+\n+static KERNEL_MMIO_VA_ALLOCATOR: IRQSafeNullLock<PageAllocator<Virtual>> =\n+    IRQSafeNullLock::new(PageAllocator::new());\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Return a reference to the kernel's MMIO virtual address allocator.\n+pub fn kernel_mmio_va_allocator() -> &'static IRQSafeNullLock<PageAllocator<Virtual>> {\n+    &KERNEL_MMIO_VA_ALLOCATOR\n+}\n+\n+impl<ATYPE: AddressType> PageAllocator<ATYPE> {\n+    /// Create an instance.\n+    pub const fn new() -> Self {\n+        Self { pool: None }\n+    }\n+\n+    /// Initialize the allocator.\n+    pub fn init(&mut self, pool: MemoryRegion<ATYPE>) {\n+        if self.pool.is_some() {\n+            warn!(\"Already initialized\");\n+            return;\n+        }\n+\n+        self.pool = Some(pool);\n+    }\n+\n+    /// Allocate a number of pages.\n+    pub fn alloc(\n+        &mut self,\n+        num_requested_pages: NonZeroUsize,\n+    ) -> Result<MemoryRegion<ATYPE>, &'static str> {\n+        if self.pool.is_none() {\n+            return Err(\"Allocator not initialized\");\n+        }\n+\n+        self.pool\n+            .as_mut()\n+            .unwrap()\n+            .take_first_n_pages(num_requested_pages)\n+    }\n+}\n\ndiff -uNr 13_exceptions_part2_peripheral_IRQs/kernel/src/memory/mmu/translation_table.rs 14_virtual_mem_part2_mmio_remap/kernel/src/memory/mmu/translation_table.rs\n--- 13_exceptions_part2_peripheral_IRQs/kernel/src/memory/mmu/translation_table.rs\n+++ 14_virtual_mem_part2_mmio_remap/kernel/src/memory/mmu/translation_table.rs\n@@ -8,7 +8,91 @@\n #[path = \"../../_arch/aarch64/memory/mmu/translation_table.rs\"]\n mod arch_translation_table;\n\n+use super::{AttributeFields, MemoryRegion};\n+use crate::memory::{Address, Physical, Virtual};\n+\n //--------------------------------------------------------------------------------------------------\n // Architectural Public Reexports\n //--------------------------------------------------------------------------------------------------\n-pub use arch_translation_table::KernelTranslationTable;\n+#[cfg(target_arch = \"aarch64\")]\n+pub use arch_translation_table::FixedSizeTranslationTable;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Translation table interfaces.\n+pub mod interface {\n+    use super::*;\n+\n+    /// Translation table operations.\n+    pub trait TranslationTable {\n+        /// Anything that needs to run before any of the other provided functions can be used.\n+        ///\n+        /// # Safety\n+        ///\n+        /// - Implementor must ensure that this function can run only once or is harmless if invoked\n+        ///   multiple times.\n+        fn init(&mut self);\n+\n+        /// The translation table's base address to be used for programming the MMU.\n+        fn phys_base_address(&self) -> Address<Physical>;\n+\n+        /// Map the given virtual memory region to the given physical memory region.\n+        ///\n+        /// # Safety\n+        ///\n+        /// - Using wrong attributes can cause multiple issues of different nature in the system.\n+        /// - It is not required that the architectural implementation prevents aliasing. That is,\n+        ///   mapping to the same physical memory using multiple virtual addresses, which would\n+        ///   break Rust's ownership assumptions. This should be protected against in the kernel's\n+        ///   generic MMU code.\n+        unsafe fn map_at(\n+            &mut self,\n+            virt_region: &MemoryRegion<Virtual>,\n+            phys_region: &MemoryRegion<Physical>,\n+            attr: &AttributeFields,\n+        ) -> Result<(), &'static str>;\n+    }\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Testing\n+//--------------------------------------------------------------------------------------------------\n+\n+#[cfg(test)]\n+mod tests {\n+    use super::*;\n+    use crate::memory::mmu::{AccessPermissions, MemAttributes, PageAddress};\n+    use arch_translation_table::MinSizeTranslationTable;\n+    use interface::TranslationTable;\n+    use test_macros::kernel_test;\n+\n+    /// Sanity checks for the TranslationTable implementation.\n+    #[kernel_test]\n+    fn translationtable_implementation_sanity() {\n+        // This will occupy a lot of space on the stack.\n+        let mut tables = MinSizeTranslationTable::new();\n+\n+        tables.init();\n+\n+        let virt_start_page_addr: PageAddress<Virtual> = PageAddress::from(0);\n+        let virt_end_exclusive_page_addr: PageAddress<Virtual> =\n+            virt_start_page_addr.checked_offset(5).unwrap();\n+\n+        let phys_start_page_addr: PageAddress<Physical> = PageAddress::from(0);\n+        let phys_end_exclusive_page_addr: PageAddress<Physical> =\n+            phys_start_page_addr.checked_offset(5).unwrap();\n+\n+        let virt_region = MemoryRegion::new(virt_start_page_addr, virt_end_exclusive_page_addr);\n+        let phys_region = MemoryRegion::new(phys_start_page_addr, phys_end_exclusive_page_addr);\n+\n+        let attr = AttributeFields {\n+            mem_attributes: MemAttributes::CacheableDRAM,\n+            acc_perms: AccessPermissions::ReadWrite,\n+            execute_never: true,\n+        };\n+\n+        unsafe { assert_eq!(tables.map_at(&virt_region, &phys_region, &attr), Ok(())) };\n+    }\n+}\n\ndiff -uNr 13_exceptions_part2_peripheral_IRQs/kernel/src/memory/mmu/types.rs 14_virtual_mem_part2_mmio_remap/kernel/src/memory/mmu/types.rs\n--- 13_exceptions_part2_peripheral_IRQs/kernel/src/memory/mmu/types.rs\n+++ 14_virtual_mem_part2_mmio_remap/kernel/src/memory/mmu/types.rs\n@@ -0,0 +1,373 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Memory Management Unit types.\n+\n+use crate::{\n+    bsp, common,\n+    memory::{Address, AddressType, Physical},\n+};\n+use core::{convert::From, iter::Step, num::NonZeroUsize, ops::Range};\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// A wrapper type around [Address] that ensures page alignment.\n+#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\n+pub struct PageAddress<ATYPE: AddressType> {\n+    inner: Address<ATYPE>,\n+}\n+\n+/// A type that describes a region of memory in quantities of pages.\n+#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\n+pub struct MemoryRegion<ATYPE: AddressType> {\n+    start: PageAddress<ATYPE>,\n+    end_exclusive: PageAddress<ATYPE>,\n+}\n+\n+/// Architecture agnostic memory attributes.\n+#[allow(missing_docs)]\n+#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\n+pub enum MemAttributes {\n+    CacheableDRAM,\n+    Device,\n+}\n+\n+/// Architecture agnostic access permissions.\n+#[allow(missing_docs)]\n+#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\n+pub enum AccessPermissions {\n+    ReadOnly,\n+    ReadWrite,\n+}\n+\n+/// Collection of memory attributes.\n+#[allow(missing_docs)]\n+#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\n+pub struct AttributeFields {\n+    pub mem_attributes: MemAttributes,\n+    pub acc_perms: AccessPermissions,\n+    pub execute_never: bool,\n+}\n+\n+/// An MMIO descriptor for use in device drivers.\n+#[derive(Copy, Clone)]\n+pub struct MMIODescriptor {\n+    start_addr: Address<Physical>,\n+    end_addr_exclusive: Address<Physical>,\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+//------------------------------------------------------------------------------\n+// PageAddress\n+//------------------------------------------------------------------------------\n+impl<ATYPE: AddressType> PageAddress<ATYPE> {\n+    /// Unwraps the value.\n+    pub fn into_inner(self) -> Address<ATYPE> {\n+        self.inner\n+    }\n+\n+    /// Calculates the offset from the page address.\n+    ///\n+    /// `count` is in units of [PageAddress]. For example, a count of 2 means `result = self + 2 *\n+    /// page_size`.\n+    pub fn checked_offset(self, count: isize) -> Option<Self> {\n+        if count == 0 {\n+            return Some(self);\n+        }\n+\n+        let delta = count\n+            .unsigned_abs()\n+            .checked_mul(bsp::memory::mmu::KernelGranule::SIZE)?;\n+        let result = if count.is_positive() {\n+            self.inner.as_usize().checked_add(delta)?\n+        } else {\n+            self.inner.as_usize().checked_sub(delta)?\n+        };\n+\n+        Some(Self {\n+            inner: Address::new(result),\n+        })\n+    }\n+}\n+\n+impl<ATYPE: AddressType> From<usize> for PageAddress<ATYPE> {\n+    fn from(addr: usize) -> Self {\n+        assert!(\n+            common::is_aligned(addr, bsp::memory::mmu::KernelGranule::SIZE),\n+            \"Input usize not page aligned\"\n+        );\n+\n+        Self {\n+            inner: Address::new(addr),\n+        }\n+    }\n+}\n+\n+impl<ATYPE: AddressType> From<Address<ATYPE>> for PageAddress<ATYPE> {\n+    fn from(addr: Address<ATYPE>) -> Self {\n+        assert!(addr.is_page_aligned(), \"Input Address not page aligned\");\n+\n+        Self { inner: addr }\n+    }\n+}\n+\n+impl<ATYPE: AddressType> Step for PageAddress<ATYPE> {\n+    fn steps_between(start: &Self, end: &Self) -> Option<usize> {\n+        if start > end {\n+            return None;\n+        }\n+\n+        // Since start <= end, do unchecked arithmetic.\n+        Some(\n+            (end.inner.as_usize() - start.inner.as_usize())\n+                >> bsp::memory::mmu::KernelGranule::SHIFT,\n+        )\n+    }\n+\n+    fn forward_checked(start: Self, count: usize) -> Option<Self> {\n+        start.checked_offset(count as isize)\n+    }\n+\n+    fn backward_checked(start: Self, count: usize) -> Option<Self> {\n+        start.checked_offset(-(count as isize))\n+    }\n+}\n+\n+//------------------------------------------------------------------------------\n+// MemoryRegion\n+//------------------------------------------------------------------------------\n+impl<ATYPE: AddressType> MemoryRegion<ATYPE> {\n+    /// Create an instance.\n+    pub fn new(start: PageAddress<ATYPE>, end_exclusive: PageAddress<ATYPE>) -> Self {\n+        assert!(start <= end_exclusive);\n+\n+        Self {\n+            start,\n+            end_exclusive,\n+        }\n+    }\n+\n+    fn as_range(&self) -> Range<PageAddress<ATYPE>> {\n+        self.into_iter()\n+    }\n+\n+    /// Returns the start page address.\n+    pub fn start_page_addr(&self) -> PageAddress<ATYPE> {\n+        self.start\n+    }\n+\n+    /// Returns the start address.\n+    pub fn start_addr(&self) -> Address<ATYPE> {\n+        self.start.into_inner()\n+    }\n+\n+    /// Returns the exclusive end page address.\n+    pub fn end_exclusive_page_addr(&self) -> PageAddress<ATYPE> {\n+        self.end_exclusive\n+    }\n+\n+    /// Returns the exclusive end page address.\n+    pub fn end_inclusive_page_addr(&self) -> PageAddress<ATYPE> {\n+        self.end_exclusive.checked_offset(-1).unwrap()\n+    }\n+\n+    /// Checks if self contains an address.\n+    pub fn contains(&self, addr: Address<ATYPE>) -> bool {\n+        let page_addr = PageAddress::from(addr.align_down_page());\n+        self.as_range().contains(&page_addr)\n+    }\n+\n+    /// Checks if there is an overlap with another memory region.\n+    pub fn overlaps(&self, other_region: &Self) -> bool {\n+        let self_range = self.as_range();\n+\n+        self_range.contains(&other_region.start_page_addr())\n+            || self_range.contains(&other_region.end_inclusive_page_addr())\n+    }\n+\n+    /// Returns the number of pages contained in this region.\n+    pub fn num_pages(&self) -> usize {\n+        PageAddress::steps_between(&self.start, &self.end_exclusive).unwrap()\n+    }\n+\n+    /// Returns the size in bytes of this region.\n+    pub fn size(&self) -> usize {\n+        // Invariant: start <= end_exclusive, so do unchecked arithmetic.\n+        let end_exclusive = self.end_exclusive.into_inner().as_usize();\n+        let start = self.start.into_inner().as_usize();\n+\n+        end_exclusive - start\n+    }\n+\n+    /// Splits the MemoryRegion like:\n+    ///\n+    /// --------------------------------------------------------------------------------\n+    /// |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |\n+    /// --------------------------------------------------------------------------------\n+    ///   ^                               ^                                       ^\n+    ///   |                               |                                       |\n+    ///   left_start     left_end_exclusive                                       |\n+    ///                                                                           |\n+    ///                                   ^                                       |\n+    ///                                   |                                       |\n+    ///                                   right_start           right_end_exclusive\n+    ///\n+    /// Left region is returned to the caller. Right region is the new region for this struct.\n+    pub fn take_first_n_pages(&mut self, num_pages: NonZeroUsize) -> Result<Self, &'static str> {\n+        let count: usize = num_pages.into();\n+\n+        let left_end_exclusive = self.start.checked_offset(count as isize);\n+        let left_end_exclusive = match left_end_exclusive {\n+            None => return Err(\"Overflow while calculating left_end_exclusive\"),\n+            Some(x) => x,\n+        };\n+\n+        if left_end_exclusive > self.end_exclusive {\n+            return Err(\"Not enough free pages\");\n+        }\n+\n+        let allocation = Self {\n+            start: self.start,\n+            end_exclusive: left_end_exclusive,\n+        };\n+        self.start = left_end_exclusive;\n+\n+        Ok(allocation)\n+    }\n+}\n+\n+impl<ATYPE: AddressType> IntoIterator for MemoryRegion<ATYPE> {\n+    type Item = PageAddress<ATYPE>;\n+    type IntoIter = Range<Self::Item>;\n+\n+    fn into_iter(self) -> Self::IntoIter {\n+        Range {\n+            start: self.start,\n+            end: self.end_exclusive,\n+        }\n+    }\n+}\n+\n+impl From<MMIODescriptor> for MemoryRegion<Physical> {\n+    fn from(desc: MMIODescriptor) -> Self {\n+        let start = PageAddress::from(desc.start_addr.align_down_page());\n+        let end_exclusive = PageAddress::from(desc.end_addr_exclusive().align_up_page());\n+\n+        Self {\n+            start,\n+            end_exclusive,\n+        }\n+    }\n+}\n+\n+//------------------------------------------------------------------------------\n+// MMIODescriptor\n+//------------------------------------------------------------------------------\n+\n+impl MMIODescriptor {\n+    /// Create an instance.\n+    pub const fn new(start_addr: Address<Physical>, size: usize) -> Self {\n+        assert!(size > 0);\n+        let end_addr_exclusive = Address::new(start_addr.as_usize() + size);\n+\n+        Self {\n+            start_addr,\n+            end_addr_exclusive,\n+        }\n+    }\n+\n+    /// Return the start address.\n+    pub const fn start_addr(&self) -> Address<Physical> {\n+        self.start_addr\n+    }\n+\n+    /// Return the exclusive end address.\n+    pub fn end_addr_exclusive(&self) -> Address<Physical> {\n+        self.end_addr_exclusive\n+    }\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Testing\n+//--------------------------------------------------------------------------------------------------\n+\n+#[cfg(test)]\n+mod tests {\n+    use super::*;\n+    use crate::memory::Virtual;\n+    use test_macros::kernel_test;\n+\n+    /// Sanity of [PageAddress] methods.\n+    #[kernel_test]\n+    fn pageaddress_type_method_sanity() {\n+        let page_addr: PageAddress<Virtual> =\n+            PageAddress::from(bsp::memory::mmu::KernelGranule::SIZE * 2);\n+\n+        assert_eq!(\n+            page_addr.checked_offset(-2),\n+            Some(PageAddress::<Virtual>::from(0))\n+        );\n+\n+        assert_eq!(\n+            page_addr.checked_offset(2),\n+            Some(PageAddress::<Virtual>::from(\n+                bsp::memory::mmu::KernelGranule::SIZE * 4\n+            ))\n+        );\n+\n+        assert_eq!(\n+            PageAddress::<Virtual>::from(0).checked_offset(0),\n+            Some(PageAddress::<Virtual>::from(0))\n+        );\n+        assert_eq!(PageAddress::<Virtual>::from(0).checked_offset(-1), None);\n+\n+        let max_page_addr = Address::<Virtual>::new(usize::MAX).align_down_page();\n+        assert_eq!(\n+            PageAddress::<Virtual>::from(max_page_addr).checked_offset(1),\n+            None\n+        );\n+\n+        let zero = PageAddress::<Virtual>::from(0);\n+        let three = PageAddress::<Virtual>::from(bsp::memory::mmu::KernelGranule::SIZE * 3);\n+        assert_eq!(PageAddress::steps_between(&zero, &three), Some(3));\n+    }\n+\n+    /// Sanity of [MemoryRegion] methods.\n+    #[kernel_test]\n+    fn memoryregion_type_method_sanity() {\n+        let zero = PageAddress::<Virtual>::from(0);\n+        let zero_region = MemoryRegion::new(zero, zero);\n+        assert_eq!(zero_region.num_pages(), 0);\n+        assert_eq!(zero_region.size(), 0);\n+\n+        let one = PageAddress::<Virtual>::from(bsp::memory::mmu::KernelGranule::SIZE);\n+        let one_region = MemoryRegion::new(zero, one);\n+        assert_eq!(one_region.num_pages(), 1);\n+        assert_eq!(one_region.size(), bsp::memory::mmu::KernelGranule::SIZE);\n+\n+        let three = PageAddress::<Virtual>::from(bsp::memory::mmu::KernelGranule::SIZE * 3);\n+        let mut three_region = MemoryRegion::new(zero, three);\n+        assert!(three_region.contains(zero.into_inner()));\n+        assert!(!three_region.contains(three.into_inner()));\n+        assert!(three_region.overlaps(&one_region));\n+\n+        let allocation = three_region\n+            .take_first_n_pages(NonZeroUsize::new(2).unwrap())\n+            .unwrap();\n+        assert_eq!(allocation.num_pages(), 2);\n+        assert_eq!(three_region.num_pages(), 1);\n+\n+        for (i, alloc) in allocation.into_iter().enumerate() {\n+            assert_eq!(\n+                alloc.into_inner().as_usize(),\n+                i * bsp::memory::mmu::KernelGranule::SIZE\n+            );\n+        }\n+    }\n+}\n\ndiff -uNr 13_exceptions_part2_peripheral_IRQs/kernel/src/memory/mmu.rs 14_virtual_mem_part2_mmio_remap/kernel/src/memory/mmu.rs\n--- 13_exceptions_part2_peripheral_IRQs/kernel/src/memory/mmu.rs\n+++ 14_virtual_mem_part2_mmio_remap/kernel/src/memory/mmu.rs\n@@ -3,30 +3,24 @@\n // Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n //! Memory Management Unit.\n-//!\n-//! In order to decouple `BSP` and `arch` parts of the MMU code (to keep them pluggable), this file\n-//! provides types for composing an architecture-agnostic description of the kernel's virtual memory\n-//! layout.\n-//!\n-//! The `BSP` provides such a description through the `bsp::memory::mmu::virt_mem_layout()`\n-//! function.\n-//!\n-//! The `MMU` driver of the `arch` code uses `bsp::memory::mmu::virt_mem_layout()` to compile and\n-//! install respective translation tables.\n\n #[cfg(target_arch = \"aarch64\")]\n #[path = \"../_arch/aarch64/memory/mmu.rs\"]\n mod arch_mmu;\n\n+mod mapping_record;\n+mod page_alloc;\n mod translation_table;\n+mod types;\n\n-use crate::common;\n-use core::{fmt, ops::RangeInclusive};\n+use crate::{\n+    bsp,\n+    memory::{Address, Physical, Virtual},\n+    synchronization, warn,\n+};\n+use core::{fmt, num::NonZeroUsize};\n\n-//--------------------------------------------------------------------------------------------------\n-// Architectural Public Reexports\n-//--------------------------------------------------------------------------------------------------\n-pub use arch_mmu::mmu;\n+pub use types::*;\n\n //--------------------------------------------------------------------------------------------------\n // Public Definitions\n@@ -46,13 +40,15 @@\n\n     /// MMU functions.\n     pub trait MMU {\n-        /// Called by the kernel during early init. Supposed to take the translation tables from the\n-        /// `BSP`-supplied `virt_mem_layout()` and install/activate them for the respective MMU.\n+        /// Turns on the MMU for the first time and enables data and instruction caching.\n         ///\n         /// # Safety\n         ///\n         /// - Changes the HW's global state.\n-        unsafe fn enable_mmu_and_caching(&self) -> Result<(), MMUEnableError>;\n+        unsafe fn enable_mmu_and_caching(\n+            &self,\n+            phys_tables_base_addr: Address<Physical>,\n+        ) -> Result<(), MMUEnableError>;\n\n         /// Returns true if the MMU is enabled, false otherwise.\n         fn is_enabled(&self) -> bool;\n@@ -65,55 +61,51 @@\n /// Describes properties of an address space.\n pub struct AddressSpace<const AS_SIZE: usize>;\n\n-/// Architecture agnostic translation types.\n-#[allow(missing_docs)]\n-#[derive(Copy, Clone)]\n-pub enum Translation {\n-    Identity,\n-    Offset(usize),\n-}\n-\n-/// Architecture agnostic memory attributes.\n-#[allow(missing_docs)]\n-#[derive(Copy, Clone)]\n-pub enum MemAttributes {\n-    CacheableDRAM,\n-    Device,\n-}\n-\n-/// Architecture agnostic access permissions.\n-#[allow(missing_docs)]\n-#[derive(Copy, Clone)]\n-pub enum AccessPermissions {\n-    ReadOnly,\n-    ReadWrite,\n-}\n-\n-/// Collection of memory attributes.\n-#[allow(missing_docs)]\n-#[derive(Copy, Clone)]\n-pub struct AttributeFields {\n-    pub mem_attributes: MemAttributes,\n-    pub acc_perms: AccessPermissions,\n-    pub execute_never: bool,\n+/// Intended to be implemented for [`AddressSpace`].\n+pub trait AssociatedTranslationTable {\n+    /// A translation table whose address range is:\n+    ///\n+    /// [AS_SIZE - 1, 0]\n+    type TableStartFromBottom;\n }\n\n-/// Architecture agnostic descriptor for a memory range.\n-#[allow(missing_docs)]\n-pub struct TranslationDescriptor {\n-    pub name: &'static str,\n-    pub virtual_range: fn() -> RangeInclusive<usize>,\n-    pub physical_range_translation: Translation,\n-    pub attribute_fields: AttributeFields,\n-}\n+//--------------------------------------------------------------------------------------------------\n+// Private Code\n+//--------------------------------------------------------------------------------------------------\n+use interface::MMU;\n+use synchronization::interface::*;\n+use translation_table::interface::TranslationTable;\n+\n+/// Query the BSP for the reserved virtual addresses for MMIO remapping and initialize the kernel's\n+/// MMIO VA allocator with it.\n+fn kernel_init_mmio_va_allocator() {\n+    let region = bsp::memory::mmu::virt_mmio_remap_region();\n+\n+    page_alloc::kernel_mmio_va_allocator().lock(|allocator| allocator.init(region));\n+}\n+\n+/// Map a region in the kernel's translation tables.\n+///\n+/// No input checks done, input is passed through to the architectural implementation.\n+///\n+/// # Safety\n+///\n+/// - See `map_at()`.\n+/// - Does not prevent aliasing.\n+unsafe fn kernel_map_at_unchecked(\n+    name: &'static str,\n+    virt_region: &MemoryRegion<Virtual>,\n+    phys_region: &MemoryRegion<Physical>,\n+    attr: &AttributeFields,\n+) -> Result<(), &'static str> {\n+    bsp::memory::mmu::kernel_translation_tables()\n+        .write(|tables| tables.map_at(virt_region, phys_region, attr))?;\n\n-/// Type for expressing the kernel's virtual memory layout.\n-pub struct KernelVirtualLayout<const NUM_SPECIAL_RANGES: usize> {\n-    /// The last (inclusive) address of the address space.\n-    max_virt_addr_inclusive: usize,\n+    if let Err(x) = mapping_record::kernel_add(name, virt_region, phys_region, attr) {\n+        warn!(\"{}\", x);\n+    }\n\n-    /// Array of descriptors for non-standard (normal cacheable DRAM) memory regions.\n-    inner: [TranslationDescriptor; NUM_SPECIAL_RANGES],\n+    Ok(())\n }\n\n //--------------------------------------------------------------------------------------------------\n@@ -133,6 +125,9 @@\n     /// The granule's size.\n     pub const SIZE: usize = Self::size_checked();\n\n+    /// The granule's mask.\n+    pub const MASK: usize = Self::SIZE - 1;\n+\n     /// The granule's shift, aka log2(size).\n     pub const SHIFT: usize = Self::SIZE.trailing_zeros() as usize;\n\n@@ -160,98 +155,147 @@\n     }\n }\n\n-impl Default for AttributeFields {\n-    fn default() -> AttributeFields {\n-        AttributeFields {\n-            mem_attributes: MemAttributes::CacheableDRAM,\n-            acc_perms: AccessPermissions::ReadWrite,\n-            execute_never: true,\n-        }\n+/// Raw mapping of a virtual to physical region in the kernel translation tables.\n+///\n+/// Prevents mapping into the MMIO range of the tables.\n+///\n+/// # Safety\n+///\n+/// - See `kernel_map_at_unchecked()`.\n+/// - Does not prevent aliasing. Currently, the callers must be trusted.\n+pub unsafe fn kernel_map_at(\n+    name: &'static str,\n+    virt_region: &MemoryRegion<Virtual>,\n+    phys_region: &MemoryRegion<Physical>,\n+    attr: &AttributeFields,\n+) -> Result<(), &'static str> {\n+    if bsp::memory::mmu::virt_mmio_remap_region().overlaps(virt_region) {\n+        return Err(\"Attempt to manually map into MMIO region\");\n     }\n-}\n\n-/// Human-readable output of a TranslationDescriptor.\n-impl fmt::Display for TranslationDescriptor {\n-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n-        // Call the function to which self.range points, and dereference the result, which causes\n-        // Rust to copy the value.\n-        let start = *(self.virtual_range)().start();\n-        let end = *(self.virtual_range)().end();\n-        let size = end - start + 1;\n-\n-        let (size, unit) = common::size_human_readable_ceil(size);\n-\n-        let attr = match self.attribute_fields.mem_attributes {\n-            MemAttributes::CacheableDRAM => \"C\",\n-            MemAttributes::Device => \"Dev\",\n-        };\n-\n-        let acc_p = match self.attribute_fields.acc_perms {\n-            AccessPermissions::ReadOnly => \"RO\",\n-            AccessPermissions::ReadWrite => \"RW\",\n-        };\n+    kernel_map_at_unchecked(name, virt_region, phys_region, attr)?;\n\n-        let xn = if self.attribute_fields.execute_never {\n-            \"PXN\"\n-        } else {\n-            \"PX\"\n-        };\n-\n-        write!(\n-            f,\n-            \"      {:#010x} - {:#010x} | {: >3} {} | {: <3} {} {: <3} | {}\",\n-            start, end, size, unit, attr, acc_p, xn, self.name\n-        )\n-    }\n+    Ok(())\n }\n\n-impl<const NUM_SPECIAL_RANGES: usize> KernelVirtualLayout<{ NUM_SPECIAL_RANGES }> {\n-    /// Create a new instance.\n-    pub const fn new(max: usize, layout: [TranslationDescriptor; NUM_SPECIAL_RANGES]) -> Self {\n-        Self {\n-            max_virt_addr_inclusive: max,\n-            inner: layout,\n-        }\n-    }\n+/// MMIO remapping in the kernel translation tables.\n+///\n+/// Typically used by device drivers.\n+///\n+/// # Safety\n+///\n+/// - Same as `kernel_map_at_unchecked()`, minus the aliasing part.\n+pub unsafe fn kernel_map_mmio(\n+    name: &'static str,\n+    mmio_descriptor: &MMIODescriptor,\n+) -> Result<Address<Virtual>, &'static str> {\n+    let phys_region = MemoryRegion::from(*mmio_descriptor);\n+    let offset_into_start_page = mmio_descriptor.start_addr().offset_into_page();\n+\n+    // Check if an identical region has been mapped for another driver. If so, reuse it.\n+    let virt_addr = if let Some(addr) =\n+        mapping_record::kernel_find_and_insert_mmio_duplicate(mmio_descriptor, name)\n+    {\n+        addr\n+    // Otherwise, allocate a new region and map it.\n+    } else {\n+        let num_pages = match NonZeroUsize::new(phys_region.num_pages()) {\n+            None => return Err(\"Requested 0 pages\"),\n+            Some(x) => x,\n+        };\n\n-    /// For a virtual address, find and return the physical output address and corresponding\n-    /// attributes.\n-    ///\n-    /// If the address is not found in `inner`, return an identity mapped default with normal\n-    /// cacheable DRAM attributes.\n-    pub fn virt_addr_properties(\n-        &self,\n-        virt_addr: usize,\n-    ) -> Result<(usize, AttributeFields), &'static str> {\n-        if virt_addr > self.max_virt_addr_inclusive {\n-            return Err(\"Address out of range\");\n-        }\n+        let virt_region =\n+            page_alloc::kernel_mmio_va_allocator().lock(|allocator| allocator.alloc(num_pages))?;\n\n-        for i in self.inner.iter() {\n-            if (i.virtual_range)().contains(&virt_addr) {\n-                let output_addr = match i.physical_range_translation {\n-                    Translation::Identity => virt_addr,\n-                    Translation::Offset(a) => a + (virt_addr - (i.virtual_range)().start()),\n-                };\n+        kernel_map_at_unchecked(\n+            name,\n+            &virt_region,\n+            &phys_region,\n+            &AttributeFields {\n+                mem_attributes: MemAttributes::Device,\n+                acc_perms: AccessPermissions::ReadWrite,\n+                execute_never: true,\n+            },\n+        )?;\n+\n+        virt_region.start_addr()\n+    };\n+\n+    Ok(virt_addr + offset_into_start_page)\n+}\n+\n+/// Map the kernel's binary. Returns the translation table's base address.\n+///\n+/// # Safety\n+///\n+/// - See [`bsp::memory::mmu::kernel_map_binary()`].\n+pub unsafe fn kernel_map_binary() -> Result<Address<Physical>, &'static str> {\n+    let phys_kernel_tables_base_addr =\n+        bsp::memory::mmu::kernel_translation_tables().write(|tables| {\n+            tables.init();\n+            tables.phys_base_address()\n+        });\n+\n+    bsp::memory::mmu::kernel_map_binary()?;\n+\n+    Ok(phys_kernel_tables_base_addr)\n+}\n+\n+/// Enable the MMU and data + instruction caching.\n+///\n+/// # Safety\n+///\n+/// - Crucial function during kernel init. Changes the the complete memory view of the processor.\n+pub unsafe fn enable_mmu_and_caching(\n+    phys_tables_base_addr: Address<Physical>,\n+) -> Result<(), MMUEnableError> {\n+    arch_mmu::mmu().enable_mmu_and_caching(phys_tables_base_addr)\n+}\n+\n+/// Finish initialization of the MMU subsystem.\n+pub fn post_enable_init() {\n+    kernel_init_mmio_va_allocator();\n+}\n+\n+/// Human-readable print of all recorded kernel mappings.\n+pub fn kernel_print_mappings() {\n+    mapping_record::kernel_print()\n+}\n\n-                return Ok((output_addr, i.attribute_fields));\n-            }\n-        }\n+//--------------------------------------------------------------------------------------------------\n+// Testing\n+//--------------------------------------------------------------------------------------------------\n\n-        Ok((virt_addr, AttributeFields::default()))\n-    }\n+#[cfg(test)]\n+mod tests {\n+    use super::*;\n+    use crate::memory::mmu::{AccessPermissions, MemAttributes, PageAddress};\n+    use test_macros::kernel_test;\n\n-    /// Print the memory layout.\n-    pub fn print_layout(&self) {\n-        use crate::info;\n+    /// Check that you cannot map into the MMIO VA range from kernel_map_at().\n+    #[kernel_test]\n+    fn no_manual_mmio_map() {\n+        let phys_start_page_addr: PageAddress<Physical> = PageAddress::from(0);\n+        let phys_end_exclusive_page_addr: PageAddress<Physical> =\n+            phys_start_page_addr.checked_offset(5).unwrap();\n+        let phys_region = MemoryRegion::new(phys_start_page_addr, phys_end_exclusive_page_addr);\n+\n+        let num_pages = NonZeroUsize::new(phys_region.num_pages()).unwrap();\n+        let virt_region = page_alloc::kernel_mmio_va_allocator()\n+            .lock(|allocator| allocator.alloc(num_pages))\n+            .unwrap();\n\n-        for i in self.inner.iter() {\n-            info!(\"{}\", i);\n-        }\n-    }\n+        let attr = AttributeFields {\n+            mem_attributes: MemAttributes::CacheableDRAM,\n+            acc_perms: AccessPermissions::ReadWrite,\n+            execute_never: true,\n+        };\n\n-    #[cfg(test)]\n-    pub fn inner(&self) -> &[TranslationDescriptor; NUM_SPECIAL_RANGES] {\n-        &self.inner\n+        unsafe {\n+            assert_eq!(\n+                kernel_map_at(\"test\", &virt_region, &phys_region, &attr),\n+                Err(\"Attempt to manually map into MMIO region\")\n+            )\n+        };\n     }\n }\n\ndiff -uNr 13_exceptions_part2_peripheral_IRQs/kernel/src/memory.rs 14_virtual_mem_part2_mmio_remap/kernel/src/memory.rs\n--- 13_exceptions_part2_peripheral_IRQs/kernel/src/memory.rs\n+++ 14_virtual_mem_part2_mmio_remap/kernel/src/memory.rs\n@@ -5,3 +5,163 @@\n //! Memory Management.\n\n pub mod mmu;\n+\n+use crate::{bsp, common};\n+use core::{\n+    fmt,\n+    marker::PhantomData,\n+    ops::{Add, Sub},\n+};\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Metadata trait for marking the type of an address.\n+pub trait AddressType: Copy + Clone + PartialOrd + PartialEq + Ord + Eq {}\n+\n+/// Zero-sized type to mark a physical address.\n+#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq)]\n+pub enum Physical {}\n+\n+/// Zero-sized type to mark a virtual address.\n+#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq)]\n+pub enum Virtual {}\n+\n+/// Generic address type.\n+#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq)]\n+pub struct Address<ATYPE: AddressType> {\n+    value: usize,\n+    _address_type: PhantomData<fn() -> ATYPE>,\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+impl AddressType for Physical {}\n+impl AddressType for Virtual {}\n+\n+impl<ATYPE: AddressType> Address<ATYPE> {\n+    /// Create an instance.\n+    pub const fn new(value: usize) -> Self {\n+        Self {\n+            value,\n+            _address_type: PhantomData,\n+        }\n+    }\n+\n+    /// Convert to usize.\n+    pub const fn as_usize(self) -> usize {\n+        self.value\n+    }\n+\n+    /// Align down to page size.\n+    #[must_use]\n+    pub const fn align_down_page(self) -> Self {\n+        let aligned = common::align_down(self.value, bsp::memory::mmu::KernelGranule::SIZE);\n+\n+        Self::new(aligned)\n+    }\n+\n+    /// Align up to page size.\n+    #[must_use]\n+    pub const fn align_up_page(self) -> Self {\n+        let aligned = common::align_up(self.value, bsp::memory::mmu::KernelGranule::SIZE);\n+\n+        Self::new(aligned)\n+    }\n+\n+    /// Checks if the address is page aligned.\n+    pub const fn is_page_aligned(&self) -> bool {\n+        common::is_aligned(self.value, bsp::memory::mmu::KernelGranule::SIZE)\n+    }\n+\n+    /// Return the address' offset into the corresponding page.\n+    pub const fn offset_into_page(&self) -> usize {\n+        self.value & bsp::memory::mmu::KernelGranule::MASK\n+    }\n+}\n+\n+impl<ATYPE: AddressType> Add<usize> for Address<ATYPE> {\n+    type Output = Self;\n+\n+    #[inline(always)]\n+    fn add(self, rhs: usize) -> Self::Output {\n+        match self.value.checked_add(rhs) {\n+            None => panic!(\"Overflow on Address::add\"),\n+            Some(x) => Self::new(x),\n+        }\n+    }\n+}\n+\n+impl<ATYPE: AddressType> Sub<Address<ATYPE>> for Address<ATYPE> {\n+    type Output = Self;\n+\n+    #[inline(always)]\n+    fn sub(self, rhs: Address<ATYPE>) -> Self::Output {\n+        match self.value.checked_sub(rhs.value) {\n+            None => panic!(\"Overflow on Address::sub\"),\n+            Some(x) => Self::new(x),\n+        }\n+    }\n+}\n+\n+impl fmt::Display for Address<Physical> {\n+    // Don't expect to see physical addresses greater than 40 bit.\n+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n+        let q3: u8 = ((self.value >> 32) & 0xff) as u8;\n+        let q2: u16 = ((self.value >> 16) & 0xffff) as u16;\n+        let q1: u16 = (self.value & 0xffff) as u16;\n+\n+        write!(f, \"0x\")?;\n+        write!(f, \"{:02x}_\", q3)?;\n+        write!(f, \"{:04x}_\", q2)?;\n+        write!(f, \"{:04x}\", q1)\n+    }\n+}\n+\n+impl fmt::Display for Address<Virtual> {\n+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n+        let q4: u16 = ((self.value >> 48) & 0xffff) as u16;\n+        let q3: u16 = ((self.value >> 32) & 0xffff) as u16;\n+        let q2: u16 = ((self.value >> 16) & 0xffff) as u16;\n+        let q1: u16 = (self.value & 0xffff) as u16;\n+\n+        write!(f, \"0x\")?;\n+        write!(f, \"{:04x}_\", q4)?;\n+        write!(f, \"{:04x}_\", q3)?;\n+        write!(f, \"{:04x}_\", q2)?;\n+        write!(f, \"{:04x}\", q1)\n+    }\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Testing\n+//--------------------------------------------------------------------------------------------------\n+\n+#[cfg(test)]\n+mod tests {\n+    use super::*;\n+    use test_macros::kernel_test;\n+\n+    /// Sanity of [Address] methods.\n+    #[kernel_test]\n+    fn address_type_method_sanity() {\n+        let addr = Address::<Virtual>::new(bsp::memory::mmu::KernelGranule::SIZE + 100);\n+\n+        assert_eq!(\n+            addr.align_down_page().as_usize(),\n+            bsp::memory::mmu::KernelGranule::SIZE\n+        );\n+\n+        assert_eq!(\n+            addr.align_up_page().as_usize(),\n+            bsp::memory::mmu::KernelGranule::SIZE * 2\n+        );\n+\n+        assert!(!addr.is_page_aligned());\n+\n+        assert_eq!(addr.offset_into_page(), 100);\n+    }\n+}\n\ndiff -uNr 13_exceptions_part2_peripheral_IRQs/kernel/tests/00_console_sanity.rs 14_virtual_mem_part2_mmio_remap/kernel/tests/00_console_sanity.rs\n--- 13_exceptions_part2_peripheral_IRQs/kernel/tests/00_console_sanity.rs\n+++ 14_virtual_mem_part2_mmio_remap/kernel/tests/00_console_sanity.rs\n@@ -11,13 +11,24 @@\n /// Console tests should time out on the I/O harness in case of panic.\n mod panic_wait_forever;\n\n-use libkernel::{bsp, console, cpu, exception, print};\n+use libkernel::{bsp, console, cpu, exception, memory, print};\n\n #[no_mangle]\n unsafe fn kernel_init() -> ! {\n     use console::console;\n\n     exception::handling_init();\n+\n+    let phys_kernel_tables_base_addr = match memory::mmu::kernel_map_binary() {\n+        Err(string) => panic!(\"Error mapping kernel binary: {}\", string),\n+        Ok(addr) => addr,\n+    };\n+\n+    if let Err(e) = memory::mmu::enable_mmu_and_caching(phys_kernel_tables_base_addr) {\n+        panic!(\"Enabling MMU failed: {}\", e);\n+    }\n+\n+    memory::mmu::post_enable_init();\n     bsp::driver::qemu_bring_up_console();\n\n     // Handshake\n\ndiff -uNr 13_exceptions_part2_peripheral_IRQs/kernel/tests/01_timer_sanity.rs 14_virtual_mem_part2_mmio_remap/kernel/tests/01_timer_sanity.rs\n--- 13_exceptions_part2_peripheral_IRQs/kernel/tests/01_timer_sanity.rs\n+++ 14_virtual_mem_part2_mmio_remap/kernel/tests/01_timer_sanity.rs\n@@ -11,12 +11,23 @@\n #![test_runner(libkernel::test_runner)]\n\n use core::time::Duration;\n-use libkernel::{bsp, cpu, exception, time};\n+use libkernel::{bsp, cpu, exception, memory, time};\n use test_macros::kernel_test;\n\n #[no_mangle]\n unsafe fn kernel_init() -> ! {\n     exception::handling_init();\n+\n+    let phys_kernel_tables_base_addr = match memory::mmu::kernel_map_binary() {\n+        Err(string) => panic!(\"Error mapping kernel binary: {}\", string),\n+        Ok(addr) => addr,\n+    };\n+\n+    if let Err(e) = memory::mmu::enable_mmu_and_caching(phys_kernel_tables_base_addr) {\n+        panic!(\"Enabling MMU failed: {}\", e);\n+    }\n+\n+    memory::mmu::post_enable_init();\n     bsp::driver::qemu_bring_up_console();\n\n     // Depending on CPU arch, some timer bring-up code could go here. Not needed for the RPi.\n\ndiff -uNr 13_exceptions_part2_peripheral_IRQs/kernel/tests/02_exception_sync_page_fault.rs 14_virtual_mem_part2_mmio_remap/kernel/tests/02_exception_sync_page_fault.rs\n--- 13_exceptions_part2_peripheral_IRQs/kernel/tests/02_exception_sync_page_fault.rs\n+++ 14_virtual_mem_part2_mmio_remap/kernel/tests/02_exception_sync_page_fault.rs\n@@ -21,19 +21,27 @@\n\n #[no_mangle]\n unsafe fn kernel_init() -> ! {\n-    use memory::mmu::interface::MMU;\n-\n     exception::handling_init();\n-    bsp::driver::qemu_bring_up_console();\n\n     // This line will be printed as the test header.\n     println!(\"Testing synchronous exception handling by causing a page fault\");\n\n-    if let Err(string) = memory::mmu::mmu().enable_mmu_and_caching() {\n-        info!(\"MMU: {}\", string);\n+    let phys_kernel_tables_base_addr = match memory::mmu::kernel_map_binary() {\n+        Err(string) => {\n+            info!(\"Error mapping kernel binary: {}\", string);\n+            cpu::qemu_exit_failure()\n+        }\n+        Ok(addr) => addr,\n+    };\n+\n+    if let Err(e) = memory::mmu::enable_mmu_and_caching(phys_kernel_tables_base_addr) {\n+        info!(\"Enabling MMU failed: {}\", e);\n         cpu::qemu_exit_failure()\n     }\n\n+    memory::mmu::post_enable_init();\n+    bsp::driver::qemu_bring_up_console();\n+\n     info!(\"Writing beyond mapped area to address 9 GiB...\");\n     let big_addr: u64 = 9 * 1024 * 1024 * 1024;\n     core::ptr::read_volatile(big_addr as *mut u64);\n\ndiff -uNr 13_exceptions_part2_peripheral_IRQs/kernel/tests/03_exception_restore_sanity.rs 14_virtual_mem_part2_mmio_remap/kernel/tests/03_exception_restore_sanity.rs\n--- 13_exceptions_part2_peripheral_IRQs/kernel/tests/03_exception_restore_sanity.rs\n+++ 14_virtual_mem_part2_mmio_remap/kernel/tests/03_exception_restore_sanity.rs\n@@ -30,19 +30,27 @@\n\n #[no_mangle]\n unsafe fn kernel_init() -> ! {\n-    use memory::mmu::interface::MMU;\n-\n     exception::handling_init();\n-    bsp::driver::qemu_bring_up_console();\n\n     // This line will be printed as the test header.\n     println!(\"Testing exception restore\");\n\n-    if let Err(string) = memory::mmu::mmu().enable_mmu_and_caching() {\n-        info!(\"MMU: {}\", string);\n+    let phys_kernel_tables_base_addr = match memory::mmu::kernel_map_binary() {\n+        Err(string) => {\n+            info!(\"Error mapping kernel binary: {}\", string);\n+            cpu::qemu_exit_failure()\n+        }\n+        Ok(addr) => addr,\n+    };\n+\n+    if let Err(e) = memory::mmu::enable_mmu_and_caching(phys_kernel_tables_base_addr) {\n+        info!(\"Enabling MMU failed: {}\", e);\n         cpu::qemu_exit_failure()\n     }\n\n+    memory::mmu::post_enable_init();\n+    bsp::driver::qemu_bring_up_console();\n+\n     info!(\"Making a dummy system call\");\n\n     // Calling this inside a function indirectly tests if the link register is restored properly.\n\ndiff -uNr 13_exceptions_part2_peripheral_IRQs/kernel/tests/04_exception_irq_sanity.rs 14_virtual_mem_part2_mmio_remap/kernel/tests/04_exception_irq_sanity.rs\n--- 13_exceptions_part2_peripheral_IRQs/kernel/tests/04_exception_irq_sanity.rs\n+++ 14_virtual_mem_part2_mmio_remap/kernel/tests/04_exception_irq_sanity.rs\n@@ -10,14 +10,25 @@\n #![reexport_test_harness_main = \"test_main\"]\n #![test_runner(libkernel::test_runner)]\n\n-use libkernel::{bsp, cpu, exception};\n+use libkernel::{bsp, cpu, exception, memory};\n use test_macros::kernel_test;\n\n #[no_mangle]\n unsafe fn kernel_init() -> ! {\n+    exception::handling_init();\n+\n+    let phys_kernel_tables_base_addr = match memory::mmu::kernel_map_binary() {\n+        Err(string) => panic!(\"Error mapping kernel binary: {}\", string),\n+        Ok(addr) => addr,\n+    };\n+\n+    if let Err(e) = memory::mmu::enable_mmu_and_caching(phys_kernel_tables_base_addr) {\n+        panic!(\"Enabling MMU failed: {}\", e);\n+    }\n+\n+    memory::mmu::post_enable_init();\n     bsp::driver::qemu_bring_up_console();\n\n-    exception::handling_init();\n     exception::asynchronous::local_irq_unmask();\n\n     test_main();\n\n```\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/Cargo.toml",
    "content": "[package]\nname = \"mingo\"\nversion = \"0.14.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n\n[features]\ndefault = []\nbsp_rpi3 = [\"tock-registers\"]\nbsp_rpi4 = [\"tock-registers\"]\ntest_build = [\"qemu-exit\"]\n\n##--------------------------------------------------------------------------------------------------\n## Dependencies\n##--------------------------------------------------------------------------------------------------\n\n[dependencies]\ntest-types = { path = \"../libraries/test-types\" }\n\n# Optional dependencies\ntock-registers = { version = \"0.8.x\", default-features = false, features = [\"register_types\"], optional = true }\nqemu-exit = { version = \"3.x.x\", optional = true }\n\n# Platform specific dependencies\n[target.'cfg(target_arch = \"aarch64\")'.dependencies]\naarch64-cpu = { version = \"9.x.x\" }\n\n##--------------------------------------------------------------------------------------------------\n## Testing\n##--------------------------------------------------------------------------------------------------\n\n[dev-dependencies]\ntest-macros = { path = \"../libraries/test-macros\" }\n\n# Unit tests are done in the library part of the kernel.\n[lib]\nname = \"libkernel\"\ntest = true\n\n# Disable unit tests for the kernel binary.\n[[bin]]\nname = \"kernel\"\npath = \"src/main.rs\"\ntest = false\n\n# List of tests without harness.\n[[test]]\nname = \"00_console_sanity\"\nharness = false\n\n[[test]]\nname = \"02_exception_sync_page_fault\"\nharness = false\n\n[[test]]\nname = \"03_exception_restore_sanity\"\nharness = false\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/build.rs",
    "content": "use std::{env, fs, process};\n\nfn main() {\n    let ld_script_path = match env::var(\"LD_SCRIPT_PATH\") {\n        Ok(var) => var,\n        _ => process::exit(0),\n    };\n\n    let files = fs::read_dir(ld_script_path).unwrap();\n    files\n        .filter_map(Result::ok)\n        .filter(|d| {\n            if let Some(e) = d.path().extension() {\n                e == \"ld\"\n            } else {\n                false\n            }\n        })\n        .for_each(|f| println!(\"cargo:rerun-if-changed={}\", f.path().display()));\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/_arch/aarch64/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural boot code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::boot::arch_boot\n\nuse aarch64_cpu::{asm, registers::*};\nuse core::arch::global_asm;\nuse tock_registers::interfaces::Writeable;\n\n// Assembly counterpart to this file.\nglobal_asm!(\n    include_str!(\"boot.s\"),\n    CONST_CURRENTEL_EL2 = const 0x8,\n    CONST_CORE_ID_MASK = const 0b11\n);\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Prepares the transition from EL2 to EL1.\n///\n/// # Safety\n///\n/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.\n/// - The HW state of EL1 must be prepared in a sound way.\n#[inline(always)]\nunsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: u64) {\n    // Enable timer counter registers for EL1.\n    CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET);\n\n    // No offset for reading the counters.\n    CNTVOFF_EL2.set(0);\n\n    // Set EL1 execution state to AArch64.\n    HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64);\n\n    // Set up a simulated exception return.\n    //\n    // First, fake a saved program status where all interrupts were masked and SP_EL1 was used as a\n    // stack pointer.\n    SPSR_EL2.write(\n        SPSR_EL2::D::Masked\n            + SPSR_EL2::A::Masked\n            + SPSR_EL2::I::Masked\n            + SPSR_EL2::F::Masked\n            + SPSR_EL2::M::EL1h,\n    );\n\n    // Second, let the link register point to kernel_init().\n    ELR_EL2.set(crate::kernel_init as *const () as u64);\n\n    // Set up SP_EL1 (stack pointer), which will be used by EL1 once we \"return\" to it. Since there\n    // are no plans to ever return to EL2, just re-use the same stack.\n    SP_EL1.set(phys_boot_core_stack_end_exclusive_addr);\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The Rust entry of the `kernel` binary.\n///\n/// The function is called from the assembly `_start` function.\n///\n/// # Safety\n///\n/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`.\n#[no_mangle]\npub unsafe extern \"C\" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! {\n    prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr);\n\n    // Use `eret` to \"return\" to EL1. This results in execution of kernel_init() in EL1.\n    asm::eret()\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/_arch/aarch64/cpu/boot.s",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2022 Andre Richter <andre.o.richter@gmail.com>\n\n//--------------------------------------------------------------------------------------------------\n// Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Load the address of a symbol into a register, PC-relative.\n//\n// The symbol must lie within +/- 4 GiB of the Program Counter.\n//\n// # Resources\n//\n// - https://sourceware.org/binutils/docs-2.36/as/AArch64_002dRelocations.html\n.macro ADR_REL register, symbol\n\tadrp\t\\register, \\symbol\n\tadd\t\\register, \\register, #:lo12:\\symbol\n.endm\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n.section .text._start\n\n//------------------------------------------------------------------------------\n// fn _start()\n//------------------------------------------------------------------------------\n_start:\n\t// Only proceed if the core executes in EL2. Park it otherwise.\n\tmrs\tx0, CurrentEL\n\tcmp\tx0, {CONST_CURRENTEL_EL2}\n\tb.ne\t.L_parking_loop\n\n\t// Only proceed on the boot core. Park it otherwise.\n\tmrs\tx1, MPIDR_EL1\n\tand\tx1, x1, {CONST_CORE_ID_MASK}\n\tldr\tx2, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs\n\tcmp\tx1, x2\n\tb.ne\t.L_parking_loop\n\n\t// If execution reaches here, it is the boot core.\n\n\t// Initialize DRAM.\n\tADR_REL\tx0, __bss_start\n\tADR_REL x1, __bss_end_exclusive\n\n.L_bss_init_loop:\n\tcmp\tx0, x1\n\tb.eq\t.L_prepare_rust\n\tstp\txzr, xzr, [x0], #16\n\tb\t.L_bss_init_loop\n\n\t// Prepare the jump to Rust code.\n.L_prepare_rust:\n\t// Set the stack pointer. This ensures that any code in EL2 that needs the stack will work.\n\tADR_REL\tx0, __boot_core_stack_end_exclusive\n\tmov\tsp, x0\n\n\t// Read the CPU's timer counter frequency and store it in ARCH_TIMER_COUNTER_FREQUENCY.\n\t// Abort if the frequency read back as 0.\n\tADR_REL\tx1, ARCH_TIMER_COUNTER_FREQUENCY // provided by aarch64/time.rs\n\tmrs\tx2, CNTFRQ_EL0\n\tcmp\tx2, xzr\n\tb.eq\t.L_parking_loop\n\tstr\tw2, [x1]\n\n\t// Jump to Rust code. x0 holds the function argument provided to _start_rust().\n\tb\t_start_rust\n\n\t// Infinitely wait for events (aka \"park the core\").\n.L_parking_loop:\n\twfe\n\tb\t.L_parking_loop\n\n.size\t_start, . - _start\n.type\t_start, function\n.global\t_start\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/_arch/aarch64/cpu/smp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural symmetric multiprocessing.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::smp::arch_smp\n\nuse aarch64_cpu::registers::*;\nuse tock_registers::interfaces::Readable;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return the executing core's id.\n#[inline(always)]\npub fn core_id<T>() -> T\nwhere\n    T: From<u8>,\n{\n    const CORE_MASK: u64 = 0b11;\n\n    T::from((MPIDR_EL1.get() & CORE_MASK) as u8)\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/_arch/aarch64/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural processor code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::arch_cpu\n\nuse aarch64_cpu::asm;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\npub use asm::nop;\n\n/// Pause execution on the core.\n#[inline(always)]\npub fn wait_forever() -> ! {\n    loop {\n        asm::wfe()\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n#[cfg(feature = \"test_build\")]\nuse qemu_exit::QEMUExit;\n\n#[cfg(feature = \"test_build\")]\nconst QEMU_EXIT_HANDLE: qemu_exit::AArch64 = qemu_exit::AArch64::new();\n\n/// Make the host QEMU binary execute `exit(1)`.\n#[cfg(feature = \"test_build\")]\npub fn qemu_exit_failure() -> ! {\n    QEMU_EXIT_HANDLE.exit_failure()\n}\n\n/// Make the host QEMU binary execute `exit(0)`.\n#[cfg(feature = \"test_build\")]\npub fn qemu_exit_success() -> ! {\n    QEMU_EXIT_HANDLE.exit_success()\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/_arch/aarch64/exception/asynchronous.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural asynchronous exception handling.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::exception::asynchronous::arch_asynchronous\n\nuse aarch64_cpu::registers::*;\nuse core::arch::asm;\nuse tock_registers::interfaces::{Readable, Writeable};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nmod daif_bits {\n    pub const IRQ: u8 = 0b0010;\n}\n\ntrait DaifField {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register>;\n}\n\nstruct Debug;\nstruct SError;\nstruct IRQ;\nstruct FIQ;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl DaifField for Debug {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::D\n    }\n}\n\nimpl DaifField for SError {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::A\n    }\n}\n\nimpl DaifField for IRQ {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::I\n    }\n}\n\nimpl DaifField for FIQ {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::F\n    }\n}\n\nfn is_masked<T>() -> bool\nwhere\n    T: DaifField,\n{\n    DAIF.is_set(T::daif_field())\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Returns whether IRQs are masked on the executing core.\npub fn is_local_irq_masked() -> bool {\n    !is_masked::<IRQ>()\n}\n\n/// Unmask IRQs on the executing core.\n///\n/// It is not needed to place an explicit instruction synchronization barrier after the `msr`.\n/// Quoting the Architecture Reference Manual for ARMv8-A, section C5.1.3:\n///\n/// \"Writes to PSTATE.{PAN, D, A, I, F} occur in program order without the need for additional\n/// synchronization.\"\n#[inline(always)]\npub fn local_irq_unmask() {\n    unsafe {\n        asm!(\n            \"msr DAIFClr, {arg}\",\n            arg = const daif_bits::IRQ,\n            options(nomem, nostack, preserves_flags)\n        );\n    }\n}\n\n/// Mask IRQs on the executing core.\n#[inline(always)]\npub fn local_irq_mask() {\n    unsafe {\n        asm!(\n            \"msr DAIFSet, {arg}\",\n            arg = const daif_bits::IRQ,\n            options(nomem, nostack, preserves_flags)\n        );\n    }\n}\n\n/// Mask IRQs on the executing core and return the previously saved interrupt mask bits (DAIF).\n#[inline(always)]\npub fn local_irq_mask_save() -> u64 {\n    let saved = DAIF.get();\n    local_irq_mask();\n\n    saved\n}\n\n/// Restore the interrupt mask bits (DAIF) using the callee's argument.\n///\n/// # Invariant\n///\n/// - No sanity checks on the input.\n#[inline(always)]\npub fn local_irq_restore(saved: u64) {\n    DAIF.set(saved);\n}\n\n/// Print the AArch64 exceptions status.\n#[rustfmt::skip]\npub fn print_state() {\n    use crate::info;\n\n    let to_mask_str = |x| -> _ {\n        if x { \"Masked\" } else { \"Unmasked\" }\n    };\n\n    info!(\"      Debug:  {}\", to_mask_str(is_masked::<Debug>()));\n    info!(\"      SError: {}\", to_mask_str(is_masked::<SError>()));\n    info!(\"      IRQ:    {}\", to_mask_str(is_masked::<IRQ>()));\n    info!(\"      FIQ:    {}\", to_mask_str(is_masked::<FIQ>()));\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/_arch/aarch64/exception.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural synchronous and asynchronous exception handling.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::exception::arch_exception\n\nuse crate::exception;\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::{arch::global_asm, cell::UnsafeCell, fmt};\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    registers::InMemoryRegister,\n};\n\n// Assembly counterpart to this file.\nglobal_asm!(include_str!(\"exception.s\"));\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Wrapper structs for memory copies of registers.\n#[repr(transparent)]\nstruct SpsrEL1(InMemoryRegister<u64, SPSR_EL1::Register>);\nstruct EsrEL1(InMemoryRegister<u64, ESR_EL1::Register>);\n\n/// The exception context as it is stored on the stack on exception entry.\n#[repr(C)]\nstruct ExceptionContext {\n    /// General Purpose Registers.\n    gpr: [u64; 30],\n\n    /// The link register, aka x30.\n    lr: u64,\n\n    /// Exception link register. The program counter at the time the exception happened.\n    elr_el1: u64,\n\n    /// Saved program status.\n    spsr_el1: SpsrEL1,\n\n    /// Exception syndrome register.\n    esr_el1: EsrEL1,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Prints verbose information about the exception and then panics.\nfn default_exception_handler(exc: &ExceptionContext) {\n    panic!(\n        \"CPU Exception!\\n\\n\\\n        {}\",\n        exc\n    );\n}\n\n//------------------------------------------------------------------------------\n// Current, EL0\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn current_el0_synchronous(_e: &mut ExceptionContext) {\n    panic!(\"Should not be here. Use of SP_EL0 in EL1 is not supported.\")\n}\n\n#[no_mangle]\nextern \"C\" fn current_el0_irq(_e: &mut ExceptionContext) {\n    panic!(\"Should not be here. Use of SP_EL0 in EL1 is not supported.\")\n}\n\n#[no_mangle]\nextern \"C\" fn current_el0_serror(_e: &mut ExceptionContext) {\n    panic!(\"Should not be here. Use of SP_EL0 in EL1 is not supported.\")\n}\n\n//------------------------------------------------------------------------------\n// Current, ELx\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn current_elx_synchronous(e: &mut ExceptionContext) {\n    #[cfg(feature = \"test_build\")]\n    {\n        const TEST_SVC_ID: u64 = 0x1337;\n\n        if let Some(ESR_EL1::EC::Value::SVC64) = e.esr_el1.exception_class() {\n            if e.esr_el1.iss() == TEST_SVC_ID {\n                return;\n            }\n        }\n    }\n\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn current_elx_irq(_e: &mut ExceptionContext) {\n    let token = unsafe { &exception::asynchronous::IRQContext::new() };\n    exception::asynchronous::irq_manager().handle_pending_irqs(token);\n}\n\n#[no_mangle]\nextern \"C\" fn current_elx_serror(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n//------------------------------------------------------------------------------\n// Lower, AArch64\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn lower_aarch64_synchronous(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch64_irq(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch64_serror(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n//------------------------------------------------------------------------------\n// Lower, AArch32\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn lower_aarch32_synchronous(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch32_irq(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch32_serror(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n//------------------------------------------------------------------------------\n// Misc\n//------------------------------------------------------------------------------\n\n/// Human readable SPSR_EL1.\n#[rustfmt::skip]\nimpl fmt::Display for SpsrEL1 {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        // Raw value.\n        writeln!(f, \"SPSR_EL1: {:#010x}\", self.0.get())?;\n\n        let to_flag_str = |x| -> _ {\n            if x { \"Set\" } else { \"Not set\" }\n         };\n\n        writeln!(f, \"      Flags:\")?;\n        writeln!(f, \"            Negative (N): {}\", to_flag_str(self.0.is_set(SPSR_EL1::N)))?;\n        writeln!(f, \"            Zero     (Z): {}\", to_flag_str(self.0.is_set(SPSR_EL1::Z)))?;\n        writeln!(f, \"            Carry    (C): {}\", to_flag_str(self.0.is_set(SPSR_EL1::C)))?;\n        writeln!(f, \"            Overflow (V): {}\", to_flag_str(self.0.is_set(SPSR_EL1::V)))?;\n\n        let to_mask_str = |x| -> _ {\n            if x { \"Masked\" } else { \"Unmasked\" }\n        };\n\n        writeln!(f, \"      Exception handling state:\")?;\n        writeln!(f, \"            Debug  (D): {}\", to_mask_str(self.0.is_set(SPSR_EL1::D)))?;\n        writeln!(f, \"            SError (A): {}\", to_mask_str(self.0.is_set(SPSR_EL1::A)))?;\n        writeln!(f, \"            IRQ    (I): {}\", to_mask_str(self.0.is_set(SPSR_EL1::I)))?;\n        writeln!(f, \"            FIQ    (F): {}\", to_mask_str(self.0.is_set(SPSR_EL1::F)))?;\n\n        write!(f, \"      Illegal Execution State (IL): {}\",\n            to_flag_str(self.0.is_set(SPSR_EL1::IL))\n        )\n    }\n}\n\nimpl EsrEL1 {\n    #[inline(always)]\n    fn exception_class(&self) -> Option<ESR_EL1::EC::Value> {\n        self.0.read_as_enum(ESR_EL1::EC)\n    }\n\n    #[cfg(feature = \"test_build\")]\n    #[inline(always)]\n    fn iss(&self) -> u64 {\n        self.0.read(ESR_EL1::ISS)\n    }\n}\n\n/// Human readable ESR_EL1.\n#[rustfmt::skip]\nimpl fmt::Display for EsrEL1 {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        // Raw print of whole register.\n        writeln!(f, \"ESR_EL1: {:#010x}\", self.0.get())?;\n\n        // Raw print of exception class.\n        write!(f, \"      Exception Class         (EC) : {:#x}\", self.0.read(ESR_EL1::EC))?;\n\n        // Exception class.\n        let ec_translation = match self.exception_class() {\n            Some(ESR_EL1::EC::Value::DataAbortCurrentEL) => \"Data Abort, current EL\",\n            _ => \"N/A\",\n        };\n        writeln!(f, \" - {}\", ec_translation)?;\n\n        // Raw print of instruction specific syndrome.\n        write!(f, \"      Instr Specific Syndrome (ISS): {:#x}\", self.0.read(ESR_EL1::ISS))\n    }\n}\n\nimpl ExceptionContext {\n    #[inline(always)]\n    fn exception_class(&self) -> Option<ESR_EL1::EC::Value> {\n        self.esr_el1.exception_class()\n    }\n\n    #[inline(always)]\n    fn fault_address_valid(&self) -> bool {\n        use ESR_EL1::EC::Value::*;\n\n        match self.exception_class() {\n            None => false,\n            Some(ec) => matches!(\n                ec,\n                InstrAbortLowerEL\n                    | InstrAbortCurrentEL\n                    | PCAlignmentFault\n                    | DataAbortLowerEL\n                    | DataAbortCurrentEL\n                    | WatchpointLowerEL\n                    | WatchpointCurrentEL\n            ),\n        }\n    }\n}\n\n/// Human readable print of the exception context.\nimpl fmt::Display for ExceptionContext {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        writeln!(f, \"{}\", self.esr_el1)?;\n\n        if self.fault_address_valid() {\n            writeln!(f, \"FAR_EL1: {:#018x}\", FAR_EL1.get() as usize)?;\n        }\n\n        writeln!(f, \"{}\", self.spsr_el1)?;\n        writeln!(f, \"ELR_EL1: {:#018x}\", self.elr_el1)?;\n        writeln!(f)?;\n        writeln!(f, \"General purpose register:\")?;\n\n        #[rustfmt::skip]\n        let alternating = |x| -> _ {\n            if x % 2 == 0 { \"   \" } else { \"\\n\" }\n        };\n\n        // Print two registers per line.\n        for (i, reg) in self.gpr.iter().enumerate() {\n            write!(f, \"      x{: <2}: {: >#018x}{}\", i, reg, alternating(i))?;\n        }\n        write!(f, \"      lr : {:#018x}\", self.lr)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse crate::exception::PrivilegeLevel;\n\n/// The processing element's current privilege level.\npub fn current_privilege_level() -> (PrivilegeLevel, &'static str) {\n    let el = CurrentEL.read_as_enum(CurrentEL::EL);\n    match el {\n        Some(CurrentEL::EL::Value::EL2) => (PrivilegeLevel::Hypervisor, \"EL2\"),\n        Some(CurrentEL::EL::Value::EL1) => (PrivilegeLevel::Kernel, \"EL1\"),\n        Some(CurrentEL::EL::Value::EL0) => (PrivilegeLevel::User, \"EL0\"),\n        _ => (PrivilegeLevel::Unknown, \"Unknown\"),\n    }\n}\n\n/// Init exception handling by setting the exception vector base address register.\n///\n/// # Safety\n///\n/// - Changes the HW state of the executing core.\n/// - The vector table and the symbol `__exception_vector_table_start` from the linker script must\n///   adhere to the alignment and size constraints demanded by the ARMv8-A Architecture Reference\n///   Manual.\npub unsafe fn handling_init() {\n    // Provided by exception.S.\n    extern \"Rust\" {\n        static __exception_vector_start: UnsafeCell<()>;\n    }\n\n    VBAR_EL1.set(__exception_vector_start.get() as u64);\n\n    // Force VBAR update to complete before next instruction.\n    barrier::isb(barrier::SY);\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/_arch/aarch64/exception.s",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2022 Andre Richter <andre.o.richter@gmail.com>\n\n//--------------------------------------------------------------------------------------------------\n// Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Call the function provided by parameter `\\handler` after saving the exception context. Provide\n/// the context as the first parameter to '\\handler'.\n.macro CALL_WITH_CONTEXT handler\n__vector_\\handler:\n\t// Make room on the stack for the exception context.\n\tsub\tsp,  sp,  #16 * 17\n\n\t// Store all general purpose registers on the stack.\n\tstp\tx0,  x1,  [sp, #16 * 0]\n\tstp\tx2,  x3,  [sp, #16 * 1]\n\tstp\tx4,  x5,  [sp, #16 * 2]\n\tstp\tx6,  x7,  [sp, #16 * 3]\n\tstp\tx8,  x9,  [sp, #16 * 4]\n\tstp\tx10, x11, [sp, #16 * 5]\n\tstp\tx12, x13, [sp, #16 * 6]\n\tstp\tx14, x15, [sp, #16 * 7]\n\tstp\tx16, x17, [sp, #16 * 8]\n\tstp\tx18, x19, [sp, #16 * 9]\n\tstp\tx20, x21, [sp, #16 * 10]\n\tstp\tx22, x23, [sp, #16 * 11]\n\tstp\tx24, x25, [sp, #16 * 12]\n\tstp\tx26, x27, [sp, #16 * 13]\n\tstp\tx28, x29, [sp, #16 * 14]\n\n\t// Add the exception link register (ELR_EL1), saved program status (SPSR_EL1) and exception\n\t// syndrome register (ESR_EL1).\n\tmrs\tx1,  ELR_EL1\n\tmrs\tx2,  SPSR_EL1\n\tmrs\tx3,  ESR_EL1\n\n\tstp\tlr,  x1,  [sp, #16 * 15]\n\tstp\tx2,  x3,  [sp, #16 * 16]\n\n\t// x0 is the first argument for the function called through `\\handler`.\n\tmov\tx0,  sp\n\n\t// Call `\\handler`.\n\tbl\t\\handler\n\n\t// After returning from exception handling code, replay the saved context and return via\n\t// `eret`.\n\tb\t__exception_restore_context\n\n.size\t__vector_\\handler, . - __vector_\\handler\n.type\t__vector_\\handler, function\n.endm\n\n.macro FIQ_SUSPEND\n1:\twfe\n\tb\t1b\n.endm\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n.section .text\n\n//------------------------------------------------------------------------------\n// The exception vector table.\n//------------------------------------------------------------------------------\n\n// Align by 2^11 bytes, as demanded by ARMv8-A. Same as ALIGN(2048) in an ld script.\n.align 11\n\n// Export a symbol for the Rust code to use.\n__exception_vector_start:\n\n// Current exception level with SP_EL0.\n//\n// .org sets the offset relative to section start.\n//\n// # Safety\n//\n// - It must be ensured that `CALL_WITH_CONTEXT` <= 0x80 bytes.\n.org 0x000\n\tCALL_WITH_CONTEXT current_el0_synchronous\n.org 0x080\n\tCALL_WITH_CONTEXT current_el0_irq\n.org 0x100\n\tFIQ_SUSPEND\n.org 0x180\n\tCALL_WITH_CONTEXT current_el0_serror\n\n// Current exception level with SP_ELx, x > 0.\n.org 0x200\n\tCALL_WITH_CONTEXT current_elx_synchronous\n.org 0x280\n\tCALL_WITH_CONTEXT current_elx_irq\n.org 0x300\n\tFIQ_SUSPEND\n.org 0x380\n\tCALL_WITH_CONTEXT current_elx_serror\n\n// Lower exception level, AArch64\n.org 0x400\n\tCALL_WITH_CONTEXT lower_aarch64_synchronous\n.org 0x480\n\tCALL_WITH_CONTEXT lower_aarch64_irq\n.org 0x500\n\tFIQ_SUSPEND\n.org 0x580\n\tCALL_WITH_CONTEXT lower_aarch64_serror\n\n// Lower exception level, AArch32\n.org 0x600\n\tCALL_WITH_CONTEXT lower_aarch32_synchronous\n.org 0x680\n\tCALL_WITH_CONTEXT lower_aarch32_irq\n.org 0x700\n\tFIQ_SUSPEND\n.org 0x780\n\tCALL_WITH_CONTEXT lower_aarch32_serror\n.org 0x800\n\n//------------------------------------------------------------------------------\n// fn __exception_restore_context()\n//------------------------------------------------------------------------------\n__exception_restore_context:\n\tldr\tw19,      [sp, #16 * 16]\n\tldp\tlr,  x20, [sp, #16 * 15]\n\n\tmsr\tSPSR_EL1, x19\n\tmsr\tELR_EL1,  x20\n\n\tldp\tx0,  x1,  [sp, #16 * 0]\n\tldp\tx2,  x3,  [sp, #16 * 1]\n\tldp\tx4,  x5,  [sp, #16 * 2]\n\tldp\tx6,  x7,  [sp, #16 * 3]\n\tldp\tx8,  x9,  [sp, #16 * 4]\n\tldp\tx10, x11, [sp, #16 * 5]\n\tldp\tx12, x13, [sp, #16 * 6]\n\tldp\tx14, x15, [sp, #16 * 7]\n\tldp\tx16, x17, [sp, #16 * 8]\n\tldp\tx18, x19, [sp, #16 * 9]\n\tldp\tx20, x21, [sp, #16 * 10]\n\tldp\tx22, x23, [sp, #16 * 11]\n\tldp\tx24, x25, [sp, #16 * 12]\n\tldp\tx26, x27, [sp, #16 * 13]\n\tldp\tx28, x29, [sp, #16 * 14]\n\n\tadd\tsp,  sp,  #16 * 17\n\n\teret\n\n.size\t__exception_restore_context, . - __exception_restore_context\n.type\t__exception_restore_context, function\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/_arch/aarch64/memory/mmu/translation_table.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural translation table.\n//!\n//! Only 64 KiB granule is supported.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::memory::mmu::translation_table::arch_translation_table\n\nuse crate::{\n    bsp,\n    memory::{\n        self,\n        mmu::{\n            arch_mmu::{Granule512MiB, Granule64KiB},\n            AccessPermissions, AttributeFields, MemAttributes, MemoryRegion, PageAddress,\n        },\n        Address, Physical, Virtual,\n    },\n};\nuse core::convert;\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields,\n    registers::InMemoryRegister,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// A table descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-15.\nregister_bitfields! {u64,\n    STAGE1_TABLE_DESCRIPTOR [\n        /// Physical address of the next descriptor.\n        NEXT_LEVEL_TABLE_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16]\n\n        TYPE  OFFSET(1) NUMBITS(1) [\n            Block = 0,\n            Table = 1\n        ],\n\n        VALID OFFSET(0) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ]\n    ]\n}\n\n// A level 3 page descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-17.\nregister_bitfields! {u64,\n    STAGE1_PAGE_DESCRIPTOR [\n        /// Unprivileged execute-never.\n        UXN      OFFSET(54) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ],\n\n        /// Privileged execute-never.\n        PXN      OFFSET(53) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ],\n\n        /// Physical address of the next table descriptor (lvl2) or the page descriptor (lvl3).\n        OUTPUT_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16]\n\n        /// Access flag.\n        AF       OFFSET(10) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ],\n\n        /// Shareability field.\n        SH       OFFSET(8) NUMBITS(2) [\n            OuterShareable = 0b10,\n            InnerShareable = 0b11\n        ],\n\n        /// Access Permissions.\n        AP       OFFSET(6) NUMBITS(2) [\n            RW_EL1 = 0b00,\n            RW_EL1_EL0 = 0b01,\n            RO_EL1 = 0b10,\n            RO_EL1_EL0 = 0b11\n        ],\n\n        /// Memory attributes index into the MAIR_EL1 register.\n        AttrIndx OFFSET(2) NUMBITS(3) [],\n\n        TYPE     OFFSET(1) NUMBITS(1) [\n            Reserved_Invalid = 0,\n            Page = 1\n        ],\n\n        VALID    OFFSET(0) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ]\n    ]\n}\n\n/// A table descriptor for 64 KiB aperture.\n///\n/// The output points to the next table.\n#[derive(Copy, Clone)]\n#[repr(C)]\nstruct TableDescriptor {\n    value: u64,\n}\n\n/// A page descriptor with 64 KiB aperture.\n///\n/// The output points to physical memory.\n#[derive(Copy, Clone)]\n#[repr(C)]\nstruct PageDescriptor {\n    value: u64,\n}\n\ntrait StartAddr {\n    fn phys_start_addr(&self) -> Address<Physical>;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB\n/// aligned, so the lvl3 is put first.\n#[repr(C)]\n#[repr(align(65536))]\npub struct FixedSizeTranslationTable<const NUM_TABLES: usize> {\n    /// Page descriptors, covering 64 KiB windows per entry.\n    lvl3: [[PageDescriptor; 8192]; NUM_TABLES],\n\n    /// Table descriptors, covering 512 MiB windows.\n    lvl2: [TableDescriptor; NUM_TABLES],\n\n    /// Have the tables been initialized?\n    initialized: bool,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n// The binary is still identity mapped, so we don't need to convert here.\nimpl<T, const N: usize> StartAddr for [T; N] {\n    fn phys_start_addr(&self) -> Address<Physical> {\n        Address::new(self as *const _ as usize)\n    }\n}\n\nimpl TableDescriptor {\n    /// Create an instance.\n    ///\n    /// Descriptor is invalid by default.\n    pub const fn new_zeroed() -> Self {\n        Self { value: 0 }\n    }\n\n    /// Create an instance pointing to the supplied address.\n    pub fn from_next_lvl_table_addr(phys_next_lvl_table_addr: Address<Physical>) -> Self {\n        let val = InMemoryRegister::<u64, STAGE1_TABLE_DESCRIPTOR::Register>::new(0);\n\n        let shifted = phys_next_lvl_table_addr.as_usize() >> Granule64KiB::SHIFT;\n        val.write(\n            STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64)\n                + STAGE1_TABLE_DESCRIPTOR::TYPE::Table\n                + STAGE1_TABLE_DESCRIPTOR::VALID::True,\n        );\n\n        TableDescriptor { value: val.get() }\n    }\n}\n\n/// Convert the kernel's generic memory attributes to HW-specific attributes of the MMU.\nimpl convert::From<AttributeFields>\n    for tock_registers::fields::FieldValue<u64, STAGE1_PAGE_DESCRIPTOR::Register>\n{\n    fn from(attribute_fields: AttributeFields) -> Self {\n        // Memory attributes.\n        let mut desc = match attribute_fields.mem_attributes {\n            MemAttributes::CacheableDRAM => {\n                STAGE1_PAGE_DESCRIPTOR::SH::InnerShareable\n                    + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::NORMAL)\n            }\n            MemAttributes::Device => {\n                STAGE1_PAGE_DESCRIPTOR::SH::OuterShareable\n                    + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::DEVICE)\n            }\n        };\n\n        // Access Permissions.\n        desc += match attribute_fields.acc_perms {\n            AccessPermissions::ReadOnly => STAGE1_PAGE_DESCRIPTOR::AP::RO_EL1,\n            AccessPermissions::ReadWrite => STAGE1_PAGE_DESCRIPTOR::AP::RW_EL1,\n        };\n\n        // The execute-never attribute is mapped to PXN in AArch64.\n        desc += if attribute_fields.execute_never {\n            STAGE1_PAGE_DESCRIPTOR::PXN::True\n        } else {\n            STAGE1_PAGE_DESCRIPTOR::PXN::False\n        };\n\n        // Always set unprivileged exectue-never as long as userspace is not implemented yet.\n        desc += STAGE1_PAGE_DESCRIPTOR::UXN::True;\n\n        desc\n    }\n}\n\nimpl PageDescriptor {\n    /// Create an instance.\n    ///\n    /// Descriptor is invalid by default.\n    pub const fn new_zeroed() -> Self {\n        Self { value: 0 }\n    }\n\n    /// Create an instance.\n    pub fn from_output_page_addr(\n        phys_output_page_addr: PageAddress<Physical>,\n        attribute_fields: &AttributeFields,\n    ) -> Self {\n        let val = InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(0);\n\n        let shifted = phys_output_page_addr.into_inner().as_usize() >> Granule64KiB::SHIFT;\n        val.write(\n            STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted as u64)\n                + STAGE1_PAGE_DESCRIPTOR::AF::True\n                + STAGE1_PAGE_DESCRIPTOR::TYPE::Page\n                + STAGE1_PAGE_DESCRIPTOR::VALID::True\n                + (*attribute_fields).into(),\n        );\n\n        Self { value: val.get() }\n    }\n\n    /// Returns the valid bit.\n    fn is_valid(&self) -> bool {\n        InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(self.value)\n            .is_set(STAGE1_PAGE_DESCRIPTOR::VALID)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<const AS_SIZE: usize> memory::mmu::AssociatedTranslationTable\n    for memory::mmu::AddressSpace<AS_SIZE>\nwhere\n    [u8; Self::SIZE >> Granule512MiB::SHIFT]: Sized,\n{\n    type TableStartFromBottom = FixedSizeTranslationTable<{ Self::SIZE >> Granule512MiB::SHIFT }>;\n}\n\nimpl<const NUM_TABLES: usize> FixedSizeTranslationTable<NUM_TABLES> {\n    /// Create an instance.\n    #[allow(clippy::assertions_on_constants)]\n    pub const fn new() -> Self {\n        assert!(bsp::memory::mmu::KernelGranule::SIZE == Granule64KiB::SIZE);\n\n        // Can't have a zero-sized address space.\n        assert!(NUM_TABLES > 0);\n\n        Self {\n            lvl3: [[PageDescriptor::new_zeroed(); 8192]; NUM_TABLES],\n            lvl2: [TableDescriptor::new_zeroed(); NUM_TABLES],\n            initialized: false,\n        }\n    }\n\n    /// Helper to calculate the lvl2 and lvl3 indices from an address.\n    #[inline(always)]\n    fn lvl2_lvl3_index_from_page_addr(\n        &self,\n        virt_page_addr: PageAddress<Virtual>,\n    ) -> Result<(usize, usize), &'static str> {\n        let addr = virt_page_addr.into_inner().as_usize();\n        let lvl2_index = addr >> Granule512MiB::SHIFT;\n        let lvl3_index = (addr & Granule512MiB::MASK) >> Granule64KiB::SHIFT;\n\n        if lvl2_index > (NUM_TABLES - 1) {\n            return Err(\"Virtual page is out of bounds of translation table\");\n        }\n\n        Ok((lvl2_index, lvl3_index))\n    }\n\n    /// Sets the PageDescriptor corresponding to the supplied page address.\n    ///\n    /// Doesn't allow overriding an already valid page.\n    #[inline(always)]\n    fn set_page_descriptor_from_page_addr(\n        &mut self,\n        virt_page_addr: PageAddress<Virtual>,\n        new_desc: &PageDescriptor,\n    ) -> Result<(), &'static str> {\n        let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from_page_addr(virt_page_addr)?;\n        let desc = &mut self.lvl3[lvl2_index][lvl3_index];\n\n        if desc.is_valid() {\n            return Err(\"Virtual page is already mapped\");\n        }\n\n        *desc = *new_desc;\n        Ok(())\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\n\nimpl<const NUM_TABLES: usize> memory::mmu::translation_table::interface::TranslationTable\n    for FixedSizeTranslationTable<NUM_TABLES>\n{\n    fn init(&mut self) {\n        if self.initialized {\n            return;\n        }\n\n        // Populate the l2 entries.\n        for (lvl2_nr, lvl2_entry) in self.lvl2.iter_mut().enumerate() {\n            let phys_table_addr = self.lvl3[lvl2_nr].phys_start_addr();\n\n            let new_desc = TableDescriptor::from_next_lvl_table_addr(phys_table_addr);\n            *lvl2_entry = new_desc;\n        }\n\n        self.initialized = true;\n    }\n\n    fn phys_base_address(&self) -> Address<Physical> {\n        self.lvl2.phys_start_addr()\n    }\n\n    unsafe fn map_at(\n        &mut self,\n        virt_region: &MemoryRegion<Virtual>,\n        phys_region: &MemoryRegion<Physical>,\n        attr: &AttributeFields,\n    ) -> Result<(), &'static str> {\n        assert!(self.initialized, \"Translation tables not initialized\");\n\n        if virt_region.size() != phys_region.size() {\n            return Err(\"Tried to map memory regions with unequal sizes\");\n        }\n\n        if phys_region.end_exclusive_page_addr() > bsp::memory::phys_addr_space_end_exclusive_addr()\n        {\n            return Err(\"Tried to map outside of physical address space\");\n        }\n\n        let iter = phys_region.into_iter().zip(virt_region.into_iter());\n        for (phys_page_addr, virt_page_addr) in iter {\n            let new_desc = PageDescriptor::from_output_page_addr(phys_page_addr, attr);\n            let virt_page = virt_page_addr;\n\n            self.set_page_descriptor_from_page_addr(virt_page, &new_desc)?;\n        }\n\n        Ok(())\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\npub type MinSizeTranslationTable = FixedSizeTranslationTable<1>;\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// Check if the size of `struct TableDescriptor` is as expected.\n    #[kernel_test]\n    fn size_of_tabledescriptor_equals_64_bit() {\n        assert_eq!(\n            core::mem::size_of::<TableDescriptor>(),\n            core::mem::size_of::<u64>()\n        );\n    }\n\n    /// Check if the size of `struct PageDescriptor` is as expected.\n    #[kernel_test]\n    fn size_of_pagedescriptor_equals_64_bit() {\n        assert_eq!(\n            core::mem::size_of::<PageDescriptor>(),\n            core::mem::size_of::<u64>()\n        );\n    }\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/_arch/aarch64/memory/mmu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management Unit Driver.\n//!\n//! Only 64 KiB granule is supported.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::memory::mmu::arch_mmu\n\nuse crate::{\n    bsp, memory,\n    memory::{mmu::TranslationGranule, Address, Physical},\n};\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::intrinsics::unlikely;\nuse tock_registers::interfaces::{ReadWriteable, Readable, Writeable};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Memory Management Unit type.\nstruct MemoryManagementUnit;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub type Granule512MiB = TranslationGranule<{ 512 * 1024 * 1024 }>;\npub type Granule64KiB = TranslationGranule<{ 64 * 1024 }>;\n\n/// Constants for indexing the MAIR_EL1.\n#[allow(dead_code)]\npub mod mair {\n    pub const DEVICE: u64 = 0;\n    pub const NORMAL: u64 = 1;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic MMU: MemoryManagementUnit = MemoryManagementUnit;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<const AS_SIZE: usize> memory::mmu::AddressSpace<AS_SIZE> {\n    /// Checks for architectural restrictions.\n    pub const fn arch_address_space_size_sanity_checks() {\n        // Size must be at least one full 512 MiB table.\n        assert!((AS_SIZE % Granule512MiB::SIZE) == 0);\n\n        // Check for 48 bit virtual address size as maximum, which is supported by any ARMv8\n        // version.\n        assert!(AS_SIZE <= (1 << 48));\n    }\n}\n\nimpl MemoryManagementUnit {\n    /// Setup function for the MAIR_EL1 register.\n    fn set_up_mair(&self) {\n        // Define the memory types being mapped.\n        MAIR_EL1.write(\n            // Attribute 1 - Cacheable normal DRAM.\n            MAIR_EL1::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc +\n        MAIR_EL1::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc +\n\n        // Attribute 0 - Device.\n        MAIR_EL1::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck,\n        );\n    }\n\n    /// Configure various settings of stage 1 of the EL1 translation regime.\n    fn configure_translation_control(&self) {\n        let t0sz = (64 - bsp::memory::mmu::KernelVirtAddrSpace::SIZE_SHIFT) as u64;\n\n        TCR_EL1.write(\n            TCR_EL1::TBI0::Used\n                + TCR_EL1::IPS::Bits_40\n                + TCR_EL1::TG0::KiB_64\n                + TCR_EL1::SH0::Inner\n                + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable\n                + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable\n                + TCR_EL1::EPD0::EnableTTBR0Walks\n                + TCR_EL1::A1::TTBR0\n                + TCR_EL1::T0SZ.val(t0sz)\n                + TCR_EL1::EPD1::DisableTTBR1Walks,\n        );\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the MMU instance.\npub fn mmu() -> &'static impl memory::mmu::interface::MMU {\n    &MMU\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse memory::mmu::MMUEnableError;\n\nimpl memory::mmu::interface::MMU for MemoryManagementUnit {\n    unsafe fn enable_mmu_and_caching(\n        &self,\n        phys_tables_base_addr: Address<Physical>,\n    ) -> Result<(), MMUEnableError> {\n        if unlikely(self.is_enabled()) {\n            return Err(MMUEnableError::AlreadyEnabled);\n        }\n\n        // Fail early if translation granule is not supported.\n        if unlikely(!ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran64::Supported)) {\n            return Err(MMUEnableError::Other(\n                \"Translation granule not supported in HW\",\n            ));\n        }\n\n        // Prepare the memory attribute indirection register.\n        self.set_up_mair();\n\n        // Set the \"Translation Table Base Register\".\n        TTBR0_EL1.set_baddr(phys_tables_base_addr.as_usize() as u64);\n\n        self.configure_translation_control();\n\n        // Switch the MMU on.\n        //\n        // First, force all previous changes to be seen before the MMU is enabled.\n        barrier::isb(barrier::SY);\n\n        // Enable the MMU and turn on data and instruction caching.\n        SCTLR_EL1.modify(SCTLR_EL1::M::Enable + SCTLR_EL1::C::Cacheable + SCTLR_EL1::I::Cacheable);\n\n        // Force MMU init to complete before next instruction.\n        barrier::isb(barrier::SY);\n\n        Ok(())\n    }\n\n    #[inline(always)]\n    fn is_enabled(&self) -> bool {\n        SCTLR_EL1.matches_all(SCTLR_EL1::M::Enable)\n    }\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/_arch/aarch64/time.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural timer primitives.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::time::arch_time\n\nuse crate::warn;\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::{\n    num::{NonZeroU128, NonZeroU32, NonZeroU64},\n    ops::{Add, Div},\n    time::Duration,\n};\nuse tock_registers::interfaces::Readable;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nconst NANOSEC_PER_SEC: NonZeroU64 = NonZeroU64::new(1_000_000_000).unwrap();\n\n#[derive(Copy, Clone, PartialOrd, PartialEq)]\nstruct GenericTimerCounterValue(u64);\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n/// Boot assembly code overwrites this value with the value of CNTFRQ_EL0 before any Rust code is\n/// executed. This given value here is just a (safe) dummy.\n#[no_mangle]\nstatic ARCH_TIMER_COUNTER_FREQUENCY: NonZeroU32 = NonZeroU32::MIN;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nfn arch_timer_counter_frequency() -> NonZeroU32 {\n    // Read volatile is needed here to prevent the compiler from optimizing\n    // ARCH_TIMER_COUNTER_FREQUENCY away.\n    //\n    // This is safe, because all the safety requirements as stated in read_volatile()'s\n    // documentation are fulfilled.\n    unsafe { core::ptr::read_volatile(&ARCH_TIMER_COUNTER_FREQUENCY) }\n}\n\nimpl GenericTimerCounterValue {\n    pub const MAX: Self = GenericTimerCounterValue(u64::MAX);\n}\n\nimpl Add for GenericTimerCounterValue {\n    type Output = Self;\n\n    fn add(self, other: Self) -> Self {\n        GenericTimerCounterValue(self.0.wrapping_add(other.0))\n    }\n}\n\nimpl From<GenericTimerCounterValue> for Duration {\n    fn from(counter_value: GenericTimerCounterValue) -> Self {\n        if counter_value.0 == 0 {\n            return Duration::ZERO;\n        }\n\n        let frequency: NonZeroU64 = arch_timer_counter_frequency().into();\n\n        // Div<NonZeroU64> implementation for u64 cannot panic.\n        let secs = counter_value.0.div(frequency);\n\n        // This is safe, because frequency can never be greater than u32::MAX, which means the\n        // largest theoretical value for sub_second_counter_value is (u32::MAX - 1). Therefore,\n        // (sub_second_counter_value * NANOSEC_PER_SEC) cannot overflow an u64.\n        //\n        // The subsequent division ensures the result fits into u32, since the max result is smaller\n        // than NANOSEC_PER_SEC. Therefore, just cast it to u32 using `as`.\n        let sub_second_counter_value = counter_value.0 % frequency;\n        let nanos = unsafe { sub_second_counter_value.unchecked_mul(u64::from(NANOSEC_PER_SEC)) }\n            .div(frequency) as u32;\n\n        Duration::new(secs, nanos)\n    }\n}\n\nfn max_duration() -> Duration {\n    Duration::from(GenericTimerCounterValue::MAX)\n}\n\nimpl TryFrom<Duration> for GenericTimerCounterValue {\n    type Error = &'static str;\n\n    fn try_from(duration: Duration) -> Result<Self, Self::Error> {\n        if duration < resolution() {\n            return Ok(GenericTimerCounterValue(0));\n        }\n\n        if duration > max_duration() {\n            return Err(\"Conversion error. Duration too big\");\n        }\n\n        let frequency: u128 = u32::from(arch_timer_counter_frequency()) as u128;\n        let duration: u128 = duration.as_nanos();\n\n        // This is safe, because frequency can never be greater than u32::MAX, and\n        // (Duration::MAX.as_nanos() * u32::MAX) < u128::MAX.\n        let counter_value =\n            unsafe { duration.unchecked_mul(frequency) }.div(NonZeroU128::from(NANOSEC_PER_SEC));\n\n        // Since we checked above that we are <= max_duration(), just cast to u64.\n        Ok(GenericTimerCounterValue(counter_value as u64))\n    }\n}\n\n#[inline(always)]\nfn read_cntpct() -> GenericTimerCounterValue {\n    // Prevent that the counter is read ahead of time due to out-of-order execution.\n    barrier::isb(barrier::SY);\n    let cnt = CNTPCT_EL0.get();\n\n    GenericTimerCounterValue(cnt)\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The timer's resolution.\npub fn resolution() -> Duration {\n    Duration::from(GenericTimerCounterValue(1))\n}\n\n/// The uptime since power-on of the device.\n///\n/// This includes time consumed by firmware and bootloaders.\npub fn uptime() -> Duration {\n    read_cntpct().into()\n}\n\n/// Spin for a given duration.\npub fn spin_for(duration: Duration) {\n    let curr_counter_value = read_cntpct();\n\n    let counter_value_delta: GenericTimerCounterValue = match duration.try_into() {\n        Err(msg) => {\n            warn!(\"spin_for: {}. Skipping\", msg);\n            return;\n        }\n        Ok(val) => val,\n    };\n    let counter_value_target = curr_counter_value + counter_value_delta;\n\n    // Busy wait.\n    //\n    // Read CNTPCT_EL0 directly to avoid the ISB that is part of [`read_cntpct`].\n    while GenericTimerCounterValue(CNTPCT_EL0.get()) < counter_value_target {}\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/bsp/device_driver/arm/gicv2/gicc.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GICC Driver - GIC CPU interface.\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    exception,\n    memory::{Address, Virtual},\n};\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields, register_structs,\n    registers::ReadWrite,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nregister_bitfields! {\n    u32,\n\n    /// CPU Interface Control Register\n    CTLR [\n        Enable OFFSET(0) NUMBITS(1) []\n    ],\n\n    /// Interrupt Priority Mask Register\n    PMR [\n        Priority OFFSET(0) NUMBITS(8) []\n    ],\n\n    /// Interrupt Acknowledge Register\n    IAR [\n        InterruptID OFFSET(0) NUMBITS(10) []\n    ],\n\n    /// End of Interrupt Register\n    EOIR [\n        EOIINTID OFFSET(0) NUMBITS(10) []\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    pub RegisterBlock {\n        (0x000 => CTLR: ReadWrite<u32, CTLR::Register>),\n        (0x004 => PMR: ReadWrite<u32, PMR::Register>),\n        (0x008 => _reserved1),\n        (0x00C => IAR: ReadWrite<u32, IAR::Register>),\n        (0x010 => EOIR: ReadWrite<u32, EOIR::Register>),\n        (0x014  => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the GIC CPU interface.\npub struct GICC {\n    registers: Registers,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GICC {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n        }\n    }\n\n    /// Accept interrupts of any priority.\n    ///\n    /// Quoting the GICv2 Architecture Specification:\n    ///\n    ///   \"Writing 255 to the GICC_PMR always sets it to the largest supported priority field\n    ///    value.\"\n    ///\n    /// # Safety\n    ///\n    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n    ///   of `&mut self`.\n    pub fn priority_accept_all(&self) {\n        self.registers.PMR.write(PMR::Priority.val(255)); // Comment in arch spec.\n    }\n\n    /// Enable the interface - start accepting IRQs.\n    ///\n    /// # Safety\n    ///\n    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n    ///   of `&mut self`.\n    pub fn enable(&self) {\n        self.registers.CTLR.write(CTLR::Enable::SET);\n    }\n\n    /// Extract the number of the highest-priority pending IRQ.\n    ///\n    /// Can only be called from IRQ context, which is ensured by taking an `IRQContext` token.\n    ///\n    /// # Safety\n    ///\n    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n    ///   of `&mut self`.\n    #[allow(clippy::trivially_copy_pass_by_ref)]\n    pub fn pending_irq_number<'irq_context>(\n        &self,\n        _ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) -> usize {\n        self.registers.IAR.read(IAR::InterruptID) as usize\n    }\n\n    /// Complete handling of the currently active IRQ.\n    ///\n    /// Can only be called from IRQ context, which is ensured by taking an `IRQContext` token.\n    ///\n    /// To be called after `pending_irq_number()`.\n    ///\n    /// # Safety\n    ///\n    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n    ///   of `&mut self`.\n    #[allow(clippy::trivially_copy_pass_by_ref)]\n    pub fn mark_comleted<'irq_context>(\n        &self,\n        irq_number: u32,\n        _ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) {\n        self.registers.EOIR.write(EOIR::EOIINTID.val(irq_number));\n    }\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/bsp/device_driver/arm/gicv2/gicd.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GICD Driver - GIC Distributor.\n//!\n//! # Glossary\n//!   - SPI - Shared Peripheral Interrupt.\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    memory::{Address, Virtual},\n    state, synchronization,\n    synchronization::IRQSafeNullLock,\n};\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields, register_structs,\n    registers::{ReadOnly, ReadWrite},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nregister_bitfields! {\n    u32,\n\n    /// Distributor Control Register\n    CTLR [\n        Enable OFFSET(0) NUMBITS(1) []\n    ],\n\n    /// Interrupt Controller Type Register\n    TYPER [\n        ITLinesNumber OFFSET(0)  NUMBITS(5) []\n    ],\n\n    /// Interrupt Processor Targets Registers\n    ITARGETSR [\n        Offset3 OFFSET(24) NUMBITS(8) [],\n        Offset2 OFFSET(16) NUMBITS(8) [],\n        Offset1 OFFSET(8)  NUMBITS(8) [],\n        Offset0 OFFSET(0)  NUMBITS(8) []\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    SharedRegisterBlock {\n        (0x000 => CTLR: ReadWrite<u32, CTLR::Register>),\n        (0x004 => TYPER: ReadOnly<u32, TYPER::Register>),\n        (0x008 => _reserved1),\n        (0x104 => ISENABLER: [ReadWrite<u32>; 31]),\n        (0x180 => _reserved2),\n        (0x820 => ITARGETSR: [ReadWrite<u32, ITARGETSR::Register>; 248]),\n        (0xC00 => @END),\n    }\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    BankedRegisterBlock {\n        (0x000 => _reserved1),\n        (0x100 => ISENABLER: ReadWrite<u32>),\n        (0x104 => _reserved2),\n        (0x800 => ITARGETSR: [ReadOnly<u32, ITARGETSR::Register>; 8]),\n        (0x820 => @END),\n    }\n}\n\n/// Abstraction for the non-banked parts of the associated MMIO registers.\ntype SharedRegisters = MMIODerefWrapper<SharedRegisterBlock>;\n\n/// Abstraction for the banked parts of the associated MMIO registers.\ntype BankedRegisters = MMIODerefWrapper<BankedRegisterBlock>;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the GIC Distributor.\npub struct GICD {\n    /// Access to shared registers is guarded with a lock.\n    shared_registers: IRQSafeNullLock<SharedRegisters>,\n\n    /// Access to banked registers is unguarded.\n    banked_registers: BankedRegisters,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl SharedRegisters {\n    /// Return the number of IRQs that this HW implements.\n    #[inline(always)]\n    fn num_irqs(&mut self) -> usize {\n        // Query number of implemented IRQs.\n        //\n        // Refer to GICv2 Architecture Specification, Section 4.3.2.\n        ((self.TYPER.read(TYPER::ITLinesNumber) as usize) + 1) * 32\n    }\n\n    /// Return a slice of the implemented ITARGETSR.\n    #[inline(always)]\n    fn implemented_itargets_slice(&mut self) -> &[ReadWrite<u32, ITARGETSR::Register>] {\n        assert!(self.num_irqs() >= 36);\n\n        // Calculate the max index of the shared ITARGETSR array.\n        //\n        // The first 32 IRQs are private, so not included in `shared_registers`. Each ITARGETS\n        // register has four entries, so shift right by two. Subtract one because we start\n        // counting at zero.\n        let spi_itargetsr_max_index = ((self.num_irqs() - 32) >> 2) - 1;\n\n        // Rust automatically inserts slice range sanity check, i.e. max >= min.\n        &self.ITARGETSR[0..spi_itargetsr_max_index]\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl GICD {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            shared_registers: IRQSafeNullLock::new(SharedRegisters::new(mmio_start_addr)),\n            banked_registers: BankedRegisters::new(mmio_start_addr),\n        }\n    }\n\n    /// Use a banked ITARGETSR to retrieve the executing core's GIC target mask.\n    ///\n    /// Quoting the GICv2 Architecture Specification:\n    ///\n    ///   \"GICD_ITARGETSR0 to GICD_ITARGETSR7 are read-only, and each field returns a value that\n    ///    corresponds only to the processor reading the register.\"\n    fn local_gic_target_mask(&self) -> u32 {\n        self.banked_registers.ITARGETSR[0].read(ITARGETSR::Offset0)\n    }\n\n    /// Route all SPIs to the boot core and enable the distributor.\n    pub fn boot_core_init(&self) {\n        assert!(\n            state::state_manager().is_init(),\n            \"Only allowed during kernel init phase\"\n        );\n\n        // Target all SPIs to the boot core only.\n        let mask = self.local_gic_target_mask();\n\n        self.shared_registers.lock(|regs| {\n            for i in regs.implemented_itargets_slice().iter() {\n                i.write(\n                    ITARGETSR::Offset3.val(mask)\n                        + ITARGETSR::Offset2.val(mask)\n                        + ITARGETSR::Offset1.val(mask)\n                        + ITARGETSR::Offset0.val(mask),\n                );\n            }\n\n            regs.CTLR.write(CTLR::Enable::SET);\n        });\n    }\n\n    /// Enable an interrupt.\n    pub fn enable(&self, irq_num: &super::IRQNumber) {\n        let irq_num = irq_num.get();\n\n        // Each bit in the u32 enable register corresponds to one IRQ number. Shift right by 5\n        // (division by 32) and arrive at the index for the respective ISENABLER[i].\n        let enable_reg_index = irq_num >> 5;\n        let enable_bit: u32 = 1u32 << (irq_num % 32);\n\n        // Check if we are handling a private or shared IRQ.\n        match irq_num {\n            // Private.\n            0..=31 => {\n                let enable_reg = &self.banked_registers.ISENABLER;\n                enable_reg.set(enable_reg.get() | enable_bit);\n            }\n            // Shared.\n            _ => {\n                let enable_reg_index_shared = enable_reg_index - 1;\n\n                self.shared_registers.lock(|regs| {\n                    let enable_reg = &regs.ISENABLER[enable_reg_index_shared];\n                    enable_reg.set(enable_reg.get() | enable_bit);\n                });\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/bsp/device_driver/arm/gicv2.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GICv2 Driver - ARM Generic Interrupt Controller v2.\n//!\n//! The following is a collection of excerpts with useful information from\n//!   - `Programmer's Guide for ARMv8-A`\n//!   - `ARM Generic Interrupt Controller Architecture Specification`\n//!\n//! # Programmer's Guide - 10.6.1 Configuration\n//!\n//! The GIC is accessed as a memory-mapped peripheral.\n//!\n//! All cores can access the common Distributor, but the CPU interface is banked, that is, each core\n//! uses the same address to access its own private CPU interface.\n//!\n//! It is not possible for a core to access the CPU interface of another core.\n//!\n//! # Architecture Specification - 10.6.2 Initialization\n//!\n//! Both the Distributor and the CPU interfaces are disabled at reset. The GIC must be initialized\n//! after reset before it can deliver interrupts to the core.\n//!\n//! In the Distributor, software must configure the priority, target, security and enable individual\n//! interrupts. The Distributor must subsequently be enabled through its control register\n//! (GICD_CTLR). For each CPU interface, software must program the priority mask and preemption\n//! settings.\n//!\n//! Each CPU interface block itself must be enabled through its control register (GICD_CTLR). This\n//! prepares the GIC to deliver interrupts to the core.\n//!\n//! Before interrupts are expected in the core, software prepares the core to take interrupts by\n//! setting a valid interrupt vector in the vector table, and clearing interrupt mask bits in\n//! PSTATE, and setting the routing controls.\n//!\n//! The entire interrupt mechanism in the system can be disabled by disabling the Distributor.\n//! Interrupt delivery to an individual core can be disabled by disabling its CPU interface.\n//! Individual interrupts can also be disabled (or enabled) in the distributor.\n//!\n//! For an interrupt to reach the core, the individual interrupt, Distributor and CPU interface must\n//! all be enabled. The interrupt also needs to be of sufficient priority, that is, higher than the\n//! core's priority mask.\n//!\n//! # Architecture Specification - 1.4.2 Interrupt types\n//!\n//! - Peripheral interrupt\n//!     - Private Peripheral Interrupt (PPI)\n//!         - This is a peripheral interrupt that is specific to a single processor.\n//!     - Shared Peripheral Interrupt (SPI)\n//!         - This is a peripheral interrupt that the Distributor can route to any of a specified\n//!           combination of processors.\n//!\n//! - Software-generated interrupt (SGI)\n//!     - This is an interrupt generated by software writing to a GICD_SGIR register in the GIC. The\n//!       system uses SGIs for interprocessor communication.\n//!     - An SGI has edge-triggered properties. The software triggering of the interrupt is\n//!       equivalent to the edge transition of the interrupt request signal.\n//!     - When an SGI occurs in a multiprocessor implementation, the CPUID field in the Interrupt\n//!       Acknowledge Register, GICC_IAR, or the Aliased Interrupt Acknowledge Register, GICC_AIAR,\n//!       identifies the processor that requested the interrupt.\n//!\n//! # Architecture Specification - 2.2.1 Interrupt IDs\n//!\n//! Interrupts from sources are identified using ID numbers. Each CPU interface can see up to 1020\n//! interrupts. The banking of SPIs and PPIs increases the total number of interrupts supported by\n//! the Distributor.\n//!\n//! The GIC assigns interrupt ID numbers ID0-ID1019 as follows:\n//!   - Interrupt numbers 32..1019 are used for SPIs.\n//!   - Interrupt numbers 0..31 are used for interrupts that are private to a CPU interface. These\n//!     interrupts are banked in the Distributor.\n//!       - A banked interrupt is one where the Distributor can have multiple interrupts with the\n//!         same ID. A banked interrupt is identified uniquely by its ID number and its associated\n//!         CPU interface number. Of the banked interrupt IDs:\n//!           - 00..15 SGIs\n//!           - 16..31 PPIs\n\nmod gicc;\nmod gicd;\n\nuse crate::{\n    bsp::{self, device_driver::common::BoundedUsize},\n    cpu, driver, exception,\n    memory::{Address, Virtual},\n    synchronization,\n    synchronization::InitStateLock,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\ntype HandlerTable = [Option<exception::asynchronous::IRQHandlerDescriptor<IRQNumber>>;\n    IRQNumber::MAX_INCLUSIVE + 1];\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Used for the associated type of trait [`exception::asynchronous::interface::IRQManager`].\npub type IRQNumber = BoundedUsize<{ GICv2::MAX_IRQ_NUMBER }>;\n\n/// Representation of the GIC.\npub struct GICv2 {\n    /// The Distributor.\n    gicd: gicd::GICD,\n\n    /// The CPU Interface.\n    gicc: gicc::GICC,\n\n    /// Stores registered IRQ handlers. Writable only during kernel init. RO afterwards.\n    handler_table: InitStateLock<HandlerTable>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GICv2 {\n    const MAX_IRQ_NUMBER: usize = 300; // Normally 1019, but keep it lower to save some space.\n\n    pub const COMPATIBLE: &'static str = \"GICv2 (ARM Generic Interrupt Controller v2)\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(\n        gicd_mmio_start_addr: Address<Virtual>,\n        gicc_mmio_start_addr: Address<Virtual>,\n    ) -> Self {\n        Self {\n            gicd: gicd::GICD::new(gicd_mmio_start_addr),\n            gicc: gicc::GICC::new(gicc_mmio_start_addr),\n            handler_table: InitStateLock::new([None; IRQNumber::MAX_INCLUSIVE + 1]),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::ReadWriteEx;\n\nimpl driver::interface::DeviceDriver for GICv2 {\n    type IRQNumberType = IRQNumber;\n\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n\n    unsafe fn init(&self) -> Result<(), &'static str> {\n        if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() {\n            self.gicd.boot_core_init();\n        }\n\n        self.gicc.priority_accept_all();\n        self.gicc.enable();\n\n        Ok(())\n    }\n}\n\nimpl exception::asynchronous::interface::IRQManager for GICv2 {\n    type IRQNumberType = IRQNumber;\n\n    fn register_handler(\n        &self,\n        irq_handler_descriptor: exception::asynchronous::IRQHandlerDescriptor<Self::IRQNumberType>,\n    ) -> Result<(), &'static str> {\n        self.handler_table.write(|table| {\n            let irq_number = irq_handler_descriptor.number().get();\n\n            if table[irq_number].is_some() {\n                return Err(\"IRQ handler already registered\");\n            }\n\n            table[irq_number] = Some(irq_handler_descriptor);\n\n            Ok(())\n        })\n    }\n\n    fn enable(&self, irq_number: &Self::IRQNumberType) {\n        self.gicd.enable(irq_number);\n    }\n\n    fn handle_pending_irqs<'irq_context>(\n        &'irq_context self,\n        ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) {\n        // Extract the highest priority pending IRQ number from the Interrupt Acknowledge Register\n        // (IAR).\n        let irq_number = self.gicc.pending_irq_number(ic);\n\n        // Guard against spurious interrupts.\n        if irq_number > GICv2::MAX_IRQ_NUMBER {\n            return;\n        }\n\n        // Call the IRQ handler. Panic if there is none.\n        self.handler_table.read(|table| {\n            match table[irq_number] {\n                None => panic!(\"No handler registered for IRQ {}\", irq_number),\n                Some(descriptor) => {\n                    // Call the IRQ handler. Panics on failure.\n                    descriptor.handler().handle().expect(\"Error handling IRQ\");\n                }\n            }\n        });\n\n        // Signal completion of handling.\n        self.gicc.mark_comleted(irq_number as u32, ic);\n    }\n\n    fn print_handler(&self) {\n        use crate::info;\n\n        info!(\"      Peripheral handler:\");\n\n        self.handler_table.read(|table| {\n            for (i, opt) in table.iter().skip(32).enumerate() {\n                if let Some(handler) = opt {\n                    info!(\"            {: >3}. {}\", i + 32, handler.name());\n                }\n            }\n        });\n    }\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/bsp/device_driver/arm.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! ARM driver top level.\n\npub mod gicv2;\n\npub use gicv2::*;\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GPIO Driver.\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    driver,\n    exception::asynchronous::IRQNumber,\n    memory::{Address, Virtual},\n    synchronization,\n    synchronization::IRQSafeNullLock,\n};\nuse tock_registers::{\n    interfaces::{ReadWriteable, Writeable},\n    register_bitfields, register_structs,\n    registers::ReadWrite,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// GPIO registers.\n//\n// Descriptions taken from\n// - https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf\n// - https://datasheets.raspberrypi.org/bcm2711/bcm2711-peripherals.pdf\nregister_bitfields! {\n    u32,\n\n    /// GPIO Function Select 1\n    GPFSEL1 [\n        /// Pin 15\n        FSEL15 OFFSET(15) NUMBITS(3) [\n            Input = 0b000,\n            Output = 0b001,\n            AltFunc0 = 0b100  // PL011 UART RX\n\n        ],\n\n        /// Pin 14\n        FSEL14 OFFSET(12) NUMBITS(3) [\n            Input = 0b000,\n            Output = 0b001,\n            AltFunc0 = 0b100  // PL011 UART TX\n        ]\n    ],\n\n    /// GPIO Pull-up/down Register\n    ///\n    /// BCM2837 only.\n    GPPUD [\n        /// Controls the actuation of the internal pull-up/down control line to ALL the GPIO pins.\n        PUD OFFSET(0) NUMBITS(2) [\n            Off = 0b00,\n            PullDown = 0b01,\n            PullUp = 0b10\n        ]\n    ],\n\n    /// GPIO Pull-up/down Clock Register 0\n    ///\n    /// BCM2837 only.\n    GPPUDCLK0 [\n        /// Pin 15\n        PUDCLK15 OFFSET(15) NUMBITS(1) [\n            NoEffect = 0,\n            AssertClock = 1\n        ],\n\n        /// Pin 14\n        PUDCLK14 OFFSET(14) NUMBITS(1) [\n            NoEffect = 0,\n            AssertClock = 1\n        ]\n    ],\n\n    /// GPIO Pull-up / Pull-down Register 0\n    ///\n    /// BCM2711 only.\n    GPIO_PUP_PDN_CNTRL_REG0 [\n        /// Pin 15\n        GPIO_PUP_PDN_CNTRL15 OFFSET(30) NUMBITS(2) [\n            NoResistor = 0b00,\n            PullUp = 0b01\n        ],\n\n        /// Pin 14\n        GPIO_PUP_PDN_CNTRL14 OFFSET(28) NUMBITS(2) [\n            NoResistor = 0b00,\n            PullUp = 0b01\n        ]\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    RegisterBlock {\n        (0x00 => _reserved1),\n        (0x04 => GPFSEL1: ReadWrite<u32, GPFSEL1::Register>),\n        (0x08 => _reserved2),\n        (0x94 => GPPUD: ReadWrite<u32, GPPUD::Register>),\n        (0x98 => GPPUDCLK0: ReadWrite<u32, GPPUDCLK0::Register>),\n        (0x9C => _reserved3),\n        (0xE4 => GPIO_PUP_PDN_CNTRL_REG0: ReadWrite<u32, GPIO_PUP_PDN_CNTRL_REG0::Register>),\n        (0xE8 => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\nstruct GPIOInner {\n    registers: Registers,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the GPIO HW.\npub struct GPIO {\n    inner: IRQSafeNullLock<GPIOInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GPIOInner {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n        }\n    }\n\n    /// Disable pull-up/down on pins 14 and 15.\n    #[cfg(feature = \"bsp_rpi3\")]\n    fn disable_pud_14_15_bcm2837(&mut self) {\n        use crate::time;\n        use core::time::Duration;\n\n        // The Linux 2837 GPIO driver waits 1 µs between the steps.\n        const DELAY: Duration = Duration::from_micros(1);\n\n        self.registers.GPPUD.write(GPPUD::PUD::Off);\n        time::time_manager().spin_for(DELAY);\n\n        self.registers\n            .GPPUDCLK0\n            .write(GPPUDCLK0::PUDCLK15::AssertClock + GPPUDCLK0::PUDCLK14::AssertClock);\n        time::time_manager().spin_for(DELAY);\n\n        self.registers.GPPUD.write(GPPUD::PUD::Off);\n        self.registers.GPPUDCLK0.set(0);\n    }\n\n    /// Disable pull-up/down on pins 14 and 15.\n    #[cfg(feature = \"bsp_rpi4\")]\n    fn disable_pud_14_15_bcm2711(&mut self) {\n        self.registers.GPIO_PUP_PDN_CNTRL_REG0.write(\n            GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL15::PullUp\n                + GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL14::PullUp,\n        );\n    }\n\n    /// Map PL011 UART as standard output.\n    ///\n    /// TX to pin 14\n    /// RX to pin 15\n    pub fn map_pl011_uart(&mut self) {\n        // Select the UART on pins 14 and 15.\n        self.registers\n            .GPFSEL1\n            .modify(GPFSEL1::FSEL15::AltFunc0 + GPFSEL1::FSEL14::AltFunc0);\n\n        // Disable pull-up/down on pins 14 and 15.\n        #[cfg(feature = \"bsp_rpi3\")]\n        self.disable_pud_14_15_bcm2837();\n\n        #[cfg(feature = \"bsp_rpi4\")]\n        self.disable_pud_14_15_bcm2711();\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GPIO {\n    pub const COMPATIBLE: &'static str = \"BCM GPIO\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            inner: IRQSafeNullLock::new(GPIOInner::new(mmio_start_addr)),\n        }\n    }\n\n    /// Concurrency safe version of `GPIOInner.map_pl011_uart()`\n    pub fn map_pl011_uart(&self) {\n        self.inner.lock(|inner| inner.map_pl011_uart())\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl driver::interface::DeviceDriver for GPIO {\n    type IRQNumberType = IRQNumber;\n\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Peripheral Interrupt Controller Driver.\n//!\n//! # Resources\n//!\n//! - <https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf>\n\nuse super::{PendingIRQs, PeripheralIRQ};\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    exception,\n    memory::{Address, Virtual},\n    synchronization,\n    synchronization::{IRQSafeNullLock, InitStateLock},\n};\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_structs,\n    registers::{ReadOnly, WriteOnly},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    WORegisterBlock {\n        (0x00 => _reserved1),\n        (0x10 => ENABLE_1: WriteOnly<u32>),\n        (0x14 => ENABLE_2: WriteOnly<u32>),\n        (0x18 => @END),\n    }\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    RORegisterBlock {\n        (0x00 => _reserved1),\n        (0x04 => PENDING_1: ReadOnly<u32>),\n        (0x08 => PENDING_2: ReadOnly<u32>),\n        (0x0c => @END),\n    }\n}\n\n/// Abstraction for the WriteOnly parts of the associated MMIO registers.\ntype WriteOnlyRegisters = MMIODerefWrapper<WORegisterBlock>;\n\n/// Abstraction for the ReadOnly parts of the associated MMIO registers.\ntype ReadOnlyRegisters = MMIODerefWrapper<RORegisterBlock>;\n\ntype HandlerTable = [Option<exception::asynchronous::IRQHandlerDescriptor<PeripheralIRQ>>;\n    PeripheralIRQ::MAX_INCLUSIVE + 1];\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the peripheral interrupt controller.\npub struct PeripheralIC {\n    /// Access to write registers is guarded with a lock.\n    wo_registers: IRQSafeNullLock<WriteOnlyRegisters>,\n\n    /// Register read access is unguarded.\n    ro_registers: ReadOnlyRegisters,\n\n    /// Stores registered IRQ handlers. Writable only during kernel init. RO afterwards.\n    handler_table: InitStateLock<HandlerTable>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PeripheralIC {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            wo_registers: IRQSafeNullLock::new(WriteOnlyRegisters::new(mmio_start_addr)),\n            ro_registers: ReadOnlyRegisters::new(mmio_start_addr),\n            handler_table: InitStateLock::new([None; PeripheralIRQ::MAX_INCLUSIVE + 1]),\n        }\n    }\n\n    /// Query the list of pending IRQs.\n    fn pending_irqs(&self) -> PendingIRQs {\n        let pending_mask: u64 = (u64::from(self.ro_registers.PENDING_2.get()) << 32)\n            | u64::from(self.ro_registers.PENDING_1.get());\n\n        PendingIRQs::new(pending_mask)\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::{Mutex, ReadWriteEx};\n\nimpl exception::asynchronous::interface::IRQManager for PeripheralIC {\n    type IRQNumberType = PeripheralIRQ;\n\n    fn register_handler(\n        &self,\n        irq_handler_descriptor: exception::asynchronous::IRQHandlerDescriptor<Self::IRQNumberType>,\n    ) -> Result<(), &'static str> {\n        self.handler_table.write(|table| {\n            let irq_number = irq_handler_descriptor.number().get();\n\n            if table[irq_number].is_some() {\n                return Err(\"IRQ handler already registered\");\n            }\n\n            table[irq_number] = Some(irq_handler_descriptor);\n\n            Ok(())\n        })\n    }\n\n    fn enable(&self, irq: &Self::IRQNumberType) {\n        self.wo_registers.lock(|regs| {\n            let enable_reg = if irq.get() <= 31 {\n                &regs.ENABLE_1\n            } else {\n                &regs.ENABLE_2\n            };\n\n            let enable_bit: u32 = 1 << (irq.get() % 32);\n\n            // Writing a 1 to a bit will set the corresponding IRQ enable bit. All other IRQ enable\n            // bits are unaffected. So we don't need read and OR'ing here.\n            enable_reg.set(enable_bit);\n        });\n    }\n\n    fn handle_pending_irqs<'irq_context>(\n        &'irq_context self,\n        _ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) {\n        self.handler_table.read(|table| {\n            for irq_number in self.pending_irqs() {\n                match table[irq_number] {\n                    None => panic!(\"No handler registered for IRQ {}\", irq_number),\n                    Some(descriptor) => {\n                        // Call the IRQ handler. Panics on failure.\n                        descriptor.handler().handle().expect(\"Error handling IRQ\");\n                    }\n                }\n            }\n        })\n    }\n\n    fn print_handler(&self) {\n        use crate::info;\n\n        info!(\"      Peripheral handler:\");\n\n        self.handler_table.read(|table| {\n            for (i, opt) in table.iter().enumerate() {\n                if let Some(handler) = opt {\n                    info!(\"            {: >3}. {}\", i, handler.name());\n                }\n            }\n        });\n    }\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Interrupt Controller Driver.\n\nmod peripheral_ic;\n\nuse crate::{\n    bsp::device_driver::common::BoundedUsize,\n    driver,\n    exception::{self, asynchronous::IRQHandlerDescriptor},\n    memory::{Address, Virtual},\n};\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Wrapper struct for a bitmask indicating pending IRQ numbers.\nstruct PendingIRQs {\n    bitmask: u64,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub type LocalIRQ = BoundedUsize<{ InterruptController::MAX_LOCAL_IRQ_NUMBER }>;\npub type PeripheralIRQ = BoundedUsize<{ InterruptController::MAX_PERIPHERAL_IRQ_NUMBER }>;\n\n/// Used for the associated type of trait [`exception::asynchronous::interface::IRQManager`].\n#[derive(Copy, Clone)]\n#[allow(missing_docs)]\npub enum IRQNumber {\n    Local(LocalIRQ),\n    Peripheral(PeripheralIRQ),\n}\n\n/// Representation of the Interrupt Controller.\npub struct InterruptController {\n    periph: peripheral_ic::PeripheralIC,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PendingIRQs {\n    pub fn new(bitmask: u64) -> Self {\n        Self { bitmask }\n    }\n}\n\nimpl Iterator for PendingIRQs {\n    type Item = usize;\n\n    fn next(&mut self) -> Option<Self::Item> {\n        if self.bitmask == 0 {\n            return None;\n        }\n\n        let next = self.bitmask.trailing_zeros() as usize;\n        self.bitmask &= self.bitmask.wrapping_sub(1);\n        Some(next)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl fmt::Display for IRQNumber {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        match self {\n            Self::Local(number) => write!(f, \"Local({})\", number),\n            Self::Peripheral(number) => write!(f, \"Peripheral({})\", number),\n        }\n    }\n}\n\nimpl InterruptController {\n    // Restrict to 3 for now. This makes future code for local_ic.rs more straight forward.\n    const MAX_LOCAL_IRQ_NUMBER: usize = 3;\n    const MAX_PERIPHERAL_IRQ_NUMBER: usize = 63;\n\n    pub const COMPATIBLE: &'static str = \"BCM Interrupt Controller\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(periph_mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            periph: peripheral_ic::PeripheralIC::new(periph_mmio_start_addr),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\n\nimpl driver::interface::DeviceDriver for InterruptController {\n    type IRQNumberType = IRQNumber;\n\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n}\n\nimpl exception::asynchronous::interface::IRQManager for InterruptController {\n    type IRQNumberType = IRQNumber;\n\n    fn register_handler(\n        &self,\n        irq_handler_descriptor: exception::asynchronous::IRQHandlerDescriptor<Self::IRQNumberType>,\n    ) -> Result<(), &'static str> {\n        match irq_handler_descriptor.number() {\n            IRQNumber::Local(_) => unimplemented!(\"Local IRQ controller not implemented.\"),\n            IRQNumber::Peripheral(pirq) => {\n                let periph_descriptor = IRQHandlerDescriptor::new(\n                    pirq,\n                    irq_handler_descriptor.name(),\n                    irq_handler_descriptor.handler(),\n                );\n\n                self.periph.register_handler(periph_descriptor)\n            }\n        }\n    }\n\n    fn enable(&self, irq: &Self::IRQNumberType) {\n        match irq {\n            IRQNumber::Local(_) => unimplemented!(\"Local IRQ controller not implemented.\"),\n            IRQNumber::Peripheral(pirq) => self.periph.enable(pirq),\n        }\n    }\n\n    fn handle_pending_irqs<'irq_context>(\n        &'irq_context self,\n        ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) {\n        // It can only be a peripheral IRQ pending because enable() does not support local IRQs yet.\n        self.periph.handle_pending_irqs(ic)\n    }\n\n    fn print_handler(&self) {\n        self.periph.print_handler();\n    }\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! PL011 UART driver.\n//!\n//! # Resources\n//!\n//! - <https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf>\n//! - <https://developer.arm.com/documentation/ddi0183/latest>\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    console, cpu, driver,\n    exception::{self, asynchronous::IRQNumber},\n    memory::{Address, Virtual},\n    synchronization,\n    synchronization::IRQSafeNullLock,\n};\nuse core::fmt;\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields, register_structs,\n    registers::{ReadOnly, ReadWrite, WriteOnly},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// PL011 UART registers.\n//\n// Descriptions taken from \"PrimeCell UART (PL011) Technical Reference Manual\" r1p5.\nregister_bitfields! {\n    u32,\n\n    /// Flag Register.\n    FR [\n        /// Transmit FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n        /// Line Control Register, LCR_H.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the transmit holding register is empty.\n        /// - If the FIFO is enabled, the TXFE bit is set when the transmit FIFO is empty.\n        /// - This bit does not indicate if there is data in the transmit shift register.\n        TXFE OFFSET(7) NUMBITS(1) [],\n\n        /// Transmit FIFO full. The meaning of this bit depends on the state of the FEN bit in the\n        /// LCR_H Register.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the transmit holding register is full.\n        /// - If the FIFO is enabled, the TXFF bit is set when the transmit FIFO is full.\n        TXFF OFFSET(5) NUMBITS(1) [],\n\n        /// Receive FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n        /// LCR_H Register.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the receive holding register is empty.\n        /// - If the FIFO is enabled, the RXFE bit is set when the receive FIFO is empty.\n        RXFE OFFSET(4) NUMBITS(1) [],\n\n        /// UART busy. If this bit is set to 1, the UART is busy transmitting data. This bit remains\n        /// set until the complete byte, including all the stop bits, has been sent from the shift\n        /// register.\n        ///\n        /// This bit is set as soon as the transmit FIFO becomes non-empty, regardless of whether\n        /// the UART is enabled or not.\n        BUSY OFFSET(3) NUMBITS(1) []\n    ],\n\n    /// Integer Baud Rate Divisor.\n    IBRD [\n        /// The integer baud rate divisor.\n        BAUD_DIVINT OFFSET(0) NUMBITS(16) []\n    ],\n\n    /// Fractional Baud Rate Divisor.\n    FBRD [\n        ///  The fractional baud rate divisor.\n        BAUD_DIVFRAC OFFSET(0) NUMBITS(6) []\n    ],\n\n    /// Line Control Register.\n    LCR_H [\n        /// Word length. These bits indicate the number of data bits transmitted or received in a\n        /// frame.\n        #[allow(clippy::enum_variant_names)]\n        WLEN OFFSET(5) NUMBITS(2) [\n            FiveBit = 0b00,\n            SixBit = 0b01,\n            SevenBit = 0b10,\n            EightBit = 0b11\n        ],\n\n        /// Enable FIFOs:\n        ///\n        /// 0 = FIFOs are disabled (character mode) that is, the FIFOs become 1-byte-deep holding\n        /// registers.\n        ///\n        /// 1 = Transmit and receive FIFO buffers are enabled (FIFO mode).\n        FEN  OFFSET(4) NUMBITS(1) [\n            FifosDisabled = 0,\n            FifosEnabled = 1\n        ]\n    ],\n\n    /// Control Register.\n    CR [\n        /// Receive enable. If this bit is set to 1, the receive section of the UART is enabled.\n        /// Data reception occurs for either UART signals or SIR signals depending on the setting of\n        /// the SIREN bit. When the UART is disabled in the middle of reception, it completes the\n        /// current character before stopping.\n        RXE OFFSET(9) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// Transmit enable. If this bit is set to 1, the transmit section of the UART is enabled.\n        /// Data transmission occurs for either UART signals, or SIR signals depending on the\n        /// setting of the SIREN bit. When the UART is disabled in the middle of transmission, it\n        /// completes the current character before stopping.\n        TXE OFFSET(8) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// UART enable:\n        ///\n        /// 0 = UART is disabled. If the UART is disabled in the middle of transmission or\n        /// reception, it completes the current character before stopping.\n        ///\n        /// 1 = The UART is enabled. Data transmission and reception occurs for either UART signals\n        /// or SIR signals depending on the setting of the SIREN bit\n        UARTEN OFFSET(0) NUMBITS(1) [\n            /// If the UART is disabled in the middle of transmission or reception, it completes the\n            /// current character before stopping.\n            Disabled = 0,\n            Enabled = 1\n        ]\n    ],\n\n    /// Interrupt FIFO Level Select Register.\n    IFLS [\n        /// Receive interrupt FIFO level select. The trigger points for the receive interrupt are as\n        /// follows.\n        RXIFLSEL OFFSET(3) NUMBITS(5) [\n            OneEigth = 0b000,\n            OneQuarter = 0b001,\n            OneHalf = 0b010,\n            ThreeQuarters = 0b011,\n            SevenEights = 0b100\n        ]\n    ],\n\n    /// Interrupt Mask Set/Clear Register.\n    IMSC [\n        /// Receive timeout interrupt mask. A read returns the current mask for the UARTRTINTR\n        /// interrupt.\n        ///\n        /// - On a write of 1, the mask of the UARTRTINTR interrupt is set.\n        /// - A write of 0 clears the mask.\n        RTIM OFFSET(6) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// Receive interrupt mask. A read returns the current mask for the UARTRXINTR interrupt.\n        ///\n        /// - On a write of 1, the mask of the UARTRXINTR interrupt is set.\n        /// - A write of 0 clears the mask.\n        RXIM OFFSET(4) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ]\n    ],\n\n    /// Masked Interrupt Status Register.\n    MIS [\n        /// Receive timeout masked interrupt status. Returns the masked interrupt state of the\n        /// UARTRTINTR interrupt.\n        RTMIS OFFSET(6) NUMBITS(1) [],\n\n        /// Receive masked interrupt status. Returns the masked interrupt state of the UARTRXINTR\n        /// interrupt.\n        RXMIS OFFSET(4) NUMBITS(1) []\n    ],\n\n    /// Interrupt Clear Register.\n    ICR [\n        /// Meta field for all pending interrupts.\n        ALL OFFSET(0) NUMBITS(11) []\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    pub RegisterBlock {\n        (0x00 => DR: ReadWrite<u32>),\n        (0x04 => _reserved1),\n        (0x18 => FR: ReadOnly<u32, FR::Register>),\n        (0x1c => _reserved2),\n        (0x24 => IBRD: WriteOnly<u32, IBRD::Register>),\n        (0x28 => FBRD: WriteOnly<u32, FBRD::Register>),\n        (0x2c => LCR_H: WriteOnly<u32, LCR_H::Register>),\n        (0x30 => CR: WriteOnly<u32, CR::Register>),\n        (0x34 => IFLS: ReadWrite<u32, IFLS::Register>),\n        (0x38 => IMSC: ReadWrite<u32, IMSC::Register>),\n        (0x3C => _reserved3),\n        (0x40 => MIS: ReadOnly<u32, MIS::Register>),\n        (0x44 => ICR: WriteOnly<u32, ICR::Register>),\n        (0x48 => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\n#[derive(PartialEq)]\nenum BlockingMode {\n    Blocking,\n    NonBlocking,\n}\n\nstruct PL011UartInner {\n    registers: Registers,\n    chars_written: usize,\n    chars_read: usize,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the UART.\npub struct PL011Uart {\n    inner: IRQSafeNullLock<PL011UartInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PL011UartInner {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n            chars_written: 0,\n            chars_read: 0,\n        }\n    }\n\n    /// Set up baud rate and characteristics.\n    ///\n    /// This results in 8N1 and 921_600 baud.\n    ///\n    /// The calculation for the BRD is (we set the clock to 48 MHz in config.txt):\n    /// `(48_000_000 / 16) / 921_600 = 3.2552083`.\n    ///\n    /// This means the integer part is `3` and goes into the `IBRD`.\n    /// The fractional part is `0.2552083`.\n    ///\n    /// `FBRD` calculation according to the PL011 Technical Reference Manual:\n    /// `INTEGER((0.2552083 * 64) + 0.5) = 16`.\n    ///\n    /// Therefore, the generated baud rate divider is: `3 + 16/64 = 3.25`. Which results in a\n    /// genrated baud rate of `48_000_000 / (16 * 3.25) = 923_077`.\n    ///\n    /// Error = `((923_077 - 921_600) / 921_600) * 100 = 0.16%`.\n    pub fn init(&mut self) {\n        // Execution can arrive here while there are still characters queued in the TX FIFO and\n        // actively being sent out by the UART hardware. If the UART is turned off in this case,\n        // those queued characters would be lost.\n        //\n        // For example, this can happen during runtime on a call to panic!(), because panic!()\n        // initializes its own UART instance and calls init().\n        //\n        // Hence, flush first to ensure all pending characters are transmitted.\n        self.flush();\n\n        // Turn the UART off temporarily.\n        self.registers.CR.set(0);\n\n        // Clear all pending interrupts.\n        self.registers.ICR.write(ICR::ALL::CLEAR);\n\n        // From the PL011 Technical Reference Manual:\n        //\n        // The LCR_H, IBRD, and FBRD registers form the single 30-bit wide LCR Register that is\n        // updated on a single write strobe generated by a LCR_H write. So, to internally update the\n        // contents of IBRD or FBRD, a LCR_H write must always be performed at the end.\n        //\n        // Set the baud rate, 8N1 and FIFO enabled.\n        self.registers.IBRD.write(IBRD::BAUD_DIVINT.val(3));\n        self.registers.FBRD.write(FBRD::BAUD_DIVFRAC.val(16));\n        self.registers\n            .LCR_H\n            .write(LCR_H::WLEN::EightBit + LCR_H::FEN::FifosEnabled);\n\n        // Set RX FIFO fill level at 1/8.\n        self.registers.IFLS.write(IFLS::RXIFLSEL::OneEigth);\n\n        // Enable RX IRQ + RX timeout IRQ.\n        self.registers\n            .IMSC\n            .write(IMSC::RXIM::Enabled + IMSC::RTIM::Enabled);\n\n        // Turn the UART on.\n        self.registers\n            .CR\n            .write(CR::UARTEN::Enabled + CR::TXE::Enabled + CR::RXE::Enabled);\n    }\n\n    /// Send a character.\n    fn write_char(&mut self, c: char) {\n        // Spin while TX FIFO full is set, waiting for an empty slot.\n        while self.registers.FR.matches_all(FR::TXFF::SET) {\n            cpu::nop();\n        }\n\n        // Write the character to the buffer.\n        self.registers.DR.set(c as u32);\n\n        self.chars_written += 1;\n    }\n\n    /// Block execution until the last buffered character has been physically put on the TX wire.\n    fn flush(&self) {\n        // Spin until the busy bit is cleared.\n        while self.registers.FR.matches_all(FR::BUSY::SET) {\n            cpu::nop();\n        }\n    }\n\n    /// Retrieve a character.\n    fn read_char_converting(&mut self, blocking_mode: BlockingMode) -> Option<char> {\n        // If RX FIFO is empty,\n        if self.registers.FR.matches_all(FR::RXFE::SET) {\n            // immediately return in non-blocking mode.\n            if blocking_mode == BlockingMode::NonBlocking {\n                return None;\n            }\n\n            // Otherwise, wait until a char was received.\n            while self.registers.FR.matches_all(FR::RXFE::SET) {\n                cpu::nop();\n            }\n        }\n\n        // Read one character.\n        let mut ret = self.registers.DR.get() as u8 as char;\n\n        // Convert carrige return to newline.\n        if ret == '\\r' {\n            ret = '\\n'\n        }\n\n        // Update statistics.\n        self.chars_read += 1;\n\n        Some(ret)\n    }\n}\n\n/// Implementing `core::fmt::Write` enables usage of the `format_args!` macros, which in turn are\n/// used to implement the `kernel`'s `print!` and `println!` macros. By implementing `write_str()`,\n/// we get `write_fmt()` automatically.\n///\n/// The function takes an `&mut self`, so it must be implemented for the inner struct.\n///\n/// See [`src/print.rs`].\n///\n/// [`src/print.rs`]: ../../print/index.html\nimpl fmt::Write for PL011UartInner {\n    fn write_str(&mut self, s: &str) -> fmt::Result {\n        for c in s.chars() {\n            self.write_char(c);\n        }\n\n        Ok(())\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PL011Uart {\n    pub const COMPATIBLE: &'static str = \"BCM PL011 UART\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            inner: IRQSafeNullLock::new(PL011UartInner::new(mmio_start_addr)),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl driver::interface::DeviceDriver for PL011Uart {\n    type IRQNumberType = IRQNumber;\n\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n\n    unsafe fn init(&self) -> Result<(), &'static str> {\n        self.inner.lock(|inner| inner.init());\n\n        Ok(())\n    }\n\n    fn register_and_enable_irq_handler(\n        &'static self,\n        irq_number: &Self::IRQNumberType,\n    ) -> Result<(), &'static str> {\n        use exception::asynchronous::{irq_manager, IRQHandlerDescriptor};\n\n        let descriptor = IRQHandlerDescriptor::new(*irq_number, Self::COMPATIBLE, self);\n\n        irq_manager().register_handler(descriptor)?;\n        irq_manager().enable(irq_number);\n\n        Ok(())\n    }\n}\n\nimpl console::interface::Write for PL011Uart {\n    /// Passthrough of `args` to the `core::fmt::Write` implementation, but guarded by a Mutex to\n    /// serialize access.\n    fn write_char(&self, c: char) {\n        self.inner.lock(|inner| inner.write_char(c));\n    }\n\n    fn write_fmt(&self, args: core::fmt::Arguments) -> fmt::Result {\n        // Fully qualified syntax for the call to `core::fmt::Write::write_fmt()` to increase\n        // readability.\n        self.inner.lock(|inner| fmt::Write::write_fmt(inner, args))\n    }\n\n    fn flush(&self) {\n        // Spin until TX FIFO empty is set.\n        self.inner.lock(|inner| inner.flush());\n    }\n}\n\nimpl console::interface::Read for PL011Uart {\n    fn read_char(&self) -> char {\n        self.inner\n            .lock(|inner| inner.read_char_converting(BlockingMode::Blocking).unwrap())\n    }\n\n    fn clear_rx(&self) {\n        // Read from the RX FIFO until it is indicating empty.\n        while self\n            .inner\n            .lock(|inner| inner.read_char_converting(BlockingMode::NonBlocking))\n            .is_some()\n        {}\n    }\n}\n\nimpl console::interface::Statistics for PL011Uart {\n    fn chars_written(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_written)\n    }\n\n    fn chars_read(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_read)\n    }\n}\n\nimpl console::interface::All for PL011Uart {}\n\nimpl exception::asynchronous::interface::IRQHandler for PL011Uart {\n    fn handle(&self) -> Result<(), &'static str> {\n        self.inner.lock(|inner| {\n            let pending = inner.registers.MIS.extract();\n\n            // Clear all pending IRQs.\n            inner.registers.ICR.write(ICR::ALL::CLEAR);\n\n            // Check for any kind of RX interrupt.\n            if pending.matches_any(MIS::RXMIS::SET + MIS::RTMIS::SET) {\n                // Echo any received characters.\n                while let Some(c) = inner.read_char_converting(BlockingMode::NonBlocking) {\n                    inner.write_char(c)\n                }\n            }\n        });\n\n        Ok(())\n    }\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/bsp/device_driver/bcm.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BCM driver top level.\n\nmod bcm2xxx_gpio;\n#[cfg(feature = \"bsp_rpi3\")]\nmod bcm2xxx_interrupt_controller;\nmod bcm2xxx_pl011_uart;\n\npub use bcm2xxx_gpio::*;\n#[cfg(feature = \"bsp_rpi3\")]\npub use bcm2xxx_interrupt_controller::*;\npub use bcm2xxx_pl011_uart::*;\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/bsp/device_driver/common.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Common device driver code.\n\nuse crate::memory::{Address, Virtual};\nuse core::{fmt, marker::PhantomData, ops};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct MMIODerefWrapper<T> {\n    start_addr: Address<Virtual>,\n    phantom: PhantomData<fn() -> T>,\n}\n\n/// A wrapper type for usize with integrated range bound check.\n#[derive(Copy, Clone)]\npub struct BoundedUsize<const MAX_INCLUSIVE: usize>(usize);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T> MMIODerefWrapper<T> {\n    /// Create an instance.\n    pub const unsafe fn new(start_addr: Address<Virtual>) -> Self {\n        Self {\n            start_addr,\n            phantom: PhantomData,\n        }\n    }\n}\n\nimpl<T> ops::Deref for MMIODerefWrapper<T> {\n    type Target = T;\n\n    fn deref(&self) -> &Self::Target {\n        unsafe { &*(self.start_addr.as_usize() as *const _) }\n    }\n}\n\nimpl<const MAX_INCLUSIVE: usize> BoundedUsize<{ MAX_INCLUSIVE }> {\n    pub const MAX_INCLUSIVE: usize = MAX_INCLUSIVE;\n\n    /// Creates a new instance if number <= MAX_INCLUSIVE.\n    pub const fn new(number: usize) -> Self {\n        assert!(number <= MAX_INCLUSIVE);\n\n        Self(number)\n    }\n\n    /// Return the wrapped number.\n    pub const fn get(self) -> usize {\n        self.0\n    }\n}\n\nimpl<const MAX_INCLUSIVE: usize> fmt::Display for BoundedUsize<{ MAX_INCLUSIVE }> {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        write!(f, \"{}\", self.0)\n    }\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/bsp/device_driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Device driver.\n\n#[cfg(feature = \"bsp_rpi4\")]\nmod arm;\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod bcm;\nmod common;\n\n#[cfg(feature = \"bsp_rpi4\")]\npub use arm::*;\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use bcm::*;\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/bsp/raspberrypi/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Processor code.\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Used by `arch` code to find the early boot core.\n#[no_mangle]\n#[link_section = \".text._start_arguments\"]\npub static BOOT_CORE_ID: u64 = 0;\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/bsp/raspberrypi/driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP driver support.\n\nuse super::{exception, memory::map::mmio};\nuse crate::{\n    bsp::device_driver,\n    console, driver as generic_driver,\n    exception::{self as generic_exception},\n    memory,\n    memory::mmu::MMIODescriptor,\n};\nuse core::{\n    mem::MaybeUninit,\n    sync::atomic::{AtomicBool, Ordering},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic mut PL011_UART: MaybeUninit<device_driver::PL011Uart> = MaybeUninit::uninit();\nstatic mut GPIO: MaybeUninit<device_driver::GPIO> = MaybeUninit::uninit();\n\n#[cfg(feature = \"bsp_rpi3\")]\nstatic mut INTERRUPT_CONTROLLER: MaybeUninit<device_driver::InterruptController> =\n    MaybeUninit::uninit();\n\n#[cfg(feature = \"bsp_rpi4\")]\nstatic mut INTERRUPT_CONTROLLER: MaybeUninit<device_driver::GICv2> = MaybeUninit::uninit();\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// This must be called only after successful init of the memory subsystem.\nunsafe fn instantiate_uart() -> Result<(), &'static str> {\n    let mmio_descriptor = MMIODescriptor::new(mmio::PL011_UART_START, mmio::PL011_UART_SIZE);\n    let virt_addr =\n        memory::mmu::kernel_map_mmio(device_driver::PL011Uart::COMPATIBLE, &mmio_descriptor)?;\n\n    PL011_UART.write(device_driver::PL011Uart::new(virt_addr));\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the UART driver.\nunsafe fn post_init_uart() -> Result<(), &'static str> {\n    console::register_console(PL011_UART.assume_init_ref());\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the memory subsystem.\nunsafe fn instantiate_gpio() -> Result<(), &'static str> {\n    let mmio_descriptor = MMIODescriptor::new(mmio::GPIO_START, mmio::GPIO_SIZE);\n    let virt_addr =\n        memory::mmu::kernel_map_mmio(device_driver::GPIO::COMPATIBLE, &mmio_descriptor)?;\n\n    GPIO.write(device_driver::GPIO::new(virt_addr));\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the GPIO driver.\nunsafe fn post_init_gpio() -> Result<(), &'static str> {\n    GPIO.assume_init_ref().map_pl011_uart();\n    Ok(())\n}\n\n/// This must be called only after successful init of the memory subsystem.\n#[cfg(feature = \"bsp_rpi3\")]\nunsafe fn instantiate_interrupt_controller() -> Result<(), &'static str> {\n    let periph_mmio_descriptor =\n        MMIODescriptor::new(mmio::PERIPHERAL_IC_START, mmio::PERIPHERAL_IC_SIZE);\n    let periph_virt_addr = memory::mmu::kernel_map_mmio(\n        device_driver::InterruptController::COMPATIBLE,\n        &periph_mmio_descriptor,\n    )?;\n\n    INTERRUPT_CONTROLLER.write(device_driver::InterruptController::new(periph_virt_addr));\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the memory subsystem.\n#[cfg(feature = \"bsp_rpi4\")]\nunsafe fn instantiate_interrupt_controller() -> Result<(), &'static str> {\n    let gicd_mmio_descriptor = MMIODescriptor::new(mmio::GICD_START, mmio::GICD_SIZE);\n    let gicd_virt_addr = memory::mmu::kernel_map_mmio(\"GICv2 GICD\", &gicd_mmio_descriptor)?;\n\n    let gicc_mmio_descriptor = MMIODescriptor::new(mmio::GICC_START, mmio::GICC_SIZE);\n    let gicc_virt_addr = memory::mmu::kernel_map_mmio(\"GICV2 GICC\", &gicc_mmio_descriptor)?;\n\n    INTERRUPT_CONTROLLER.write(device_driver::GICv2::new(gicd_virt_addr, gicc_virt_addr));\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the interrupt controller driver.\nunsafe fn post_init_interrupt_controller() -> Result<(), &'static str> {\n    generic_exception::asynchronous::register_irq_manager(INTERRUPT_CONTROLLER.assume_init_ref());\n\n    Ok(())\n}\n\n/// Function needs to ensure that driver registration happens only after correct instantiation.\nunsafe fn driver_uart() -> Result<(), &'static str> {\n    instantiate_uart()?;\n\n    let uart_descriptor = generic_driver::DeviceDriverDescriptor::new(\n        PL011_UART.assume_init_ref(),\n        Some(post_init_uart),\n        Some(exception::asynchronous::irq_map::PL011_UART),\n    );\n    generic_driver::driver_manager().register_driver(uart_descriptor);\n\n    Ok(())\n}\n\n/// Function needs to ensure that driver registration happens only after correct instantiation.\nunsafe fn driver_gpio() -> Result<(), &'static str> {\n    instantiate_gpio()?;\n\n    let gpio_descriptor = generic_driver::DeviceDriverDescriptor::new(\n        GPIO.assume_init_ref(),\n        Some(post_init_gpio),\n        None,\n    );\n    generic_driver::driver_manager().register_driver(gpio_descriptor);\n\n    Ok(())\n}\n\n/// Function needs to ensure that driver registration happens only after correct instantiation.\nunsafe fn driver_interrupt_controller() -> Result<(), &'static str> {\n    instantiate_interrupt_controller()?;\n\n    let interrupt_controller_descriptor = generic_driver::DeviceDriverDescriptor::new(\n        INTERRUPT_CONTROLLER.assume_init_ref(),\n        Some(post_init_interrupt_controller),\n        None,\n    );\n    generic_driver::driver_manager().register_driver(interrupt_controller_descriptor);\n\n    Ok(())\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Initialize the driver subsystem.\n///\n/// # Safety\n///\n/// See child function calls.\npub unsafe fn init() -> Result<(), &'static str> {\n    static INIT_DONE: AtomicBool = AtomicBool::new(false);\n    if INIT_DONE.load(Ordering::Relaxed) {\n        return Err(\"Init already done\");\n    }\n\n    driver_uart()?;\n    driver_gpio()?;\n    driver_interrupt_controller()?;\n\n    INIT_DONE.store(true, Ordering::Relaxed);\n    Ok(())\n}\n\n/// Minimal code needed to bring up the console in QEMU (for testing only). This is often less steps\n/// than on real hardware due to QEMU's abstractions.\n#[cfg(feature = \"test_build\")]\npub fn qemu_bring_up_console() {\n    use crate::cpu;\n\n    unsafe {\n        instantiate_uart().unwrap_or_else(|_| cpu::qemu_exit_failure());\n        console::register_console(PL011_UART.assume_init_ref());\n    };\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/bsp/raspberrypi/exception/asynchronous.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP asynchronous exception handling.\n\nuse crate::bsp;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Export for reuse in generic asynchronous.rs.\npub use bsp::device_driver::IRQNumber;\n\n#[cfg(feature = \"bsp_rpi3\")]\npub(in crate::bsp) mod irq_map {\n    use super::bsp::device_driver::{IRQNumber, PeripheralIRQ};\n\n    pub const PL011_UART: IRQNumber = IRQNumber::Peripheral(PeripheralIRQ::new(57));\n}\n\n#[cfg(feature = \"bsp_rpi4\")]\npub(in crate::bsp) mod irq_map {\n    use super::bsp::device_driver::IRQNumber;\n\n    pub const PL011_UART: IRQNumber = IRQNumber::new(153);\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/bsp/raspberrypi/exception.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP synchronous and asynchronous exception handling.\n\npub mod asynchronous;\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/bsp/raspberrypi/kernel.ld",
    "content": "/* SPDX-License-Identifier: MIT OR Apache-2.0\n *\n * Copyright (c) 2018-2022 Andre Richter <andre.o.richter@gmail.com>\n */\n\nPAGE_SIZE = 64K;\nPAGE_MASK = PAGE_SIZE - 1;\n\n__rpi_phys_dram_start_addr = 0;\n\n/* The physical address at which the the kernel binary will be loaded by the Raspberry's firmware */\n__rpi_phys_binary_load_addr = 0x80000;\n\n\nENTRY(__rpi_phys_binary_load_addr)\n\n/* Flags:\n *     4 == R\n *     5 == RX\n *     6 == RW\n *\n * Segments are marked PT_LOAD below so that the ELF file provides virtual and physical addresses.\n * It doesn't mean all of them need actually be loaded.\n */\nPHDRS\n{\n    segment_boot_core_stack PT_LOAD FLAGS(6);\n    segment_code            PT_LOAD FLAGS(5);\n    segment_data            PT_LOAD FLAGS(6);\n}\n\nSECTIONS\n{\n    . =  __rpi_phys_dram_start_addr;\n\n    /***********************************************************************************************\n    * Boot Core Stack\n    ***********************************************************************************************/\n    .boot_core_stack (NOLOAD) :\n    {\n        __boot_core_stack_start = .;         /*   ^             */\n                                             /*   | stack       */\n        . += __rpi_phys_binary_load_addr;    /*   | growth      */\n                                             /*   | direction   */\n        __boot_core_stack_end_exclusive = .; /*   |             */\n    } :segment_boot_core_stack\n\n    ASSERT((. & PAGE_MASK) == 0, \"End of boot core stack is not page aligned\")\n\n    /***********************************************************************************************\n    * Code + RO Data + Global Offset Table\n    ***********************************************************************************************/\n    __code_start = .;\n    .text :\n    {\n        KEEP(*(.text._start))\n        *(.text._start_arguments) /* Constants (or statics in Rust speak) read by _start(). */\n        *(.text._start_rust)      /* The Rust entry point */\n        *(.text*)                 /* Everything else */\n    } :segment_code\n\n    .rodata : ALIGN(8) { *(.rodata*) } :segment_code\n\n    . = ALIGN(PAGE_SIZE);\n    __code_end_exclusive = .;\n\n    /***********************************************************************************************\n    * Data + BSS\n    ***********************************************************************************************/\n    __data_start = .;\n    .data : { *(.data*) } :segment_data\n\n    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */\n    .bss (NOLOAD) : ALIGN(16)\n    {\n        __bss_start = .;\n        *(.bss*);\n        . = ALIGN(16);\n        __bss_end_exclusive = .;\n    } :segment_data\n\n    . = ALIGN(PAGE_SIZE);\n    __data_end_exclusive = .;\n\n    /***********************************************************************************************\n    * MMIO Remap Reserved\n    ***********************************************************************************************/\n    __mmio_remap_start = .;\n    . += 8 * 1024 * 1024;\n    __mmio_remap_end_exclusive = .;\n\n    ASSERT((. & PAGE_MASK) == 0, \"MMIO remap reservation is not page aligned\")\n\n    /***********************************************************************************************\n    * Misc\n    ***********************************************************************************************/\n    .got : { *(.got*) }\n    ASSERT(SIZEOF(.got) == 0, \"Relocation support not expected\")\n\n    /DISCARD/ : { *(.comment*) }\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/bsp/raspberrypi/memory/mmu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Memory Management Unit.\n\nuse crate::{\n    memory::{\n        mmu::{\n            self as generic_mmu, AccessPermissions, AddressSpace, AssociatedTranslationTable,\n            AttributeFields, MemAttributes, MemoryRegion, PageAddress, TranslationGranule,\n        },\n        Physical, Virtual,\n    },\n    synchronization::InitStateLock,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\ntype KernelTranslationTable =\n    <KernelVirtAddrSpace as AssociatedTranslationTable>::TableStartFromBottom;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// The translation granule chosen by this BSP. This will be used everywhere else in the kernel to\n/// derive respective data structures and their sizes. For example, the `crate::memory::mmu::Page`.\npub type KernelGranule = TranslationGranule<{ 64 * 1024 }>;\n\n/// The kernel's virtual address space defined by this BSP.\npub type KernelVirtAddrSpace = AddressSpace<{ 1024 * 1024 * 1024 }>;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n/// The kernel translation tables.\n///\n/// It is mandatory that InitStateLock is transparent.\n///\n/// That is, `size_of(InitStateLock<KernelTranslationTable>) == size_of(KernelTranslationTable)`.\n/// There is a unit tests that checks this porperty.\nstatic KERNEL_TABLES: InitStateLock<KernelTranslationTable> =\n    InitStateLock::new(KernelTranslationTable::new());\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Helper function for calculating the number of pages the given parameter spans.\nconst fn size_to_num_pages(size: usize) -> usize {\n    assert!(size > 0);\n    assert!(size % KernelGranule::SIZE == 0);\n\n    size >> KernelGranule::SHIFT\n}\n\n/// The code pages of the kernel binary.\nfn virt_code_region() -> MemoryRegion<Virtual> {\n    let num_pages = size_to_num_pages(super::code_size());\n\n    let start_page_addr = super::virt_code_start();\n    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n\n    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n}\n\n/// The data pages of the kernel binary.\nfn virt_data_region() -> MemoryRegion<Virtual> {\n    let num_pages = size_to_num_pages(super::data_size());\n\n    let start_page_addr = super::virt_data_start();\n    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n\n    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n}\n\n/// The boot core stack pages.\nfn virt_boot_core_stack_region() -> MemoryRegion<Virtual> {\n    let num_pages = size_to_num_pages(super::boot_core_stack_size());\n\n    let start_page_addr = super::virt_boot_core_stack_start();\n    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n\n    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n}\n\n// The binary is still identity mapped, so use this trivial conversion function for mapping below.\n\nfn kernel_virt_to_phys_region(virt_region: MemoryRegion<Virtual>) -> MemoryRegion<Physical> {\n    MemoryRegion::new(\n        PageAddress::from(virt_region.start_page_addr().into_inner().as_usize()),\n        PageAddress::from(\n            virt_region\n                .end_exclusive_page_addr()\n                .into_inner()\n                .as_usize(),\n        ),\n    )\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the kernel's translation tables.\npub fn kernel_translation_tables() -> &'static InitStateLock<KernelTranslationTable> {\n    &KERNEL_TABLES\n}\n\n/// The MMIO remap pages.\npub fn virt_mmio_remap_region() -> MemoryRegion<Virtual> {\n    let num_pages = size_to_num_pages(super::mmio_remap_size());\n\n    let start_page_addr = super::virt_mmio_remap_start();\n    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n\n    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n}\n\n/// Map the kernel binary.\n///\n/// # Safety\n///\n/// - Any miscalculation or attribute error will likely be fatal. Needs careful manual checking.\npub unsafe fn kernel_map_binary() -> Result<(), &'static str> {\n    generic_mmu::kernel_map_at(\n        \"Kernel boot-core stack\",\n        &virt_boot_core_stack_region(),\n        &kernel_virt_to_phys_region(virt_boot_core_stack_region()),\n        &AttributeFields {\n            mem_attributes: MemAttributes::CacheableDRAM,\n            acc_perms: AccessPermissions::ReadWrite,\n            execute_never: true,\n        },\n    )?;\n\n    generic_mmu::kernel_map_at(\n        \"Kernel code and RO data\",\n        &virt_code_region(),\n        &kernel_virt_to_phys_region(virt_code_region()),\n        &AttributeFields {\n            mem_attributes: MemAttributes::CacheableDRAM,\n            acc_perms: AccessPermissions::ReadOnly,\n            execute_never: false,\n        },\n    )?;\n\n    generic_mmu::kernel_map_at(\n        \"Kernel data and bss\",\n        &virt_data_region(),\n        &kernel_virt_to_phys_region(virt_data_region()),\n        &AttributeFields {\n            mem_attributes: MemAttributes::CacheableDRAM,\n            acc_perms: AccessPermissions::ReadWrite,\n            execute_never: true,\n        },\n    )?;\n\n    Ok(())\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use core::{cell::UnsafeCell, ops::Range};\n    use test_macros::kernel_test;\n\n    /// Check alignment of the kernel's virtual memory layout sections.\n    #[kernel_test]\n    fn virt_mem_layout_sections_are_64KiB_aligned() {\n        for i in [\n            virt_boot_core_stack_region,\n            virt_code_region,\n            virt_data_region,\n        ]\n        .iter()\n        {\n            let start = i().start_page_addr().into_inner();\n            let end_exclusive = i().end_exclusive_page_addr().into_inner();\n\n            assert!(start.is_page_aligned());\n            assert!(end_exclusive.is_page_aligned());\n            assert!(end_exclusive >= start);\n        }\n    }\n\n    /// Ensure the kernel's virtual memory layout is free of overlaps.\n    #[kernel_test]\n    fn virt_mem_layout_has_no_overlaps() {\n        let layout = [\n            virt_boot_core_stack_region(),\n            virt_code_region(),\n            virt_data_region(),\n        ];\n\n        for (i, first_range) in layout.iter().enumerate() {\n            for second_range in layout.iter().skip(i + 1) {\n                assert!(!first_range.overlaps(second_range))\n            }\n        }\n    }\n\n    /// Check if KERNEL_TABLES is in .bss.\n    #[kernel_test]\n    fn kernel_tables_in_bss() {\n        extern \"Rust\" {\n            static __bss_start: UnsafeCell<u64>;\n            static __bss_end_exclusive: UnsafeCell<u64>;\n        }\n\n        let bss_range = unsafe {\n            Range {\n                start: __bss_start.get(),\n                end: __bss_end_exclusive.get(),\n            }\n        };\n        let kernel_tables_addr = &KERNEL_TABLES as *const _ as usize as *mut u64;\n\n        assert!(bss_range.contains(&kernel_tables_addr));\n    }\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/bsp/raspberrypi/memory.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Memory Management.\n//!\n//! The physical memory layout.\n//!\n//! The Raspberry's firmware copies the kernel binary to 0x8_0000. The preceding region will be used\n//! as the boot core's stack.\n//!\n//! +---------------------------------------+\n//! |                                       | boot_core_stack_start @ 0x0\n//! |                                       |                                ^\n//! | Boot-core Stack                       |                                | stack\n//! |                                       |                                | growth\n//! |                                       |                                | direction\n//! +---------------------------------------+\n//! |                                       | code_start @ 0x8_0000 == boot_core_stack_end_exclusive\n//! | .text                                 |\n//! | .rodata                               |\n//! | .got                                  |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       | data_start == code_end_exclusive\n//! | .data                                 |\n//! | .bss                                  |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       | data_end_exclusive\n//! |                                       |\n//!\n//!\n//!\n//!\n//!\n//! The virtual memory layout is as follows:\n//!\n//! +---------------------------------------+\n//! |                                       | boot_core_stack_start @ 0x0\n//! |                                       |                                ^\n//! | Boot-core Stack                       |                                | stack\n//! |                                       |                                | growth\n//! |                                       |                                | direction\n//! +---------------------------------------+\n//! |                                       | code_start @ 0x8_0000 == boot_core_stack_end_exclusive\n//! | .text                                 |\n//! | .rodata                               |\n//! | .got                                  |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       | data_start == code_end_exclusive\n//! | .data                                 |\n//! | .bss                                  |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       |  mmio_remap_start == data_end_exclusive\n//! | VA region for MMIO remapping          |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       |  mmio_remap_end_exclusive\n//! |                                       |\npub mod mmu;\n\nuse crate::memory::{mmu::PageAddress, Address, Physical, Virtual};\nuse core::cell::UnsafeCell;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Symbols from the linker script.\nextern \"Rust\" {\n    static __code_start: UnsafeCell<()>;\n    static __code_end_exclusive: UnsafeCell<()>;\n\n    static __data_start: UnsafeCell<()>;\n    static __data_end_exclusive: UnsafeCell<()>;\n\n    static __mmio_remap_start: UnsafeCell<()>;\n    static __mmio_remap_end_exclusive: UnsafeCell<()>;\n\n    static __boot_core_stack_start: UnsafeCell<()>;\n    static __boot_core_stack_end_exclusive: UnsafeCell<()>;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// The board's physical memory map.\n#[rustfmt::skip]\npub(super) mod map {\n    use super::*;\n\n    /// Physical devices.\n    #[cfg(feature = \"bsp_rpi3\")]\n    pub mod mmio {\n        use super::*;\n\n        pub const PERIPHERAL_IC_START: Address<Physical> = Address::new(0x3F00_B200);\n        pub const PERIPHERAL_IC_SIZE:  usize             =              0x24;\n\n        pub const GPIO_START:          Address<Physical> = Address::new(0x3F20_0000);\n        pub const GPIO_SIZE:           usize             =              0xA0;\n\n        pub const PL011_UART_START:    Address<Physical> = Address::new(0x3F20_1000);\n        pub const PL011_UART_SIZE:     usize             =              0x48;\n\n        pub const END:                 Address<Physical> = Address::new(0x4001_0000);\n    }\n\n    /// Physical devices.\n    #[cfg(feature = \"bsp_rpi4\")]\n    pub mod mmio {\n        use super::*;\n\n        pub const GPIO_START:       Address<Physical> = Address::new(0xFE20_0000);\n        pub const GPIO_SIZE:        usize             =              0xA0;\n\n        pub const PL011_UART_START: Address<Physical> = Address::new(0xFE20_1000);\n        pub const PL011_UART_SIZE:  usize             =              0x48;\n\n        pub const GICD_START:       Address<Physical> = Address::new(0xFF84_1000);\n        pub const GICD_SIZE:        usize             =              0x824;\n\n        pub const GICC_START:       Address<Physical> = Address::new(0xFF84_2000);\n        pub const GICC_SIZE:        usize             =              0x14;\n\n        pub const END:              Address<Physical> = Address::new(0xFF85_0000);\n    }\n\n    pub const END: Address<Physical> = mmio::END;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Start page address of the code segment.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn virt_code_start() -> PageAddress<Virtual> {\n    PageAddress::from(unsafe { __code_start.get() as usize })\n}\n\n/// Size of the code segment.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn code_size() -> usize {\n    unsafe { (__code_end_exclusive.get() as usize) - (__code_start.get() as usize) }\n}\n\n/// Start page address of the data segment.\n#[inline(always)]\nfn virt_data_start() -> PageAddress<Virtual> {\n    PageAddress::from(unsafe { __data_start.get() as usize })\n}\n\n/// Size of the data segment.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn data_size() -> usize {\n    unsafe { (__data_end_exclusive.get() as usize) - (__data_start.get() as usize) }\n}\n\n/// Start page address of the MMIO remap reservation.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn virt_mmio_remap_start() -> PageAddress<Virtual> {\n    PageAddress::from(unsafe { __mmio_remap_start.get() as usize })\n}\n\n/// Size of the MMIO remap reservation.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn mmio_remap_size() -> usize {\n    unsafe { (__mmio_remap_end_exclusive.get() as usize) - (__mmio_remap_start.get() as usize) }\n}\n\n/// Start page address of the boot core's stack.\n#[inline(always)]\nfn virt_boot_core_stack_start() -> PageAddress<Virtual> {\n    PageAddress::from(unsafe { __boot_core_stack_start.get() as usize })\n}\n\n/// Size of the boot core's stack.\n#[inline(always)]\nfn boot_core_stack_size() -> usize {\n    unsafe {\n        (__boot_core_stack_end_exclusive.get() as usize) - (__boot_core_stack_start.get() as usize)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Exclusive end address of the physical address space.\n#[inline(always)]\npub fn phys_addr_space_end_exclusive_addr() -> PageAddress<Physical> {\n    PageAddress::from(map::END)\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/bsp/raspberrypi.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Top-level BSP file for the Raspberry Pi 3 and 4.\n\npub mod cpu;\npub mod driver;\npub mod exception;\npub mod memory;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Board identification.\npub fn board_name() -> &'static str {\n    #[cfg(feature = \"bsp_rpi3\")]\n    {\n        \"Raspberry Pi 3\"\n    }\n\n    #[cfg(feature = \"bsp_rpi4\")]\n    {\n        \"Raspberry Pi 4\"\n    }\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/bsp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Conditional reexporting of Board Support Packages.\n\nmod device_driver;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod raspberrypi;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use raspberrypi::*;\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/common.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! General purpose code.\n\n/// Check if a value is aligned to a given size.\n#[inline(always)]\npub const fn is_aligned(value: usize, alignment: usize) -> bool {\n    assert!(alignment.is_power_of_two());\n\n    (value & (alignment - 1)) == 0\n}\n\n/// Align down.\n#[inline(always)]\npub const fn align_down(value: usize, alignment: usize) -> usize {\n    assert!(alignment.is_power_of_two());\n\n    value & !(alignment - 1)\n}\n\n/// Align up.\n#[inline(always)]\npub const fn align_up(value: usize, alignment: usize) -> usize {\n    assert!(alignment.is_power_of_two());\n\n    (value + alignment - 1) & !(alignment - 1)\n}\n\n/// Convert a size into human readable format.\npub const fn size_human_readable_ceil(size: usize) -> (usize, &'static str) {\n    const KIB: usize = 1024;\n    const MIB: usize = 1024 * 1024;\n    const GIB: usize = 1024 * 1024 * 1024;\n\n    if (size / GIB) > 0 {\n        (size.div_ceil(GIB), \"GiB\")\n    } else if (size / MIB) > 0 {\n        (size.div_ceil(MIB), \"MiB\")\n    } else if (size / KIB) > 0 {\n        (size.div_ceil(KIB), \"KiB\")\n    } else {\n        (size, \"Byte\")\n    }\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/console/null_console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Null console.\n\nuse super::interface;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct NullConsole;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\npub static NULL_CONSOLE: NullConsole = NullConsole {};\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl interface::Write for NullConsole {\n    fn write_char(&self, _c: char) {}\n\n    fn write_fmt(&self, _args: fmt::Arguments) -> fmt::Result {\n        fmt::Result::Ok(())\n    }\n\n    fn flush(&self) {}\n}\n\nimpl interface::Read for NullConsole {\n    fn clear_rx(&self) {}\n}\n\nimpl interface::Statistics for NullConsole {}\nimpl interface::All for NullConsole {}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! System console.\n\nmod null_console;\n\nuse crate::synchronization;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Console interfaces.\npub mod interface {\n    use core::fmt;\n\n    /// Console write functions.\n    pub trait Write {\n        /// Write a single character.\n        fn write_char(&self, c: char);\n\n        /// Write a Rust format string.\n        fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result;\n\n        /// Block until the last buffered character has been physically put on the TX wire.\n        fn flush(&self);\n    }\n\n    /// Console read functions.\n    pub trait Read {\n        /// Read a single character.\n        fn read_char(&self) -> char {\n            ' '\n        }\n\n        /// Clear RX buffers, if any.\n        fn clear_rx(&self);\n    }\n\n    /// Console statistics.\n    pub trait Statistics {\n        /// Return the number of characters written.\n        fn chars_written(&self) -> usize {\n            0\n        }\n\n        /// Return the number of characters read.\n        fn chars_read(&self) -> usize {\n            0\n        }\n    }\n\n    /// Trait alias for a full-fledged console.\n    pub trait All: Write + Read + Statistics {}\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic CUR_CONSOLE: InitStateLock<&'static (dyn interface::All + Sync)> =\n    InitStateLock::new(&null_console::NULL_CONSOLE);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::{interface::ReadWriteEx, InitStateLock};\n\n/// Register a new console.\npub fn register_console(new_console: &'static (dyn interface::All + Sync)) {\n    CUR_CONSOLE.write(|con| *con = new_console);\n}\n\n/// Return a reference to the currently registered console.\n///\n/// This is the global console used by all printing macros.\npub fn console() -> &'static dyn interface::All {\n    CUR_CONSOLE.read(|con| *con)\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Boot code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/cpu/boot.rs\"]\nmod arch_boot;\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/cpu/smp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Symmetric multiprocessing.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/cpu/smp.rs\"]\nmod arch_smp;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_smp::core_id;\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Processor code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/cpu.rs\"]\nmod arch_cpu;\n\nmod boot;\n\npub mod smp;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_cpu::{nop, wait_forever};\n\n#[cfg(feature = \"test_build\")]\npub use arch_cpu::{qemu_exit_failure, qemu_exit_success};\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Driver support.\n\nuse crate::{\n    exception, info,\n    synchronization::{interface::ReadWriteEx, InitStateLock},\n};\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nconst NUM_DRIVERS: usize = 5;\n\nstruct DriverManagerInner<T>\nwhere\n    T: 'static,\n{\n    next_index: usize,\n    descriptors: [Option<DeviceDriverDescriptor<T>>; NUM_DRIVERS],\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Driver interfaces.\npub mod interface {\n    /// Device Driver functions.\n    pub trait DeviceDriver {\n        /// Different interrupt controllers might use different types for IRQ number.\n        type IRQNumberType: super::fmt::Display;\n\n        /// Return a compatibility string for identifying the driver.\n        fn compatible(&self) -> &'static str;\n\n        /// Called by the kernel to bring up the device.\n        ///\n        /// # Safety\n        ///\n        /// - During init, drivers might do stuff with system-wide impact.\n        unsafe fn init(&self) -> Result<(), &'static str> {\n            Ok(())\n        }\n\n        /// Called by the kernel to register and enable the device's IRQ handler.\n        ///\n        /// Rust's type system will prevent a call to this function unless the calling instance\n        /// itself has static lifetime.\n        fn register_and_enable_irq_handler(\n            &'static self,\n            irq_number: &Self::IRQNumberType,\n        ) -> Result<(), &'static str> {\n            panic!(\n                \"Attempt to enable IRQ {} for device {}, but driver does not support this\",\n                irq_number,\n                self.compatible()\n            )\n        }\n    }\n}\n\n/// Tpye to be used as an optional callback after a driver's init() has run.\npub type DeviceDriverPostInitCallback = unsafe fn() -> Result<(), &'static str>;\n\n/// A descriptor for device drivers.\n#[derive(Copy, Clone)]\npub struct DeviceDriverDescriptor<T>\nwhere\n    T: 'static,\n{\n    device_driver: &'static (dyn interface::DeviceDriver<IRQNumberType = T> + Sync),\n    post_init_callback: Option<DeviceDriverPostInitCallback>,\n    irq_number: Option<T>,\n}\n\n/// Provides device driver management functions.\npub struct DriverManager<T>\nwhere\n    T: 'static,\n{\n    inner: InitStateLock<DriverManagerInner<T>>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic DRIVER_MANAGER: DriverManager<exception::asynchronous::IRQNumber> = DriverManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T> DriverManagerInner<T>\nwhere\n    T: 'static + Copy,\n{\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            next_index: 0,\n            descriptors: [None; NUM_DRIVERS],\n        }\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T> DeviceDriverDescriptor<T> {\n    /// Create an instance.\n    pub fn new(\n        device_driver: &'static (dyn interface::DeviceDriver<IRQNumberType = T> + Sync),\n        post_init_callback: Option<DeviceDriverPostInitCallback>,\n        irq_number: Option<T>,\n    ) -> Self {\n        Self {\n            device_driver,\n            post_init_callback,\n            irq_number,\n        }\n    }\n}\n\n/// Return a reference to the global DriverManager.\npub fn driver_manager() -> &'static DriverManager<exception::asynchronous::IRQNumber> {\n    &DRIVER_MANAGER\n}\n\nimpl<T> DriverManager<T>\nwhere\n    T: fmt::Display + Copy,\n{\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            inner: InitStateLock::new(DriverManagerInner::new()),\n        }\n    }\n\n    /// Register a device driver with the kernel.\n    pub fn register_driver(&self, descriptor: DeviceDriverDescriptor<T>) {\n        self.inner.write(|inner| {\n            inner.descriptors[inner.next_index] = Some(descriptor);\n            inner.next_index += 1;\n        })\n    }\n\n    /// Helper for iterating over registered drivers.\n    fn for_each_descriptor<'a>(&'a self, f: impl FnMut(&'a DeviceDriverDescriptor<T>)) {\n        self.inner.read(|inner| {\n            inner\n                .descriptors\n                .iter()\n                .filter_map(|x| x.as_ref())\n                .for_each(f)\n        })\n    }\n\n    /// Fully initialize all drivers and their interrupts handlers.\n    ///\n    /// # Safety\n    ///\n    /// - During init, drivers might do stuff with system-wide impact.\n    pub unsafe fn init_drivers_and_irqs(&self) {\n        self.for_each_descriptor(|descriptor| {\n            // 1. Initialize driver.\n            if let Err(x) = descriptor.device_driver.init() {\n                panic!(\n                    \"Error initializing driver: {}: {}\",\n                    descriptor.device_driver.compatible(),\n                    x\n                );\n            }\n\n            // 2. Call corresponding post init callback.\n            if let Some(callback) = &descriptor.post_init_callback {\n                if let Err(x) = callback() {\n                    panic!(\n                        \"Error during driver post-init callback: {}: {}\",\n                        descriptor.device_driver.compatible(),\n                        x\n                    );\n                }\n            }\n        });\n\n        // 3. After all post-init callbacks were done, the interrupt controller should be\n        //    registered and functional. So let drivers register with it now.\n        self.for_each_descriptor(|descriptor| {\n            if let Some(irq_number) = &descriptor.irq_number {\n                if let Err(x) = descriptor\n                    .device_driver\n                    .register_and_enable_irq_handler(irq_number)\n                {\n                    panic!(\n                        \"Error during driver interrupt handler registration: {}: {}\",\n                        descriptor.device_driver.compatible(),\n                        x\n                    );\n                }\n            }\n        });\n    }\n\n    /// Enumerate all registered device drivers.\n    pub fn enumerate(&self) {\n        let mut i: usize = 1;\n        self.for_each_descriptor(|descriptor| {\n            info!(\"      {}. {}\", i, descriptor.device_driver.compatible());\n\n            i += 1;\n        });\n    }\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/exception/asynchronous/null_irq_manager.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Null IRQ Manager.\n\nuse super::{interface, IRQContext, IRQHandlerDescriptor};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct NullIRQManager;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\npub static NULL_IRQ_MANAGER: NullIRQManager = NullIRQManager {};\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl interface::IRQManager for NullIRQManager {\n    type IRQNumberType = super::IRQNumber;\n\n    fn register_handler(\n        &self,\n        _descriptor: IRQHandlerDescriptor<Self::IRQNumberType>,\n    ) -> Result<(), &'static str> {\n        panic!(\"No IRQ Manager registered yet\");\n    }\n\n    fn enable(&self, _irq_number: &Self::IRQNumberType) {\n        panic!(\"No IRQ Manager registered yet\");\n    }\n\n    fn handle_pending_irqs<'irq_context>(&'irq_context self, _ic: &IRQContext<'irq_context>) {\n        panic!(\"No IRQ Manager registered yet\");\n    }\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/exception/asynchronous.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Asynchronous exception handling.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/exception/asynchronous.rs\"]\nmod arch_asynchronous;\nmod null_irq_manager;\n\nuse crate::{bsp, synchronization};\nuse core::marker::PhantomData;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_asynchronous::{\n    is_local_irq_masked, local_irq_mask, local_irq_mask_save, local_irq_restore, local_irq_unmask,\n    print_state,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Interrupt number as defined by the BSP.\npub type IRQNumber = bsp::exception::asynchronous::IRQNumber;\n\n/// Interrupt descriptor.\n#[derive(Copy, Clone)]\npub struct IRQHandlerDescriptor<T>\nwhere\n    T: Copy,\n{\n    /// The IRQ number.\n    number: T,\n\n    /// Descriptive name.\n    name: &'static str,\n\n    /// Reference to handler trait object.\n    handler: &'static (dyn interface::IRQHandler + Sync),\n}\n\n/// IRQContext token.\n///\n/// An instance of this type indicates that the local core is currently executing in IRQ\n/// context, aka executing an interrupt vector or subcalls of it.\n///\n/// Concept and implementation derived from the `CriticalSection` introduced in\n/// <https://github.com/rust-embedded/bare-metal>\n#[derive(Clone, Copy)]\npub struct IRQContext<'irq_context> {\n    _0: PhantomData<&'irq_context ()>,\n}\n\n/// Asynchronous exception handling interfaces.\npub mod interface {\n\n    /// Implemented by types that handle IRQs.\n    pub trait IRQHandler {\n        /// Called when the corresponding interrupt is asserted.\n        fn handle(&self) -> Result<(), &'static str>;\n    }\n\n    /// IRQ management functions.\n    ///\n    /// The `BSP` is supposed to supply one global instance. Typically implemented by the\n    /// platform's interrupt controller.\n    pub trait IRQManager {\n        /// The IRQ number type depends on the implementation.\n        type IRQNumberType: Copy;\n\n        /// Register a handler.\n        fn register_handler(\n            &self,\n            irq_handler_descriptor: super::IRQHandlerDescriptor<Self::IRQNumberType>,\n        ) -> Result<(), &'static str>;\n\n        /// Enable an interrupt in the controller.\n        fn enable(&self, irq_number: &Self::IRQNumberType);\n\n        /// Handle pending interrupts.\n        ///\n        /// This function is called directly from the CPU's IRQ exception vector. On AArch64,\n        /// this means that the respective CPU core has disabled exception handling.\n        /// This function can therefore not be preempted and runs start to finish.\n        ///\n        /// Takes an IRQContext token to ensure it can only be called from IRQ context.\n        #[allow(clippy::trivially_copy_pass_by_ref)]\n        fn handle_pending_irqs<'irq_context>(\n            &'irq_context self,\n            ic: &super::IRQContext<'irq_context>,\n        );\n\n        /// Print list of registered handlers.\n        fn print_handler(&self) {}\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic CUR_IRQ_MANAGER: InitStateLock<\n    &'static (dyn interface::IRQManager<IRQNumberType = IRQNumber> + Sync),\n> = InitStateLock::new(&null_irq_manager::NULL_IRQ_MANAGER);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::{interface::ReadWriteEx, InitStateLock};\n\nimpl<T> IRQHandlerDescriptor<T>\nwhere\n    T: Copy,\n{\n    /// Create an instance.\n    pub const fn new(\n        number: T,\n        name: &'static str,\n        handler: &'static (dyn interface::IRQHandler + Sync),\n    ) -> Self {\n        Self {\n            number,\n            name,\n            handler,\n        }\n    }\n\n    /// Return the number.\n    pub const fn number(&self) -> T {\n        self.number\n    }\n\n    /// Return the name.\n    pub const fn name(&self) -> &'static str {\n        self.name\n    }\n\n    /// Return the handler.\n    pub const fn handler(&self) -> &'static (dyn interface::IRQHandler + Sync) {\n        self.handler\n    }\n}\n\nimpl<'irq_context> IRQContext<'irq_context> {\n    /// Creates an IRQContext token.\n    ///\n    /// # Safety\n    ///\n    /// - This must only be called when the current core is in an interrupt context and will not\n    ///   live beyond the end of it. That is, creation is allowed in interrupt vector functions. For\n    ///   example, in the ARMv8-A case, in `extern \"C\" fn current_elx_irq()`.\n    /// - Note that the lifetime `'irq_context` of the returned instance is unconstrained. User code\n    ///   must not be able to influence the lifetime picked for this type, since that might cause it\n    ///   to be inferred to `'static`.\n    #[inline(always)]\n    pub unsafe fn new() -> Self {\n        IRQContext { _0: PhantomData }\n    }\n}\n\n/// Executes the provided closure while IRQs are masked on the executing core.\n///\n/// While the function temporarily changes the HW state of the executing core, it restores it to the\n/// previous state before returning, so this is deemed safe.\n#[inline(always)]\npub fn exec_with_irq_masked<T>(f: impl FnOnce() -> T) -> T {\n    let saved = local_irq_mask_save();\n    let ret = f();\n    local_irq_restore(saved);\n\n    ret\n}\n\n/// Register a new IRQ manager.\npub fn register_irq_manager(\n    new_manager: &'static (dyn interface::IRQManager<IRQNumberType = IRQNumber> + Sync),\n) {\n    CUR_IRQ_MANAGER.write(|manager| *manager = new_manager);\n}\n\n/// Return a reference to the currently registered IRQ manager.\n///\n/// This is the IRQ manager used by the architectural interrupt handling code.\npub fn irq_manager() -> &'static dyn interface::IRQManager<IRQNumberType = IRQNumber> {\n    CUR_IRQ_MANAGER.read(|manager| *manager)\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/exception.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Synchronous and asynchronous exception handling.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/exception.rs\"]\nmod arch_exception;\n\npub mod asynchronous;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_exception::{current_privilege_level, handling_init};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Kernel privilege levels.\n#[allow(missing_docs)]\n#[derive(Eq, PartialEq)]\npub enum PrivilegeLevel {\n    User,\n    Kernel,\n    Hypervisor,\n    Unknown,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// Libkernel unit tests must execute in kernel mode.\n    #[kernel_test]\n    fn test_runner_executes_in_kernel_mode() {\n        let (level, _) = current_privilege_level();\n\n        assert!(level == PrivilegeLevel::Kernel)\n    }\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/lib.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n// Rust embedded logo for `make doc`.\n#![doc(\n    html_logo_url = \"https://raw.githubusercontent.com/rust-embedded/wg/master/assets/logo/ewg-logo-blue-white-on-transparent.png\"\n)]\n\n//! The `kernel` library.\n//!\n//! Used to compose the final kernel binary.\n//!\n//! # Code organization and architecture\n//!\n//! The code is divided into different *modules*, each representing a typical **subsystem** of the\n//! `kernel`. Top-level module files of subsystems reside directly in the `src` folder. For example,\n//! `src/memory.rs` contains code that is concerned with all things memory management.\n//!\n//! ## Visibility of processor architecture code\n//!\n//! Some of the `kernel`'s subsystems depend on low-level code that is specific to the target\n//! processor architecture. For each supported processor architecture, there exists a subfolder in\n//! `src/_arch`, for example, `src/_arch/aarch64`.\n//!\n//! The architecture folders mirror the subsystem modules laid out in `src`. For example,\n//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go\n//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in\n//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic\n//! module's name prefixed with `arch_`.\n//!\n//! For example, this is the top of `src/memory/mmu.rs`:\n//!\n//! ```\n//! #[cfg(target_arch = \"aarch64\")]\n//! #[path = \"../_arch/aarch64/memory/mmu.rs\"]\n//! mod arch_mmu;\n//! ```\n//!\n//! Often times, items from the `arch_ module` will be publicly reexported by the parent module.\n//! This way, each architecture specific module can provide its implementation of an item, while the\n//! caller must not be concerned which architecture has been conditionally compiled.\n//!\n//! ## BSP code\n//!\n//! `BSP` stands for Board Support Package. `BSP` code is organized under `src/bsp.rs` and contains\n//! target board specific definitions and functions. These are things such as the board's memory map\n//! or instances of drivers for devices that are featured on the respective board.\n//!\n//! Just like processor architecture code, the `BSP` code's module structure tries to mirror the\n//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is\n//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`.\n//!\n//! ## Kernel interfaces\n//!\n//! Both `arch` and `bsp` contain code that is conditionally compiled depending on the actual target\n//! and board for which the kernel is compiled. For example, the `interrupt controller` hardware of\n//! the `Raspberry Pi 3` and the `Raspberry Pi 4` is different, but we want the rest of the `kernel`\n//! code to play nicely with any of the two without much hassle.\n//!\n//! In order to provide a clean abstraction between `arch`, `bsp` and `generic kernel code`,\n//! `interface` traits are provided *whenever possible* and *where it makes sense*. They are defined\n//! in the respective subsystem module and help to enforce the idiom of *program to an interface,\n//! not an implementation*. For example, there will be a common IRQ handling interface which the two\n//! different interrupt controller `drivers` of both Raspberrys will implement, and only export the\n//! interface to the rest of the `kernel`.\n//!\n//! ```\n//!         +-------------------+\n//!         | Interface (Trait) |\n//!         |                   |\n//!         +--+-------------+--+\n//!            ^             ^\n//!            |             |\n//!            |             |\n//! +----------+--+       +--+----------+\n//! | kernel code |       |  bsp code   |\n//! |             |       |  arch code  |\n//! +-------------+       +-------------+\n//! ```\n//!\n//! # Summary\n//!\n//! For a logical `kernel` subsystem, corresponding code can be distributed over several physical\n//! locations. Here is an example for the **memory** subsystem:\n//!\n//! - `src/memory.rs` and `src/memory/**/*`\n//!   - Common code that is agnostic of target processor architecture and `BSP` characteristics.\n//!     - Example: A function to zero a chunk of memory.\n//!   - Interfaces for the memory subsystem that are implemented by `arch` or `BSP` code.\n//!     - Example: An `MMU` interface that defines `MMU` function prototypes.\n//! - `src/bsp/__board_name__/memory.rs` and `src/bsp/__board_name__/memory/**/*`\n//!   - `BSP` specific code.\n//!   - Example: The board's memory map (physical addresses of DRAM and MMIO devices).\n//! - `src/_arch/__arch_name__/memory.rs` and `src/_arch/__arch_name__/memory/**/*`\n//!   - Processor architecture specific code.\n//!   - Example: Implementation of the `MMU` interface for the `__arch_name__` processor\n//!     architecture.\n//!\n//! From a namespace perspective, **memory** subsystem code lives in:\n//!\n//! - `crate::memory::*`\n//! - `crate::bsp::memory::*`\n//!\n//! # Boot flow\n//!\n//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.\n//!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.\n//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.\n\n#![allow(clippy::upper_case_acronyms)]\n#![allow(incomplete_features)]\n#![feature(asm_const)]\n#![feature(const_option)]\n#![feature(core_intrinsics)]\n#![feature(format_args_nl)]\n#![feature(generic_const_exprs)]\n#![feature(int_roundings)]\n#![feature(is_sorted)]\n#![feature(linkage)]\n#![feature(nonzero_min_max)]\n#![feature(panic_info_message)]\n#![feature(step_trait)]\n#![feature(trait_alias)]\n#![feature(unchecked_math)]\n#![no_std]\n// Testing\n#![cfg_attr(test, no_main)]\n#![feature(custom_test_frameworks)]\n#![reexport_test_harness_main = \"test_main\"]\n#![test_runner(crate::test_runner)]\n\nmod panic_wait;\nmod synchronization;\n\npub mod bsp;\npub mod common;\npub mod console;\npub mod cpu;\npub mod driver;\npub mod exception;\npub mod memory;\npub mod print;\npub mod state;\npub mod time;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Version string.\npub fn version() -> &'static str {\n    concat!(\n        env!(\"CARGO_PKG_NAME\"),\n        \" version \",\n        env!(\"CARGO_PKG_VERSION\")\n    )\n}\n\n#[cfg(not(test))]\nextern \"Rust\" {\n    fn kernel_init() -> !;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n/// The default runner for unit tests.\npub fn test_runner(tests: &[&test_types::UnitTest]) {\n    // This line will be printed as the test header.\n    println!(\"Running {} tests\", tests.len());\n\n    for (i, test) in tests.iter().enumerate() {\n        print!(\"{:>3}. {:.<58}\", i + 1, test.name);\n\n        // Run the actual test.\n        (test.test_func)();\n\n        // Failed tests call panic!(). Execution reaches here only if the test has passed.\n        println!(\"[ok]\")\n    }\n}\n\n/// The `kernel_init()` for unit tests.\n#[cfg(test)]\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n\n    let phys_kernel_tables_base_addr = match memory::mmu::kernel_map_binary() {\n        Err(string) => panic!(\"Error mapping kernel binary: {}\", string),\n        Ok(addr) => addr,\n    };\n\n    if let Err(e) = memory::mmu::enable_mmu_and_caching(phys_kernel_tables_base_addr) {\n        panic!(\"Enabling MMU failed: {}\", e);\n    }\n\n    memory::mmu::post_enable_init();\n    bsp::driver::qemu_bring_up_console();\n\n    test_main();\n\n    cpu::qemu_exit_success()\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/main.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n// Rust embedded logo for `make doc`.\n#![doc(\n    html_logo_url = \"https://raw.githubusercontent.com/rust-embedded/wg/master/assets/logo/ewg-logo-blue-white-on-transparent.png\"\n)]\n\n//! The `kernel` binary.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\nuse libkernel::{bsp, cpu, driver, exception, info, memory, state, time};\n\n/// Early init code.\n///\n/// # Safety\n///\n/// - Only a single core must be active and running this function.\n/// - The init calls in this function must appear in the correct order:\n///     - MMU + Data caching must be activated at the earliest. Without it, any atomic operations,\n///       e.g. the yet-to-be-introduced spinlocks in the device drivers (which currently employ\n///       IRQSafeNullLocks instead of spinlocks), will fail to work (properly) on the RPi SoCs.\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n\n    let phys_kernel_tables_base_addr = match memory::mmu::kernel_map_binary() {\n        Err(string) => panic!(\"Error mapping kernel binary: {}\", string),\n        Ok(addr) => addr,\n    };\n\n    if let Err(e) = memory::mmu::enable_mmu_and_caching(phys_kernel_tables_base_addr) {\n        panic!(\"Enabling MMU failed: {}\", e);\n    }\n\n    memory::mmu::post_enable_init();\n\n    // Initialize the BSP driver subsystem.\n    if let Err(x) = bsp::driver::init() {\n        panic!(\"Error initializing BSP driver subsystem: {}\", x);\n    }\n\n    // Initialize all device drivers.\n    driver::driver_manager().init_drivers_and_irqs();\n\n    // Unmask interrupts on the boot CPU core.\n    exception::asynchronous::local_irq_unmask();\n\n    // Announce conclusion of the kernel_init() phase.\n    state::state_manager().transition_to_single_core_main();\n\n    // Transition from unsafe to safe.\n    kernel_main()\n}\n\n/// The main function running after the early init.\nfn kernel_main() -> ! {\n    info!(\"{}\", libkernel::version());\n    info!(\"Booting on: {}\", bsp::board_name());\n\n    info!(\"MMU online:\");\n    memory::mmu::kernel_print_mappings();\n\n    let (_, privilege_level) = exception::current_privilege_level();\n    info!(\"Current privilege level: {}\", privilege_level);\n\n    info!(\"Exception handling state:\");\n    exception::asynchronous::print_state();\n\n    info!(\n        \"Architectural timer resolution: {} ns\",\n        time::time_manager().resolution().as_nanos()\n    );\n\n    info!(\"Drivers loaded:\");\n    driver::driver_manager().enumerate();\n\n    info!(\"Registered IRQ handlers:\");\n    exception::asynchronous::irq_manager().print_handler();\n\n    info!(\"Echoing input now\");\n    cpu::wait_forever();\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/memory/mmu/mapping_record.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A record of mapped pages.\n\nuse super::{\n    AccessPermissions, Address, AttributeFields, MMIODescriptor, MemAttributes, MemoryRegion,\n    Physical, Virtual,\n};\nuse crate::{bsp, common, info, synchronization, synchronization::InitStateLock, warn};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Type describing a virtual memory mapping.\n#[allow(missing_docs)]\n#[derive(Copy, Clone)]\nstruct MappingRecordEntry {\n    pub users: [Option<&'static str>; 5],\n    pub phys_start_addr: Address<Physical>,\n    pub virt_start_addr: Address<Virtual>,\n    pub num_pages: usize,\n    pub attribute_fields: AttributeFields,\n}\n\nstruct MappingRecord {\n    inner: [Option<MappingRecordEntry>; 12],\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic KERNEL_MAPPING_RECORD: InitStateLock<MappingRecord> =\n    InitStateLock::new(MappingRecord::new());\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl MappingRecordEntry {\n    pub fn new(\n        name: &'static str,\n        virt_region: &MemoryRegion<Virtual>,\n        phys_region: &MemoryRegion<Physical>,\n        attr: &AttributeFields,\n    ) -> Self {\n        Self {\n            users: [Some(name), None, None, None, None],\n            phys_start_addr: phys_region.start_addr(),\n            virt_start_addr: virt_region.start_addr(),\n            num_pages: phys_region.num_pages(),\n            attribute_fields: *attr,\n        }\n    }\n\n    fn find_next_free_user(&mut self) -> Result<&mut Option<&'static str>, &'static str> {\n        if let Some(x) = self.users.iter_mut().find(|x| x.is_none()) {\n            return Ok(x);\n        };\n\n        Err(\"Storage for user info exhausted\")\n    }\n\n    pub fn add_user(&mut self, user: &'static str) -> Result<(), &'static str> {\n        let x = self.find_next_free_user()?;\n        *x = Some(user);\n        Ok(())\n    }\n}\n\nimpl MappingRecord {\n    pub const fn new() -> Self {\n        Self { inner: [None; 12] }\n    }\n\n    fn size(&self) -> usize {\n        self.inner.iter().filter(|x| x.is_some()).count()\n    }\n\n    fn sort(&mut self) {\n        let upper_bound_exclusive = self.size();\n        let entries = &mut self.inner[0..upper_bound_exclusive];\n\n        if !entries.is_sorted_by_key(|item| item.unwrap().virt_start_addr) {\n            entries.sort_unstable_by_key(|item| item.unwrap().virt_start_addr)\n        }\n    }\n\n    fn find_next_free(&mut self) -> Result<&mut Option<MappingRecordEntry>, &'static str> {\n        if let Some(x) = self.inner.iter_mut().find(|x| x.is_none()) {\n            return Ok(x);\n        }\n\n        Err(\"Storage for mapping info exhausted\")\n    }\n\n    fn find_duplicate(\n        &mut self,\n        phys_region: &MemoryRegion<Physical>,\n    ) -> Option<&mut MappingRecordEntry> {\n        self.inner\n            .iter_mut()\n            .filter_map(|x| x.as_mut())\n            .filter(|x| x.attribute_fields.mem_attributes == MemAttributes::Device)\n            .find(|x| {\n                if x.phys_start_addr != phys_region.start_addr() {\n                    return false;\n                }\n\n                if x.num_pages != phys_region.num_pages() {\n                    return false;\n                }\n\n                true\n            })\n    }\n\n    pub fn add(\n        &mut self,\n        name: &'static str,\n        virt_region: &MemoryRegion<Virtual>,\n        phys_region: &MemoryRegion<Physical>,\n        attr: &AttributeFields,\n    ) -> Result<(), &'static str> {\n        let x = self.find_next_free()?;\n\n        *x = Some(MappingRecordEntry::new(\n            name,\n            virt_region,\n            phys_region,\n            attr,\n        ));\n\n        self.sort();\n\n        Ok(())\n    }\n\n    pub fn print(&self) {\n        info!(\"      -------------------------------------------------------------------------------------------------------------------------------------------\");\n        info!(\n            \"      {:^44}     {:^30}   {:^7}   {:^9}   {:^35}\",\n            \"Virtual\", \"Physical\", \"Size\", \"Attr\", \"Entity\"\n        );\n        info!(\"      -------------------------------------------------------------------------------------------------------------------------------------------\");\n\n        for i in self.inner.iter().flatten() {\n            let size = i.num_pages * bsp::memory::mmu::KernelGranule::SIZE;\n            let virt_start = i.virt_start_addr;\n            let virt_end_inclusive = virt_start + (size - 1);\n            let phys_start = i.phys_start_addr;\n            let phys_end_inclusive = phys_start + (size - 1);\n\n            let (size, unit) = common::size_human_readable_ceil(size);\n\n            let attr = match i.attribute_fields.mem_attributes {\n                MemAttributes::CacheableDRAM => \"C\",\n                MemAttributes::Device => \"Dev\",\n            };\n\n            let acc_p = match i.attribute_fields.acc_perms {\n                AccessPermissions::ReadOnly => \"RO\",\n                AccessPermissions::ReadWrite => \"RW\",\n            };\n\n            let xn = if i.attribute_fields.execute_never {\n                \"XN\"\n            } else {\n                \"X\"\n            };\n\n            info!(\n                \"      {}..{} --> {}..{} | {:>3} {} | {:<3} {} {:<2} | {}\",\n                virt_start,\n                virt_end_inclusive,\n                phys_start,\n                phys_end_inclusive,\n                size,\n                unit,\n                attr,\n                acc_p,\n                xn,\n                i.users[0].unwrap()\n            );\n\n            for k in i.users[1..].iter() {\n                if let Some(additional_user) = *k {\n                    info!(\n                        \"                                                                                                            | {}\",\n                        additional_user\n                    );\n                }\n            }\n        }\n\n        info!(\"      -------------------------------------------------------------------------------------------------------------------------------------------\");\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::interface::ReadWriteEx;\n\n/// Add an entry to the mapping info record.\npub fn kernel_add(\n    name: &'static str,\n    virt_region: &MemoryRegion<Virtual>,\n    phys_region: &MemoryRegion<Physical>,\n    attr: &AttributeFields,\n) -> Result<(), &'static str> {\n    KERNEL_MAPPING_RECORD.write(|mr| mr.add(name, virt_region, phys_region, attr))\n}\n\npub fn kernel_find_and_insert_mmio_duplicate(\n    mmio_descriptor: &MMIODescriptor,\n    new_user: &'static str,\n) -> Option<Address<Virtual>> {\n    let phys_region: MemoryRegion<Physical> = (*mmio_descriptor).into();\n\n    KERNEL_MAPPING_RECORD.write(|mr| {\n        let dup = mr.find_duplicate(&phys_region)?;\n\n        if let Err(x) = dup.add_user(new_user) {\n            warn!(\"{}\", x);\n        }\n\n        Some(dup.virt_start_addr)\n    })\n}\n\n/// Human-readable print of all recorded kernel mappings.\npub fn kernel_print() {\n    KERNEL_MAPPING_RECORD.read(|mr| mr.print());\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/memory/mmu/page_alloc.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Page allocation.\n\nuse super::MemoryRegion;\nuse crate::{\n    memory::{AddressType, Virtual},\n    synchronization::IRQSafeNullLock,\n    warn,\n};\nuse core::num::NonZeroUsize;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// A page allocator that can be lazyily initialized.\npub struct PageAllocator<ATYPE: AddressType> {\n    pool: Option<MemoryRegion<ATYPE>>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic KERNEL_MMIO_VA_ALLOCATOR: IRQSafeNullLock<PageAllocator<Virtual>> =\n    IRQSafeNullLock::new(PageAllocator::new());\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the kernel's MMIO virtual address allocator.\npub fn kernel_mmio_va_allocator() -> &'static IRQSafeNullLock<PageAllocator<Virtual>> {\n    &KERNEL_MMIO_VA_ALLOCATOR\n}\n\nimpl<ATYPE: AddressType> PageAllocator<ATYPE> {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self { pool: None }\n    }\n\n    /// Initialize the allocator.\n    pub fn init(&mut self, pool: MemoryRegion<ATYPE>) {\n        if self.pool.is_some() {\n            warn!(\"Already initialized\");\n            return;\n        }\n\n        self.pool = Some(pool);\n    }\n\n    /// Allocate a number of pages.\n    pub fn alloc(\n        &mut self,\n        num_requested_pages: NonZeroUsize,\n    ) -> Result<MemoryRegion<ATYPE>, &'static str> {\n        if self.pool.is_none() {\n            return Err(\"Allocator not initialized\");\n        }\n\n        self.pool\n            .as_mut()\n            .unwrap()\n            .take_first_n_pages(num_requested_pages)\n    }\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/memory/mmu/translation_table.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Translation table.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../../_arch/aarch64/memory/mmu/translation_table.rs\"]\nmod arch_translation_table;\n\nuse super::{AttributeFields, MemoryRegion};\nuse crate::memory::{Address, Physical, Virtual};\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\n#[cfg(target_arch = \"aarch64\")]\npub use arch_translation_table::FixedSizeTranslationTable;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Translation table interfaces.\npub mod interface {\n    use super::*;\n\n    /// Translation table operations.\n    pub trait TranslationTable {\n        /// Anything that needs to run before any of the other provided functions can be used.\n        ///\n        /// # Safety\n        ///\n        /// - Implementor must ensure that this function can run only once or is harmless if invoked\n        ///   multiple times.\n        fn init(&mut self);\n\n        /// The translation table's base address to be used for programming the MMU.\n        fn phys_base_address(&self) -> Address<Physical>;\n\n        /// Map the given virtual memory region to the given physical memory region.\n        ///\n        /// # Safety\n        ///\n        /// - Using wrong attributes can cause multiple issues of different nature in the system.\n        /// - It is not required that the architectural implementation prevents aliasing. That is,\n        ///   mapping to the same physical memory using multiple virtual addresses, which would\n        ///   break Rust's ownership assumptions. This should be protected against in the kernel's\n        ///   generic MMU code.\n        unsafe fn map_at(\n            &mut self,\n            virt_region: &MemoryRegion<Virtual>,\n            phys_region: &MemoryRegion<Physical>,\n            attr: &AttributeFields,\n        ) -> Result<(), &'static str>;\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use crate::memory::mmu::{AccessPermissions, MemAttributes, PageAddress};\n    use arch_translation_table::MinSizeTranslationTable;\n    use interface::TranslationTable;\n    use test_macros::kernel_test;\n\n    /// Sanity checks for the TranslationTable implementation.\n    #[kernel_test]\n    fn translationtable_implementation_sanity() {\n        // This will occupy a lot of space on the stack.\n        let mut tables = MinSizeTranslationTable::new();\n\n        tables.init();\n\n        let virt_start_page_addr: PageAddress<Virtual> = PageAddress::from(0);\n        let virt_end_exclusive_page_addr: PageAddress<Virtual> =\n            virt_start_page_addr.checked_offset(5).unwrap();\n\n        let phys_start_page_addr: PageAddress<Physical> = PageAddress::from(0);\n        let phys_end_exclusive_page_addr: PageAddress<Physical> =\n            phys_start_page_addr.checked_offset(5).unwrap();\n\n        let virt_region = MemoryRegion::new(virt_start_page_addr, virt_end_exclusive_page_addr);\n        let phys_region = MemoryRegion::new(phys_start_page_addr, phys_end_exclusive_page_addr);\n\n        let attr = AttributeFields {\n            mem_attributes: MemAttributes::CacheableDRAM,\n            acc_perms: AccessPermissions::ReadWrite,\n            execute_never: true,\n        };\n\n        unsafe { assert_eq!(tables.map_at(&virt_region, &phys_region, &attr), Ok(())) };\n    }\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/memory/mmu/types.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management Unit types.\n\nuse crate::{\n    bsp, common,\n    memory::{Address, AddressType, Physical},\n};\nuse core::{convert::From, iter::Step, num::NonZeroUsize, ops::Range};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// A wrapper type around [Address] that ensures page alignment.\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub struct PageAddress<ATYPE: AddressType> {\n    inner: Address<ATYPE>,\n}\n\n/// A type that describes a region of memory in quantities of pages.\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub struct MemoryRegion<ATYPE: AddressType> {\n    start: PageAddress<ATYPE>,\n    end_exclusive: PageAddress<ATYPE>,\n}\n\n/// Architecture agnostic memory attributes.\n#[allow(missing_docs)]\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub enum MemAttributes {\n    CacheableDRAM,\n    Device,\n}\n\n/// Architecture agnostic access permissions.\n#[allow(missing_docs)]\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub enum AccessPermissions {\n    ReadOnly,\n    ReadWrite,\n}\n\n/// Collection of memory attributes.\n#[allow(missing_docs)]\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub struct AttributeFields {\n    pub mem_attributes: MemAttributes,\n    pub acc_perms: AccessPermissions,\n    pub execute_never: bool,\n}\n\n/// An MMIO descriptor for use in device drivers.\n#[derive(Copy, Clone)]\npub struct MMIODescriptor {\n    start_addr: Address<Physical>,\n    end_addr_exclusive: Address<Physical>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n//------------------------------------------------------------------------------\n// PageAddress\n//------------------------------------------------------------------------------\nimpl<ATYPE: AddressType> PageAddress<ATYPE> {\n    /// Unwraps the value.\n    pub fn into_inner(self) -> Address<ATYPE> {\n        self.inner\n    }\n\n    /// Calculates the offset from the page address.\n    ///\n    /// `count` is in units of [PageAddress]. For example, a count of 2 means `result = self + 2 *\n    /// page_size`.\n    pub fn checked_offset(self, count: isize) -> Option<Self> {\n        if count == 0 {\n            return Some(self);\n        }\n\n        let delta = count\n            .unsigned_abs()\n            .checked_mul(bsp::memory::mmu::KernelGranule::SIZE)?;\n        let result = if count.is_positive() {\n            self.inner.as_usize().checked_add(delta)?\n        } else {\n            self.inner.as_usize().checked_sub(delta)?\n        };\n\n        Some(Self {\n            inner: Address::new(result),\n        })\n    }\n}\n\nimpl<ATYPE: AddressType> From<usize> for PageAddress<ATYPE> {\n    fn from(addr: usize) -> Self {\n        assert!(\n            common::is_aligned(addr, bsp::memory::mmu::KernelGranule::SIZE),\n            \"Input usize not page aligned\"\n        );\n\n        Self {\n            inner: Address::new(addr),\n        }\n    }\n}\n\nimpl<ATYPE: AddressType> From<Address<ATYPE>> for PageAddress<ATYPE> {\n    fn from(addr: Address<ATYPE>) -> Self {\n        assert!(addr.is_page_aligned(), \"Input Address not page aligned\");\n\n        Self { inner: addr }\n    }\n}\n\nimpl<ATYPE: AddressType> Step for PageAddress<ATYPE> {\n    fn steps_between(start: &Self, end: &Self) -> Option<usize> {\n        if start > end {\n            return None;\n        }\n\n        // Since start <= end, do unchecked arithmetic.\n        Some(\n            (end.inner.as_usize() - start.inner.as_usize())\n                >> bsp::memory::mmu::KernelGranule::SHIFT,\n        )\n    }\n\n    fn forward_checked(start: Self, count: usize) -> Option<Self> {\n        start.checked_offset(count as isize)\n    }\n\n    fn backward_checked(start: Self, count: usize) -> Option<Self> {\n        start.checked_offset(-(count as isize))\n    }\n}\n\n//------------------------------------------------------------------------------\n// MemoryRegion\n//------------------------------------------------------------------------------\nimpl<ATYPE: AddressType> MemoryRegion<ATYPE> {\n    /// Create an instance.\n    pub fn new(start: PageAddress<ATYPE>, end_exclusive: PageAddress<ATYPE>) -> Self {\n        assert!(start <= end_exclusive);\n\n        Self {\n            start,\n            end_exclusive,\n        }\n    }\n\n    fn as_range(&self) -> Range<PageAddress<ATYPE>> {\n        self.into_iter()\n    }\n\n    /// Returns the start page address.\n    pub fn start_page_addr(&self) -> PageAddress<ATYPE> {\n        self.start\n    }\n\n    /// Returns the start address.\n    pub fn start_addr(&self) -> Address<ATYPE> {\n        self.start.into_inner()\n    }\n\n    /// Returns the exclusive end page address.\n    pub fn end_exclusive_page_addr(&self) -> PageAddress<ATYPE> {\n        self.end_exclusive\n    }\n\n    /// Returns the exclusive end page address.\n    pub fn end_inclusive_page_addr(&self) -> PageAddress<ATYPE> {\n        self.end_exclusive.checked_offset(-1).unwrap()\n    }\n\n    /// Checks if self contains an address.\n    pub fn contains(&self, addr: Address<ATYPE>) -> bool {\n        let page_addr = PageAddress::from(addr.align_down_page());\n        self.as_range().contains(&page_addr)\n    }\n\n    /// Checks if there is an overlap with another memory region.\n    pub fn overlaps(&self, other_region: &Self) -> bool {\n        let self_range = self.as_range();\n\n        self_range.contains(&other_region.start_page_addr())\n            || self_range.contains(&other_region.end_inclusive_page_addr())\n    }\n\n    /// Returns the number of pages contained in this region.\n    pub fn num_pages(&self) -> usize {\n        PageAddress::steps_between(&self.start, &self.end_exclusive).unwrap()\n    }\n\n    /// Returns the size in bytes of this region.\n    pub fn size(&self) -> usize {\n        // Invariant: start <= end_exclusive, so do unchecked arithmetic.\n        let end_exclusive = self.end_exclusive.into_inner().as_usize();\n        let start = self.start.into_inner().as_usize();\n\n        end_exclusive - start\n    }\n\n    /// Splits the MemoryRegion like:\n    ///\n    /// --------------------------------------------------------------------------------\n    /// |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |\n    /// --------------------------------------------------------------------------------\n    ///   ^                               ^                                       ^\n    ///   |                               |                                       |\n    ///   left_start     left_end_exclusive                                       |\n    ///                                                                           |\n    ///                                   ^                                       |\n    ///                                   |                                       |\n    ///                                   right_start           right_end_exclusive\n    ///\n    /// Left region is returned to the caller. Right region is the new region for this struct.\n    pub fn take_first_n_pages(&mut self, num_pages: NonZeroUsize) -> Result<Self, &'static str> {\n        let count: usize = num_pages.into();\n\n        let left_end_exclusive = self.start.checked_offset(count as isize);\n        let left_end_exclusive = match left_end_exclusive {\n            None => return Err(\"Overflow while calculating left_end_exclusive\"),\n            Some(x) => x,\n        };\n\n        if left_end_exclusive > self.end_exclusive {\n            return Err(\"Not enough free pages\");\n        }\n\n        let allocation = Self {\n            start: self.start,\n            end_exclusive: left_end_exclusive,\n        };\n        self.start = left_end_exclusive;\n\n        Ok(allocation)\n    }\n}\n\nimpl<ATYPE: AddressType> IntoIterator for MemoryRegion<ATYPE> {\n    type Item = PageAddress<ATYPE>;\n    type IntoIter = Range<Self::Item>;\n\n    fn into_iter(self) -> Self::IntoIter {\n        Range {\n            start: self.start,\n            end: self.end_exclusive,\n        }\n    }\n}\n\nimpl From<MMIODescriptor> for MemoryRegion<Physical> {\n    fn from(desc: MMIODescriptor) -> Self {\n        let start = PageAddress::from(desc.start_addr.align_down_page());\n        let end_exclusive = PageAddress::from(desc.end_addr_exclusive().align_up_page());\n\n        Self {\n            start,\n            end_exclusive,\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// MMIODescriptor\n//------------------------------------------------------------------------------\n\nimpl MMIODescriptor {\n    /// Create an instance.\n    pub const fn new(start_addr: Address<Physical>, size: usize) -> Self {\n        assert!(size > 0);\n        let end_addr_exclusive = Address::new(start_addr.as_usize() + size);\n\n        Self {\n            start_addr,\n            end_addr_exclusive,\n        }\n    }\n\n    /// Return the start address.\n    pub const fn start_addr(&self) -> Address<Physical> {\n        self.start_addr\n    }\n\n    /// Return the exclusive end address.\n    pub fn end_addr_exclusive(&self) -> Address<Physical> {\n        self.end_addr_exclusive\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use crate::memory::Virtual;\n    use test_macros::kernel_test;\n\n    /// Sanity of [PageAddress] methods.\n    #[kernel_test]\n    fn pageaddress_type_method_sanity() {\n        let page_addr: PageAddress<Virtual> =\n            PageAddress::from(bsp::memory::mmu::KernelGranule::SIZE * 2);\n\n        assert_eq!(\n            page_addr.checked_offset(-2),\n            Some(PageAddress::<Virtual>::from(0))\n        );\n\n        assert_eq!(\n            page_addr.checked_offset(2),\n            Some(PageAddress::<Virtual>::from(\n                bsp::memory::mmu::KernelGranule::SIZE * 4\n            ))\n        );\n\n        assert_eq!(\n            PageAddress::<Virtual>::from(0).checked_offset(0),\n            Some(PageAddress::<Virtual>::from(0))\n        );\n        assert_eq!(PageAddress::<Virtual>::from(0).checked_offset(-1), None);\n\n        let max_page_addr = Address::<Virtual>::new(usize::MAX).align_down_page();\n        assert_eq!(\n            PageAddress::<Virtual>::from(max_page_addr).checked_offset(1),\n            None\n        );\n\n        let zero = PageAddress::<Virtual>::from(0);\n        let three = PageAddress::<Virtual>::from(bsp::memory::mmu::KernelGranule::SIZE * 3);\n        assert_eq!(PageAddress::steps_between(&zero, &three), Some(3));\n    }\n\n    /// Sanity of [MemoryRegion] methods.\n    #[kernel_test]\n    fn memoryregion_type_method_sanity() {\n        let zero = PageAddress::<Virtual>::from(0);\n        let zero_region = MemoryRegion::new(zero, zero);\n        assert_eq!(zero_region.num_pages(), 0);\n        assert_eq!(zero_region.size(), 0);\n\n        let one = PageAddress::<Virtual>::from(bsp::memory::mmu::KernelGranule::SIZE);\n        let one_region = MemoryRegion::new(zero, one);\n        assert_eq!(one_region.num_pages(), 1);\n        assert_eq!(one_region.size(), bsp::memory::mmu::KernelGranule::SIZE);\n\n        let three = PageAddress::<Virtual>::from(bsp::memory::mmu::KernelGranule::SIZE * 3);\n        let mut three_region = MemoryRegion::new(zero, three);\n        assert!(three_region.contains(zero.into_inner()));\n        assert!(!three_region.contains(three.into_inner()));\n        assert!(three_region.overlaps(&one_region));\n\n        let allocation = three_region\n            .take_first_n_pages(NonZeroUsize::new(2).unwrap())\n            .unwrap();\n        assert_eq!(allocation.num_pages(), 2);\n        assert_eq!(three_region.num_pages(), 1);\n\n        for (i, alloc) in allocation.into_iter().enumerate() {\n            assert_eq!(\n                alloc.into_inner().as_usize(),\n                i * bsp::memory::mmu::KernelGranule::SIZE\n            );\n        }\n    }\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/memory/mmu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management Unit.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/memory/mmu.rs\"]\nmod arch_mmu;\n\nmod mapping_record;\nmod page_alloc;\nmod translation_table;\nmod types;\n\nuse crate::{\n    bsp,\n    memory::{Address, Physical, Virtual},\n    synchronization, warn,\n};\nuse core::{fmt, num::NonZeroUsize};\n\npub use types::*;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// MMU enable errors variants.\n#[allow(missing_docs)]\n#[derive(Debug)]\npub enum MMUEnableError {\n    AlreadyEnabled,\n    Other(&'static str),\n}\n\n/// Memory Management interfaces.\npub mod interface {\n    use super::*;\n\n    /// MMU functions.\n    pub trait MMU {\n        /// Turns on the MMU for the first time and enables data and instruction caching.\n        ///\n        /// # Safety\n        ///\n        /// - Changes the HW's global state.\n        unsafe fn enable_mmu_and_caching(\n            &self,\n            phys_tables_base_addr: Address<Physical>,\n        ) -> Result<(), MMUEnableError>;\n\n        /// Returns true if the MMU is enabled, false otherwise.\n        fn is_enabled(&self) -> bool;\n    }\n}\n\n/// Describes the characteristics of a translation granule.\npub struct TranslationGranule<const GRANULE_SIZE: usize>;\n\n/// Describes properties of an address space.\npub struct AddressSpace<const AS_SIZE: usize>;\n\n/// Intended to be implemented for [`AddressSpace`].\npub trait AssociatedTranslationTable {\n    /// A translation table whose address range is:\n    ///\n    /// [AS_SIZE - 1, 0]\n    type TableStartFromBottom;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\nuse interface::MMU;\nuse synchronization::interface::*;\nuse translation_table::interface::TranslationTable;\n\n/// Query the BSP for the reserved virtual addresses for MMIO remapping and initialize the kernel's\n/// MMIO VA allocator with it.\nfn kernel_init_mmio_va_allocator() {\n    let region = bsp::memory::mmu::virt_mmio_remap_region();\n\n    page_alloc::kernel_mmio_va_allocator().lock(|allocator| allocator.init(region));\n}\n\n/// Map a region in the kernel's translation tables.\n///\n/// No input checks done, input is passed through to the architectural implementation.\n///\n/// # Safety\n///\n/// - See `map_at()`.\n/// - Does not prevent aliasing.\nunsafe fn kernel_map_at_unchecked(\n    name: &'static str,\n    virt_region: &MemoryRegion<Virtual>,\n    phys_region: &MemoryRegion<Physical>,\n    attr: &AttributeFields,\n) -> Result<(), &'static str> {\n    bsp::memory::mmu::kernel_translation_tables()\n        .write(|tables| tables.map_at(virt_region, phys_region, attr))?;\n\n    if let Err(x) = mapping_record::kernel_add(name, virt_region, phys_region, attr) {\n        warn!(\"{}\", x);\n    }\n\n    Ok(())\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl fmt::Display for MMUEnableError {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        match self {\n            MMUEnableError::AlreadyEnabled => write!(f, \"MMU is already enabled\"),\n            MMUEnableError::Other(x) => write!(f, \"{}\", x),\n        }\n    }\n}\n\nimpl<const GRANULE_SIZE: usize> TranslationGranule<GRANULE_SIZE> {\n    /// The granule's size.\n    pub const SIZE: usize = Self::size_checked();\n\n    /// The granule's mask.\n    pub const MASK: usize = Self::SIZE - 1;\n\n    /// The granule's shift, aka log2(size).\n    pub const SHIFT: usize = Self::SIZE.trailing_zeros() as usize;\n\n    const fn size_checked() -> usize {\n        assert!(GRANULE_SIZE.is_power_of_two());\n\n        GRANULE_SIZE\n    }\n}\n\nimpl<const AS_SIZE: usize> AddressSpace<AS_SIZE> {\n    /// The address space size.\n    pub const SIZE: usize = Self::size_checked();\n\n    /// The address space shift, aka log2(size).\n    pub const SIZE_SHIFT: usize = Self::SIZE.trailing_zeros() as usize;\n\n    const fn size_checked() -> usize {\n        assert!(AS_SIZE.is_power_of_two());\n\n        // Check for architectural restrictions as well.\n        Self::arch_address_space_size_sanity_checks();\n\n        AS_SIZE\n    }\n}\n\n/// Raw mapping of a virtual to physical region in the kernel translation tables.\n///\n/// Prevents mapping into the MMIO range of the tables.\n///\n/// # Safety\n///\n/// - See `kernel_map_at_unchecked()`.\n/// - Does not prevent aliasing. Currently, the callers must be trusted.\npub unsafe fn kernel_map_at(\n    name: &'static str,\n    virt_region: &MemoryRegion<Virtual>,\n    phys_region: &MemoryRegion<Physical>,\n    attr: &AttributeFields,\n) -> Result<(), &'static str> {\n    if bsp::memory::mmu::virt_mmio_remap_region().overlaps(virt_region) {\n        return Err(\"Attempt to manually map into MMIO region\");\n    }\n\n    kernel_map_at_unchecked(name, virt_region, phys_region, attr)?;\n\n    Ok(())\n}\n\n/// MMIO remapping in the kernel translation tables.\n///\n/// Typically used by device drivers.\n///\n/// # Safety\n///\n/// - Same as `kernel_map_at_unchecked()`, minus the aliasing part.\npub unsafe fn kernel_map_mmio(\n    name: &'static str,\n    mmio_descriptor: &MMIODescriptor,\n) -> Result<Address<Virtual>, &'static str> {\n    let phys_region = MemoryRegion::from(*mmio_descriptor);\n    let offset_into_start_page = mmio_descriptor.start_addr().offset_into_page();\n\n    // Check if an identical region has been mapped for another driver. If so, reuse it.\n    let virt_addr = if let Some(addr) =\n        mapping_record::kernel_find_and_insert_mmio_duplicate(mmio_descriptor, name)\n    {\n        addr\n    // Otherwise, allocate a new region and map it.\n    } else {\n        let num_pages = match NonZeroUsize::new(phys_region.num_pages()) {\n            None => return Err(\"Requested 0 pages\"),\n            Some(x) => x,\n        };\n\n        let virt_region =\n            page_alloc::kernel_mmio_va_allocator().lock(|allocator| allocator.alloc(num_pages))?;\n\n        kernel_map_at_unchecked(\n            name,\n            &virt_region,\n            &phys_region,\n            &AttributeFields {\n                mem_attributes: MemAttributes::Device,\n                acc_perms: AccessPermissions::ReadWrite,\n                execute_never: true,\n            },\n        )?;\n\n        virt_region.start_addr()\n    };\n\n    Ok(virt_addr + offset_into_start_page)\n}\n\n/// Map the kernel's binary. Returns the translation table's base address.\n///\n/// # Safety\n///\n/// - See [`bsp::memory::mmu::kernel_map_binary()`].\npub unsafe fn kernel_map_binary() -> Result<Address<Physical>, &'static str> {\n    let phys_kernel_tables_base_addr =\n        bsp::memory::mmu::kernel_translation_tables().write(|tables| {\n            tables.init();\n            tables.phys_base_address()\n        });\n\n    bsp::memory::mmu::kernel_map_binary()?;\n\n    Ok(phys_kernel_tables_base_addr)\n}\n\n/// Enable the MMU and data + instruction caching.\n///\n/// # Safety\n///\n/// - Crucial function during kernel init. Changes the the complete memory view of the processor.\npub unsafe fn enable_mmu_and_caching(\n    phys_tables_base_addr: Address<Physical>,\n) -> Result<(), MMUEnableError> {\n    arch_mmu::mmu().enable_mmu_and_caching(phys_tables_base_addr)\n}\n\n/// Finish initialization of the MMU subsystem.\npub fn post_enable_init() {\n    kernel_init_mmio_va_allocator();\n}\n\n/// Human-readable print of all recorded kernel mappings.\npub fn kernel_print_mappings() {\n    mapping_record::kernel_print()\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use crate::memory::mmu::{AccessPermissions, MemAttributes, PageAddress};\n    use test_macros::kernel_test;\n\n    /// Check that you cannot map into the MMIO VA range from kernel_map_at().\n    #[kernel_test]\n    fn no_manual_mmio_map() {\n        let phys_start_page_addr: PageAddress<Physical> = PageAddress::from(0);\n        let phys_end_exclusive_page_addr: PageAddress<Physical> =\n            phys_start_page_addr.checked_offset(5).unwrap();\n        let phys_region = MemoryRegion::new(phys_start_page_addr, phys_end_exclusive_page_addr);\n\n        let num_pages = NonZeroUsize::new(phys_region.num_pages()).unwrap();\n        let virt_region = page_alloc::kernel_mmio_va_allocator()\n            .lock(|allocator| allocator.alloc(num_pages))\n            .unwrap();\n\n        let attr = AttributeFields {\n            mem_attributes: MemAttributes::CacheableDRAM,\n            acc_perms: AccessPermissions::ReadWrite,\n            execute_never: true,\n        };\n\n        unsafe {\n            assert_eq!(\n                kernel_map_at(\"test\", &virt_region, &phys_region, &attr),\n                Err(\"Attempt to manually map into MMIO region\")\n            )\n        };\n    }\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/memory.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management.\n\npub mod mmu;\n\nuse crate::{bsp, common};\nuse core::{\n    fmt,\n    marker::PhantomData,\n    ops::{Add, Sub},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Metadata trait for marking the type of an address.\npub trait AddressType: Copy + Clone + PartialOrd + PartialEq + Ord + Eq {}\n\n/// Zero-sized type to mark a physical address.\n#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq)]\npub enum Physical {}\n\n/// Zero-sized type to mark a virtual address.\n#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq)]\npub enum Virtual {}\n\n/// Generic address type.\n#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq)]\npub struct Address<ATYPE: AddressType> {\n    value: usize,\n    _address_type: PhantomData<fn() -> ATYPE>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl AddressType for Physical {}\nimpl AddressType for Virtual {}\n\nimpl<ATYPE: AddressType> Address<ATYPE> {\n    /// Create an instance.\n    pub const fn new(value: usize) -> Self {\n        Self {\n            value,\n            _address_type: PhantomData,\n        }\n    }\n\n    /// Convert to usize.\n    pub const fn as_usize(self) -> usize {\n        self.value\n    }\n\n    /// Align down to page size.\n    #[must_use]\n    pub const fn align_down_page(self) -> Self {\n        let aligned = common::align_down(self.value, bsp::memory::mmu::KernelGranule::SIZE);\n\n        Self::new(aligned)\n    }\n\n    /// Align up to page size.\n    #[must_use]\n    pub const fn align_up_page(self) -> Self {\n        let aligned = common::align_up(self.value, bsp::memory::mmu::KernelGranule::SIZE);\n\n        Self::new(aligned)\n    }\n\n    /// Checks if the address is page aligned.\n    pub const fn is_page_aligned(&self) -> bool {\n        common::is_aligned(self.value, bsp::memory::mmu::KernelGranule::SIZE)\n    }\n\n    /// Return the address' offset into the corresponding page.\n    pub const fn offset_into_page(&self) -> usize {\n        self.value & bsp::memory::mmu::KernelGranule::MASK\n    }\n}\n\nimpl<ATYPE: AddressType> Add<usize> for Address<ATYPE> {\n    type Output = Self;\n\n    #[inline(always)]\n    fn add(self, rhs: usize) -> Self::Output {\n        match self.value.checked_add(rhs) {\n            None => panic!(\"Overflow on Address::add\"),\n            Some(x) => Self::new(x),\n        }\n    }\n}\n\nimpl<ATYPE: AddressType> Sub<Address<ATYPE>> for Address<ATYPE> {\n    type Output = Self;\n\n    #[inline(always)]\n    fn sub(self, rhs: Address<ATYPE>) -> Self::Output {\n        match self.value.checked_sub(rhs.value) {\n            None => panic!(\"Overflow on Address::sub\"),\n            Some(x) => Self::new(x),\n        }\n    }\n}\n\nimpl fmt::Display for Address<Physical> {\n    // Don't expect to see physical addresses greater than 40 bit.\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        let q3: u8 = ((self.value >> 32) & 0xff) as u8;\n        let q2: u16 = ((self.value >> 16) & 0xffff) as u16;\n        let q1: u16 = (self.value & 0xffff) as u16;\n\n        write!(f, \"0x\")?;\n        write!(f, \"{:02x}_\", q3)?;\n        write!(f, \"{:04x}_\", q2)?;\n        write!(f, \"{:04x}\", q1)\n    }\n}\n\nimpl fmt::Display for Address<Virtual> {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        let q4: u16 = ((self.value >> 48) & 0xffff) as u16;\n        let q3: u16 = ((self.value >> 32) & 0xffff) as u16;\n        let q2: u16 = ((self.value >> 16) & 0xffff) as u16;\n        let q1: u16 = (self.value & 0xffff) as u16;\n\n        write!(f, \"0x\")?;\n        write!(f, \"{:04x}_\", q4)?;\n        write!(f, \"{:04x}_\", q3)?;\n        write!(f, \"{:04x}_\", q2)?;\n        write!(f, \"{:04x}\", q1)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// Sanity of [Address] methods.\n    #[kernel_test]\n    fn address_type_method_sanity() {\n        let addr = Address::<Virtual>::new(bsp::memory::mmu::KernelGranule::SIZE + 100);\n\n        assert_eq!(\n            addr.align_down_page().as_usize(),\n            bsp::memory::mmu::KernelGranule::SIZE\n        );\n\n        assert_eq!(\n            addr.align_up_page().as_usize(),\n            bsp::memory::mmu::KernelGranule::SIZE * 2\n        );\n\n        assert!(!addr.is_page_aligned());\n\n        assert_eq!(addr.offset_into_page(), 100);\n    }\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/panic_wait.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A panic handler that infinitely waits.\n\nuse crate::{cpu, exception, println};\nuse core::panic::PanicInfo;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// The point of exit for `libkernel`.\n///\n/// It is linked weakly, so that the integration tests can overload its standard behavior.\n#[linkage = \"weak\"]\n#[no_mangle]\nfn _panic_exit() -> ! {\n    #[cfg(not(feature = \"test_build\"))]\n    {\n        cpu::wait_forever()\n    }\n\n    #[cfg(feature = \"test_build\")]\n    {\n        cpu::qemu_exit_failure()\n    }\n}\n\n/// Stop immediately if called a second time.\n///\n/// # Note\n///\n/// Using atomics here relieves us from needing to use `unsafe` for the static variable.\n///\n/// On `AArch64`, which is the only implemented architecture at the time of writing this,\n/// [`AtomicBool::load`] and [`AtomicBool::store`] are lowered to ordinary load and store\n/// instructions. They are therefore safe to use even with MMU + caching deactivated.\n///\n/// [`AtomicBool::load`]: core::sync::atomic::AtomicBool::load\n/// [`AtomicBool::store`]: core::sync::atomic::AtomicBool::store\nfn panic_prevent_reenter() {\n    use core::sync::atomic::{AtomicBool, Ordering};\n\n    #[cfg(not(target_arch = \"aarch64\"))]\n    compile_error!(\"Add the target_arch to above's check if the following code is safe to use\");\n\n    static PANIC_IN_PROGRESS: AtomicBool = AtomicBool::new(false);\n\n    if !PANIC_IN_PROGRESS.load(Ordering::Relaxed) {\n        PANIC_IN_PROGRESS.store(true, Ordering::Relaxed);\n\n        return;\n    }\n\n    _panic_exit()\n}\n\n#[panic_handler]\nfn panic(info: &PanicInfo) -> ! {\n    exception::asynchronous::local_irq_mask();\n\n    // Protect against panic infinite loops if any of the following code panics itself.\n    panic_prevent_reenter();\n\n    let timestamp = crate::time::time_manager().uptime();\n    let (location, line, column) = match info.location() {\n        Some(loc) => (loc.file(), loc.line(), loc.column()),\n        _ => (\"???\", 0, 0),\n    };\n\n    println!(\n        \"[  {:>3}.{:06}] Kernel panic!\\n\\n\\\n        Panic location:\\n      File '{}', line {}, column {}\\n\\n\\\n        {}\",\n        timestamp.as_secs(),\n        timestamp.subsec_micros(),\n        location,\n        line,\n        column,\n        info.message().unwrap_or(&format_args!(\"\")),\n    );\n\n    _panic_exit()\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/print.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Printing.\n\nuse crate::console;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n#[doc(hidden)]\npub fn _print(args: fmt::Arguments) {\n    console::console().write_fmt(args).unwrap();\n}\n\n/// Prints without a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! print {\n    ($($arg:tt)*) => ($crate::print::_print(format_args!($($arg)*)));\n}\n\n/// Prints with a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! println {\n    () => ($crate::print!(\"\\n\"));\n    ($($arg:tt)*) => ({\n        $crate::print::_print(format_args_nl!($($arg)*));\n    })\n}\n\n/// Prints an info, with a newline.\n#[macro_export]\nmacro_rules! info {\n    ($string:expr) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[  {:>3}.{:06}] \", $string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n        ));\n    });\n    ($format_string:expr, $($arg:tt)*) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[  {:>3}.{:06}] \", $format_string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n            $($arg)*\n        ));\n    })\n}\n\n/// Prints a warning, with a newline.\n#[macro_export]\nmacro_rules! warn {\n    ($string:expr) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[W {:>3}.{:06}] \", $string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n        ));\n    });\n    ($format_string:expr, $($arg:tt)*) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[W {:>3}.{:06}] \", $format_string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n            $($arg)*\n        ));\n    })\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/state.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! State information about the kernel itself.\n\nuse core::sync::atomic::{AtomicU8, Ordering};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Different stages in the kernel execution.\n#[derive(Copy, Clone, Eq, PartialEq)]\nenum State {\n    /// The kernel starts booting in this state.\n    Init,\n\n    /// The kernel transitions to this state when jumping to `kernel_main()` (at the end of\n    /// `kernel_init()`, after all init calls are done).\n    SingleCoreMain,\n\n    /// The kernel transitions to this state when it boots the secondary cores, aka switches\n    /// exectution mode to symmetric multiprocessing (SMP).\n    MultiCoreMain,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Maintains the kernel state and state transitions.\npub struct StateManager(AtomicU8);\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic STATE_MANAGER: StateManager = StateManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the global StateManager.\npub fn state_manager() -> &'static StateManager {\n    &STATE_MANAGER\n}\n\nimpl StateManager {\n    const INIT: u8 = 0;\n    const SINGLE_CORE_MAIN: u8 = 1;\n    const MULTI_CORE_MAIN: u8 = 2;\n\n    /// Create a new instance.\n    pub const fn new() -> Self {\n        Self(AtomicU8::new(Self::INIT))\n    }\n\n    /// Return the current state.\n    fn state(&self) -> State {\n        let state = self.0.load(Ordering::Acquire);\n\n        match state {\n            Self::INIT => State::Init,\n            Self::SINGLE_CORE_MAIN => State::SingleCoreMain,\n            Self::MULTI_CORE_MAIN => State::MultiCoreMain,\n            _ => panic!(\"Invalid KERNEL_STATE\"),\n        }\n    }\n\n    /// Return if the kernel is init state.\n    pub fn is_init(&self) -> bool {\n        self.state() == State::Init\n    }\n\n    /// Transition from Init to SingleCoreMain.\n    pub fn transition_to_single_core_main(&self) {\n        if self\n            .0\n            .compare_exchange(\n                Self::INIT,\n                Self::SINGLE_CORE_MAIN,\n                Ordering::Acquire,\n                Ordering::Relaxed,\n            )\n            .is_err()\n        {\n            panic!(\"transition_to_single_core_main() called while state != Init\");\n        }\n    }\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/synchronization.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Synchronization primitives.\n//!\n//! # Resources\n//!\n//!   - <https://doc.rust-lang.org/book/ch16-04-extensible-concurrency-sync-and-send.html>\n//!   - <https://stackoverflow.com/questions/59428096/understanding-the-send-trait>\n//!   - <https://doc.rust-lang.org/std/cell/index.html>\n\nuse core::cell::UnsafeCell;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Synchronization interfaces.\npub mod interface {\n\n    /// Any object implementing this trait guarantees exclusive access to the data wrapped within\n    /// the Mutex for the duration of the provided closure.\n    pub trait Mutex {\n        /// The type of the data that is wrapped by this mutex.\n        type Data;\n\n        /// Locks the mutex and grants the closure temporary mutable access to the wrapped data.\n        fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R;\n    }\n\n    /// A reader-writer exclusion type.\n    ///\n    /// The implementing object allows either a number of readers or at most one writer at any point\n    /// in time.\n    pub trait ReadWriteEx {\n        /// The type of encapsulated data.\n        type Data;\n\n        /// Grants temporary mutable access to the encapsulated data.\n        fn write<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R;\n\n        /// Grants temporary immutable access to the encapsulated data.\n        fn read<'a, R>(&'a self, f: impl FnOnce(&'a Self::Data) -> R) -> R;\n    }\n}\n\n/// A pseudo-lock for teaching purposes.\n///\n/// In contrast to a real Mutex implementation, does not protect against concurrent access from\n/// other cores to the contained data. This part is preserved for later lessons.\n///\n/// The lock will only be used as long as it is safe to do so, i.e. as long as the kernel is\n/// executing on a single core.\npub struct IRQSafeNullLock<T>\nwhere\n    T: ?Sized,\n{\n    data: UnsafeCell<T>,\n}\n\n/// A pseudo-lock that is RW during the single-core kernel init phase and RO afterwards.\n///\n/// Intended to encapsulate data that is populated during kernel init when no concurrency exists.\npub struct InitStateLock<T>\nwhere\n    T: ?Sized,\n{\n    data: UnsafeCell<T>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nunsafe impl<T> Send for IRQSafeNullLock<T> where T: ?Sized + Send {}\nunsafe impl<T> Sync for IRQSafeNullLock<T> where T: ?Sized + Send {}\n\nimpl<T> IRQSafeNullLock<T> {\n    /// Create an instance.\n    pub const fn new(data: T) -> Self {\n        Self {\n            data: UnsafeCell::new(data),\n        }\n    }\n}\n\nunsafe impl<T> Send for InitStateLock<T> where T: ?Sized + Send {}\nunsafe impl<T> Sync for InitStateLock<T> where T: ?Sized + Send {}\n\nimpl<T> InitStateLock<T> {\n    /// Create an instance.\n    pub const fn new(data: T) -> Self {\n        Self {\n            data: UnsafeCell::new(data),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse crate::{exception, state};\n\nimpl<T> interface::Mutex for IRQSafeNullLock<T> {\n    type Data = T;\n\n    fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R {\n        // In a real lock, there would be code encapsulating this line that ensures that this\n        // mutable reference will ever only be given out once at a time.\n        let data = unsafe { &mut *self.data.get() };\n\n        // Execute the closure while IRQs are masked.\n        exception::asynchronous::exec_with_irq_masked(|| f(data))\n    }\n}\n\nimpl<T> interface::ReadWriteEx for InitStateLock<T> {\n    type Data = T;\n\n    fn write<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R {\n        assert!(\n            state::state_manager().is_init(),\n            \"InitStateLock::write called after kernel init phase\"\n        );\n        assert!(\n            !exception::asynchronous::is_local_irq_masked(),\n            \"InitStateLock::write called with IRQs unmasked\"\n        );\n\n        let data = unsafe { &mut *self.data.get() };\n\n        f(data)\n    }\n\n    fn read<'a, R>(&'a self, f: impl FnOnce(&'a Self::Data) -> R) -> R {\n        let data = unsafe { &*self.data.get() };\n\n        f(data)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// InitStateLock must be transparent.\n    #[kernel_test]\n    fn init_state_lock_is_transparent() {\n        use core::mem::size_of;\n\n        assert_eq!(size_of::<InitStateLock<u64>>(), size_of::<u64>());\n    }\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/src/time.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Timer primitives.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/time.rs\"]\nmod arch_time;\n\nuse core::time::Duration;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Provides time management functions.\npub struct TimeManager;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic TIME_MANAGER: TimeManager = TimeManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the global TimeManager.\npub fn time_manager() -> &'static TimeManager {\n    &TIME_MANAGER\n}\n\nimpl TimeManager {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self\n    }\n\n    /// The timer's resolution.\n    pub fn resolution(&self) -> Duration {\n        arch_time::resolution()\n    }\n\n    /// The uptime since power-on of the device.\n    ///\n    /// This includes time consumed by firmware and bootloaders.\n    pub fn uptime(&self) -> Duration {\n        arch_time::uptime()\n    }\n\n    /// Spin for a given duration.\n    pub fn spin_for(&self, duration: Duration) {\n        arch_time::spin_for(duration)\n    }\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/tests/00_console_sanity.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'console_io_test'\n\n# Verify sending and receiving works as expected.\nclass TxRxHandshakeTest < SubtestBase\n    def name\n        'Transmit and Receive handshake'\n    end\n\n    def run(qemu_out, qemu_in)\n        qemu_in.write_nonblock('ABC')\n        expect_or_raise(qemu_out, 'OK1234')\n    end\nend\n\n# Check for correct TX statistics implementation. Depends on test 1 being run first.\nclass TxStatisticsTest < SubtestBase\n    def name\n        'Transmit statistics'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, '6')\n    end\nend\n\n# Check for correct RX statistics implementation. Depends on test 1 being run first.\nclass RxStatisticsTest < SubtestBase\n    def name\n        'Receive statistics'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, '3')\n    end\nend\n\n## -------------------------------------------------------------------------------------------------\n## Test registration\n## -------------------------------------------------------------------------------------------------\ndef subtest_collection\n    [TxRxHandshakeTest.new, TxStatisticsTest.new, RxStatisticsTest.new]\nend\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/tests/00_console_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Console sanity tests - RX, TX and statistics.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Console tests should time out on the I/O harness in case of panic.\nmod panic_wait_forever;\n\nuse libkernel::{bsp, console, cpu, exception, memory, print};\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    use console::console;\n\n    exception::handling_init();\n\n    let phys_kernel_tables_base_addr = match memory::mmu::kernel_map_binary() {\n        Err(string) => panic!(\"Error mapping kernel binary: {}\", string),\n        Ok(addr) => addr,\n    };\n\n    if let Err(e) = memory::mmu::enable_mmu_and_caching(phys_kernel_tables_base_addr) {\n        panic!(\"Enabling MMU failed: {}\", e);\n    }\n\n    memory::mmu::post_enable_init();\n    bsp::driver::qemu_bring_up_console();\n\n    // Handshake\n    assert_eq!(console().read_char(), 'A');\n    assert_eq!(console().read_char(), 'B');\n    assert_eq!(console().read_char(), 'C');\n    print!(\"OK1234\");\n\n    // 6\n    print!(\"{}\", console().chars_written());\n\n    // 3\n    print!(\"{}\", console().chars_read());\n\n    // The QEMU process running this test will be closed by the I/O test harness.\n    cpu::wait_forever();\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/tests/01_timer_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Timer sanity tests.\n\n#![feature(custom_test_frameworks)]\n#![no_main]\n#![no_std]\n#![reexport_test_harness_main = \"test_main\"]\n#![test_runner(libkernel::test_runner)]\n\nuse core::time::Duration;\nuse libkernel::{bsp, cpu, exception, memory, time};\nuse test_macros::kernel_test;\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n\n    let phys_kernel_tables_base_addr = match memory::mmu::kernel_map_binary() {\n        Err(string) => panic!(\"Error mapping kernel binary: {}\", string),\n        Ok(addr) => addr,\n    };\n\n    if let Err(e) = memory::mmu::enable_mmu_and_caching(phys_kernel_tables_base_addr) {\n        panic!(\"Enabling MMU failed: {}\", e);\n    }\n\n    memory::mmu::post_enable_init();\n    bsp::driver::qemu_bring_up_console();\n\n    // Depending on CPU arch, some timer bring-up code could go here. Not needed for the RPi.\n\n    test_main();\n\n    cpu::qemu_exit_success()\n}\n\n/// Simple check that the timer is running.\n#[kernel_test]\nfn timer_is_counting() {\n    assert!(time::time_manager().uptime().as_nanos() > 0)\n}\n\n/// Timer resolution must be sufficient.\n#[kernel_test]\nfn timer_resolution_is_sufficient() {\n    assert!(time::time_manager().resolution().as_nanos() > 0);\n    assert!(time::time_manager().resolution().as_nanos() < 100)\n}\n\n/// Sanity check spin_for() implementation.\n#[kernel_test]\nfn spin_accuracy_check_1_second() {\n    let t1 = time::time_manager().uptime();\n    time::time_manager().spin_for(Duration::from_secs(1));\n    let t2 = time::time_manager().uptime();\n\n    assert_eq!((t2 - t1).as_secs(), 1)\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/tests/02_exception_sync_page_fault.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Page faults must result in synchronous exceptions.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Overwrites libkernel's `panic_wait::_panic_exit()` so that it returns a \"success\" code.\n///\n/// In this test, reaching the panic is a success, because it is called from the synchronous\n/// exception handler, which is what this test wants to achieve.\n///\n/// It also means that this integration test can not use any other code that calls panic!() directly\n/// or indirectly.\nmod panic_exit_success;\n\nuse libkernel::{bsp, cpu, exception, info, memory, println};\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n\n    // This line will be printed as the test header.\n    println!(\"Testing synchronous exception handling by causing a page fault\");\n\n    let phys_kernel_tables_base_addr = match memory::mmu::kernel_map_binary() {\n        Err(string) => {\n            info!(\"Error mapping kernel binary: {}\", string);\n            cpu::qemu_exit_failure()\n        }\n        Ok(addr) => addr,\n    };\n\n    if let Err(e) = memory::mmu::enable_mmu_and_caching(phys_kernel_tables_base_addr) {\n        info!(\"Enabling MMU failed: {}\", e);\n        cpu::qemu_exit_failure()\n    }\n\n    memory::mmu::post_enable_init();\n    bsp::driver::qemu_bring_up_console();\n\n    info!(\"Writing beyond mapped area to address 9 GiB...\");\n    let big_addr: u64 = 9 * 1024 * 1024 * 1024;\n    core::ptr::read_volatile(big_addr as *mut u64);\n\n    // If execution reaches here, the memory access above did not cause a page fault exception.\n    cpu::qemu_exit_failure()\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/tests/03_exception_restore_sanity.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'console_io_test'\n\n# Verify that exception restore works.\nclass ExceptionRestoreTest < SubtestBase\n    def name\n        'Exception restore'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, 'Back from system call!')\n    end\nend\n\n## -------------------------------------------------------------------------------------------------\n## Test registration\n## -------------------------------------------------------------------------------------------------\ndef subtest_collection\n    [ExceptionRestoreTest.new]\nend\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/tests/03_exception_restore_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A simple sanity test to see if exception restore code works.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Console tests should time out on the I/O harness in case of panic.\nmod panic_wait_forever;\n\nuse core::arch::asm;\nuse libkernel::{bsp, cpu, exception, info, memory, println};\n\n#[inline(never)]\nfn nested_system_call() {\n    #[cfg(target_arch = \"aarch64\")]\n    unsafe {\n        asm!(\"svc #0x1337\", options(nomem, nostack, preserves_flags));\n    }\n\n    #[cfg(not(target_arch = \"aarch64\"))]\n    {\n        info!(\"Not supported yet\");\n        cpu::wait_forever();\n    }\n}\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n\n    // This line will be printed as the test header.\n    println!(\"Testing exception restore\");\n\n    let phys_kernel_tables_base_addr = match memory::mmu::kernel_map_binary() {\n        Err(string) => {\n            info!(\"Error mapping kernel binary: {}\", string);\n            cpu::qemu_exit_failure()\n        }\n        Ok(addr) => addr,\n    };\n\n    if let Err(e) = memory::mmu::enable_mmu_and_caching(phys_kernel_tables_base_addr) {\n        info!(\"Enabling MMU failed: {}\", e);\n        cpu::qemu_exit_failure()\n    }\n\n    memory::mmu::post_enable_init();\n    bsp::driver::qemu_bring_up_console();\n\n    info!(\"Making a dummy system call\");\n\n    // Calling this inside a function indirectly tests if the link register is restored properly.\n    nested_system_call();\n\n    info!(\"Back from system call!\");\n\n    // The QEMU process running this test will be closed by the I/O test harness.\n    cpu::wait_forever();\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/tests/04_exception_irq_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! IRQ handling sanity tests.\n\n#![feature(custom_test_frameworks)]\n#![no_main]\n#![no_std]\n#![reexport_test_harness_main = \"test_main\"]\n#![test_runner(libkernel::test_runner)]\n\nuse libkernel::{bsp, cpu, exception, memory};\nuse test_macros::kernel_test;\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n\n    let phys_kernel_tables_base_addr = match memory::mmu::kernel_map_binary() {\n        Err(string) => panic!(\"Error mapping kernel binary: {}\", string),\n        Ok(addr) => addr,\n    };\n\n    if let Err(e) = memory::mmu::enable_mmu_and_caching(phys_kernel_tables_base_addr) {\n        panic!(\"Enabling MMU failed: {}\", e);\n    }\n\n    memory::mmu::post_enable_init();\n    bsp::driver::qemu_bring_up_console();\n\n    exception::asynchronous::local_irq_unmask();\n\n    test_main();\n\n    cpu::qemu_exit_success()\n}\n\n/// Check that IRQ masking works.\n#[kernel_test]\nfn local_irq_mask_works() {\n    // Precondition: IRQs are unmasked.\n    assert!(exception::asynchronous::is_local_irq_masked());\n\n    exception::asynchronous::local_irq_mask();\n    assert!(!exception::asynchronous::is_local_irq_masked());\n\n    // Restore earlier state.\n    exception::asynchronous::local_irq_unmask();\n}\n\n/// Check that IRQ unmasking works.\n#[kernel_test]\nfn local_irq_unmask_works() {\n    // Precondition: IRQs are masked.\n    exception::asynchronous::local_irq_mask();\n    assert!(!exception::asynchronous::is_local_irq_masked());\n\n    exception::asynchronous::local_irq_unmask();\n    assert!(exception::asynchronous::is_local_irq_masked());\n}\n\n/// Check that IRQ mask save is saving \"something\".\n#[kernel_test]\nfn local_irq_mask_save_works() {\n    // Precondition: IRQs are unmasked.\n    assert!(exception::asynchronous::is_local_irq_masked());\n\n    let first = exception::asynchronous::local_irq_mask_save();\n    assert!(!exception::asynchronous::is_local_irq_masked());\n\n    let second = exception::asynchronous::local_irq_mask_save();\n    assert_ne!(first, second);\n\n    exception::asynchronous::local_irq_restore(first);\n    assert!(exception::asynchronous::is_local_irq_masked());\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/tests/boot_test_string.rb",
    "content": "# frozen_string_literal: true\n\nEXPECTED_PRINT = 'Echoing input now'\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/tests/panic_exit_success/mod.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n/// Overwrites libkernel's `panic_wait::_panic_exit()` with the QEMU-exit version.\n#[no_mangle]\nfn _panic_exit() -> ! {\n    libkernel::cpu::qemu_exit_success()\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/kernel/tests/panic_wait_forever/mod.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n/// Overwrites libkernel's `panic_wait::_panic_exit()` with wait_forever.\n#[no_mangle]\nfn _panic_exit() -> ! {\n    libkernel::cpu::wait_forever()\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/libraries/test-macros/Cargo.toml",
    "content": "[package]\nname = \"test-macros\"\nversion = \"0.1.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n\n[lib]\nproc-macro = true\n\n[dependencies]\nproc-macro2 = \"1.x\"\nquote = \"1.x\"\nsyn = { version = \"1.x\", features = [\"full\"] }\ntest-types = { path = \"../test-types\" }\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/libraries/test-macros/src/lib.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\nuse proc_macro::TokenStream;\nuse proc_macro2::Span;\nuse quote::quote;\nuse syn::{parse_macro_input, Ident, ItemFn};\n\n#[proc_macro_attribute]\npub fn kernel_test(_attr: TokenStream, input: TokenStream) -> TokenStream {\n    let f = parse_macro_input!(input as ItemFn);\n\n    let test_name = &format!(\"{}\", f.sig.ident);\n    let test_ident = Ident::new(\n        &format!(\"{}_TEST_CONTAINER\", f.sig.ident.to_string().to_uppercase()),\n        Span::call_site(),\n    );\n    let test_code_block = f.block;\n\n    quote!(\n        #[test_case]\n        const #test_ident: test_types::UnitTest = test_types::UnitTest {\n            name: #test_name,\n            test_func: || #test_code_block,\n        };\n    )\n    .into()\n}\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/libraries/test-types/Cargo.toml",
    "content": "[package]\nname = \"test-types\"\nversion = \"0.1.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n"
  },
  {
    "path": "14_virtual_mem_part2_mmio_remap/libraries/test-types/src/lib.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Types for the `custom_test_frameworks` implementation.\n\n#![no_std]\n\n/// Unit test container.\npub struct UnitTest {\n    /// Name of the test.\n    pub name: &'static str,\n\n    /// Function pointer to the test.\n    pub test_func: fn(),\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/.cargo/config.toml",
    "content": "[target.'cfg(target_os = \"none\")']\nrunner = \"target/kernel_test_runner.sh\"\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/.vscode/settings.json",
    "content": "{\n    \"editor.formatOnSave\": true,\n    \"editor.rulers\": [100],\n    \"rust-analyzer.cargo.target\": \"aarch64-unknown-none-softfloat\",\n    \"rust-analyzer.cargo.features\": [\"bsp_rpi3\"],\n    \"rust-analyzer.checkOnSave.allTargets\": false,\n    \"rust-analyzer.checkOnSave.extraArgs\": [\"--lib\", \"--bins\"],\n    \"rust-analyzer.lens.debug\": false,\n    \"rust-analyzer.lens.run\": false\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/Cargo.toml",
    "content": "[workspace]\n\nmembers = [\n        \"libraries/*\",\n        \"kernel\"\n]\n\n[profile.release]\nlto = true\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/Makefile",
    "content": "## SPDX-License-Identifier: MIT OR Apache-2.0\n##\n## Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\ninclude ../common/docker.mk\ninclude ../common/format.mk\ninclude ../common/operating_system.mk\n\n##--------------------------------------------------------------------------------------------------\n## Optional, user-provided configuration values\n##--------------------------------------------------------------------------------------------------\n\n# Default to the RPi3.\nBSP ?= rpi3\n\n# Default to a serial device name that is common in Linux.\nDEV_SERIAL ?= /dev/ttyUSB0\n\n# Optional integration test name.\nifdef TEST\n    TEST_ARG = --test $(TEST)\nelse\n    TEST_ARG = --test '*'\nendif\n\n\n\n##--------------------------------------------------------------------------------------------------\n## BSP-specific configuration values\n##--------------------------------------------------------------------------------------------------\nQEMU_MISSING_STRING = \"This board is not yet supported for QEMU.\"\n\nifeq ($(BSP),rpi3)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE = raspi3\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    QEMU_TEST_ARGS    = $(QEMU_RELEASE_ARGS) -semihosting\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    OPENOCD_ARG       = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi3.cfg\n    JTAG_BOOT_IMAGE   = ../X1_JTAG_boot/jtag_boot_rpi3.img\n    LD_SCRIPT_PATH    = $(shell pwd)/kernel/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a53\nelse ifeq ($(BSP),rpi4)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE =\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    QEMU_TEST_ARGS    = $(QEMU_RELEASE_ARGS) -semihosting\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    OPENOCD_ARG       = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi4.cfg\n    JTAG_BOOT_IMAGE   = ../X1_JTAG_boot/jtag_boot_rpi4.img\n    LD_SCRIPT_PATH    = $(shell pwd)/kernel/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a72\nendif\n\n# Export for build.rs.\nexport LD_SCRIPT_PATH\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets and Prerequisites\n##--------------------------------------------------------------------------------------------------\nKERNEL_MANIFEST      = kernel/Cargo.toml\nKERNEL_LINKER_SCRIPT = kernel.ld\nLAST_BUILD_CONFIG    = target/$(BSP).build_config\n\nKERNEL_ELF_RAW      = target/$(TARGET)/release/kernel\n# This parses cargo's dep-info file.\n# https://doc.rust-lang.org/cargo/guide/build-cache.html#dep-info-files\nKERNEL_ELF_RAW_DEPS = $(filter-out %: ,$(file < $(KERNEL_ELF_RAW).d)) $(KERNEL_MANIFEST) $(LAST_BUILD_CONFIG)\n\n##------------------------------------------------------------------------------\n## Translation tables\n##------------------------------------------------------------------------------\nTT_TOOL_PATH = tools/translation_table_tool\n\nKERNEL_ELF_TTABLES      = target/$(TARGET)/release/kernel+ttables\nKERNEL_ELF_TTABLES_DEPS = $(KERNEL_ELF_RAW) $(wildcard $(TT_TOOL_PATH)/*)\n\nKERNEL_ELF = $(KERNEL_ELF_TTABLES)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Command building blocks\n##--------------------------------------------------------------------------------------------------\nRUSTFLAGS = $(RUSTC_MISC_ARGS)                   \\\n    -C link-arg=--library-path=$(LD_SCRIPT_PATH) \\\n    -C link-arg=--script=$(KERNEL_LINKER_SCRIPT)\n\nRUSTFLAGS_PEDANTIC = $(RUSTFLAGS) \\\n    -D warnings                   \\\n    -D missing_docs\n\nFEATURES      = --features bsp_$(BSP)\nCOMPILER_ARGS = --target=$(TARGET) \\\n    $(FEATURES)                    \\\n    --release\n\nRUSTC_CMD   = cargo rustc $(COMPILER_ARGS) --manifest-path $(KERNEL_MANIFEST)\nDOC_CMD     = cargo doc $(COMPILER_ARGS)\nCLIPPY_CMD  = cargo clippy $(COMPILER_ARGS)\nTEST_CMD    = cargo test $(COMPILER_ARGS) --manifest-path $(KERNEL_MANIFEST)\nOBJCOPY_CMD = rust-objcopy \\\n    --strip-all            \\\n    -O binary\n\nEXEC_QEMU          = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE)\nEXEC_TT_TOOL       = ruby $(TT_TOOL_PATH)/main.rb\nEXEC_TEST_DISPATCH = ruby ../common/tests/dispatch.rb\nEXEC_MINIPUSH      = ruby ../common/serial/minipush.rb\n\n##------------------------------------------------------------------------------\n## Dockerization\n##------------------------------------------------------------------------------\nDOCKER_CMD            = docker run -t --rm -v $(shell pwd):/work/tutorial -w /work/tutorial\nDOCKER_CMD_INTERACT   = $(DOCKER_CMD) -i\nDOCKER_ARG_DIR_COMMON = -v $(shell pwd)/../common:/work/common\nDOCKER_ARG_DIR_JTAG   = -v $(shell pwd)/../X1_JTAG_boot:/work/X1_JTAG_boot\nDOCKER_ARG_DEV        = --privileged -v /dev:/dev\nDOCKER_ARG_NET        = --network host\n\n# DOCKER_IMAGE defined in include file (see top of this file).\nDOCKER_QEMU  = $(DOCKER_CMD_INTERACT) $(DOCKER_IMAGE)\nDOCKER_TOOLS = $(DOCKER_CMD) $(DOCKER_IMAGE)\nDOCKER_TEST  = $(DOCKER_CMD) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\nDOCKER_GDB   = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_NET) $(DOCKER_IMAGE)\n\n# Dockerize commands, which require USB device passthrough, only on Linux.\nifeq ($(shell uname -s),Linux)\n    DOCKER_CMD_DEV = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_DEV)\n\n    DOCKER_CHAINBOOT = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\n    DOCKER_JTAGBOOT  = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_ARG_DIR_JTAG) $(DOCKER_IMAGE)\n    DOCKER_OPENOCD   = $(DOCKER_CMD_DEV) $(DOCKER_ARG_NET) $(DOCKER_IMAGE)\nelse\n    DOCKER_OPENOCD   = echo \"Not yet supported on non-Linux systems.\"; \\#\nendif\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: all doc qemu chainboot clippy clean readelf objdump nm check\n\nall: $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Save the configuration as a file, so make understands if it changed.\n##------------------------------------------------------------------------------\n$(LAST_BUILD_CONFIG):\n\t@rm -f target/*.build_config\n\t@mkdir -p target\n\t@touch $(LAST_BUILD_CONFIG)\n\n##------------------------------------------------------------------------------\n## Compile the kernel ELF\n##------------------------------------------------------------------------------\n$(KERNEL_ELF_RAW): $(KERNEL_ELF_RAW_DEPS)\n\t$(call color_header, \"Compiling kernel ELF - $(BSP)\")\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(RUSTC_CMD)\n\n##------------------------------------------------------------------------------\n## Precompute the kernel translation tables and patch them into the kernel ELF\n##------------------------------------------------------------------------------\n$(KERNEL_ELF_TTABLES): $(KERNEL_ELF_TTABLES_DEPS)\n\t$(call color_header, \"Precomputing kernel translation tables and patching kernel ELF\")\n\t@cp $(KERNEL_ELF_RAW) $(KERNEL_ELF_TTABLES)\n\t@$(DOCKER_TOOLS) $(EXEC_TT_TOOL) $(BSP) $(KERNEL_ELF_TTABLES)\n\n##------------------------------------------------------------------------------\n## Generate the stripped kernel binary\n##------------------------------------------------------------------------------\n$(KERNEL_BIN): $(KERNEL_ELF_TTABLES)\n\t$(call color_header, \"Generating stripped binary\")\n\t@$(OBJCOPY_CMD) $(KERNEL_ELF_TTABLES) $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Name\")\n\t@echo $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Size\")\n\t$(call disk_usage_KiB, $(KERNEL_BIN))\n\n##------------------------------------------------------------------------------\n## Generate the documentation\n##------------------------------------------------------------------------------\ndoc:\n\t$(call color_header, \"Generating docs\")\n\t@$(DOC_CMD) --document-private-items --open\n\n##------------------------------------------------------------------------------\n## Run the kernel in QEMU\n##------------------------------------------------------------------------------\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\nqemu:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\nqemu: $(KERNEL_BIN)\n\t$(call color_header, \"Launching QEMU\")\n\t@$(DOCKER_QEMU) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\nendif\n\n##------------------------------------------------------------------------------\n## Push the kernel to the real HW target\n##------------------------------------------------------------------------------\nchainboot: $(KERNEL_BIN)\n\t@$(DOCKER_CHAINBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run clippy\n##------------------------------------------------------------------------------\nclippy:\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(CLIPPY_CMD)\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(CLIPPY_CMD) --features test_build --tests \\\n                --manifest-path $(KERNEL_MANIFEST)\n\n##------------------------------------------------------------------------------\n## Clean\n##------------------------------------------------------------------------------\nclean:\n\trm -rf target $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run readelf\n##------------------------------------------------------------------------------\nreadelf: $(KERNEL_ELF)\n\t$(call color_header, \"Launching readelf\")\n\t@$(DOCKER_TOOLS) $(READELF_BINARY) --headers $(KERNEL_ELF)\n\n##------------------------------------------------------------------------------\n## Run objdump\n##------------------------------------------------------------------------------\nobjdump: $(KERNEL_ELF)\n\t$(call color_header, \"Launching objdump\")\n\t@$(DOCKER_TOOLS) $(OBJDUMP_BINARY) --disassemble --demangle \\\n                --section .text   \\\n                --section .rodata \\\n                $(KERNEL_ELF) | rustfilt\n\n##------------------------------------------------------------------------------\n## Run nm\n##------------------------------------------------------------------------------\nnm: $(KERNEL_ELF)\n\t$(call color_header, \"Launching nm\")\n\t@$(DOCKER_TOOLS) $(NM_BINARY) --demangle --print-size $(KERNEL_ELF) | sort | rustfilt\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Debugging targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: jtagboot openocd gdb gdb-opt0\n\n##------------------------------------------------------------------------------\n## Push the JTAG boot image to the real HW target\n##------------------------------------------------------------------------------\njtagboot:\n\t@$(DOCKER_JTAGBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(JTAG_BOOT_IMAGE)\n\n##------------------------------------------------------------------------------\n## Start OpenOCD session\n##------------------------------------------------------------------------------\nopenocd:\n\t$(call color_header, \"Launching OpenOCD\")\n\t@$(DOCKER_OPENOCD) openocd $(OPENOCD_ARG)\n\n##------------------------------------------------------------------------------\n## Start GDB session\n##------------------------------------------------------------------------------\ngdb: RUSTC_MISC_ARGS += -C debuginfo=2\ngdb-opt0: RUSTC_MISC_ARGS += -C debuginfo=2 -C opt-level=0\ngdb gdb-opt0: $(KERNEL_ELF)\n\t$(call color_header, \"Launching GDB\")\n\t@$(DOCKER_GDB) gdb-multiarch -q $(KERNEL_ELF)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Testing targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: test test_boot test_unit test_integration\n\ntest_unit test_integration: FEATURES += --features test_build\n\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\ntest_boot test_unit test_integration test:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\n##------------------------------------------------------------------------------\n## Run boot test\n##------------------------------------------------------------------------------\ntest_boot: $(KERNEL_BIN)\n\t$(call color_header, \"Boot test - $(BSP)\")\n\t@$(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Helpers for unit and integration test targets\n##------------------------------------------------------------------------------\ndefine KERNEL_TEST_RUNNER\n#!/usr/bin/env bash\n\n    # The cargo test runner seems to change into the crate under test's directory. Therefore, ensure\n    # this script executes from the root.\n    cd $(shell pwd)\n\n    TEST_ELF=$$(echo $$1 | sed -e 's/.*target/target/g')\n    TEST_BINARY=$$(echo $$1.img | sed -e 's/.*target/target/g')\n\n    $(DOCKER_TOOLS) $(EXEC_TT_TOOL) $(BSP) $$TEST_ELF > /dev/null\n    $(OBJCOPY_CMD) $$TEST_ELF $$TEST_BINARY\n    $(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_TEST_ARGS) -kernel $$TEST_BINARY\nendef\n\nexport KERNEL_TEST_RUNNER\n\ndefine test_prepare\n    @mkdir -p target\n    @echo \"$$KERNEL_TEST_RUNNER\" > target/kernel_test_runner.sh\n    @chmod +x target/kernel_test_runner.sh\nendef\n\n##------------------------------------------------------------------------------\n## Run unit test(s)\n##------------------------------------------------------------------------------\ntest_unit:\n\t$(call color_header, \"Compiling unit test(s) - $(BSP)\")\n\t$(call test_prepare)\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(TEST_CMD) --lib\n\n##------------------------------------------------------------------------------\n## Run integration test(s)\n##------------------------------------------------------------------------------\ntest_integration:\n\t$(call color_header, \"Compiling integration test(s) - $(BSP)\")\n\t$(call test_prepare)\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(TEST_CMD) $(TEST_ARG)\n\ntest: test_boot test_unit test_integration\n\nendif\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/README.md",
    "content": "# Tutorial 15 - Virtual Memory Part 3: Precomputed Translation Tables\n\n## tl;dr\n\n- We are making the next baby-steps towards mapping the kernel to the most significant area of the\n  virtual memory space.\n- Instead of dynamically computing the kernel's translation tables during runtime while booting, we\n  are precomputing them in advance just after kernel compilation, and patch them into the kernel's\n  binary ahead of time.\n- For now, we are still `identity-mapping` the kernel binary.\n  - However, after this tutorial, we have all the infrastructure in place to easily map it\n    elsewhere.\n\n## Table of Contents\n\n- [Introduction](#introduction)\n- [When Load Address != Link Address, Funny Things Can Happen](#when-load-address--link-address-funny-things-can-happen)\n  * [Interim Conclusion](#interim-conclusion)\n  * [Doing the Same Thing - Expecting Different Results](#doing-the-same-thing---expecting-different-results)\n- [Position-Independent Code (PIC)](#position-independent-code-pic)\n  * [Using PIC during kernel startup](#using-pic-during-kernel-startup)\n- [Precomputed Translation Tables](#precomputed-translation-tables)\n- [Implementation](#implementation)\n  * [Preparing the Kernel Tables](#preparing-the-kernel-tables)\n  * [Turning on the MMU Before Switching to EL1](#turning-on-the-mmu-before-switching-to-el1)\n  * [The Translation Table Tool](#the-translation-table-tool)\n  * [Other changes](#other-changes)\n- [Discussion](#discussion)\n- [Test it](#test-it)\n- [Diff to previous](#diff-to-previous)\n\n## Introduction\n\nThis tutorial is another preparatory step for our overall goal of mapping the kernel to the most\nsignificant area of the virtual memory space.\n\nThe reasoning of why we want to do this was given in the previous tutorial's introduction. But lets\nfor a quick moment think about what it actually means in practice: Currently, the kernel's binary is\nloaded by the Raspberry's firmware at address `0x8_0000`.\n\nIn decimal, this address is at `512 KiB`, and therefore well within the _least significant part_ of\nthe address space. Let's have a look at the picture from the [ARM Cortex-A Series Programmer’s Guide\nfor ARMv8-A] again to understand in which virtual address space region the kernel would ideally be\nmapped to:\n\n<p align=\"center\">\n    <img src=\"../doc/15_kernel_user_address_space_partitioning.png\" height=\"500\" align=\"center\">\n</p>\n\nAs we can see, the architecture proposes somewhere between addresses `0xffff_0000_0000_0000` and\n`0xffff_ffff_ffff_ffff`. Once we succeed in mapping the kernel there, the whole lower range between\n`0x0` and `0xffff_ffff_ffff` would be free for future applications to use.\n\n[ARM Cortex-A Series Programmer’s Guide for ARMv8-A]: https://developer.arm.com/documentation/den0024/latest/\n\nNow, how can we get there?\n\n## When Load Address != Link Address, Funny Things Can Happen\n\nImagine that, using the linker script, we link the kernel so that its `_start()` function is located\nat address `0xffff_0000_0000_0000`. What hasn't changed is that the Raspberry's firmware will still\nload the kernel binary at address `0x8_0000`, and the kernel will still start executing from there\nwith the `MMU` disabled.\n\nSo one of the very first things the kernel must achieve during its boot to function correctly, is to\nsomehow enable the `MMU` together with `translation tables` that account for the address offset\n(`0xffff_0000_0000_0000 -> 0x8_0000`). In previous tutorials, we already generated translation\ntables during the kernel's boot, so lets quickly remember how we did that:\n\nIn `src/bsp/__board_name__/memory/mmu.rs` we have a static (or \"global\" in non-Rust speak) instance\nof `struct KernelTranslationTable`:\n\n```rust\nstatic KERNEL_TABLES: InitStateLock<KernelTranslationTable> =\n    InitStateLock::new(KernelTranslationTable::new());\n```\n\nIn other parts of the kernel code, this instance would be referenced one way or the other, and its\nmember functions would be called, for example, when mapping a range of pages. At the end of the day,\nafter multiple layers of indirection, what happens at the most basic level is that a `piece of code`\nmanipulates some `global data`. So part of the job of the code is to retrieve the data's constant\naddress before it can manipulate it.\n\nLet's simplify the address-retrieval to the most basic code example possible. The example will be\npresented as `C` code. Don't ask yet why `C` is chosen. It will get clear as the tutorial develops.\n\n```c\n#include <stdint.h>\n\nuint64_t global_data_word = 0x11223344;\n\nuint64_t* get_address_of_global(void) {\n     return &global_data_word;\n}\n```\n\nLet's compile and link this using the following linker script:\n\n```ld.s\nSECTIONS\n{\n    . =  0x80000;\n\n    .text : {\n        QUAD(0); /* Intentional fill word */\n        QUAD(0); /* Intentional fill word */\n        KEEP(*(.text*))\n    }\n    .got    : ALIGN(8)   { *(.got) }\n    .data   : ALIGN(64K) {\n        QUAD(0); /* Intentional fill word */\n        *(.data*)\n    }\n}\n```\n\nHere are the compilation steps and the corresponding `objdump` for `AArch64`:\n\n```console\n$ clang --target=aarch64-none-elf -Iinclude -Wall -c start.c -o start.o\n$ ld.lld start.o -T kernel.ld -o example.elf\n```\n\n```c-objdump\nDisassembly of section .text:\n\n0000000000080010 get_address_of_global:\n   80010: 80 00 00 90                  \tadrp\tx0, #0x10000\n   80014: 00 20 00 91                  \tadd\tx0, x0, #0x8\n   80018: c0 03 5f d6                  \tret\n\nDisassembly of section .data:\n\n0000000000090008 global_data_word:\n   90008: 44 33 22 11\n   9000c: 00 00 00 00\n```\n\nAs you can see, the address of function `get_address_of_global()` is `0x8_0010` and\n`global_data_word` got address `0x9_0008`. In the function body, the compiler emitted an [`ADRP`]\nand `ADD` instruction pair, which means that the global's address is calculated as a `PC-relative\noffset`. `PC` means program counter, aka the current position of where the CPU core is currently\nexecuting from.\n\nWithout going in too much detail, what the instruction basically does is: It retrieves the `4 KiB`\npage address that belongs to the program counter's (PC) current position (PC is at `0x8_0010`, so\nthe page address is `0x8_0000`), and adds `0x1_0000`. So after the `ADRP` instruction, register `x0`\nholds the value `0x9_0000`. To this value, `8` is added in the next instruction, resulting in the\noverall address of `0x9_0008`, which is exactly where `global_data_word` is located. This works,\nbecause after linking a `static executable binary` like we do since `tutorial 01`, relative\npositions of code and data are fixed, and not supposed to change during runtime.\n\n[`ADRP`]: https://developer.arm.com/documentation/dui0802/b/A64-General-Instructions/ADRP\n\nIf the Raspberry's firmware now loads this binary at address `0x8_0000`, as always, we can be sure\nthat our function returns the correct address of our global data word.\n\nNow lets link this to the most significant area of memory:\n\n```ld.s\nSECTIONS\n{\n    . =  0xffff000000000000; /* <--- Only line changed in the linker script! */\n\n    .text : {\n\n    /* omitted for brevity */\n}\n```\n\nAnd compile again:\n\n```c-objdump\nDisassembly of section .text:\n\nffff000000000010 get_address_of_global:\nffff000000000010: 80 00 00 90          \tadrp\tx0, #0x10000\nffff000000000014: 00 20 00 91          \tadd\tx0, x0, #0x8\nffff000000000018: c0 03 5f d6          \tret\n\nDisassembly of section .data:\n\nffff000000010008 global_data_word:\nffff000000010008: 44 33 22 11\nffff00000001000c: 00 00 00 00\n```\n\nAnd let the Raspberry's firmware load the binary at address `0x8_0000` again (we couldn't load it to\n`0xffff_0000_0000_0000` even if we wanted to. That address is `15 Exbibyte`. A Raspberry Pi with\nthat much RAM won't exist for some time to come 😉).\n\nLet's try to answer the same question again: Would `get_address_of_global()` return the value for\n`global_data_word` that we expect to see (`0xffff_0000_0001_0008` as shown in the objdump)? This\ntime, the answer is **no**. It would again return `0x9_0008`.\n\nWhy is that? Don't let yourself be distracted by the addresses the `objdump` above is showing. When\nthe Raspberry's firmware loads this binary at `0x8_0000`, then the Program Counter value when\n`get_address_of_global()` executes is again `0x8_0010`. So **the PC-relative calculation** will not\nresult in the expected value, which would be the **absolute** (alternatively: **link-time**) address\nof `global_data_word`.\n\n### Interim Conclusion\n\nWhat have we learned so far? We wrote a little piece of code in a high-level language that retrieves\nan address, and we naively expected to retrieve an **absolute** address.\n\nBut compiler and linker conspired against us, and machine code was emitted that uses a PC-relative\naddressing scheme, so our expectation is not matched when **load address != link address**. If you\ncompile for `AArch64`, you'll see relative addressing schemes a lot, because it is natural to the\narchitecture.\n\nIf you now say: Wait a second, how is this a problem? It actually helps! After all, since the code\nis loaded at address `0x8_0000`, this relative addressing scheme will ensure that the processor\naccesses the global data word at the correct address!\n\nYes, in this particular, constrained demo case, it worked out for us. But have a look at the\nfollowing.\n\n### Doing the Same Thing - Expecting Different Results\n\nLet's take a quick detour and see what happens if we compile **the exactly same code** for the\n`x86_64` processor architecture. First when linked to `0x8_0000`:\n\n```c-objdump\nDisassembly of section .text:\n\n0000000000080070 get_address_of_global:\n   80070: 55                            push    rbp\n   80071: 48 89 e5                      mov     rbp, rsp\n   80074: 48 b8 08 00 09 00 00 00 00 00 movabs  rax, 0x90008\n   8007e: 5d                            pop     rbp\n   8007f: c3                            ret\n\nDisassembly of section .data:\n\nffff000000010008 global_data_word:\nffff000000010008: 44 33 22 11\nffff00000001000c: 00 00 00 00\n```\n\nAnd now linked to `0xffff_0000_0000_0000`:\n\n```c-objdump\nDisassembly of section .text:\n\nffff000000000070 get_address_of_global:\nffff000000000070: 55                            push    rbp\nffff000000000071: 48 89 e5                      mov     rbp, rsp\nffff000000000074: 48 b8 08 00 01 00 00 00 ff ff movabs  rax, 0xffff000000010008\nffff00000000007e: 5d                            pop     rbp\nffff00000000007f: c3                            ret\n\nDisassembly of section .data:\n\nffff000000010008 global_data_word:\nffff000000010008: 44 33 22 11\nffff00000001000c: 00 00 00 00\n```\n\nBoth times, the `movabs` instruction gets emitted. It means that the address is put into the target\nregister using hardcoded `immediate values`. PC-relative address calculation is not used here.\nHence, this code would return the `absolute` address in both cases. Which means in the second case,\neven when the binary would be loaded at `0x8_0000`, the return value would be\n`0xffff_0000_0001_0008`.\n\n**In summary, we get a different result for the same piece of `C` code, depending on the target\nprocessor architecture**. What do we learn from this little detour?\n\nFirst, you cannot naively compile and run `Rust` or `C` statically linked binaries when there will\nbe a **load address != link address** situation. You'll run into undefined behavior very fast. It is\nkinda expected and obvious, but hopefully it helped to see it fail in action.\n\nFurthermore, it is important to understand that there are of course ways to load a symbol's absolute\naddress into `AArch64` registers using `immediate values` as well. Likewise, you can also do\nPC-relative addressing in `x86`. We just looked at a tiny example. Maybe the next line of code would\nbe compiled into the opposite behavior on the two architectures, so that the `x86` code would do a\nPC-relative calculation while the `AArch64` code goes for absolute.\n\nAt the end of the day, what is needed to solve our task at hand (bringup of virtual memory, while\nbeing linked to one address and executing from another), is tight control over the machine\ninstructions that get emitted for **those pieces of code** that generate the `translation tables`\nand enable the `MMU`.\n\nWhat we need is called [position-independent code].\n\n[position-independent code]: https://en.wikipedia.org/wiki/Position-independent_code\n\n> Much low-level stuff in this tutorial, isn't it? This was a lot to digest already, but we're far\n> from finished. So take a minute or two and clear your mind before we continue. 🧘\n\n## Position-Independent Code (PIC)\n\nAs describend by Wikipedia, position-independent code\n\n> is a body of machine code that, being placed somewhere in the primary memory, **executes\n> properly** regardless of its absolute address.\n\nYour safest bet is to write the pieces that need to be position-independent in `assembly`, because\nthis gives you full control over when relative or absolute addresses are being generated or used.\nYou will see this approach often in big projects like the Linux kernel, for example. The downside of\nthat approach is that the programmer needs good domain knowledge.\n\nIf you feel more adventurous and don't want to go completely without high-level code, you can try to\nmake use of suitable compiler flags such as `-fpic`, and only use `assembly` where absolutely\nneeded. Here is the [`-fpic` description for GCC]:\n\n> -fpic\n>\n> Generate position-independent code (PIC) suitable for use in a shared library, if supported for\n> the target machine. Such code accesses all constant addresses through a global offset table (GOT).\n> The dynamic loader resolves the GOT entries when the program starts (the dynamic loader is not\n> part of GCC; it is part of the operating system).\n\n[`-fpic` description for GCC]: https://gcc.gnu.org/onlinedocs/gcc/Code-Gen-Options.html#Code-Gen-Options\n\nHowever, it is very important to understand that this flag **is not** a ready-made solution for our\nparticular problem (and wasn't invented for that case either). There is a hint in the quoted text\nabove that gives it away: \"_The dynamic loader resolves the GOT entries when the program starts (the\ndynamic loader is not part of GCC; it is part of the operating system)_\".\n\nWell, we are a booting kernel, and not some (userspace) program running on top of an operating\nsystem. Hence, there is no dynamic loader available. However, it is still possible to benefit from\n`-fpic` even in our case. Lets have a look at what happens if we compile the earlier piece of `C`\ncode for `AArch64` using `-fpic`, still linking the output to the most signifcant part of the memory\nspace:\n\n```console\n$ clang --target=aarch64-none-elf -Iinclude -Wall -fpic -c start.c -o start.o\n$ ld.lld start.o -T kernel.ld -o example.elf\n```\n\n```c-objdump\nDisassembly of section .text:\n\nffff000000000010 get_address_of_global:\nffff000000000010: 00 00 00 90          \tadrp\tx0, #0x0\nffff000000000014: 00 28 40 f9          \tldr\tx0, [x0, #0x50]\nffff000000000018: c0 03 5f d6          \tret\n\nDisassembly of section .got:\n\nffff000000000050 .got:\nffff000000000050: 08 00 01 00\nffff000000000054: 00 00 ff ff\n\nDisassembly of section .data:\n\nffff000000010008 global_data_word:\nffff000000010008: 44 33 22 11\nffff00000001000c: 00 00 00 00\n```\n\nWhat changed compared to earlier is that `get_address_of_global()` now indirects through the `Global\nOffset Table`, as has been promised by the compiler's documentation. Specifically,\n`get_address_of_global()` addresses the `GOT` using PC-relative addressing (distance from code to\n`GOT` must always be fixed), and loads the first 64 bit word from the start of the `GOT`, which\nhappens to be `0xffff_0000_0001_0008`.\n\nOkay okay okay... So when we use `-fpic`, we get the **absolute** address of `global_data_word` even\non `AArch64` now. How does this help when the code executes from `0x8_0000`?\n\nWell, this is the part where the `dynamic loader` quoted above would come into picture if this was a\nuserspace program: \"_The dynamic loader resolves the GOT entries when the program starts_\". The\n`-fpic` flag is normally used to compile _shared libraries_. Suppose we have a program that uses one\nor more shared library. For various reasons, it happens that the shared library is loaded at a\ndifferent address than where the userspace program would initially expect it. In our example,\n`global_data_word` could be supplied by such a shared library, and the userspace program is only\nreferencing it. The dynamic loader would know where the shared library was loaded into memory, and\ntherefore know the real address of `global_data_word`. So before the userspace program starts, the\nloader would overwrite the `GOT` entry with the correct location. Et voilà, the compiled high-level\ncode would execute properly.\n\n### Using PIC during kernel startup\n\nIf you think about it, our problem is a special case of what we just learned. We have a single\nstatically linked binary, where everything is dislocated by a fixed offset. In our case, it is\n`0xffff_0000_0000_0000 - 0x8_0000 = 0x0fff_efff_ffff8_0000`. If we write some PIC-`assembly` code\nwhich loops over the `GOT` and subtracts `0x0fff_efff_ffff8_0000` from every entry as the very first\nthing when our kernel boots, any high-level code compiled with `-fpic` would work correctly\nafterwards.\n\nMoreover, this approach would be portable! Here's the output of our code compiled with `-fpic` for\n`x86_64`:\n\n```c-objdump\nDisassembly of section .text:\n\nffff000000000070 get_address_of_global:\nffff000000000070: 55                    push    rbp\nffff000000000071: 48 89 e5              mov     rbp, rsp\nffff000000000074: 48 8b 05 2d 00 00 00  mov     rax, qword ptr [rip + 0x2d]\nffff00000000007b: 5d                    pop     rbp\nffff00000000007c: c3                    ret\nffff00000000007d: 0f 1f 00              nop     dword ptr [rax]\n\nDisassembly of section .got:\n\nffff0000000000a8 .got:\nffff0000000000a8: 08 00 01 00\nffff0000000000ac: 00 00 ff ff\n\nDisassembly of section .data:\n\nffff000000010008 global_data_word:\nffff000000010008: 44 33 22 11\nffff00000001000c: 00 00 00 00\n```\n\nAs you can see, the `x86_64` code indirects through the `GOT` now same as the `AArch64` code.\n\nOf course, indirecting through the `GOT` would be detrimental to performance, so you would restrict\n`-fpic` compilation only to the code that is needed to enable the `MMU`. Everything else can be\ncompiled `non-relocatable` as always, because the translation tables naturally resolve the **load\naddress != link address** situation once they are live.\n\nWith `C/C++` compilers, this can be done rather easily. The compilers support compilation of\nPIC-code on a per-[translation-unit] basis. Think of it as telling the compiler to compile this `.c`\nfile as `PIC`, but this other `.c` file not.\n\n[translation-unit]: https://en.wikipedia.org/wiki/Translation_unit_(programming)\n\nWith `Rust`, unfortunately, the [relocation model] can only be set on a per-`crate` basis at the\nmoment (IINM), so that makes it difficult for us to put this approach to use.\n\n[relocation model]: https://doc.rust-lang.org/rustc/codegen-options/index.html#relocation-model\n\n## Precomputed Translation Tables\n\nAs we have just seen, going the `-fpic` way isn't a feasible solution at the time of writing this\ntext. On the other hand, writing the code to set up the initial page tables in `assembly` isn't that\nattractive either, because writing larger pieces of assembly is an error-prone and delicate task.\n\nFortunately, there is a third way. We are writing an embedded kernel, and therefore the execution\nenvironment is way more static and deterministic as compared to a general-purpose kernel that can be\ndeployed on a wide variety of targets. Specifically, for the Raspberrypi, we exactly know the **load\naddress** of the kernel in advance, and we know about the capabilities of the `MMU`. So there is\nnothing stopping us from precomputing the kernel's translation tables ahead of time.\n\nA disadvantage of this approach is an increased binary size, but this is not a deal breaker in our\ncase.\n\n## Implementation\n\nAs stated in the initial `tl;dr`, we're not yet mapping the kernel to the most significant area of\nvirtual memory. This tutorial will keep the binary `identity-mapped`, and focuses only on the\ninfrastructure changes which enable the kernel to use `precomputed translation tables`. The actual\nswitch to high memory will happen in the next tutorial.\n\nThe changes needed are as follows:\n\n1. Make preparations so that precomputed tables are supported by the kernel's memory subsystem code.\n2. Change the boot code of the kernel so that the `MMU` is enabled with the precomputed tables as\n   soon as possible.\n3. Write a `translation table tool` that precomputes the translation tables from the generated\n   `kernel.elf` file, and patches the tables back into the same.\n\n### Preparing the Kernel Tables\n\nThe tables must be linked into the `.data` section now so that they become part of the final binary.\nThis is ensured using an attribute on the table's instance definition in\n`bsp/__board_name__/memory/mmu.rs`:\n\n```rust\n#[link_section = \".data\"]\n#[no_mangle]\nstatic KERNEL_TABLES: InitStateLock<KernelTranslationTable> =\n    InitStateLock::new(KernelTranslationTable::new_for_precompute());\n```\n\nThe `new_for_precompute()` is a new constructor in the the respective `_arch` code that ensures some\nstruct members that are not the translation table entries themselves are initialized properly for\nthe precompute use-case. The additional `#[no_mangle]` is added because we will need to parse the\nsymbol from the `translation table tool`, and this is easier with unmangled names.\n\nIn the `BSP` code, there is also a new file called `kernel_virt_addr_space_size.ld`, which contains\nthe kernel's virtual address space size. This file gets included in both, the `kernel.ld` linker\nscript and `mmu.rs`. We need this value both as a symbol in the kernel's ELF (for the `translation\ntable tool` to parse it later) and as a constant in the `Rust` code. This inclusion approach is just\na convenience hack that turned out working well.\n\nOne critical parameter that the kernel's boot code needs in order to enable the precomputed tables\nis the `translation table base address` which must be programmed into the MMU's `TTBR` register. To\nmake it accessible easily, it is added to the `.text._start_arguments` section. The definition is\njust below the definition of the kernel table instance in the `BSP` code:\n\n```rust\n/// This value is needed during early boot for MMU setup.\n///\n/// This will be patched to the correct value by the \"translation table tool\" after linking. This\n/// given value here is just a dummy.\n#[link_section = \".text._start_arguments\"]\n#[no_mangle]\nstatic PHYS_KERNEL_TABLES_BASE_ADDR: u64 = 0xCCCCAAAAFFFFEEEE;\n```\n\n### Turning on the MMU Before Switching to EL1\n\nSince the Raspberry Pi starts execution in the `EL2` privilege level, one of the first things we do\nduring boot since `tutorial 09` is to context-switch to the appropriate `EL1`. The `EL2` boot code\nis a great place to set up virtual memory for `EL1`. It will allow execution in `EL1` to start with\nvirtual memory enabled since the very first instruction. The tweaks to `boot.s` are minimal:\n\n```asm\n// Load the base address of the kernel's translation tables.\nldr\tx0, PHYS_KERNEL_TABLES_BASE_ADDR // provided by bsp/__board_name__/memory/mmu.rs\n\n// Set the stack pointer. This ensures that any code in EL2 that needs the stack will work.\nADR_REL\tx1, __boot_core_stack_end_exclusive\nmov\tsp, x1\n\n// Jump to Rust code. x0 and x1 hold the function arguments provided to _start_rust().\nb\t_start_rust\n```\n\nIn addition to the stack's address, we are now reading _the value_ of\n`PHYS_KERNEL_TABLES_BASE_ADDR`. The `ldr` instruction addresses the value-to-be-read using a\nPC-relative offset, so this is a `position-independent` operation and will therefore be future\nproof. The retrieved value is supplied as an argument to function `_start_rust()`, which is defined\nin `_arch/__arch_name__/cpu/boot.rs`:\n\n```rust\n#[no_mangle]\npub unsafe extern \"C\" fn _start_rust(\n    phys_kernel_tables_base_addr: u64,\n    phys_boot_core_stack_end_exclusive_addr: u64,\n) -> ! {\n    prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr);\n\n    // Turn on the MMU for EL1.\n    let addr = Address::new(phys_kernel_tables_base_addr as usize);\n    memory::mmu::enable_mmu_and_caching(addr).unwrap();\n\n    // Use `eret` to \"return\" to EL1. This results in execution of kernel_init() in EL1.\n    asm::eret()\n}\n```\n\nYou can also see that we now turn on the `MMU` just before returning to `EL1`. That's basically it\nalready, the only missing piece that's left is the offline computation of the translation tables.\n\n### The Translation Table Tool\n\nThe tool for translation table computation is located in the folder\n`$ROOT/tools/translation_table_tool`. For ease of use, it is written in `Ruby` 💎. The code is\norganized into `BSP` and `arch` parts just like the kernel's `Rust` code, and also has a class for\nprocessing the kernel `ELF` file:\n\n```console\n$ tree tools/translation_table_tool\n\ntools/translation_table_tool\n├── arch.rb\n├── bsp.rb\n├── generic.rb\n├── kernel_elf.rb\n└── main.rb\n\n0 directories, 5 files\n```\n\nEspecially the `arch` part, which deals with compiling the translation table entries, will contain\nsome overlap with the `Rust` code present in `_arch/aarch64/memory/mmu/translation_table.rs`. It\nmight have been possible to write this tool in Rust as well, and borrow/share these pieces of code\nwith the kernel. But in the end, I found it not worth the effort for the few lines of code.\n\nIn the `Makefile`, the tool is invoked after compiling and linking the kernel, and before the\n`stripped binary` is generated. It's command line arguments are the target `BSP` type and the path\nto the kernel's `ELF` file:\n\n```Makefile\nTT_TOOL_PATH = tools/translation_table_tool\n\nKERNEL_ELF_RAW      = target/$(TARGET)/release/kernel\n# [...]\n\nKERNEL_ELF_TTABLES      = target/$(TARGET)/release/kernel+ttables\n# [...]\n\nEXEC_TT_TOOL       = ruby $(TT_TOOL_PATH)/main.rb\n# [...]\n\n##------------------------------------------------------------------------------\n## Compile the kernel ELF\n##------------------------------------------------------------------------------\n$(KERNEL_ELF_RAW): $(KERNEL_ELF_RAW_DEPS)\n\t$(call color_header, \"Compiling kernel ELF - $(BSP)\")\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(RUSTC_CMD)\n\n##------------------------------------------------------------------------------\n## Precompute the kernel translation tables and patch them into the kernel ELF\n##------------------------------------------------------------------------------\n$(KERNEL_ELF_TTABLES): $(KERNEL_ELF_TTABLES_DEPS)\n\t$(call color_header, \"Precomputing kernel translation tables and patching kernel ELF\")\n\t@cp $(KERNEL_ELF_RAW) $(KERNEL_ELF_TTABLES)\n\t@$(DOCKER_TOOLS) $(EXEC_TT_TOOL) $(TARGET) $(BSP) $(KERNEL_ELF_TTABLES)\n\n##------------------------------------------------------------------------------\n## Generate the stripped kernel binary\n##------------------------------------------------------------------------------\n$(KERNEL_BIN): $(KERNEL_ELF_TTABLES)\n\t$(call color_header, \"Generating stripped binary\")\n\t@$(OBJCOPY_CMD) $(KERNEL_ELF_TTABLES) $(KERNEL_BIN)\n```\n\nIn `main.rb`, the `KERNEL_ELF` instance for handling the `ELF` file is created first, followed by\n`BSP` and `arch` parts:\n\n```ruby\nKERNEL_ELF = KernelELF.new(kernel_elf_path)\n\nBSP = case BSP_TYPE\n      when :rpi3, :rpi4\n          RaspberryPi.new\n      else\n          raise\n      end\n\nTRANSLATION_TABLES = case KERNEL_ELF.machine\n                     when :AArch64\n                         Arch::ARMv8::TranslationTable.new\n                     else\n                         raise\n                     end\n\nkernel_map_binary\n```\n\nFinally, the function `kernel_map_binary` is called, which kicks of a sequence of interactions\nbetween the `KERNEL_ELF`, `BSP` and `TRANSLATION_TABLES` instances:\n\n```ruby\ndef kernel_map_binary\n    mapping_descriptors = KERNEL_ELF.generate_mapping_descriptors\n\n    # omitted\n\n    mapping_descriptors.each do |i|\n        print 'Generating'.rjust(12).green.bold\n        print ' '\n        puts i\n\n        TRANSLATION_TABLES.map_at(i.virt_region, i.phys_region, i.attributes)\n    end\n\n    # omitted\nend\n```\n\nThe `generate_mapping_descriptors` method internally uses the\n[rbelftools](https://github.com/david942j/rbelftools) gem to parse the kernel's `ELF`. It extracts\ninformation about the various segments that comprise the kernel, as well as segment characteristics\nlike their `virtual` and `physical` addresses (aka the mapping; still identity-mapped in this case)\nand the memory attributes.\n\nPart of this information can be cross-checked using the `make readelf` target if you're curious:\n\n```console\n$ make readelf\n\nProgram Headers:\n  Type           Offset             VirtAddr           PhysAddr\n                 FileSiz            MemSiz              Flags  Align\n  LOAD           0x0000000000010000 0x0000000000000000 0x0000000000000000\n                 0x0000000000000000 0x0000000000080000  RW     0x10000\n  LOAD           0x0000000000010000 0x0000000000080000 0x0000000000080000\n                 0x000000000000cae8 0x000000000000cae8  R E    0x10000\n  LOAD           0x0000000000020000 0x0000000000090000 0x0000000000090000\n                 0x0000000000030dc0 0x0000000000030de0  RW     0x10000\n\n Section to Segment mapping:\n  Segment Sections...\n   00     .boot_core_stack\n   01     .text .rodata\n   02     .data .bss\n\n```\n\nThe output of `generate_mapping_descriptors` is then fed into the `map_at()` method of the\n`TRANSLATION_TABLE` instance. For it to work properly, `TRANSLATION_TABLE` needs knowledge about\n**location** and **layout** of the kernel's table structure. Location will be queried from the `BSP`\ncode, which itself retrieves it by querying `KERNEL_ELF` for the `BSP`-specific `KERNEL_TABLES`\nsymbol. The layout, on the other hand, is hardcoded and as such must be kept in sync with the\nstructure definition in `translation_tables.rs`.\n\nFinally, after the mappings have been created, it is time to _patch_ them back into the kernel ELF\nfile. This is initiated from `main.rb` again:\n\n```ruby\nkernel_patch_tables(kernel_elf_path)\nkernel_patch_base_addr(kernel_elf_path)\n```\n\nThe tool prints some information on the fly. Here's the console output of a successful run:\n\n```console\n$ make\n\nCompiling kernel - rpi3\n    Finished release [optimized] target(s) in 0.00s\n\nPrecomputing kernel translation tables and patching kernel ELF\n             ------------------------------------------------------------------------------------\n                 Sections          Virt Start Addr         Phys Start Addr       Size      Attr\n             ------------------------------------------------------------------------------------\n  Generating .boot_core_stack | 0x0000_0000_0000_0000 | 0x0000_0000_0000_0000 | 512 KiB | C RW XN\n  Generating .text .rodata    | 0x0000_0000_0008_0000 | 0x0000_0000_0008_0000 |  64 KiB | C RO X\n  Generating .data .bss       | 0x0000_0000_0009_0000 | 0x0000_0000_0009_0000 | 256 KiB | C RW XN\n             ------------------------------------------------------------------------------------\n    Patching Kernel table struct at ELF file offset 0x2_0000\n    Patching Kernel tables physical base address start argument to value 0xb_0000 at ELF file offset 0x1_0068\n    Finished in 0.16s\n\n```\n\nPlease note how **only** the kernel binary is precomputed! Thanks to the changes made in the last\ntutorial, anything else, like `MMIO-remapping`, can and will happen lazily during runtime.\n\n### Other changes\n\nTwo more things that changed in this tutorial, but won't be explained in detail:\n\n- Since virtual memory in `EL1` is now active from the start, any attempt to convert from a kernel\n  `Address<Virtual>` to `Address<Physical>` is now done using the function\n  `mmu::try_kernel_virt_addr_to_phys_addr()`, which internally uses a new function that has been\n  added to the `TranslationTable` interface. If there is no valid virtual-to-physical mapping\n  present in the tables, an `Err()` is returned.\n- The precomputed translation table mappings won't automatically have entries in the kernel's\n  `mapping info record`, which is used to print mapping info during boot. Mapping record entries are\n  not computed offline in order to reduce complexity. For this reason, the `BSP` code, which in\n  earlier tutorials would have called `kernel_map_at()` (which implicitly would have generated\n  mapping record entries), now only calls `kernel_add_mapping_record()`, since the mappings are\n  already in place.\n\n## Discussion\n\nIt is understood that there is room for optimizations in the presented approach. For example, the\ngenerated kernel binary contains the _complete_ array of translation tables for the whole kernel\nvirtual address space. However, most of the entries are empty initially, because the kernel binary\nonly occupies a small area in the tables. It would make sense to add some smarts so that only the\nnon-zero entries are packed into binary.\n\nOn the other hand, this would add complexity to the code. The increased size doesn't hurt too much\nyet, so the reduced complexity in the code is a tradeoff made willingly to keep everything concise\nand focused on the high-level concepts.\n\n## Test it\n\n```console\n$ make chainboot\n[...]\n\nPrecomputing kernel translation tables and patching kernel ELF\n             ------------------------------------------------------------------------------------\n                 Sections          Virt Start Addr         Phys Start Addr       Size      Attr\n             ------------------------------------------------------------------------------------\n  Generating .boot_core_stack | 0x0000_0000_0000_0000 | 0x0000_0000_0000_0000 | 512 KiB | C RW XN\n  Generating .text .rodata    | 0x0000_0000_0008_0000 | 0x0000_0000_0008_0000 |  64 KiB | C RO X\n  Generating .data .bss       | 0x0000_0000_0009_0000 | 0x0000_0000_0009_0000 | 256 KiB | C RW XN\n             ------------------------------------------------------------------------------------\n    Patching Kernel table struct at ELF file offset 0x2_0000\n    Patching Kernel tables physical base address start argument to value 0xb_0000 at ELF file offset 0x1_0068\n    Finished in 0.15s\n\nMinipush 1.0\n\n[MP] ⏳ Waiting for /dev/ttyUSB0\n[MP] ✅ Serial connected\n[MP] 🔌 Please power the target now\n\n __  __ _      _ _                 _\n|  \\/  (_)_ _ (_) |   ___  __ _ __| |\n| |\\/| | | ' \\| | |__/ _ \\/ _` / _` |\n|_|  |_|_|_||_|_|____\\___/\\__,_\\__,_|\n\n           Raspberry Pi 3\n\n[ML] Requesting binary\n[MP] ⏩ Pushing 257 KiB ======================================🦀 100% 128 KiB/s Time: 00:00:02\n[ML] Loaded! Executing the payload now\n\n[    2.866917] mingo version 0.15.0\n[    2.867125] Booting on: Raspberry Pi 3\n[    2.867580] MMU online:\n[    2.867872]       -------------------------------------------------------------------------------------------------------------------------------------------\n[    2.869616]                         Virtual                                   Physical               Size       Attr                    Entity\n[    2.871360]       -------------------------------------------------------------------------------------------------------------------------------------------\n[    2.873105]       0x0000_0000_0000_0000..0x0000_0000_0007_ffff --> 0x00_0000_0000..0x00_0007_ffff | 512 KiB | C   RW XN | Kernel boot-core stack\n[    2.874709]       0x0000_0000_0008_0000..0x0000_0000_0008_ffff --> 0x00_0008_0000..0x00_0008_ffff |  64 KiB | C   RO X  | Kernel code and RO data\n[    2.876322]       0x0000_0000_0009_0000..0x0000_0000_000c_ffff --> 0x00_0009_0000..0x00_000c_ffff | 256 KiB | C   RW XN | Kernel data and bss\n[    2.877893]       0x0000_0000_000d_0000..0x0000_0000_000d_ffff --> 0x00_3f20_0000..0x00_3f20_ffff |  64 KiB | Dev RW XN | BCM PL011 UART\n[    2.879410]                                                                                                             | BCM GPIO\n[    2.880861]       0x0000_0000_000e_0000..0x0000_0000_000e_ffff --> 0x00_3f00_0000..0x00_3f00_ffff |  64 KiB | Dev RW XN | BCM Interrupt Controller\n[    2.882487]       -------------------------------------------------------------------------------------------------------------------------------------------\n```\n\n## Diff to previous\n```diff\n\ndiff -uNr 14_virtual_mem_part2_mmio_remap/kernel/Cargo.toml 15_virtual_mem_part3_precomputed_tables/kernel/Cargo.toml\n--- 14_virtual_mem_part2_mmio_remap/kernel/Cargo.toml\n+++ 15_virtual_mem_part3_precomputed_tables/kernel/Cargo.toml\n@@ -1,6 +1,6 @@\n [package]\n name = \"mingo\"\n-version = \"0.14.0\"\n+version = \"0.15.0\"\n authors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\n edition = \"2021\"\n\n\ndiff -uNr 14_virtual_mem_part2_mmio_remap/kernel/src/_arch/aarch64/cpu/boot.rs 15_virtual_mem_part3_precomputed_tables/kernel/src/_arch/aarch64/cpu/boot.rs\n--- 14_virtual_mem_part2_mmio_remap/kernel/src/_arch/aarch64/cpu/boot.rs\n+++ 15_virtual_mem_part3_precomputed_tables/kernel/src/_arch/aarch64/cpu/boot.rs\n@@ -11,6 +11,7 @@\n //!\n //! crate::cpu::boot::arch_boot\n\n+use crate::{memory, memory::Address};\n use aarch64_cpu::{asm, registers::*};\n use core::arch::global_asm;\n use tock_registers::interfaces::Writeable;\n@@ -75,9 +76,16 @@\n ///\n /// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`.\n #[no_mangle]\n-pub unsafe extern \"C\" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! {\n+pub unsafe extern \"C\" fn _start_rust(\n+    phys_kernel_tables_base_addr: u64,\n+    phys_boot_core_stack_end_exclusive_addr: u64,\n+) -> ! {\n     prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr);\n\n+    // Turn on the MMU for EL1.\n+    let addr = Address::new(phys_kernel_tables_base_addr as usize);\n+    memory::mmu::enable_mmu_and_caching(addr).unwrap();\n+\n     // Use `eret` to \"return\" to EL1. This results in execution of kernel_init() in EL1.\n     asm::eret()\n }\n\ndiff -uNr 14_virtual_mem_part2_mmio_remap/kernel/src/_arch/aarch64/cpu/boot.s 15_virtual_mem_part3_precomputed_tables/kernel/src/_arch/aarch64/cpu/boot.s\n--- 14_virtual_mem_part2_mmio_remap/kernel/src/_arch/aarch64/cpu/boot.s\n+++ 15_virtual_mem_part3_precomputed_tables/kernel/src/_arch/aarch64/cpu/boot.s\n@@ -53,19 +53,22 @@\n\n \t// Prepare the jump to Rust code.\n .L_prepare_rust:\n+\t// Load the base address of the kernel's translation tables.\n+\tldr\tx0, PHYS_KERNEL_TABLES_BASE_ADDR // provided by bsp/__board_name__/memory/mmu.rs\n+\n \t// Set the stack pointer. This ensures that any code in EL2 that needs the stack will work.\n-\tADR_REL\tx0, __boot_core_stack_end_exclusive\n-\tmov\tsp, x0\n+\tADR_REL\tx1, __boot_core_stack_end_exclusive\n+\tmov\tsp, x1\n\n \t// Read the CPU's timer counter frequency and store it in ARCH_TIMER_COUNTER_FREQUENCY.\n \t// Abort if the frequency read back as 0.\n-\tADR_REL\tx1, ARCH_TIMER_COUNTER_FREQUENCY // provided by aarch64/time.rs\n-\tmrs\tx2, CNTFRQ_EL0\n-\tcmp\tx2, xzr\n+\tADR_REL\tx2, ARCH_TIMER_COUNTER_FREQUENCY // provided by aarch64/time.rs\n+\tmrs\tx3, CNTFRQ_EL0\n+\tcmp\tx3, xzr\n \tb.eq\t.L_parking_loop\n-\tstr\tw2, [x1]\n+\tstr\tw3, [x2]\n\n-\t// Jump to Rust code. x0 holds the function argument provided to _start_rust().\n+\t// Jump to Rust code. x0 and x1 hold the function arguments provided to _start_rust().\n \tb\t_start_rust\n\n \t// Infinitely wait for events (aka \"park the core\").\n\ndiff -uNr 14_virtual_mem_part2_mmio_remap/kernel/src/_arch/aarch64/memory/mmu/translation_table.rs 15_virtual_mem_part3_precomputed_tables/kernel/src/_arch/aarch64/memory/mmu/translation_table.rs\n--- 14_virtual_mem_part2_mmio_remap/kernel/src/_arch/aarch64/memory/mmu/translation_table.rs\n+++ 15_virtual_mem_part3_precomputed_tables/kernel/src/_arch/aarch64/memory/mmu/translation_table.rs\n@@ -125,7 +125,7 @@\n }\n\n trait StartAddr {\n-    fn phys_start_addr(&self) -> Address<Physical>;\n+    fn virt_start_addr(&self) -> Address<Virtual>;\n }\n\n //--------------------------------------------------------------------------------------------------\n@@ -151,9 +151,8 @@\n // Private Code\n //--------------------------------------------------------------------------------------------------\n\n-// The binary is still identity mapped, so we don't need to convert here.\n impl<T, const N: usize> StartAddr for [T; N] {\n-    fn phys_start_addr(&self) -> Address<Physical> {\n+    fn virt_start_addr(&self) -> Address<Virtual> {\n         Address::new(self as *const _ as usize)\n     }\n }\n@@ -218,6 +217,35 @@\n     }\n }\n\n+/// Convert the HW-specific attributes of the MMU to kernel's generic memory attributes.\n+impl convert::TryFrom<InMemoryRegister<u64, STAGE1_PAGE_DESCRIPTOR::Register>> for AttributeFields {\n+    type Error = &'static str;\n+\n+    fn try_from(\n+        desc: InMemoryRegister<u64, STAGE1_PAGE_DESCRIPTOR::Register>,\n+    ) -> Result<AttributeFields, Self::Error> {\n+        let mem_attributes = match desc.read(STAGE1_PAGE_DESCRIPTOR::AttrIndx) {\n+            memory::mmu::arch_mmu::mair::NORMAL => MemAttributes::CacheableDRAM,\n+            memory::mmu::arch_mmu::mair::DEVICE => MemAttributes::Device,\n+            _ => return Err(\"Unexpected memory attribute\"),\n+        };\n+\n+        let acc_perms = match desc.read_as_enum(STAGE1_PAGE_DESCRIPTOR::AP) {\n+            Some(STAGE1_PAGE_DESCRIPTOR::AP::Value::RO_EL1) => AccessPermissions::ReadOnly,\n+            Some(STAGE1_PAGE_DESCRIPTOR::AP::Value::RW_EL1) => AccessPermissions::ReadWrite,\n+            _ => return Err(\"Unexpected access permission\"),\n+        };\n+\n+        let execute_never = desc.read(STAGE1_PAGE_DESCRIPTOR::PXN) > 0;\n+\n+        Ok(AttributeFields {\n+            mem_attributes,\n+            acc_perms,\n+            execute_never,\n+        })\n+    }\n+}\n+\n impl PageDescriptor {\n     /// Create an instance.\n     ///\n@@ -250,6 +278,19 @@\n         InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(self.value)\n             .is_set(STAGE1_PAGE_DESCRIPTOR::VALID)\n     }\n+\n+    /// Returns the output page.\n+    fn output_page_addr(&self) -> PageAddress<Physical> {\n+        let shifted = InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(self.value)\n+            .read(STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB) as usize;\n+\n+        PageAddress::from(shifted << Granule64KiB::SHIFT)\n+    }\n+\n+    /// Returns the attributes.\n+    fn try_attributes(&self) -> Result<AttributeFields, &'static str> {\n+        InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(self.value).try_into()\n+    }\n }\n\n //--------------------------------------------------------------------------------------------------\n@@ -267,7 +308,7 @@\n impl<const NUM_TABLES: usize> FixedSizeTranslationTable<NUM_TABLES> {\n     /// Create an instance.\n     #[allow(clippy::assertions_on_constants)]\n-    pub const fn new() -> Self {\n+    const fn _new(for_precompute: bool) -> Self {\n         assert!(bsp::memory::mmu::KernelGranule::SIZE == Granule64KiB::SIZE);\n\n         // Can't have a zero-sized address space.\n@@ -276,10 +317,19 @@\n         Self {\n             lvl3: [[PageDescriptor::new_zeroed(); 8192]; NUM_TABLES],\n             lvl2: [TableDescriptor::new_zeroed(); NUM_TABLES],\n-            initialized: false,\n+            initialized: for_precompute,\n         }\n     }\n\n+    pub const fn new_for_precompute() -> Self {\n+        Self::_new(true)\n+    }\n+\n+    #[cfg(test)]\n+    pub fn new_for_runtime() -> Self {\n+        Self::_new(false)\n+    }\n+\n     /// Helper to calculate the lvl2 and lvl3 indices from an address.\n     #[inline(always)]\n     fn lvl2_lvl3_index_from_page_addr(\n@@ -297,6 +347,18 @@\n         Ok((lvl2_index, lvl3_index))\n     }\n\n+    /// Returns the PageDescriptor corresponding to the supplied page address.\n+    #[inline(always)]\n+    fn page_descriptor_from_page_addr(\n+        &self,\n+        virt_page_addr: PageAddress<Virtual>,\n+    ) -> Result<&PageDescriptor, &'static str> {\n+        let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from_page_addr(virt_page_addr)?;\n+        let desc = &self.lvl3[lvl2_index][lvl3_index];\n+\n+        Ok(desc)\n+    }\n+\n     /// Sets the PageDescriptor corresponding to the supplied page address.\n     ///\n     /// Doesn't allow overriding an already valid page.\n@@ -325,24 +387,23 @@\n impl<const NUM_TABLES: usize> memory::mmu::translation_table::interface::TranslationTable\n     for FixedSizeTranslationTable<NUM_TABLES>\n {\n-    fn init(&mut self) {\n+    fn init(&mut self) -> Result<(), &'static str> {\n         if self.initialized {\n-            return;\n+            return Ok(());\n         }\n\n         // Populate the l2 entries.\n         for (lvl2_nr, lvl2_entry) in self.lvl2.iter_mut().enumerate() {\n-            let phys_table_addr = self.lvl3[lvl2_nr].phys_start_addr();\n+            let virt_table_addr = self.lvl3[lvl2_nr].virt_start_addr();\n+            let phys_table_addr = memory::mmu::try_kernel_virt_addr_to_phys_addr(virt_table_addr)?;\n\n             let new_desc = TableDescriptor::from_next_lvl_table_addr(phys_table_addr);\n             *lvl2_entry = new_desc;\n         }\n\n         self.initialized = true;\n-    }\n\n-    fn phys_base_address(&self) -> Address<Physical> {\n-        self.lvl2.phys_start_addr()\n+        Ok(())\n     }\n\n     unsafe fn map_at(\n@@ -372,6 +433,45 @@\n\n         Ok(())\n     }\n+\n+    fn try_virt_page_addr_to_phys_page_addr(\n+        &self,\n+        virt_page_addr: PageAddress<Virtual>,\n+    ) -> Result<PageAddress<Physical>, &'static str> {\n+        let page_desc = self.page_descriptor_from_page_addr(virt_page_addr)?;\n+\n+        if !page_desc.is_valid() {\n+            return Err(\"Page marked invalid\");\n+        }\n+\n+        Ok(page_desc.output_page_addr())\n+    }\n+\n+    fn try_page_attributes(\n+        &self,\n+        virt_page_addr: PageAddress<Virtual>,\n+    ) -> Result<AttributeFields, &'static str> {\n+        let page_desc = self.page_descriptor_from_page_addr(virt_page_addr)?;\n+\n+        if !page_desc.is_valid() {\n+            return Err(\"Page marked invalid\");\n+        }\n+\n+        page_desc.try_attributes()\n+    }\n+\n+    /// Try to translate a virtual address to a physical address.\n+    ///\n+    /// Will only succeed if there exists a valid mapping for the input address.\n+    fn try_virt_addr_to_phys_addr(\n+        &self,\n+        virt_addr: Address<Virtual>,\n+    ) -> Result<Address<Physical>, &'static str> {\n+        let virt_page = PageAddress::from(virt_addr.align_down_page());\n+        let phys_page = self.try_virt_page_addr_to_phys_page_addr(virt_page)?;\n+\n+        Ok(phys_page.into_inner() + virt_addr.offset_into_page())\n+    }\n }\n\n //--------------------------------------------------------------------------------------------------\n\ndiff -uNr 14_virtual_mem_part2_mmio_remap/kernel/src/bsp/raspberrypi/kernel.ld 15_virtual_mem_part3_precomputed_tables/kernel/src/bsp/raspberrypi/kernel.ld\n--- 14_virtual_mem_part2_mmio_remap/kernel/src/bsp/raspberrypi/kernel.ld\n+++ 15_virtual_mem_part3_precomputed_tables/kernel/src/bsp/raspberrypi/kernel.ld\n@@ -3,6 +3,8 @@\n  * Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n  */\n\n+INCLUDE kernel_virt_addr_space_size.ld;\n+\n PAGE_SIZE = 64K;\n PAGE_MASK = PAGE_SIZE - 1;\n\n@@ -89,7 +91,7 @@\n     . += 8 * 1024 * 1024;\n     __mmio_remap_end_exclusive = .;\n\n-    ASSERT((. & PAGE_MASK) == 0, \"MMIO remap reservation is not page aligned\")\n+    ASSERT((. & PAGE_MASK) == 0, \"End of boot core stack is not page aligned\")\n\n     /***********************************************************************************************\n     * Misc\n\ndiff -uNr 14_virtual_mem_part2_mmio_remap/kernel/src/bsp/raspberrypi/kernel_virt_addr_space_size.ld 15_virtual_mem_part3_precomputed_tables/kernel/src/bsp/raspberrypi/kernel_virt_addr_space_size.ld\n--- 14_virtual_mem_part2_mmio_remap/kernel/src/bsp/raspberrypi/kernel_virt_addr_space_size.ld\n+++ 15_virtual_mem_part3_precomputed_tables/kernel/src/bsp/raspberrypi/kernel_virt_addr_space_size.ld\n@@ -0,0 +1 @@\n+__kernel_virt_addr_space_size = 1024 * 1024 * 1024\n\ndiff -uNr 14_virtual_mem_part2_mmio_remap/kernel/src/bsp/raspberrypi/memory/mmu.rs 15_virtual_mem_part3_precomputed_tables/kernel/src/bsp/raspberrypi/memory/mmu.rs\n--- 14_virtual_mem_part2_mmio_remap/kernel/src/bsp/raspberrypi/memory/mmu.rs\n+++ 15_virtual_mem_part3_precomputed_tables/kernel/src/bsp/raspberrypi/memory/mmu.rs\n@@ -7,8 +7,8 @@\n use crate::{\n     memory::{\n         mmu::{\n-            self as generic_mmu, AccessPermissions, AddressSpace, AssociatedTranslationTable,\n-            AttributeFields, MemAttributes, MemoryRegion, PageAddress, TranslationGranule,\n+            self as generic_mmu, AddressSpace, AssociatedTranslationTable, AttributeFields,\n+            MemoryRegion, PageAddress, TranslationGranule,\n         },\n         Physical, Virtual,\n     },\n@@ -31,7 +31,7 @@\n pub type KernelGranule = TranslationGranule<{ 64 * 1024 }>;\n\n /// The kernel's virtual address space defined by this BSP.\n-pub type KernelVirtAddrSpace = AddressSpace<{ 1024 * 1024 * 1024 }>;\n+pub type KernelVirtAddrSpace = AddressSpace<{ kernel_virt_addr_space_size() }>;\n\n //--------------------------------------------------------------------------------------------------\n // Global instances\n@@ -43,13 +43,35 @@\n ///\n /// That is, `size_of(InitStateLock<KernelTranslationTable>) == size_of(KernelTranslationTable)`.\n /// There is a unit tests that checks this porperty.\n+#[link_section = \".data\"]\n+#[no_mangle]\n static KERNEL_TABLES: InitStateLock<KernelTranslationTable> =\n-    InitStateLock::new(KernelTranslationTable::new());\n+    InitStateLock::new(KernelTranslationTable::new_for_precompute());\n+\n+/// This value is needed during early boot for MMU setup.\n+///\n+/// This will be patched to the correct value by the \"translation table tool\" after linking. This\n+/// given value here is just a dummy.\n+#[link_section = \".text._start_arguments\"]\n+#[no_mangle]\n+static PHYS_KERNEL_TABLES_BASE_ADDR: u64 = 0xCCCCAAAAFFFFEEEE;\n\n //--------------------------------------------------------------------------------------------------\n // Private Code\n //--------------------------------------------------------------------------------------------------\n\n+/// This is a hack for retrieving the value for the kernel's virtual address space size as a\n+/// constant from a common place, since it is needed as a compile-time/link-time constant in both,\n+/// the linker script and the Rust sources.\n+#[allow(clippy::needless_late_init)]\n+const fn kernel_virt_addr_space_size() -> usize {\n+    let __kernel_virt_addr_space_size;\n+\n+    include!(\"../kernel_virt_addr_space_size.ld\");\n+\n+    __kernel_virt_addr_space_size\n+}\n+\n /// Helper function for calculating the number of pages the given parameter spans.\n const fn size_to_num_pages(size: usize) -> usize {\n     assert!(size > 0);\n@@ -88,18 +110,22 @@\n     MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n }\n\n-// The binary is still identity mapped, so use this trivial conversion function for mapping below.\n-\n+// There is no reason to expect the following conversions to fail, since they were generated offline\n+// by the `translation table tool`. If it doesn't work, a panic due to the unwraps is justified.\n fn kernel_virt_to_phys_region(virt_region: MemoryRegion<Virtual>) -> MemoryRegion<Physical> {\n-    MemoryRegion::new(\n-        PageAddress::from(virt_region.start_page_addr().into_inner().as_usize()),\n-        PageAddress::from(\n-            virt_region\n-                .end_exclusive_page_addr()\n-                .into_inner()\n-                .as_usize(),\n-        ),\n-    )\n+    let phys_start_page_addr =\n+        generic_mmu::try_kernel_virt_page_addr_to_phys_page_addr(virt_region.start_page_addr())\n+            .unwrap();\n+\n+    let phys_end_exclusive_page_addr = phys_start_page_addr\n+        .checked_offset(virt_region.num_pages() as isize)\n+        .unwrap();\n+\n+    MemoryRegion::new(phys_start_page_addr, phys_end_exclusive_page_addr)\n+}\n+\n+fn kernel_page_attributes(virt_page_addr: PageAddress<Virtual>) -> AttributeFields {\n+    generic_mmu::try_kernel_page_attributes(virt_page_addr).unwrap()\n }\n\n //--------------------------------------------------------------------------------------------------\n@@ -121,109 +147,33 @@\n     MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n }\n\n-/// Map the kernel binary.\n+/// Add mapping records for the kernel binary.\n ///\n-/// # Safety\n-///\n-/// - Any miscalculation or attribute error will likely be fatal. Needs careful manual checking.\n-pub unsafe fn kernel_map_binary() -> Result<(), &'static str> {\n-    generic_mmu::kernel_map_at(\n+/// The actual translation table entries for the kernel binary are generated using the offline\n+/// `translation table tool` and patched into the kernel binary. This function just adds the mapping\n+/// record entries.\n+pub fn kernel_add_mapping_records_for_precomputed() {\n+    let virt_boot_core_stack_region = virt_boot_core_stack_region();\n+    generic_mmu::kernel_add_mapping_record(\n         \"Kernel boot-core stack\",\n-        &virt_boot_core_stack_region(),\n-        &kernel_virt_to_phys_region(virt_boot_core_stack_region()),\n-        &AttributeFields {\n-            mem_attributes: MemAttributes::CacheableDRAM,\n-            acc_perms: AccessPermissions::ReadWrite,\n-            execute_never: true,\n-        },\n-    )?;\n+        &virt_boot_core_stack_region,\n+        &kernel_virt_to_phys_region(virt_boot_core_stack_region),\n+        &kernel_page_attributes(virt_boot_core_stack_region.start_page_addr()),\n+    );\n\n-    generic_mmu::kernel_map_at(\n+    let virt_code_region = virt_code_region();\n+    generic_mmu::kernel_add_mapping_record(\n         \"Kernel code and RO data\",\n-        &virt_code_region(),\n-        &kernel_virt_to_phys_region(virt_code_region()),\n-        &AttributeFields {\n-            mem_attributes: MemAttributes::CacheableDRAM,\n-            acc_perms: AccessPermissions::ReadOnly,\n-            execute_never: false,\n-        },\n-    )?;\n+        &virt_code_region,\n+        &kernel_virt_to_phys_region(virt_code_region),\n+        &kernel_page_attributes(virt_code_region.start_page_addr()),\n+    );\n\n-    generic_mmu::kernel_map_at(\n+    let virt_data_region = virt_data_region();\n+    generic_mmu::kernel_add_mapping_record(\n         \"Kernel data and bss\",\n-        &virt_data_region(),\n-        &kernel_virt_to_phys_region(virt_data_region()),\n-        &AttributeFields {\n-            mem_attributes: MemAttributes::CacheableDRAM,\n-            acc_perms: AccessPermissions::ReadWrite,\n-            execute_never: true,\n-        },\n-    )?;\n-\n-    Ok(())\n-}\n-\n-//--------------------------------------------------------------------------------------------------\n-// Testing\n-//--------------------------------------------------------------------------------------------------\n-\n-#[cfg(test)]\n-mod tests {\n-    use super::*;\n-    use core::{cell::UnsafeCell, ops::Range};\n-    use test_macros::kernel_test;\n-\n-    /// Check alignment of the kernel's virtual memory layout sections.\n-    #[kernel_test]\n-    fn virt_mem_layout_sections_are_64KiB_aligned() {\n-        for i in [\n-            virt_boot_core_stack_region,\n-            virt_code_region,\n-            virt_data_region,\n-        ]\n-        .iter()\n-        {\n-            let start = i().start_page_addr().into_inner();\n-            let end_exclusive = i().end_exclusive_page_addr().into_inner();\n-\n-            assert!(start.is_page_aligned());\n-            assert!(end_exclusive.is_page_aligned());\n-            assert!(end_exclusive >= start);\n-        }\n-    }\n-\n-    /// Ensure the kernel's virtual memory layout is free of overlaps.\n-    #[kernel_test]\n-    fn virt_mem_layout_has_no_overlaps() {\n-        let layout = [\n-            virt_boot_core_stack_region(),\n-            virt_code_region(),\n-            virt_data_region(),\n-        ];\n-\n-        for (i, first_range) in layout.iter().enumerate() {\n-            for second_range in layout.iter().skip(i + 1) {\n-                assert!(!first_range.overlaps(second_range))\n-            }\n-        }\n-    }\n-\n-    /// Check if KERNEL_TABLES is in .bss.\n-    #[kernel_test]\n-    fn kernel_tables_in_bss() {\n-        extern \"Rust\" {\n-            static __bss_start: UnsafeCell<u64>;\n-            static __bss_end_exclusive: UnsafeCell<u64>;\n-        }\n-\n-        let bss_range = unsafe {\n-            Range {\n-                start: __bss_start.get(),\n-                end: __bss_end_exclusive.get(),\n-            }\n-        };\n-        let kernel_tables_addr = &KERNEL_TABLES as *const _ as usize as *mut u64;\n-\n-        assert!(bss_range.contains(&kernel_tables_addr));\n-    }\n+        &virt_data_region,\n+        &kernel_virt_to_phys_region(virt_data_region),\n+        &kernel_page_attributes(virt_data_region.start_page_addr()),\n+    );\n }\n\ndiff -uNr 14_virtual_mem_part2_mmio_remap/kernel/src/lib.rs 15_virtual_mem_part3_precomputed_tables/kernel/src/lib.rs\n--- 14_virtual_mem_part2_mmio_remap/kernel/src/lib.rs\n+++ 15_virtual_mem_part3_precomputed_tables/kernel/src/lib.rs\n@@ -187,17 +187,7 @@\n #[no_mangle]\n unsafe fn kernel_init() -> ! {\n     exception::handling_init();\n-\n-    let phys_kernel_tables_base_addr = match memory::mmu::kernel_map_binary() {\n-        Err(string) => panic!(\"Error mapping kernel binary: {}\", string),\n-        Ok(addr) => addr,\n-    };\n-\n-    if let Err(e) = memory::mmu::enable_mmu_and_caching(phys_kernel_tables_base_addr) {\n-        panic!(\"Enabling MMU failed: {}\", e);\n-    }\n-\n-    memory::mmu::post_enable_init();\n+    memory::init();\n     bsp::driver::qemu_bring_up_console();\n\n     test_main();\n\ndiff -uNr 14_virtual_mem_part2_mmio_remap/kernel/src/main.rs 15_virtual_mem_part3_precomputed_tables/kernel/src/main.rs\n--- 14_virtual_mem_part2_mmio_remap/kernel/src/main.rs\n+++ 15_virtual_mem_part3_precomputed_tables/kernel/src/main.rs\n@@ -17,27 +17,16 @@\n\n /// Early init code.\n ///\n+/// When this code runs, virtual memory is already enabled.\n+///\n /// # Safety\n ///\n /// - Only a single core must be active and running this function.\n-/// - The init calls in this function must appear in the correct order:\n-///     - MMU + Data caching must be activated at the earliest. Without it, any atomic operations,\n-///       e.g. the yet-to-be-introduced spinlocks in the device drivers (which currently employ\n-///       IRQSafeNullLocks instead of spinlocks), will fail to work (properly) on the RPi SoCs.\n+/// - Printing will not work until the respective driver's MMIO is remapped.\n #[no_mangle]\n unsafe fn kernel_init() -> ! {\n     exception::handling_init();\n-\n-    let phys_kernel_tables_base_addr = match memory::mmu::kernel_map_binary() {\n-        Err(string) => panic!(\"Error mapping kernel binary: {}\", string),\n-        Ok(addr) => addr,\n-    };\n-\n-    if let Err(e) = memory::mmu::enable_mmu_and_caching(phys_kernel_tables_base_addr) {\n-        panic!(\"Enabling MMU failed: {}\", e);\n-    }\n-\n-    memory::mmu::post_enable_init();\n+    memory::init();\n\n     // Initialize the BSP driver subsystem.\n     if let Err(x) = bsp::driver::init() {\n@@ -47,6 +36,8 @@\n     // Initialize all device drivers.\n     driver::driver_manager().init_drivers_and_irqs();\n\n+    bsp::memory::mmu::kernel_add_mapping_records_for_precomputed();\n+\n     // Unmask interrupts on the boot CPU core.\n     exception::asynchronous::local_irq_unmask();\n\n\ndiff -uNr 14_virtual_mem_part2_mmio_remap/kernel/src/memory/mmu/translation_table.rs 15_virtual_mem_part3_precomputed_tables/kernel/src/memory/mmu/translation_table.rs\n--- 14_virtual_mem_part2_mmio_remap/kernel/src/memory/mmu/translation_table.rs\n+++ 15_virtual_mem_part3_precomputed_tables/kernel/src/memory/mmu/translation_table.rs\n@@ -23,6 +23,8 @@\n\n /// Translation table interfaces.\n pub mod interface {\n+    use crate::memory::mmu::PageAddress;\n+\n     use super::*;\n\n     /// Translation table operations.\n@@ -33,10 +35,7 @@\n         ///\n         /// - Implementor must ensure that this function can run only once or is harmless if invoked\n         ///   multiple times.\n-        fn init(&mut self);\n-\n-        /// The translation table's base address to be used for programming the MMU.\n-        fn phys_base_address(&self) -> Address<Physical>;\n+        fn init(&mut self) -> Result<(), &'static str>;\n\n         /// Map the given virtual memory region to the given physical memory region.\n         ///\n@@ -53,6 +52,30 @@\n             phys_region: &MemoryRegion<Physical>,\n             attr: &AttributeFields,\n         ) -> Result<(), &'static str>;\n+\n+        /// Try to translate a virtual page address to a physical page address.\n+        ///\n+        /// Will only succeed if there exists a valid mapping for the input page.\n+        fn try_virt_page_addr_to_phys_page_addr(\n+            &self,\n+            virt_page_addr: PageAddress<Virtual>,\n+        ) -> Result<PageAddress<Physical>, &'static str>;\n+\n+        /// Try to get the attributes of a page.\n+        ///\n+        /// Will only succeed if there exists a valid mapping for the input page.\n+        fn try_page_attributes(\n+            &self,\n+            virt_page_addr: PageAddress<Virtual>,\n+        ) -> Result<AttributeFields, &'static str>;\n+\n+        /// Try to translate a virtual address to a physical address.\n+        ///\n+        /// Will only succeed if there exists a valid mapping for the input address.\n+        fn try_virt_addr_to_phys_addr(\n+            &self,\n+            virt_addr: Address<Virtual>,\n+        ) -> Result<Address<Physical>, &'static str>;\n     }\n }\n\n@@ -72,9 +95,9 @@\n     #[kernel_test]\n     fn translationtable_implementation_sanity() {\n         // This will occupy a lot of space on the stack.\n-        let mut tables = MinSizeTranslationTable::new();\n+        let mut tables = MinSizeTranslationTable::new_for_runtime();\n\n-        tables.init();\n+        assert_eq!(tables.init(), Ok(()));\n\n         let virt_start_page_addr: PageAddress<Virtual> = PageAddress::from(0);\n         let virt_end_exclusive_page_addr: PageAddress<Virtual> =\n@@ -94,5 +117,21 @@\n         };\n\n         unsafe { assert_eq!(tables.map_at(&virt_region, &phys_region, &attr), Ok(())) };\n+\n+        assert_eq!(\n+            tables.try_virt_page_addr_to_phys_page_addr(virt_start_page_addr),\n+            Ok(phys_start_page_addr)\n+        );\n+\n+        assert_eq!(\n+            tables.try_page_attributes(virt_start_page_addr.checked_offset(6).unwrap()),\n+            Err(\"Page marked invalid\")\n+        );\n+\n+        assert_eq!(tables.try_page_attributes(virt_start_page_addr), Ok(attr));\n+\n+        let virt_addr = virt_start_page_addr.into_inner() + 0x100;\n+        let phys_addr = phys_start_page_addr.into_inner() + 0x100;\n+        assert_eq!(tables.try_virt_addr_to_phys_addr(virt_addr), Ok(phys_addr));\n     }\n }\n\ndiff -uNr 14_virtual_mem_part2_mmio_remap/kernel/src/memory/mmu.rs 15_virtual_mem_part3_precomputed_tables/kernel/src/memory/mmu.rs\n--- 14_virtual_mem_part2_mmio_remap/kernel/src/memory/mmu.rs\n+++ 15_virtual_mem_part3_precomputed_tables/kernel/src/memory/mmu.rs\n@@ -16,7 +16,8 @@\n use crate::{\n     bsp,\n     memory::{Address, Physical, Virtual},\n-    synchronization, warn,\n+    synchronization::{self, interface::Mutex},\n+    warn,\n };\n use core::{fmt, num::NonZeroUsize};\n\n@@ -73,17 +74,9 @@\n // Private Code\n //--------------------------------------------------------------------------------------------------\n use interface::MMU;\n-use synchronization::interface::*;\n+use synchronization::interface::ReadWriteEx;\n use translation_table::interface::TranslationTable;\n\n-/// Query the BSP for the reserved virtual addresses for MMIO remapping and initialize the kernel's\n-/// MMIO VA allocator with it.\n-fn kernel_init_mmio_va_allocator() {\n-    let region = bsp::memory::mmu::virt_mmio_remap_region();\n-\n-    page_alloc::kernel_mmio_va_allocator().lock(|allocator| allocator.init(region));\n-}\n-\n /// Map a region in the kernel's translation tables.\n ///\n /// No input checks done, input is passed through to the architectural implementation.\n@@ -101,13 +94,21 @@\n     bsp::memory::mmu::kernel_translation_tables()\n         .write(|tables| tables.map_at(virt_region, phys_region, attr))?;\n\n-    if let Err(x) = mapping_record::kernel_add(name, virt_region, phys_region, attr) {\n-        warn!(\"{}\", x);\n-    }\n+    kernel_add_mapping_record(name, virt_region, phys_region, attr);\n\n     Ok(())\n }\n\n+/// Try to translate a kernel virtual address to a physical address.\n+///\n+/// Will only succeed if there exists a valid mapping for the input address.\n+fn try_kernel_virt_addr_to_phys_addr(\n+    virt_addr: Address<Virtual>,\n+) -> Result<Address<Physical>, &'static str> {\n+    bsp::memory::mmu::kernel_translation_tables()\n+        .read(|tables| tables.try_virt_addr_to_phys_addr(virt_addr))\n+}\n+\n //--------------------------------------------------------------------------------------------------\n // Public Code\n //--------------------------------------------------------------------------------------------------\n@@ -155,27 +156,24 @@\n     }\n }\n\n-/// Raw mapping of a virtual to physical region in the kernel translation tables.\n-///\n-/// Prevents mapping into the MMIO range of the tables.\n-///\n-/// # Safety\n-///\n-/// - See `kernel_map_at_unchecked()`.\n-/// - Does not prevent aliasing. Currently, the callers must be trusted.\n-pub unsafe fn kernel_map_at(\n+/// Query the BSP for the reserved virtual addresses for MMIO remapping and initialize the kernel's\n+/// MMIO VA allocator with it.\n+pub fn kernel_init_mmio_va_allocator() {\n+    let region = bsp::memory::mmu::virt_mmio_remap_region();\n+\n+    page_alloc::kernel_mmio_va_allocator().lock(|allocator| allocator.init(region));\n+}\n+\n+/// Add an entry to the mapping info record.\n+pub fn kernel_add_mapping_record(\n     name: &'static str,\n     virt_region: &MemoryRegion<Virtual>,\n     phys_region: &MemoryRegion<Physical>,\n     attr: &AttributeFields,\n-) -> Result<(), &'static str> {\n-    if bsp::memory::mmu::virt_mmio_remap_region().overlaps(virt_region) {\n-        return Err(\"Attempt to manually map into MMIO region\");\n+) {\n+    if let Err(x) = mapping_record::kernel_add(name, virt_region, phys_region, attr) {\n+        warn!(\"{}\", x);\n     }\n-\n-    kernel_map_at_unchecked(name, virt_region, phys_region, attr)?;\n-\n-    Ok(())\n }\n\n /// MMIO remapping in the kernel translation tables.\n@@ -224,21 +222,29 @@\n     Ok(virt_addr + offset_into_start_page)\n }\n\n-/// Map the kernel's binary. Returns the translation table's base address.\n-///\n-/// # Safety\n+/// Try to translate a kernel virtual page address to a physical page address.\n ///\n-/// - See [`bsp::memory::mmu::kernel_map_binary()`].\n-pub unsafe fn kernel_map_binary() -> Result<Address<Physical>, &'static str> {\n-    let phys_kernel_tables_base_addr =\n-        bsp::memory::mmu::kernel_translation_tables().write(|tables| {\n-            tables.init();\n-            tables.phys_base_address()\n-        });\n+/// Will only succeed if there exists a valid mapping for the input page.\n+pub fn try_kernel_virt_page_addr_to_phys_page_addr(\n+    virt_page_addr: PageAddress<Virtual>,\n+) -> Result<PageAddress<Physical>, &'static str> {\n+    bsp::memory::mmu::kernel_translation_tables()\n+        .read(|tables| tables.try_virt_page_addr_to_phys_page_addr(virt_page_addr))\n+}\n\n-    bsp::memory::mmu::kernel_map_binary()?;\n+/// Try to get the attributes of a kernel page.\n+///\n+/// Will only succeed if there exists a valid mapping for the input page.\n+pub fn try_kernel_page_attributes(\n+    virt_page_addr: PageAddress<Virtual>,\n+) -> Result<AttributeFields, &'static str> {\n+    bsp::memory::mmu::kernel_translation_tables()\n+        .read(|tables| tables.try_page_attributes(virt_page_addr))\n+}\n\n-    Ok(phys_kernel_tables_base_addr)\n+/// Human-readable print of all recorded kernel mappings.\n+pub fn kernel_print_mappings() {\n+    mapping_record::kernel_print()\n }\n\n /// Enable the MMU and data + instruction caching.\n@@ -246,56 +252,9 @@\n /// # Safety\n ///\n /// - Crucial function during kernel init. Changes the the complete memory view of the processor.\n+#[inline(always)]\n pub unsafe fn enable_mmu_and_caching(\n     phys_tables_base_addr: Address<Physical>,\n ) -> Result<(), MMUEnableError> {\n     arch_mmu::mmu().enable_mmu_and_caching(phys_tables_base_addr)\n }\n-\n-/// Finish initialization of the MMU subsystem.\n-pub fn post_enable_init() {\n-    kernel_init_mmio_va_allocator();\n-}\n-\n-/// Human-readable print of all recorded kernel mappings.\n-pub fn kernel_print_mappings() {\n-    mapping_record::kernel_print()\n-}\n-\n-//--------------------------------------------------------------------------------------------------\n-// Testing\n-//--------------------------------------------------------------------------------------------------\n-\n-#[cfg(test)]\n-mod tests {\n-    use super::*;\n-    use crate::memory::mmu::{AccessPermissions, MemAttributes, PageAddress};\n-    use test_macros::kernel_test;\n-\n-    /// Check that you cannot map into the MMIO VA range from kernel_map_at().\n-    #[kernel_test]\n-    fn no_manual_mmio_map() {\n-        let phys_start_page_addr: PageAddress<Physical> = PageAddress::from(0);\n-        let phys_end_exclusive_page_addr: PageAddress<Physical> =\n-            phys_start_page_addr.checked_offset(5).unwrap();\n-        let phys_region = MemoryRegion::new(phys_start_page_addr, phys_end_exclusive_page_addr);\n-\n-        let num_pages = NonZeroUsize::new(phys_region.num_pages()).unwrap();\n-        let virt_region = page_alloc::kernel_mmio_va_allocator()\n-            .lock(|allocator| allocator.alloc(num_pages))\n-            .unwrap();\n-\n-        let attr = AttributeFields {\n-            mem_attributes: MemAttributes::CacheableDRAM,\n-            acc_perms: AccessPermissions::ReadWrite,\n-            execute_never: true,\n-        };\n-\n-        unsafe {\n-            assert_eq!(\n-                kernel_map_at(\"test\", &virt_region, &phys_region, &attr),\n-                Err(\"Attempt to manually map into MMIO region\")\n-            )\n-        };\n-    }\n-}\n\ndiff -uNr 14_virtual_mem_part2_mmio_remap/kernel/src/memory.rs 15_virtual_mem_part3_precomputed_tables/kernel/src/memory.rs\n--- 14_virtual_mem_part2_mmio_remap/kernel/src/memory.rs\n+++ 15_virtual_mem_part3_precomputed_tables/kernel/src/memory.rs\n@@ -136,6 +136,11 @@\n     }\n }\n\n+/// Initialize the memory subsystem.\n+pub fn init() {\n+    mmu::kernel_init_mmio_va_allocator();\n+}\n+\n //--------------------------------------------------------------------------------------------------\n // Testing\n //--------------------------------------------------------------------------------------------------\n\ndiff -uNr 14_virtual_mem_part2_mmio_remap/kernel/tests/00_console_sanity.rs 15_virtual_mem_part3_precomputed_tables/kernel/tests/00_console_sanity.rs\n--- 14_virtual_mem_part2_mmio_remap/kernel/tests/00_console_sanity.rs\n+++ 15_virtual_mem_part3_precomputed_tables/kernel/tests/00_console_sanity.rs\n@@ -18,17 +18,7 @@\n     use console::console;\n\n     exception::handling_init();\n-\n-    let phys_kernel_tables_base_addr = match memory::mmu::kernel_map_binary() {\n-        Err(string) => panic!(\"Error mapping kernel binary: {}\", string),\n-        Ok(addr) => addr,\n-    };\n-\n-    if let Err(e) = memory::mmu::enable_mmu_and_caching(phys_kernel_tables_base_addr) {\n-        panic!(\"Enabling MMU failed: {}\", e);\n-    }\n-\n-    memory::mmu::post_enable_init();\n+    memory::init();\n     bsp::driver::qemu_bring_up_console();\n\n     // Handshake\n\ndiff -uNr 14_virtual_mem_part2_mmio_remap/kernel/tests/01_timer_sanity.rs 15_virtual_mem_part3_precomputed_tables/kernel/tests/01_timer_sanity.rs\n--- 14_virtual_mem_part2_mmio_remap/kernel/tests/01_timer_sanity.rs\n+++ 15_virtual_mem_part3_precomputed_tables/kernel/tests/01_timer_sanity.rs\n@@ -17,17 +17,7 @@\n #[no_mangle]\n unsafe fn kernel_init() -> ! {\n     exception::handling_init();\n-\n-    let phys_kernel_tables_base_addr = match memory::mmu::kernel_map_binary() {\n-        Err(string) => panic!(\"Error mapping kernel binary: {}\", string),\n-        Ok(addr) => addr,\n-    };\n-\n-    if let Err(e) = memory::mmu::enable_mmu_and_caching(phys_kernel_tables_base_addr) {\n-        panic!(\"Enabling MMU failed: {}\", e);\n-    }\n-\n-    memory::mmu::post_enable_init();\n+    memory::init();\n     bsp::driver::qemu_bring_up_console();\n\n     // Depending on CPU arch, some timer bring-up code could go here. Not needed for the RPi.\n\ndiff -uNr 14_virtual_mem_part2_mmio_remap/kernel/tests/02_exception_sync_page_fault.rs 15_virtual_mem_part3_precomputed_tables/kernel/tests/02_exception_sync_page_fault.rs\n--- 14_virtual_mem_part2_mmio_remap/kernel/tests/02_exception_sync_page_fault.rs\n+++ 15_virtual_mem_part3_precomputed_tables/kernel/tests/02_exception_sync_page_fault.rs\n@@ -22,26 +22,12 @@\n #[no_mangle]\n unsafe fn kernel_init() -> ! {\n     exception::handling_init();\n+    memory::init();\n+    bsp::driver::qemu_bring_up_console();\n\n     // This line will be printed as the test header.\n     println!(\"Testing synchronous exception handling by causing a page fault\");\n\n-    let phys_kernel_tables_base_addr = match memory::mmu::kernel_map_binary() {\n-        Err(string) => {\n-            info!(\"Error mapping kernel binary: {}\", string);\n-            cpu::qemu_exit_failure()\n-        }\n-        Ok(addr) => addr,\n-    };\n-\n-    if let Err(e) = memory::mmu::enable_mmu_and_caching(phys_kernel_tables_base_addr) {\n-        info!(\"Enabling MMU failed: {}\", e);\n-        cpu::qemu_exit_failure()\n-    }\n-\n-    memory::mmu::post_enable_init();\n-    bsp::driver::qemu_bring_up_console();\n-\n     info!(\"Writing beyond mapped area to address 9 GiB...\");\n     let big_addr: u64 = 9 * 1024 * 1024 * 1024;\n     core::ptr::read_volatile(big_addr as *mut u64);\n\ndiff -uNr 14_virtual_mem_part2_mmio_remap/kernel/tests/03_exception_restore_sanity.rs 15_virtual_mem_part3_precomputed_tables/kernel/tests/03_exception_restore_sanity.rs\n--- 14_virtual_mem_part2_mmio_remap/kernel/tests/03_exception_restore_sanity.rs\n+++ 15_virtual_mem_part3_precomputed_tables/kernel/tests/03_exception_restore_sanity.rs\n@@ -31,26 +31,12 @@\n #[no_mangle]\n unsafe fn kernel_init() -> ! {\n     exception::handling_init();\n+    memory::init();\n+    bsp::driver::qemu_bring_up_console();\n\n     // This line will be printed as the test header.\n     println!(\"Testing exception restore\");\n\n-    let phys_kernel_tables_base_addr = match memory::mmu::kernel_map_binary() {\n-        Err(string) => {\n-            info!(\"Error mapping kernel binary: {}\", string);\n-            cpu::qemu_exit_failure()\n-        }\n-        Ok(addr) => addr,\n-    };\n-\n-    if let Err(e) = memory::mmu::enable_mmu_and_caching(phys_kernel_tables_base_addr) {\n-        info!(\"Enabling MMU failed: {}\", e);\n-        cpu::qemu_exit_failure()\n-    }\n-\n-    memory::mmu::post_enable_init();\n-    bsp::driver::qemu_bring_up_console();\n-\n     info!(\"Making a dummy system call\");\n\n     // Calling this inside a function indirectly tests if the link register is restored properly.\n\ndiff -uNr 14_virtual_mem_part2_mmio_remap/kernel/tests/04_exception_irq_sanity.rs 15_virtual_mem_part3_precomputed_tables/kernel/tests/04_exception_irq_sanity.rs\n--- 14_virtual_mem_part2_mmio_remap/kernel/tests/04_exception_irq_sanity.rs\n+++ 15_virtual_mem_part3_precomputed_tables/kernel/tests/04_exception_irq_sanity.rs\n@@ -15,20 +15,10 @@\n\n #[no_mangle]\n unsafe fn kernel_init() -> ! {\n-    exception::handling_init();\n-\n-    let phys_kernel_tables_base_addr = match memory::mmu::kernel_map_binary() {\n-        Err(string) => panic!(\"Error mapping kernel binary: {}\", string),\n-        Ok(addr) => addr,\n-    };\n-\n-    if let Err(e) = memory::mmu::enable_mmu_and_caching(phys_kernel_tables_base_addr) {\n-        panic!(\"Enabling MMU failed: {}\", e);\n-    }\n-\n-    memory::mmu::post_enable_init();\n+    memory::init();\n     bsp::driver::qemu_bring_up_console();\n\n+    exception::handling_init();\n     exception::asynchronous::local_irq_unmask();\n\n     test_main();\n\ndiff -uNr 14_virtual_mem_part2_mmio_remap/Makefile 15_virtual_mem_part3_precomputed_tables/Makefile\n--- 14_virtual_mem_part2_mmio_remap/Makefile\n+++ 15_virtual_mem_part3_precomputed_tables/Makefile\n@@ -72,10 +72,20 @@\n KERNEL_LINKER_SCRIPT = kernel.ld\n LAST_BUILD_CONFIG    = target/$(BSP).build_config\n\n-KERNEL_ELF      = target/$(TARGET)/release/kernel\n+KERNEL_ELF_RAW      = target/$(TARGET)/release/kernel\n # This parses cargo's dep-info file.\n # https://doc.rust-lang.org/cargo/guide/build-cache.html#dep-info-files\n-KERNEL_ELF_DEPS = $(filter-out modulo: ,$(file < $(KERNEL_ELF).d)) $(KERNEL_MANIFEST) $(LAST_BUILD_CONFIG)\n+KERNEL_ELF_RAW_DEPS = $(filter-out modulo: ,$(file < $(KERNEL_ELF_RAW).d)) $(KERNEL_MANIFEST) $(LAST_BUILD_CONFIG)\n+\n+##------------------------------------------------------------------------------\n+## Translation tables\n+##------------------------------------------------------------------------------\n+TT_TOOL_PATH = tools/translation_table_tool\n+\n+KERNEL_ELF_TTABLES      = target/$(TARGET)/release/kernel+ttables\n+KERNEL_ELF_TTABLES_DEPS = $(KERNEL_ELF_RAW) $(wildcard $(TT_TOOL_PATH)/*)\n+\n+KERNEL_ELF = $(KERNEL_ELF_TTABLES)\n\n\n\n@@ -104,6 +114,7 @@\n     -O binary\n\n EXEC_QEMU          = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE)\n+EXEC_TT_TOOL       = ruby $(TT_TOOL_PATH)/main.rb\n EXEC_TEST_DISPATCH = ruby ../common/tests/dispatch.rb\n EXEC_MINIPUSH      = ruby ../common/serial/minipush.rb\n\n@@ -154,16 +165,24 @@\n ##------------------------------------------------------------------------------\n ## Compile the kernel ELF\n ##------------------------------------------------------------------------------\n-$(KERNEL_ELF): $(KERNEL_ELF_DEPS)\n+$(KERNEL_ELF_RAW): $(KERNEL_ELF_RAW_DEPS)\n \t$(call color_header, \"Compiling kernel ELF - $(BSP)\")\n \t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(RUSTC_CMD)\n\n ##------------------------------------------------------------------------------\n+## Precompute the kernel translation tables and patch them into the kernel ELF\n+##------------------------------------------------------------------------------\n+$(KERNEL_ELF_TTABLES): $(KERNEL_ELF_TTABLES_DEPS)\n+\t$(call color_header, \"Precomputing kernel translation tables and patching kernel ELF\")\n+\t@cp $(KERNEL_ELF_RAW) $(KERNEL_ELF_TTABLES)\n+\t@$(DOCKER_TOOLS) $(EXEC_TT_TOOL) $(BSP) $(KERNEL_ELF_TTABLES)\n+\n+##------------------------------------------------------------------------------\n ## Generate the stripped kernel binary\n ##------------------------------------------------------------------------------\n-$(KERNEL_BIN): $(KERNEL_ELF)\n+$(KERNEL_BIN): $(KERNEL_ELF_TTABLES)\n \t$(call color_header, \"Generating stripped binary\")\n-\t@$(OBJCOPY_CMD) $(KERNEL_ELF) $(KERNEL_BIN)\n+\t@$(OBJCOPY_CMD) $(KERNEL_ELF_TTABLES) $(KERNEL_BIN)\n \t$(call color_progress_prefix, \"Name\")\n \t@echo $(KERNEL_BIN)\n \t$(call color_progress_prefix, \"Size\")\n@@ -301,6 +320,7 @@\n     TEST_ELF=$$(echo $$1 | sed -e 's/.*target/target/g')\n     TEST_BINARY=$$(echo $$1.img | sed -e 's/.*target/target/g')\n\n+    $(DOCKER_TOOLS) $(EXEC_TT_TOOL) $(BSP) $$TEST_ELF > /dev/null\n     $(OBJCOPY_CMD) $$TEST_ELF $$TEST_BINARY\n     $(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_TEST_ARGS) -kernel $$TEST_BINARY\n endef\n\ndiff -uNr 14_virtual_mem_part2_mmio_remap/tools/translation_table_tool/arch.rb 15_virtual_mem_part3_precomputed_tables/tools/translation_table_tool/arch.rb\n--- 14_virtual_mem_part2_mmio_remap/tools/translation_table_tool/arch.rb\n+++ 15_virtual_mem_part3_precomputed_tables/tools/translation_table_tool/arch.rb\n@@ -0,0 +1,312 @@\n+# frozen_string_literal: true\n+\n+# SPDX-License-Identifier: MIT OR Apache-2.0\n+#\n+# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+# Bitfield manipulation.\n+class BitField\n+    def initialize\n+        @value = 0\n+    end\n+\n+    def self.attr_bitfield(name, offset, num_bits)\n+        define_method(\"#{name}=\") do |bits|\n+            mask = (2**num_bits) - 1\n+\n+            raise \"Input out of range: #{name} = 0x#{bits.to_s(16)}\" if (bits & ~mask).positive?\n+\n+            # Clear bitfield\n+            @value &= ~(mask << offset)\n+\n+            # Set it\n+            @value |= (bits << offset)\n+        end\n+    end\n+\n+    def to_i\n+        @value\n+    end\n+\n+    def size_in_byte\n+        8\n+    end\n+end\n+\n+# An array class that knows its memory location.\n+class CArray < Array\n+    attr_reader :phys_start_addr\n+\n+    def initialize(phys_start_addr, size, &block)\n+        @phys_start_addr = phys_start_addr\n+\n+        super(size, &block)\n+    end\n+\n+    def size_in_byte\n+        inject(0) { |sum, n| sum + n.size_in_byte }\n+    end\n+end\n+\n+#---------------------------------------------------------------------------------------------------\n+# Arch::\n+#---------------------------------------------------------------------------------------------------\n+module Arch\n+#---------------------------------------------------------------------------------------------------\n+# Arch::ARMv8\n+#---------------------------------------------------------------------------------------------------\n+module ARMv8\n+# ARMv8 Table Descriptor.\n+class Stage1TableDescriptor < BitField\n+    module NextLevelTableAddr\n+        OFFSET = 16\n+        NUMBITS = 32\n+    end\n+\n+    module Type\n+        OFFSET = 1\n+        NUMBITS = 1\n+\n+        BLOCK = 0\n+        TABLE = 1\n+    end\n+\n+    module Valid\n+        OFFSET = 0\n+        NUMBITS = 1\n+\n+        FALSE = 0\n+        TRUE = 1\n+    end\n+\n+    attr_bitfield(:__next_level_table_addr, NextLevelTableAddr::OFFSET, NextLevelTableAddr::NUMBITS)\n+    attr_bitfield(:type, Type::OFFSET, Type::NUMBITS)\n+    attr_bitfield(:valid, Valid::OFFSET, Valid::NUMBITS)\n+\n+    def next_level_table_addr=(addr)\n+        addr >>= Granule64KiB::SHIFT\n+\n+        self.__next_level_table_addr = addr\n+    end\n+\n+    private :__next_level_table_addr=\n+end\n+\n+# ARMv8 level 3 page descriptor.\n+class Stage1PageDescriptor < BitField\n+    module UXN\n+        OFFSET = 54\n+        NUMBITS = 1\n+\n+        FALSE = 0\n+        TRUE = 1\n+    end\n+\n+    module PXN\n+        OFFSET = 53\n+        NUMBITS = 1\n+\n+        FALSE = 0\n+        TRUE = 1\n+    end\n+\n+    module OutputAddr\n+        OFFSET = 16\n+        NUMBITS = 32\n+    end\n+\n+    module AF\n+        OFFSET = 10\n+        NUMBITS = 1\n+\n+        FALSE = 0\n+        TRUE = 1\n+    end\n+\n+    module SH\n+        OFFSET = 8\n+        NUMBITS = 2\n+\n+        INNER_SHAREABLE = 0b11\n+    end\n+\n+    module AP\n+        OFFSET = 6\n+        NUMBITS = 2\n+\n+        RW_EL1 = 0b00\n+        RO_EL1 = 0b10\n+    end\n+\n+    module AttrIndx\n+        OFFSET = 2\n+        NUMBITS = 3\n+    end\n+\n+    module Type\n+        OFFSET = 1\n+        NUMBITS = 1\n+\n+        RESERVED_INVALID = 0\n+        PAGE = 1\n+    end\n+\n+    module Valid\n+        OFFSET = 0\n+        NUMBITS = 1\n+\n+        FALSE = 0\n+        TRUE = 1\n+    end\n+\n+    attr_bitfield(:uxn, UXN::OFFSET, UXN::NUMBITS)\n+    attr_bitfield(:pxn, PXN::OFFSET, PXN::NUMBITS)\n+    attr_bitfield(:__output_addr, OutputAddr::OFFSET, OutputAddr::NUMBITS)\n+    attr_bitfield(:af, AF::OFFSET, AF::NUMBITS)\n+    attr_bitfield(:sh, SH::OFFSET, SH::NUMBITS)\n+    attr_bitfield(:ap, AP::OFFSET, AP::NUMBITS)\n+    attr_bitfield(:attr_indx, AttrIndx::OFFSET, AttrIndx::NUMBITS)\n+    attr_bitfield(:type, Type::OFFSET, Type::NUMBITS)\n+    attr_bitfield(:valid, Valid::OFFSET, Valid::NUMBITS)\n+\n+    def output_addr=(addr)\n+        addr >>= Granule64KiB::SHIFT\n+\n+        self.__output_addr = addr\n+    end\n+\n+    private :__output_addr=\n+end\n+\n+# Translation table representing the structure defined in translation_table.rs.\n+class TranslationTable\n+    module MAIR\n+        NORMAL = 1\n+    end\n+\n+    def initialize\n+        do_sanity_checks\n+\n+        num_lvl2_tables = BSP.kernel_virt_addr_space_size >> Granule512MiB::SHIFT\n+\n+        @lvl3 = new_lvl3(num_lvl2_tables, BSP.phys_addr_of_kernel_tables)\n+\n+        @lvl2_phys_start_addr = @lvl3.phys_start_addr + @lvl3.size_in_byte\n+        @lvl2 = new_lvl2(num_lvl2_tables, @lvl2_phys_start_addr)\n+\n+        populate_lvl2_entries\n+    end\n+\n+    def map_at(virt_region, phys_region, attributes)\n+        return if virt_region.empty?\n+\n+        raise if virt_region.size != phys_region.size\n+        raise if phys_region.last > BSP.phys_addr_space_end_page\n+\n+        virt_region.zip(phys_region).each do |virt_page, phys_page|\n+            desc = page_descriptor_from(virt_page)\n+            set_lvl3_entry(desc, phys_page, attributes)\n+        end\n+    end\n+\n+    def to_binary\n+        data = @lvl3.flatten.map(&:to_i) + @lvl2.map(&:to_i)\n+        data.pack('Q<*') # \"Q\" == uint64_t, \"<\" == little endian\n+    end\n+\n+    def phys_tables_base_addr_binary\n+        [@lvl2_phys_start_addr].pack('Q<*') # \"Q\" == uint64_t, \"<\" == little endian\n+    end\n+\n+    def phys_tables_base_addr\n+        @lvl2_phys_start_addr\n+    end\n+\n+    private\n+\n+    def do_sanity_checks\n+        raise unless BSP.kernel_granule::SIZE == Granule64KiB::SIZE\n+        raise unless (BSP.kernel_virt_addr_space_size modulo Granule512MiB::SIZE).zero?\n+    end\n+\n+    def new_lvl3(num_lvl2_tables, start_addr)\n+        CArray.new(start_addr, num_lvl2_tables) do\n+            temp = CArray.new(start_addr, 8192) do\n+                Stage1PageDescriptor.new\n+            end\n+            start_addr += temp.size_in_byte\n+\n+            temp\n+        end\n+    end\n+\n+    def new_lvl2(num_lvl2_tables, start_addr)\n+        CArray.new(start_addr, num_lvl2_tables) do\n+            Stage1TableDescriptor.new\n+        end\n+    end\n+\n+    def populate_lvl2_entries\n+        @lvl2.each_with_index do |descriptor, i|\n+            descriptor.next_level_table_addr = @lvl3[i].phys_start_addr\n+            descriptor.type = Stage1TableDescriptor::Type::TABLE\n+            descriptor.valid = Stage1TableDescriptor::Valid::TRUE\n+        end\n+    end\n+\n+    def lvl2_lvl3_index_from(addr)\n+        lvl2_index = addr >> Granule512MiB::SHIFT\n+        lvl3_index = (addr & Granule512MiB::MASK) >> Granule64KiB::SHIFT\n+\n+        raise unless lvl2_index < @lvl2.size\n+\n+        [lvl2_index, lvl3_index]\n+    end\n+\n+    def page_descriptor_from(virt_addr)\n+        lvl2_index, lvl3_index = lvl2_lvl3_index_from(virt_addr)\n+\n+        @lvl3[lvl2_index][lvl3_index]\n+    end\n+\n+    # rubocop:disable Metrics/MethodLength\n+    def set_attributes(desc, attributes)\n+        case attributes.mem_attributes\n+        when :CacheableDRAM\n+            desc.sh = Stage1PageDescriptor::SH::INNER_SHAREABLE\n+            desc.attr_indx = MAIR::NORMAL\n+        else\n+            raise 'Invalid input'\n+        end\n+\n+        desc.ap = case attributes.acc_perms\n+                  when :ReadOnly\n+                      Stage1PageDescriptor::AP::RO_EL1\n+                  when :ReadWrite\n+                      Stage1PageDescriptor::AP::RW_EL1\n+                  else\n+                      raise 'Invalid input'\n+\n+                  end\n+\n+        desc.pxn = if attributes.execute_never\n+                       Stage1PageDescriptor::PXN::TRUE\n+                   else\n+                       Stage1PageDescriptor::PXN::FALSE\n+                   end\n+\n+        desc.uxn = Stage1PageDescriptor::UXN::TRUE\n+    end\n+    # rubocop:enable Metrics/MethodLength\n+\n+    def set_lvl3_entry(desc, output_addr, attributes)\n+        desc.output_addr = output_addr\n+        desc.af = Stage1PageDescriptor::AF::TRUE\n+        desc.type = Stage1PageDescriptor::Type::PAGE\n+        desc.valid = Stage1PageDescriptor::Valid::TRUE\n+\n+        set_attributes(desc, attributes)\n+    end\n+end\n+end\n+end\n\ndiff -uNr 14_virtual_mem_part2_mmio_remap/tools/translation_table_tool/bsp.rb 15_virtual_mem_part3_precomputed_tables/tools/translation_table_tool/bsp.rb\n--- 14_virtual_mem_part2_mmio_remap/tools/translation_table_tool/bsp.rb\n+++ 15_virtual_mem_part3_precomputed_tables/tools/translation_table_tool/bsp.rb\n@@ -0,0 +1,49 @@\n+# frozen_string_literal: true\n+\n+# SPDX-License-Identifier: MIT OR Apache-2.0\n+#\n+# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+# Raspberry Pi 3 + 4\n+class RaspberryPi\n+    attr_reader :kernel_granule, :kernel_virt_addr_space_size\n+\n+    MEMORY_SRC = File.read('kernel/src/bsp/raspberrypi/memory.rs').split(\"\\n\")\n+\n+    def initialize\n+        @kernel_granule = Granule64KiB\n+\n+        @kernel_virt_addr_space_size = KERNEL_ELF.symbol_value('__kernel_virt_addr_space_size')\n+\n+        @virt_addr_of_kernel_tables = KERNEL_ELF.symbol_value('KERNEL_TABLES')\n+        @virt_addr_of_phys_kernel_tables_base_addr = KERNEL_ELF.symbol_value(\n+            'PHYS_KERNEL_TABLES_BASE_ADDR'\n+        )\n+    end\n+\n+    def phys_addr_of_kernel_tables\n+        KERNEL_ELF.virt_to_phys(@virt_addr_of_kernel_tables)\n+    end\n+\n+    def kernel_tables_offset_in_file\n+        KERNEL_ELF.virt_addr_to_file_offset(@virt_addr_of_kernel_tables)\n+    end\n+\n+    def phys_kernel_tables_base_addr_offset_in_file\n+        KERNEL_ELF.virt_addr_to_file_offset(@virt_addr_of_phys_kernel_tables_base_addr)\n+    end\n+\n+    def phys_addr_space_end_page\n+        x = MEMORY_SRC.grep(/pub const END/)\n+        x = case BSP_TYPE\n+            when :rpi3\n+                x[0]\n+            when :rpi4\n+                x[1]\n+            else\n+                raise\n+            end\n+\n+        x.scan(/\\d+/).join.to_i(16)\n+    end\n+end\n\ndiff -uNr 14_virtual_mem_part2_mmio_remap/tools/translation_table_tool/generic.rb 15_virtual_mem_part3_precomputed_tables/tools/translation_table_tool/generic.rb\n--- 14_virtual_mem_part2_mmio_remap/tools/translation_table_tool/generic.rb\n+++ 15_virtual_mem_part3_precomputed_tables/tools/translation_table_tool/generic.rb\n@@ -0,0 +1,179 @@\n+# frozen_string_literal: true\n+\n+# SPDX-License-Identifier: MIT OR Apache-2.0\n+#\n+# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+module Granule64KiB\n+    SIZE = 64 * 1024\n+    SHIFT = Math.log2(SIZE).to_i\n+end\n+\n+module Granule512MiB\n+    SIZE = 512 * 1024 * 1024\n+    SHIFT = Math.log2(SIZE).to_i\n+    MASK = SIZE - 1\n+end\n+\n+# Monkey-patch Integer with some helper functions.\n+class Integer\n+    def power_of_two?\n+        self[0].zero?\n+    end\n+\n+    def aligned?(alignment)\n+        raise unless alignment.power_of_two?\n+\n+        (self & (alignment - 1)).zero?\n+    end\n+\n+    def align_up(alignment)\n+        raise unless alignment.power_of_two?\n+\n+        (self + alignment - 1) & ~(alignment - 1)\n+    end\n+\n+    def to_hex_underscore(with_leading_zeros: false)\n+        fmt = with_leading_zeros ? 'modulo016x' : 'modulox'\n+        value = format(fmt, self).to_s.reverse.scan(/.{4}|.+/).join('_').reverse\n+\n+        format('0xmodulos', value)\n+    end\n+end\n+\n+# An array where each value is the start address of a Page.\n+class MemoryRegion < Array\n+    def initialize(start_addr, size, granule_size)\n+        raise unless start_addr.aligned?(granule_size)\n+        raise unless size.positive?\n+        raise unless (size modulo granule_size).zero?\n+\n+        num_pages = size / granule_size\n+        super(num_pages) do |i|\n+            (i * granule_size) + start_addr\n+        end\n+    end\n+end\n+\n+# Collection of memory attributes.\n+class AttributeFields\n+    attr_reader :mem_attributes, :acc_perms, :execute_never\n+\n+    def initialize(mem_attributes, acc_perms, execute_never)\n+        @mem_attributes = mem_attributes\n+        @acc_perms = acc_perms\n+        @execute_never = execute_never\n+    end\n+\n+    def to_s\n+        x = case @mem_attributes\n+            when :CacheableDRAM\n+                'C'\n+            else\n+                '?'\n+            end\n+\n+        y = case @acc_perms\n+            when :ReadWrite\n+                'RW'\n+            when :ReadOnly\n+                'RO'\n+            else\n+                '??'\n+            end\n+\n+        z = @execute_never ? 'XN' : 'X '\n+\n+        \"#{x} #{y} #{z}\"\n+    end\n+end\n+\n+# A container that describes a virt-to-phys region mapping.\n+class MappingDescriptor\n+    @max_section_name_length = 'Sections'.length\n+\n+    class << self\n+        attr_accessor :max_section_name_length\n+\n+        def update_max_section_name_length(length)\n+            @max_section_name_length = [@max_section_name_length, length].max\n+        end\n+    end\n+\n+    attr_reader :name, :virt_region, :phys_region, :attributes\n+\n+    def initialize(name, virt_region, phys_region, attributes)\n+        @name = name\n+        @virt_region = virt_region\n+        @phys_region = phys_region\n+        @attributes = attributes\n+    end\n+\n+    def to_s\n+        name = @name.ljust(self.class.max_section_name_length)\n+        virt_start = @virt_region.first.to_hex_underscore(with_leading_zeros: true)\n+        phys_start = @phys_region.first.to_hex_underscore(with_leading_zeros: true)\n+        size = ((@virt_region.size * 65_536) / 1024).to_s.rjust(3)\n+\n+        \"#{name} | #{virt_start} | #{phys_start} | #{size} KiB | #{@attributes}\"\n+    end\n+\n+    def self.print_divider\n+        print '             '\n+        print '-' * max_section_name_length\n+        puts '--------------------------------------------------------------------'\n+    end\n+\n+    def self.print_header\n+        print_divider\n+        print '             '\n+        print 'Sections'.center(max_section_name_length)\n+        print '   '\n+        print 'Virt Start Addr'.center(21)\n+        print '   '\n+        print 'Phys Start Addr'.center(21)\n+        print '   '\n+        print 'Size'.center(7)\n+        print '   '\n+        print 'Attr'.center(7)\n+        puts\n+        print_divider\n+    end\n+end\n+\n+def kernel_map_binary\n+    mapping_descriptors = KERNEL_ELF.generate_mapping_descriptors\n+\n+    # Generate_mapping_descriptors updates the header being printed with this call. So it must come\n+    # afterwards.\n+    MappingDescriptor.print_header\n+\n+    mapping_descriptors.each do |i|\n+        print 'Generating'.rjust(12).green.bold\n+        print ' '\n+        puts i\n+\n+        TRANSLATION_TABLES.map_at(i.virt_region, i.phys_region, i.attributes)\n+    end\n+\n+    MappingDescriptor.print_divider\n+end\n+\n+def kernel_patch_tables(kernel_elf_path)\n+    print 'Patching'.rjust(12).green.bold\n+    print ' Kernel table struct at ELF file offset '\n+    puts BSP.kernel_tables_offset_in_file.to_hex_underscore\n+\n+    File.binwrite(kernel_elf_path, TRANSLATION_TABLES.to_binary, BSP.kernel_tables_offset_in_file)\n+end\n+\n+def kernel_patch_base_addr(kernel_elf_path)\n+    print 'Patching'.rjust(12).green.bold\n+    print ' Kernel tables physical base address start argument to value '\n+    print TRANSLATION_TABLES.phys_tables_base_addr.to_hex_underscore\n+    print ' at ELF file offset '\n+    puts BSP.phys_kernel_tables_base_addr_offset_in_file.to_hex_underscore\n+\n+    File.binwrite(kernel_elf_path, TRANSLATION_TABLES.phys_tables_base_addr_binary,\n+                  BSP.phys_kernel_tables_base_addr_offset_in_file)\n+end\n\ndiff -uNr 14_virtual_mem_part2_mmio_remap/tools/translation_table_tool/kernel_elf.rb 15_virtual_mem_part3_precomputed_tables/tools/translation_table_tool/kernel_elf.rb\n--- 14_virtual_mem_part2_mmio_remap/tools/translation_table_tool/kernel_elf.rb\n+++ 15_virtual_mem_part3_precomputed_tables/tools/translation_table_tool/kernel_elf.rb\n@@ -0,0 +1,96 @@\n+# frozen_string_literal: true\n+\n+# SPDX-License-Identifier: MIT OR Apache-2.0\n+#\n+# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+# KernelELF\n+class KernelELF\n+    SECTION_FLAG_ALLOC = 2\n+\n+    def initialize(kernel_elf_path)\n+        @elf = ELFTools::ELFFile.new(File.open(kernel_elf_path))\n+        @symtab_section = @elf.section_by_name('.symtab')\n+    end\n+\n+    def machine\n+        @elf.machine.to_sym\n+    end\n+\n+    def symbol_value(symbol_name)\n+        @symtab_section.symbol_by_name(symbol_name).header.st_value\n+    end\n+\n+    def segment_containing_virt_addr(virt_addr)\n+        @elf.each_segments do |segment|\n+            return segment if segment.vma_in?(virt_addr)\n+        end\n+    end\n+\n+    def virt_to_phys(virt_addr)\n+        segment = segment_containing_virt_addr(virt_addr)\n+        translation_offset = segment.header.p_vaddr - segment.header.p_paddr\n+\n+        virt_addr - translation_offset\n+    end\n+\n+    def virt_addr_to_file_offset(virt_addr)\n+        segment = segment_containing_virt_addr(virt_addr)\n+        segment.vma_to_offset(virt_addr)\n+    end\n+\n+    def sections_in_segment(segment)\n+        head = segment.mem_head\n+        tail = segment.mem_tail\n+\n+        sections = @elf.each_sections.select do |section|\n+            file_offset = section.header.sh_addr\n+            flags = section.header.sh_flags\n+\n+            file_offset >= head && file_offset < tail && (flags & SECTION_FLAG_ALLOC != 0)\n+        end\n+\n+        sections.map(&:name).join(' ')\n+    end\n+\n+    def select_load_segments\n+        @elf.each_segments.select do |segment|\n+            segment.instance_of?(ELFTools::Segments::LoadSegment)\n+        end\n+    end\n+\n+    def segment_get_acc_perms(segment)\n+        if segment.readable? && segment.writable?\n+            :ReadWrite\n+        elsif segment.readable?\n+            :ReadOnly\n+        else\n+            :Invalid\n+        end\n+    end\n+\n+    def update_max_section_name_length(descriptors)\n+        MappingDescriptor.update_max_section_name_length(descriptors.map { |i| i.name.size }.max)\n+    end\n+\n+    def generate_mapping_descriptors\n+        descriptors = select_load_segments.map do |segment|\n+            # Assume each segment is page aligned.\n+            size = segment.mem_size.align_up(BSP.kernel_granule::SIZE)\n+            virt_start_addr = segment.header.p_vaddr\n+            phys_start_addr = segment.header.p_paddr\n+            acc_perms = segment_get_acc_perms(segment)\n+            execute_never = !segment.executable?\n+            section_names = sections_in_segment(segment)\n+\n+            virt_region = MemoryRegion.new(virt_start_addr, size, BSP.kernel_granule::SIZE)\n+            phys_region = MemoryRegion.new(phys_start_addr, size, BSP.kernel_granule::SIZE)\n+            attributes = AttributeFields.new(:CacheableDRAM, acc_perms, execute_never)\n+\n+            MappingDescriptor.new(section_names, virt_region, phys_region, attributes)\n+        end\n+\n+        update_max_section_name_length(descriptors)\n+        descriptors\n+    end\n+end\n\ndiff -uNr 14_virtual_mem_part2_mmio_remap/tools/translation_table_tool/main.rb 15_virtual_mem_part3_precomputed_tables/tools/translation_table_tool/main.rb\n--- 14_virtual_mem_part2_mmio_remap/tools/translation_table_tool/main.rb\n+++ 15_virtual_mem_part3_precomputed_tables/tools/translation_table_tool/main.rb\n@@ -0,0 +1,46 @@\n+#!/usr/bin/env ruby\n+# frozen_string_literal: true\n+\n+# SPDX-License-Identifier: MIT OR Apache-2.0\n+#\n+# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+require 'rubygems'\n+require 'bundler/setup'\n+require 'colorize'\n+require 'elftools'\n+\n+require_relative 'generic'\n+require_relative 'kernel_elf'\n+require_relative 'bsp'\n+require_relative 'arch'\n+\n+BSP_TYPE = ARGV[0].to_sym\n+kernel_elf_path = ARGV[1]\n+\n+start = Time.now\n+\n+KERNEL_ELF = KernelELF.new(kernel_elf_path)\n+\n+BSP = case BSP_TYPE\n+      when :rpi3, :rpi4\n+          RaspberryPi.new\n+      else\n+          raise\n+      end\n+\n+TRANSLATION_TABLES = case KERNEL_ELF.machine\n+                     when :AArch64\n+                         Arch::ARMv8::TranslationTable.new\n+                     else\n+                         raise\n+                     end\n+\n+kernel_map_binary\n+kernel_patch_tables(kernel_elf_path)\n+kernel_patch_base_addr(kernel_elf_path)\n+\n+elapsed = Time.now - start\n+\n+print 'Finished'.rjust(12).green.bold\n+puts \" in #{elapsed.round(2)}s\"\n\n```\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/Cargo.toml",
    "content": "[package]\nname = \"mingo\"\nversion = \"0.15.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n\n[features]\ndefault = []\nbsp_rpi3 = [\"tock-registers\"]\nbsp_rpi4 = [\"tock-registers\"]\ntest_build = [\"qemu-exit\"]\n\n##-------------------------------------------------------------------------------------------------\n## Dependencies\n##-------------------------------------------------------------------------------------------------\n\n[dependencies]\ntest-types = { path = \"../libraries/test-types\" }\n\n# Optional dependencies\ntock-registers = { version = \"0.8.x\", default-features = false, features = [\"register_types\"], optional = true }\nqemu-exit = { version = \"3.x.x\", optional = true }\n\n# Platform specific dependencies\n[target.'cfg(target_arch = \"aarch64\")'.dependencies]\naarch64-cpu = { version = \"9.x.x\" }\n\n##-------------------------------------------------------------------------------------------------\n## Testing\n##-------------------------------------------------------------------------------------------------\n\n[dev-dependencies]\ntest-macros = { path = \"../libraries/test-macros\" }\n\n# Unit tests are done in the library part of the kernel.\n[lib]\nname = \"libkernel\"\ntest = true\n\n# Disable unit tests for the kernel binary.\n[[bin]]\nname = \"kernel\"\npath = \"src/main.rs\"\ntest = false\n\n# List of tests without harness.\n[[test]]\nname = \"00_console_sanity\"\nharness = false\n\n[[test]]\nname = \"02_exception_sync_page_fault\"\nharness = false\n\n[[test]]\nname = \"03_exception_restore_sanity\"\nharness = false\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/build.rs",
    "content": "use std::{env, fs, process};\n\nfn main() {\n    let ld_script_path = match env::var(\"LD_SCRIPT_PATH\") {\n        Ok(var) => var,\n        _ => process::exit(0),\n    };\n\n    let files = fs::read_dir(ld_script_path).unwrap();\n    files\n        .filter_map(Result::ok)\n        .filter(|d| {\n            if let Some(e) = d.path().extension() {\n                e == \"ld\"\n            } else {\n                false\n            }\n        })\n        .for_each(|f| println!(\"cargo:rerun-if-changed={}\", f.path().display()));\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/_arch/aarch64/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural boot code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::boot::arch_boot\n\nuse crate::{memory, memory::Address};\nuse aarch64_cpu::{asm, registers::*};\nuse core::arch::global_asm;\nuse tock_registers::interfaces::Writeable;\n\n// Assembly counterpart to this file.\nglobal_asm!(\n    include_str!(\"boot.s\"),\n    CONST_CURRENTEL_EL2 = const 0x8,\n    CONST_CORE_ID_MASK = const 0b11\n);\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Prepares the transition from EL2 to EL1.\n///\n/// # Safety\n///\n/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.\n/// - The HW state of EL1 must be prepared in a sound way.\n#[inline(always)]\nunsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: u64) {\n    // Enable timer counter registers for EL1.\n    CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET);\n\n    // No offset for reading the counters.\n    CNTVOFF_EL2.set(0);\n\n    // Set EL1 execution state to AArch64.\n    HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64);\n\n    // Set up a simulated exception return.\n    //\n    // First, fake a saved program status where all interrupts were masked and SP_EL1 was used as a\n    // stack pointer.\n    SPSR_EL2.write(\n        SPSR_EL2::D::Masked\n            + SPSR_EL2::A::Masked\n            + SPSR_EL2::I::Masked\n            + SPSR_EL2::F::Masked\n            + SPSR_EL2::M::EL1h,\n    );\n\n    // Second, let the link register point to kernel_init().\n    ELR_EL2.set(crate::kernel_init as *const () as u64);\n\n    // Set up SP_EL1 (stack pointer), which will be used by EL1 once we \"return\" to it. Since there\n    // are no plans to ever return to EL2, just re-use the same stack.\n    SP_EL1.set(phys_boot_core_stack_end_exclusive_addr);\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The Rust entry of the `kernel` binary.\n///\n/// The function is called from the assembly `_start` function.\n///\n/// # Safety\n///\n/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`.\n#[no_mangle]\npub unsafe extern \"C\" fn _start_rust(\n    phys_kernel_tables_base_addr: u64,\n    phys_boot_core_stack_end_exclusive_addr: u64,\n) -> ! {\n    prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr);\n\n    // Turn on the MMU for EL1.\n    let addr = Address::new(phys_kernel_tables_base_addr as usize);\n    memory::mmu::enable_mmu_and_caching(addr).unwrap();\n\n    // Use `eret` to \"return\" to EL1. This results in execution of kernel_init() in EL1.\n    asm::eret()\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/_arch/aarch64/cpu/boot.s",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2022 Andre Richter <andre.o.richter@gmail.com>\n\n//--------------------------------------------------------------------------------------------------\n// Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Load the address of a symbol into a register, PC-relative.\n//\n// The symbol must lie within +/- 4 GiB of the Program Counter.\n//\n// # Resources\n//\n// - https://sourceware.org/binutils/docs-2.36/as/AArch64_002dRelocations.html\n.macro ADR_REL register, symbol\n\tadrp\t\\register, \\symbol\n\tadd\t\\register, \\register, #:lo12:\\symbol\n.endm\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n.section .text._start\n\n//------------------------------------------------------------------------------\n// fn _start()\n//------------------------------------------------------------------------------\n_start:\n\t// Only proceed if the core executes in EL2. Park it otherwise.\n\tmrs\tx0, CurrentEL\n\tcmp\tx0, {CONST_CURRENTEL_EL2}\n\tb.ne\t.L_parking_loop\n\n\t// Only proceed on the boot core. Park it otherwise.\n\tmrs\tx1, MPIDR_EL1\n\tand\tx1, x1, {CONST_CORE_ID_MASK}\n\tldr\tx2, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs\n\tcmp\tx1, x2\n\tb.ne\t.L_parking_loop\n\n\t// If execution reaches here, it is the boot core.\n\n\t// Initialize DRAM.\n\tADR_REL\tx0, __bss_start\n\tADR_REL x1, __bss_end_exclusive\n\n.L_bss_init_loop:\n\tcmp\tx0, x1\n\tb.eq\t.L_prepare_rust\n\tstp\txzr, xzr, [x0], #16\n\tb\t.L_bss_init_loop\n\n\t// Prepare the jump to Rust code.\n.L_prepare_rust:\n\t// Load the base address of the kernel's translation tables.\n\tldr\tx0, PHYS_KERNEL_TABLES_BASE_ADDR // provided by bsp/__board_name__/memory/mmu.rs\n\n\t// Set the stack pointer. This ensures that any code in EL2 that needs the stack will work.\n\tADR_REL\tx1, __boot_core_stack_end_exclusive\n\tmov\tsp, x1\n\n\t// Read the CPU's timer counter frequency and store it in ARCH_TIMER_COUNTER_FREQUENCY.\n\t// Abort if the frequency read back as 0.\n\tADR_REL\tx2, ARCH_TIMER_COUNTER_FREQUENCY // provided by aarch64/time.rs\n\tmrs\tx3, CNTFRQ_EL0\n\tcmp\tx3, xzr\n\tb.eq\t.L_parking_loop\n\tstr\tw3, [x2]\n\n\t// Jump to Rust code. x0 and x1 hold the function arguments provided to _start_rust().\n\tb\t_start_rust\n\n\t// Infinitely wait for events (aka \"park the core\").\n.L_parking_loop:\n\twfe\n\tb\t.L_parking_loop\n\n.size\t_start, . - _start\n.type\t_start, function\n.global\t_start\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/_arch/aarch64/cpu/smp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural symmetric multiprocessing.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::smp::arch_smp\n\nuse aarch64_cpu::registers::*;\nuse tock_registers::interfaces::Readable;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return the executing core's id.\n#[inline(always)]\npub fn core_id<T>() -> T\nwhere\n    T: From<u8>,\n{\n    const CORE_MASK: u64 = 0b11;\n\n    T::from((MPIDR_EL1.get() & CORE_MASK) as u8)\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/_arch/aarch64/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural processor code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::arch_cpu\n\nuse aarch64_cpu::asm;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\npub use asm::nop;\n\n/// Pause execution on the core.\n#[inline(always)]\npub fn wait_forever() -> ! {\n    loop {\n        asm::wfe()\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n#[cfg(feature = \"test_build\")]\nuse qemu_exit::QEMUExit;\n\n#[cfg(feature = \"test_build\")]\nconst QEMU_EXIT_HANDLE: qemu_exit::AArch64 = qemu_exit::AArch64::new();\n\n/// Make the host QEMU binary execute `exit(1)`.\n#[cfg(feature = \"test_build\")]\npub fn qemu_exit_failure() -> ! {\n    QEMU_EXIT_HANDLE.exit_failure()\n}\n\n/// Make the host QEMU binary execute `exit(0)`.\n#[cfg(feature = \"test_build\")]\npub fn qemu_exit_success() -> ! {\n    QEMU_EXIT_HANDLE.exit_success()\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/_arch/aarch64/exception/asynchronous.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural asynchronous exception handling.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::exception::asynchronous::arch_asynchronous\n\nuse aarch64_cpu::registers::*;\nuse core::arch::asm;\nuse tock_registers::interfaces::{Readable, Writeable};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nmod daif_bits {\n    pub const IRQ: u8 = 0b0010;\n}\n\ntrait DaifField {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register>;\n}\n\nstruct Debug;\nstruct SError;\nstruct IRQ;\nstruct FIQ;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl DaifField for Debug {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::D\n    }\n}\n\nimpl DaifField for SError {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::A\n    }\n}\n\nimpl DaifField for IRQ {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::I\n    }\n}\n\nimpl DaifField for FIQ {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::F\n    }\n}\n\nfn is_masked<T>() -> bool\nwhere\n    T: DaifField,\n{\n    DAIF.is_set(T::daif_field())\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Returns whether IRQs are masked on the executing core.\npub fn is_local_irq_masked() -> bool {\n    !is_masked::<IRQ>()\n}\n\n/// Unmask IRQs on the executing core.\n///\n/// It is not needed to place an explicit instruction synchronization barrier after the `msr`.\n/// Quoting the Architecture Reference Manual for ARMv8-A, section C5.1.3:\n///\n/// \"Writes to PSTATE.{PAN, D, A, I, F} occur in program order without the need for additional\n/// synchronization.\"\n#[inline(always)]\npub fn local_irq_unmask() {\n    unsafe {\n        asm!(\n            \"msr DAIFClr, {arg}\",\n            arg = const daif_bits::IRQ,\n            options(nomem, nostack, preserves_flags)\n        );\n    }\n}\n\n/// Mask IRQs on the executing core.\n#[inline(always)]\npub fn local_irq_mask() {\n    unsafe {\n        asm!(\n            \"msr DAIFSet, {arg}\",\n            arg = const daif_bits::IRQ,\n            options(nomem, nostack, preserves_flags)\n        );\n    }\n}\n\n/// Mask IRQs on the executing core and return the previously saved interrupt mask bits (DAIF).\n#[inline(always)]\npub fn local_irq_mask_save() -> u64 {\n    let saved = DAIF.get();\n    local_irq_mask();\n\n    saved\n}\n\n/// Restore the interrupt mask bits (DAIF) using the callee's argument.\n///\n/// # Invariant\n///\n/// - No sanity checks on the input.\n#[inline(always)]\npub fn local_irq_restore(saved: u64) {\n    DAIF.set(saved);\n}\n\n/// Print the AArch64 exceptions status.\n#[rustfmt::skip]\npub fn print_state() {\n    use crate::info;\n\n    let to_mask_str = |x| -> _ {\n        if x { \"Masked\" } else { \"Unmasked\" }\n    };\n\n    info!(\"      Debug:  {}\", to_mask_str(is_masked::<Debug>()));\n    info!(\"      SError: {}\", to_mask_str(is_masked::<SError>()));\n    info!(\"      IRQ:    {}\", to_mask_str(is_masked::<IRQ>()));\n    info!(\"      FIQ:    {}\", to_mask_str(is_masked::<FIQ>()));\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/_arch/aarch64/exception.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural synchronous and asynchronous exception handling.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::exception::arch_exception\n\nuse crate::exception;\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::{arch::global_asm, cell::UnsafeCell, fmt};\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    registers::InMemoryRegister,\n};\n\n// Assembly counterpart to this file.\nglobal_asm!(include_str!(\"exception.s\"));\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Wrapper structs for memory copies of registers.\n#[repr(transparent)]\nstruct SpsrEL1(InMemoryRegister<u64, SPSR_EL1::Register>);\nstruct EsrEL1(InMemoryRegister<u64, ESR_EL1::Register>);\n\n/// The exception context as it is stored on the stack on exception entry.\n#[repr(C)]\nstruct ExceptionContext {\n    /// General Purpose Registers.\n    gpr: [u64; 30],\n\n    /// The link register, aka x30.\n    lr: u64,\n\n    /// Exception link register. The program counter at the time the exception happened.\n    elr_el1: u64,\n\n    /// Saved program status.\n    spsr_el1: SpsrEL1,\n\n    /// Exception syndrome register.\n    esr_el1: EsrEL1,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Prints verbose information about the exception and then panics.\nfn default_exception_handler(exc: &ExceptionContext) {\n    panic!(\n        \"CPU Exception!\\n\\n\\\n        {}\",\n        exc\n    );\n}\n\n//------------------------------------------------------------------------------\n// Current, EL0\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn current_el0_synchronous(_e: &mut ExceptionContext) {\n    panic!(\"Should not be here. Use of SP_EL0 in EL1 is not supported.\")\n}\n\n#[no_mangle]\nextern \"C\" fn current_el0_irq(_e: &mut ExceptionContext) {\n    panic!(\"Should not be here. Use of SP_EL0 in EL1 is not supported.\")\n}\n\n#[no_mangle]\nextern \"C\" fn current_el0_serror(_e: &mut ExceptionContext) {\n    panic!(\"Should not be here. Use of SP_EL0 in EL1 is not supported.\")\n}\n\n//------------------------------------------------------------------------------\n// Current, ELx\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn current_elx_synchronous(e: &mut ExceptionContext) {\n    #[cfg(feature = \"test_build\")]\n    {\n        const TEST_SVC_ID: u64 = 0x1337;\n\n        if let Some(ESR_EL1::EC::Value::SVC64) = e.esr_el1.exception_class() {\n            if e.esr_el1.iss() == TEST_SVC_ID {\n                return;\n            }\n        }\n    }\n\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn current_elx_irq(_e: &mut ExceptionContext) {\n    let token = unsafe { &exception::asynchronous::IRQContext::new() };\n    exception::asynchronous::irq_manager().handle_pending_irqs(token);\n}\n\n#[no_mangle]\nextern \"C\" fn current_elx_serror(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n//------------------------------------------------------------------------------\n// Lower, AArch64\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn lower_aarch64_synchronous(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch64_irq(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch64_serror(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n//------------------------------------------------------------------------------\n// Lower, AArch32\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn lower_aarch32_synchronous(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch32_irq(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch32_serror(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n//------------------------------------------------------------------------------\n// Misc\n//------------------------------------------------------------------------------\n\n/// Human readable SPSR_EL1.\n#[rustfmt::skip]\nimpl fmt::Display for SpsrEL1 {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        // Raw value.\n        writeln!(f, \"SPSR_EL1: {:#010x}\", self.0.get())?;\n\n        let to_flag_str = |x| -> _ {\n            if x { \"Set\" } else { \"Not set\" }\n         };\n\n        writeln!(f, \"      Flags:\")?;\n        writeln!(f, \"            Negative (N): {}\", to_flag_str(self.0.is_set(SPSR_EL1::N)))?;\n        writeln!(f, \"            Zero     (Z): {}\", to_flag_str(self.0.is_set(SPSR_EL1::Z)))?;\n        writeln!(f, \"            Carry    (C): {}\", to_flag_str(self.0.is_set(SPSR_EL1::C)))?;\n        writeln!(f, \"            Overflow (V): {}\", to_flag_str(self.0.is_set(SPSR_EL1::V)))?;\n\n        let to_mask_str = |x| -> _ {\n            if x { \"Masked\" } else { \"Unmasked\" }\n        };\n\n        writeln!(f, \"      Exception handling state:\")?;\n        writeln!(f, \"            Debug  (D): {}\", to_mask_str(self.0.is_set(SPSR_EL1::D)))?;\n        writeln!(f, \"            SError (A): {}\", to_mask_str(self.0.is_set(SPSR_EL1::A)))?;\n        writeln!(f, \"            IRQ    (I): {}\", to_mask_str(self.0.is_set(SPSR_EL1::I)))?;\n        writeln!(f, \"            FIQ    (F): {}\", to_mask_str(self.0.is_set(SPSR_EL1::F)))?;\n\n        write!(f, \"      Illegal Execution State (IL): {}\",\n            to_flag_str(self.0.is_set(SPSR_EL1::IL))\n        )\n    }\n}\n\nimpl EsrEL1 {\n    #[inline(always)]\n    fn exception_class(&self) -> Option<ESR_EL1::EC::Value> {\n        self.0.read_as_enum(ESR_EL1::EC)\n    }\n\n    #[cfg(feature = \"test_build\")]\n    #[inline(always)]\n    fn iss(&self) -> u64 {\n        self.0.read(ESR_EL1::ISS)\n    }\n}\n\n/// Human readable ESR_EL1.\n#[rustfmt::skip]\nimpl fmt::Display for EsrEL1 {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        // Raw print of whole register.\n        writeln!(f, \"ESR_EL1: {:#010x}\", self.0.get())?;\n\n        // Raw print of exception class.\n        write!(f, \"      Exception Class         (EC) : {:#x}\", self.0.read(ESR_EL1::EC))?;\n\n        // Exception class.\n        let ec_translation = match self.exception_class() {\n            Some(ESR_EL1::EC::Value::DataAbortCurrentEL) => \"Data Abort, current EL\",\n            _ => \"N/A\",\n        };\n        writeln!(f, \" - {}\", ec_translation)?;\n\n        // Raw print of instruction specific syndrome.\n        write!(f, \"      Instr Specific Syndrome (ISS): {:#x}\", self.0.read(ESR_EL1::ISS))\n    }\n}\n\nimpl ExceptionContext {\n    #[inline(always)]\n    fn exception_class(&self) -> Option<ESR_EL1::EC::Value> {\n        self.esr_el1.exception_class()\n    }\n\n    #[inline(always)]\n    fn fault_address_valid(&self) -> bool {\n        use ESR_EL1::EC::Value::*;\n\n        match self.exception_class() {\n            None => false,\n            Some(ec) => matches!(\n                ec,\n                InstrAbortLowerEL\n                    | InstrAbortCurrentEL\n                    | PCAlignmentFault\n                    | DataAbortLowerEL\n                    | DataAbortCurrentEL\n                    | WatchpointLowerEL\n                    | WatchpointCurrentEL\n            ),\n        }\n    }\n}\n\n/// Human readable print of the exception context.\nimpl fmt::Display for ExceptionContext {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        writeln!(f, \"{}\", self.esr_el1)?;\n\n        if self.fault_address_valid() {\n            writeln!(f, \"FAR_EL1: {:#018x}\", FAR_EL1.get() as usize)?;\n        }\n\n        writeln!(f, \"{}\", self.spsr_el1)?;\n        writeln!(f, \"ELR_EL1: {:#018x}\", self.elr_el1)?;\n        writeln!(f)?;\n        writeln!(f, \"General purpose register:\")?;\n\n        #[rustfmt::skip]\n        let alternating = |x| -> _ {\n            if x % 2 == 0 { \"   \" } else { \"\\n\" }\n        };\n\n        // Print two registers per line.\n        for (i, reg) in self.gpr.iter().enumerate() {\n            write!(f, \"      x{: <2}: {: >#018x}{}\", i, reg, alternating(i))?;\n        }\n        write!(f, \"      lr : {:#018x}\", self.lr)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse crate::exception::PrivilegeLevel;\n\n/// The processing element's current privilege level.\npub fn current_privilege_level() -> (PrivilegeLevel, &'static str) {\n    let el = CurrentEL.read_as_enum(CurrentEL::EL);\n    match el {\n        Some(CurrentEL::EL::Value::EL2) => (PrivilegeLevel::Hypervisor, \"EL2\"),\n        Some(CurrentEL::EL::Value::EL1) => (PrivilegeLevel::Kernel, \"EL1\"),\n        Some(CurrentEL::EL::Value::EL0) => (PrivilegeLevel::User, \"EL0\"),\n        _ => (PrivilegeLevel::Unknown, \"Unknown\"),\n    }\n}\n\n/// Init exception handling by setting the exception vector base address register.\n///\n/// # Safety\n///\n/// - Changes the HW state of the executing core.\n/// - The vector table and the symbol `__exception_vector_table_start` from the linker script must\n///   adhere to the alignment and size constraints demanded by the ARMv8-A Architecture Reference\n///   Manual.\npub unsafe fn handling_init() {\n    // Provided by exception.S.\n    extern \"Rust\" {\n        static __exception_vector_start: UnsafeCell<()>;\n    }\n\n    VBAR_EL1.set(__exception_vector_start.get() as u64);\n\n    // Force VBAR update to complete before next instruction.\n    barrier::isb(barrier::SY);\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/_arch/aarch64/exception.s",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2022 Andre Richter <andre.o.richter@gmail.com>\n\n//--------------------------------------------------------------------------------------------------\n// Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Call the function provided by parameter `\\handler` after saving the exception context. Provide\n/// the context as the first parameter to '\\handler'.\n.macro CALL_WITH_CONTEXT handler\n__vector_\\handler:\n\t// Make room on the stack for the exception context.\n\tsub\tsp,  sp,  #16 * 17\n\n\t// Store all general purpose registers on the stack.\n\tstp\tx0,  x1,  [sp, #16 * 0]\n\tstp\tx2,  x3,  [sp, #16 * 1]\n\tstp\tx4,  x5,  [sp, #16 * 2]\n\tstp\tx6,  x7,  [sp, #16 * 3]\n\tstp\tx8,  x9,  [sp, #16 * 4]\n\tstp\tx10, x11, [sp, #16 * 5]\n\tstp\tx12, x13, [sp, #16 * 6]\n\tstp\tx14, x15, [sp, #16 * 7]\n\tstp\tx16, x17, [sp, #16 * 8]\n\tstp\tx18, x19, [sp, #16 * 9]\n\tstp\tx20, x21, [sp, #16 * 10]\n\tstp\tx22, x23, [sp, #16 * 11]\n\tstp\tx24, x25, [sp, #16 * 12]\n\tstp\tx26, x27, [sp, #16 * 13]\n\tstp\tx28, x29, [sp, #16 * 14]\n\n\t// Add the exception link register (ELR_EL1), saved program status (SPSR_EL1) and exception\n\t// syndrome register (ESR_EL1).\n\tmrs\tx1,  ELR_EL1\n\tmrs\tx2,  SPSR_EL1\n\tmrs\tx3,  ESR_EL1\n\n\tstp\tlr,  x1,  [sp, #16 * 15]\n\tstp\tx2,  x3,  [sp, #16 * 16]\n\n\t// x0 is the first argument for the function called through `\\handler`.\n\tmov\tx0,  sp\n\n\t// Call `\\handler`.\n\tbl\t\\handler\n\n\t// After returning from exception handling code, replay the saved context and return via\n\t// `eret`.\n\tb\t__exception_restore_context\n\n.size\t__vector_\\handler, . - __vector_\\handler\n.type\t__vector_\\handler, function\n.endm\n\n.macro FIQ_SUSPEND\n1:\twfe\n\tb\t1b\n.endm\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n.section .text\n\n//------------------------------------------------------------------------------\n// The exception vector table.\n//------------------------------------------------------------------------------\n\n// Align by 2^11 bytes, as demanded by ARMv8-A. Same as ALIGN(2048) in an ld script.\n.align 11\n\n// Export a symbol for the Rust code to use.\n__exception_vector_start:\n\n// Current exception level with SP_EL0.\n//\n// .org sets the offset relative to section start.\n//\n// # Safety\n//\n// - It must be ensured that `CALL_WITH_CONTEXT` <= 0x80 bytes.\n.org 0x000\n\tCALL_WITH_CONTEXT current_el0_synchronous\n.org 0x080\n\tCALL_WITH_CONTEXT current_el0_irq\n.org 0x100\n\tFIQ_SUSPEND\n.org 0x180\n\tCALL_WITH_CONTEXT current_el0_serror\n\n// Current exception level with SP_ELx, x > 0.\n.org 0x200\n\tCALL_WITH_CONTEXT current_elx_synchronous\n.org 0x280\n\tCALL_WITH_CONTEXT current_elx_irq\n.org 0x300\n\tFIQ_SUSPEND\n.org 0x380\n\tCALL_WITH_CONTEXT current_elx_serror\n\n// Lower exception level, AArch64\n.org 0x400\n\tCALL_WITH_CONTEXT lower_aarch64_synchronous\n.org 0x480\n\tCALL_WITH_CONTEXT lower_aarch64_irq\n.org 0x500\n\tFIQ_SUSPEND\n.org 0x580\n\tCALL_WITH_CONTEXT lower_aarch64_serror\n\n// Lower exception level, AArch32\n.org 0x600\n\tCALL_WITH_CONTEXT lower_aarch32_synchronous\n.org 0x680\n\tCALL_WITH_CONTEXT lower_aarch32_irq\n.org 0x700\n\tFIQ_SUSPEND\n.org 0x780\n\tCALL_WITH_CONTEXT lower_aarch32_serror\n.org 0x800\n\n//------------------------------------------------------------------------------\n// fn __exception_restore_context()\n//------------------------------------------------------------------------------\n__exception_restore_context:\n\tldr\tw19,      [sp, #16 * 16]\n\tldp\tlr,  x20, [sp, #16 * 15]\n\n\tmsr\tSPSR_EL1, x19\n\tmsr\tELR_EL1,  x20\n\n\tldp\tx0,  x1,  [sp, #16 * 0]\n\tldp\tx2,  x3,  [sp, #16 * 1]\n\tldp\tx4,  x5,  [sp, #16 * 2]\n\tldp\tx6,  x7,  [sp, #16 * 3]\n\tldp\tx8,  x9,  [sp, #16 * 4]\n\tldp\tx10, x11, [sp, #16 * 5]\n\tldp\tx12, x13, [sp, #16 * 6]\n\tldp\tx14, x15, [sp, #16 * 7]\n\tldp\tx16, x17, [sp, #16 * 8]\n\tldp\tx18, x19, [sp, #16 * 9]\n\tldp\tx20, x21, [sp, #16 * 10]\n\tldp\tx22, x23, [sp, #16 * 11]\n\tldp\tx24, x25, [sp, #16 * 12]\n\tldp\tx26, x27, [sp, #16 * 13]\n\tldp\tx28, x29, [sp, #16 * 14]\n\n\tadd\tsp,  sp,  #16 * 17\n\n\teret\n\n.size\t__exception_restore_context, . - __exception_restore_context\n.type\t__exception_restore_context, function\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/_arch/aarch64/memory/mmu/translation_table.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural translation table.\n//!\n//! Only 64 KiB granule is supported.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::memory::mmu::translation_table::arch_translation_table\n\nuse crate::{\n    bsp,\n    memory::{\n        self,\n        mmu::{\n            arch_mmu::{Granule512MiB, Granule64KiB},\n            AccessPermissions, AttributeFields, MemAttributes, MemoryRegion, PageAddress,\n        },\n        Address, Physical, Virtual,\n    },\n};\nuse core::convert;\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields,\n    registers::InMemoryRegister,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// A table descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-15.\nregister_bitfields! {u64,\n    STAGE1_TABLE_DESCRIPTOR [\n        /// Physical address of the next descriptor.\n        NEXT_LEVEL_TABLE_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16]\n\n        TYPE  OFFSET(1) NUMBITS(1) [\n            Block = 0,\n            Table = 1\n        ],\n\n        VALID OFFSET(0) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ]\n    ]\n}\n\n// A level 3 page descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-17.\nregister_bitfields! {u64,\n    STAGE1_PAGE_DESCRIPTOR [\n        /// Unprivileged execute-never.\n        UXN      OFFSET(54) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ],\n\n        /// Privileged execute-never.\n        PXN      OFFSET(53) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ],\n\n        /// Physical address of the next table descriptor (lvl2) or the page descriptor (lvl3).\n        OUTPUT_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16]\n\n        /// Access flag.\n        AF       OFFSET(10) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ],\n\n        /// Shareability field.\n        SH       OFFSET(8) NUMBITS(2) [\n            OuterShareable = 0b10,\n            InnerShareable = 0b11\n        ],\n\n        /// Access Permissions.\n        AP       OFFSET(6) NUMBITS(2) [\n            RW_EL1 = 0b00,\n            RW_EL1_EL0 = 0b01,\n            RO_EL1 = 0b10,\n            RO_EL1_EL0 = 0b11\n        ],\n\n        /// Memory attributes index into the MAIR_EL1 register.\n        AttrIndx OFFSET(2) NUMBITS(3) [],\n\n        TYPE     OFFSET(1) NUMBITS(1) [\n            Reserved_Invalid = 0,\n            Page = 1\n        ],\n\n        VALID    OFFSET(0) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ]\n    ]\n}\n\n/// A table descriptor for 64 KiB aperture.\n///\n/// The output points to the next table.\n#[derive(Copy, Clone)]\n#[repr(C)]\nstruct TableDescriptor {\n    value: u64,\n}\n\n/// A page descriptor with 64 KiB aperture.\n///\n/// The output points to physical memory.\n#[derive(Copy, Clone)]\n#[repr(C)]\nstruct PageDescriptor {\n    value: u64,\n}\n\ntrait StartAddr {\n    fn virt_start_addr(&self) -> Address<Virtual>;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB\n/// aligned, so the lvl3 is put first.\n#[repr(C)]\n#[repr(align(65536))]\npub struct FixedSizeTranslationTable<const NUM_TABLES: usize> {\n    /// Page descriptors, covering 64 KiB windows per entry.\n    lvl3: [[PageDescriptor; 8192]; NUM_TABLES],\n\n    /// Table descriptors, covering 512 MiB windows.\n    lvl2: [TableDescriptor; NUM_TABLES],\n\n    /// Have the tables been initialized?\n    initialized: bool,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T, const N: usize> StartAddr for [T; N] {\n    fn virt_start_addr(&self) -> Address<Virtual> {\n        Address::new(self as *const _ as usize)\n    }\n}\n\nimpl TableDescriptor {\n    /// Create an instance.\n    ///\n    /// Descriptor is invalid by default.\n    pub const fn new_zeroed() -> Self {\n        Self { value: 0 }\n    }\n\n    /// Create an instance pointing to the supplied address.\n    pub fn from_next_lvl_table_addr(phys_next_lvl_table_addr: Address<Physical>) -> Self {\n        let val = InMemoryRegister::<u64, STAGE1_TABLE_DESCRIPTOR::Register>::new(0);\n\n        let shifted = phys_next_lvl_table_addr.as_usize() >> Granule64KiB::SHIFT;\n        val.write(\n            STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64)\n                + STAGE1_TABLE_DESCRIPTOR::TYPE::Table\n                + STAGE1_TABLE_DESCRIPTOR::VALID::True,\n        );\n\n        TableDescriptor { value: val.get() }\n    }\n}\n\n/// Convert the kernel's generic memory attributes to HW-specific attributes of the MMU.\nimpl convert::From<AttributeFields>\n    for tock_registers::fields::FieldValue<u64, STAGE1_PAGE_DESCRIPTOR::Register>\n{\n    fn from(attribute_fields: AttributeFields) -> Self {\n        // Memory attributes.\n        let mut desc = match attribute_fields.mem_attributes {\n            MemAttributes::CacheableDRAM => {\n                STAGE1_PAGE_DESCRIPTOR::SH::InnerShareable\n                    + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::NORMAL)\n            }\n            MemAttributes::Device => {\n                STAGE1_PAGE_DESCRIPTOR::SH::OuterShareable\n                    + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::DEVICE)\n            }\n        };\n\n        // Access Permissions.\n        desc += match attribute_fields.acc_perms {\n            AccessPermissions::ReadOnly => STAGE1_PAGE_DESCRIPTOR::AP::RO_EL1,\n            AccessPermissions::ReadWrite => STAGE1_PAGE_DESCRIPTOR::AP::RW_EL1,\n        };\n\n        // The execute-never attribute is mapped to PXN in AArch64.\n        desc += if attribute_fields.execute_never {\n            STAGE1_PAGE_DESCRIPTOR::PXN::True\n        } else {\n            STAGE1_PAGE_DESCRIPTOR::PXN::False\n        };\n\n        // Always set unprivileged exectue-never as long as userspace is not implemented yet.\n        desc += STAGE1_PAGE_DESCRIPTOR::UXN::True;\n\n        desc\n    }\n}\n\n/// Convert the HW-specific attributes of the MMU to kernel's generic memory attributes.\nimpl convert::TryFrom<InMemoryRegister<u64, STAGE1_PAGE_DESCRIPTOR::Register>> for AttributeFields {\n    type Error = &'static str;\n\n    fn try_from(\n        desc: InMemoryRegister<u64, STAGE1_PAGE_DESCRIPTOR::Register>,\n    ) -> Result<AttributeFields, Self::Error> {\n        let mem_attributes = match desc.read(STAGE1_PAGE_DESCRIPTOR::AttrIndx) {\n            memory::mmu::arch_mmu::mair::NORMAL => MemAttributes::CacheableDRAM,\n            memory::mmu::arch_mmu::mair::DEVICE => MemAttributes::Device,\n            _ => return Err(\"Unexpected memory attribute\"),\n        };\n\n        let acc_perms = match desc.read_as_enum(STAGE1_PAGE_DESCRIPTOR::AP) {\n            Some(STAGE1_PAGE_DESCRIPTOR::AP::Value::RO_EL1) => AccessPermissions::ReadOnly,\n            Some(STAGE1_PAGE_DESCRIPTOR::AP::Value::RW_EL1) => AccessPermissions::ReadWrite,\n            _ => return Err(\"Unexpected access permission\"),\n        };\n\n        let execute_never = desc.read(STAGE1_PAGE_DESCRIPTOR::PXN) > 0;\n\n        Ok(AttributeFields {\n            mem_attributes,\n            acc_perms,\n            execute_never,\n        })\n    }\n}\n\nimpl PageDescriptor {\n    /// Create an instance.\n    ///\n    /// Descriptor is invalid by default.\n    pub const fn new_zeroed() -> Self {\n        Self { value: 0 }\n    }\n\n    /// Create an instance.\n    pub fn from_output_page_addr(\n        phys_output_page_addr: PageAddress<Physical>,\n        attribute_fields: &AttributeFields,\n    ) -> Self {\n        let val = InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(0);\n\n        let shifted = phys_output_page_addr.into_inner().as_usize() >> Granule64KiB::SHIFT;\n        val.write(\n            STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted as u64)\n                + STAGE1_PAGE_DESCRIPTOR::AF::True\n                + STAGE1_PAGE_DESCRIPTOR::TYPE::Page\n                + STAGE1_PAGE_DESCRIPTOR::VALID::True\n                + (*attribute_fields).into(),\n        );\n\n        Self { value: val.get() }\n    }\n\n    /// Returns the valid bit.\n    fn is_valid(&self) -> bool {\n        InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(self.value)\n            .is_set(STAGE1_PAGE_DESCRIPTOR::VALID)\n    }\n\n    /// Returns the output page.\n    fn output_page_addr(&self) -> PageAddress<Physical> {\n        let shifted = InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(self.value)\n            .read(STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB) as usize;\n\n        PageAddress::from(shifted << Granule64KiB::SHIFT)\n    }\n\n    /// Returns the attributes.\n    fn try_attributes(&self) -> Result<AttributeFields, &'static str> {\n        InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(self.value).try_into()\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<const AS_SIZE: usize> memory::mmu::AssociatedTranslationTable\n    for memory::mmu::AddressSpace<AS_SIZE>\nwhere\n    [u8; Self::SIZE >> Granule512MiB::SHIFT]: Sized,\n{\n    type TableStartFromBottom = FixedSizeTranslationTable<{ Self::SIZE >> Granule512MiB::SHIFT }>;\n}\n\nimpl<const NUM_TABLES: usize> FixedSizeTranslationTable<NUM_TABLES> {\n    /// Create an instance.\n    #[allow(clippy::assertions_on_constants)]\n    const fn _new(for_precompute: bool) -> Self {\n        assert!(bsp::memory::mmu::KernelGranule::SIZE == Granule64KiB::SIZE);\n\n        // Can't have a zero-sized address space.\n        assert!(NUM_TABLES > 0);\n\n        Self {\n            lvl3: [[PageDescriptor::new_zeroed(); 8192]; NUM_TABLES],\n            lvl2: [TableDescriptor::new_zeroed(); NUM_TABLES],\n            initialized: for_precompute,\n        }\n    }\n\n    pub const fn new_for_precompute() -> Self {\n        Self::_new(true)\n    }\n\n    #[cfg(test)]\n    pub fn new_for_runtime() -> Self {\n        Self::_new(false)\n    }\n\n    /// Helper to calculate the lvl2 and lvl3 indices from an address.\n    #[inline(always)]\n    fn lvl2_lvl3_index_from_page_addr(\n        &self,\n        virt_page_addr: PageAddress<Virtual>,\n    ) -> Result<(usize, usize), &'static str> {\n        let addr = virt_page_addr.into_inner().as_usize();\n        let lvl2_index = addr >> Granule512MiB::SHIFT;\n        let lvl3_index = (addr & Granule512MiB::MASK) >> Granule64KiB::SHIFT;\n\n        if lvl2_index > (NUM_TABLES - 1) {\n            return Err(\"Virtual page is out of bounds of translation table\");\n        }\n\n        Ok((lvl2_index, lvl3_index))\n    }\n\n    /// Returns the PageDescriptor corresponding to the supplied page address.\n    #[inline(always)]\n    fn page_descriptor_from_page_addr(\n        &self,\n        virt_page_addr: PageAddress<Virtual>,\n    ) -> Result<&PageDescriptor, &'static str> {\n        let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from_page_addr(virt_page_addr)?;\n        let desc = &self.lvl3[lvl2_index][lvl3_index];\n\n        Ok(desc)\n    }\n\n    /// Sets the PageDescriptor corresponding to the supplied page address.\n    ///\n    /// Doesn't allow overriding an already valid page.\n    #[inline(always)]\n    fn set_page_descriptor_from_page_addr(\n        &mut self,\n        virt_page_addr: PageAddress<Virtual>,\n        new_desc: &PageDescriptor,\n    ) -> Result<(), &'static str> {\n        let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from_page_addr(virt_page_addr)?;\n        let desc = &mut self.lvl3[lvl2_index][lvl3_index];\n\n        if desc.is_valid() {\n            return Err(\"Virtual page is already mapped\");\n        }\n\n        *desc = *new_desc;\n        Ok(())\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\n\nimpl<const NUM_TABLES: usize> memory::mmu::translation_table::interface::TranslationTable\n    for FixedSizeTranslationTable<NUM_TABLES>\n{\n    fn init(&mut self) -> Result<(), &'static str> {\n        if self.initialized {\n            return Ok(());\n        }\n\n        // Populate the l2 entries.\n        for (lvl2_nr, lvl2_entry) in self.lvl2.iter_mut().enumerate() {\n            let virt_table_addr = self.lvl3[lvl2_nr].virt_start_addr();\n            let phys_table_addr = memory::mmu::try_kernel_virt_addr_to_phys_addr(virt_table_addr)?;\n\n            let new_desc = TableDescriptor::from_next_lvl_table_addr(phys_table_addr);\n            *lvl2_entry = new_desc;\n        }\n\n        self.initialized = true;\n\n        Ok(())\n    }\n\n    unsafe fn map_at(\n        &mut self,\n        virt_region: &MemoryRegion<Virtual>,\n        phys_region: &MemoryRegion<Physical>,\n        attr: &AttributeFields,\n    ) -> Result<(), &'static str> {\n        assert!(self.initialized, \"Translation tables not initialized\");\n\n        if virt_region.size() != phys_region.size() {\n            return Err(\"Tried to map memory regions with unequal sizes\");\n        }\n\n        if phys_region.end_exclusive_page_addr() > bsp::memory::phys_addr_space_end_exclusive_addr()\n        {\n            return Err(\"Tried to map outside of physical address space\");\n        }\n\n        let iter = phys_region.into_iter().zip(virt_region.into_iter());\n        for (phys_page_addr, virt_page_addr) in iter {\n            let new_desc = PageDescriptor::from_output_page_addr(phys_page_addr, attr);\n            let virt_page = virt_page_addr;\n\n            self.set_page_descriptor_from_page_addr(virt_page, &new_desc)?;\n        }\n\n        Ok(())\n    }\n\n    fn try_virt_page_addr_to_phys_page_addr(\n        &self,\n        virt_page_addr: PageAddress<Virtual>,\n    ) -> Result<PageAddress<Physical>, &'static str> {\n        let page_desc = self.page_descriptor_from_page_addr(virt_page_addr)?;\n\n        if !page_desc.is_valid() {\n            return Err(\"Page marked invalid\");\n        }\n\n        Ok(page_desc.output_page_addr())\n    }\n\n    fn try_page_attributes(\n        &self,\n        virt_page_addr: PageAddress<Virtual>,\n    ) -> Result<AttributeFields, &'static str> {\n        let page_desc = self.page_descriptor_from_page_addr(virt_page_addr)?;\n\n        if !page_desc.is_valid() {\n            return Err(\"Page marked invalid\");\n        }\n\n        page_desc.try_attributes()\n    }\n\n    /// Try to translate a virtual address to a physical address.\n    ///\n    /// Will only succeed if there exists a valid mapping for the input address.\n    fn try_virt_addr_to_phys_addr(\n        &self,\n        virt_addr: Address<Virtual>,\n    ) -> Result<Address<Physical>, &'static str> {\n        let virt_page = PageAddress::from(virt_addr.align_down_page());\n        let phys_page = self.try_virt_page_addr_to_phys_page_addr(virt_page)?;\n\n        Ok(phys_page.into_inner() + virt_addr.offset_into_page())\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\npub type MinSizeTranslationTable = FixedSizeTranslationTable<1>;\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// Check if the size of `struct TableDescriptor` is as expected.\n    #[kernel_test]\n    fn size_of_tabledescriptor_equals_64_bit() {\n        assert_eq!(\n            core::mem::size_of::<TableDescriptor>(),\n            core::mem::size_of::<u64>()\n        );\n    }\n\n    /// Check if the size of `struct PageDescriptor` is as expected.\n    #[kernel_test]\n    fn size_of_pagedescriptor_equals_64_bit() {\n        assert_eq!(\n            core::mem::size_of::<PageDescriptor>(),\n            core::mem::size_of::<u64>()\n        );\n    }\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/_arch/aarch64/memory/mmu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management Unit Driver.\n//!\n//! Only 64 KiB granule is supported.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::memory::mmu::arch_mmu\n\nuse crate::{\n    bsp, memory,\n    memory::{mmu::TranslationGranule, Address, Physical},\n};\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::intrinsics::unlikely;\nuse tock_registers::interfaces::{ReadWriteable, Readable, Writeable};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Memory Management Unit type.\nstruct MemoryManagementUnit;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub type Granule512MiB = TranslationGranule<{ 512 * 1024 * 1024 }>;\npub type Granule64KiB = TranslationGranule<{ 64 * 1024 }>;\n\n/// Constants for indexing the MAIR_EL1.\n#[allow(dead_code)]\npub mod mair {\n    pub const DEVICE: u64 = 0;\n    pub const NORMAL: u64 = 1;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic MMU: MemoryManagementUnit = MemoryManagementUnit;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<const AS_SIZE: usize> memory::mmu::AddressSpace<AS_SIZE> {\n    /// Checks for architectural restrictions.\n    pub const fn arch_address_space_size_sanity_checks() {\n        // Size must be at least one full 512 MiB table.\n        assert!((AS_SIZE % Granule512MiB::SIZE) == 0);\n\n        // Check for 48 bit virtual address size as maximum, which is supported by any ARMv8\n        // version.\n        assert!(AS_SIZE <= (1 << 48));\n    }\n}\n\nimpl MemoryManagementUnit {\n    /// Setup function for the MAIR_EL1 register.\n    fn set_up_mair(&self) {\n        // Define the memory types being mapped.\n        MAIR_EL1.write(\n            // Attribute 1 - Cacheable normal DRAM.\n            MAIR_EL1::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc +\n        MAIR_EL1::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc +\n\n        // Attribute 0 - Device.\n        MAIR_EL1::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck,\n        );\n    }\n\n    /// Configure various settings of stage 1 of the EL1 translation regime.\n    fn configure_translation_control(&self) {\n        let t0sz = (64 - bsp::memory::mmu::KernelVirtAddrSpace::SIZE_SHIFT) as u64;\n\n        TCR_EL1.write(\n            TCR_EL1::TBI0::Used\n                + TCR_EL1::IPS::Bits_40\n                + TCR_EL1::TG0::KiB_64\n                + TCR_EL1::SH0::Inner\n                + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable\n                + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable\n                + TCR_EL1::EPD0::EnableTTBR0Walks\n                + TCR_EL1::A1::TTBR0\n                + TCR_EL1::T0SZ.val(t0sz)\n                + TCR_EL1::EPD1::DisableTTBR1Walks,\n        );\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the MMU instance.\npub fn mmu() -> &'static impl memory::mmu::interface::MMU {\n    &MMU\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse memory::mmu::MMUEnableError;\n\nimpl memory::mmu::interface::MMU for MemoryManagementUnit {\n    unsafe fn enable_mmu_and_caching(\n        &self,\n        phys_tables_base_addr: Address<Physical>,\n    ) -> Result<(), MMUEnableError> {\n        if unlikely(self.is_enabled()) {\n            return Err(MMUEnableError::AlreadyEnabled);\n        }\n\n        // Fail early if translation granule is not supported.\n        if unlikely(!ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran64::Supported)) {\n            return Err(MMUEnableError::Other(\n                \"Translation granule not supported in HW\",\n            ));\n        }\n\n        // Prepare the memory attribute indirection register.\n        self.set_up_mair();\n\n        // Set the \"Translation Table Base Register\".\n        TTBR0_EL1.set_baddr(phys_tables_base_addr.as_usize() as u64);\n\n        self.configure_translation_control();\n\n        // Switch the MMU on.\n        //\n        // First, force all previous changes to be seen before the MMU is enabled.\n        barrier::isb(barrier::SY);\n\n        // Enable the MMU and turn on data and instruction caching.\n        SCTLR_EL1.modify(SCTLR_EL1::M::Enable + SCTLR_EL1::C::Cacheable + SCTLR_EL1::I::Cacheable);\n\n        // Force MMU init to complete before next instruction.\n        barrier::isb(barrier::SY);\n\n        Ok(())\n    }\n\n    #[inline(always)]\n    fn is_enabled(&self) -> bool {\n        SCTLR_EL1.matches_all(SCTLR_EL1::M::Enable)\n    }\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/_arch/aarch64/time.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural timer primitives.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::time::arch_time\n\nuse crate::warn;\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::{\n    num::{NonZeroU128, NonZeroU32, NonZeroU64},\n    ops::{Add, Div},\n    time::Duration,\n};\nuse tock_registers::interfaces::Readable;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nconst NANOSEC_PER_SEC: NonZeroU64 = NonZeroU64::new(1_000_000_000).unwrap();\n\n#[derive(Copy, Clone, PartialOrd, PartialEq)]\nstruct GenericTimerCounterValue(u64);\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n/// Boot assembly code overwrites this value with the value of CNTFRQ_EL0 before any Rust code is\n/// executed. This given value here is just a (safe) dummy.\n#[no_mangle]\nstatic ARCH_TIMER_COUNTER_FREQUENCY: NonZeroU32 = NonZeroU32::MIN;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nfn arch_timer_counter_frequency() -> NonZeroU32 {\n    // Read volatile is needed here to prevent the compiler from optimizing\n    // ARCH_TIMER_COUNTER_FREQUENCY away.\n    //\n    // This is safe, because all the safety requirements as stated in read_volatile()'s\n    // documentation are fulfilled.\n    unsafe { core::ptr::read_volatile(&ARCH_TIMER_COUNTER_FREQUENCY) }\n}\n\nimpl GenericTimerCounterValue {\n    pub const MAX: Self = GenericTimerCounterValue(u64::MAX);\n}\n\nimpl Add for GenericTimerCounterValue {\n    type Output = Self;\n\n    fn add(self, other: Self) -> Self {\n        GenericTimerCounterValue(self.0.wrapping_add(other.0))\n    }\n}\n\nimpl From<GenericTimerCounterValue> for Duration {\n    fn from(counter_value: GenericTimerCounterValue) -> Self {\n        if counter_value.0 == 0 {\n            return Duration::ZERO;\n        }\n\n        let frequency: NonZeroU64 = arch_timer_counter_frequency().into();\n\n        // Div<NonZeroU64> implementation for u64 cannot panic.\n        let secs = counter_value.0.div(frequency);\n\n        // This is safe, because frequency can never be greater than u32::MAX, which means the\n        // largest theoretical value for sub_second_counter_value is (u32::MAX - 1). Therefore,\n        // (sub_second_counter_value * NANOSEC_PER_SEC) cannot overflow an u64.\n        //\n        // The subsequent division ensures the result fits into u32, since the max result is smaller\n        // than NANOSEC_PER_SEC. Therefore, just cast it to u32 using `as`.\n        let sub_second_counter_value = counter_value.0 % frequency;\n        let nanos = unsafe { sub_second_counter_value.unchecked_mul(u64::from(NANOSEC_PER_SEC)) }\n            .div(frequency) as u32;\n\n        Duration::new(secs, nanos)\n    }\n}\n\nfn max_duration() -> Duration {\n    Duration::from(GenericTimerCounterValue::MAX)\n}\n\nimpl TryFrom<Duration> for GenericTimerCounterValue {\n    type Error = &'static str;\n\n    fn try_from(duration: Duration) -> Result<Self, Self::Error> {\n        if duration < resolution() {\n            return Ok(GenericTimerCounterValue(0));\n        }\n\n        if duration > max_duration() {\n            return Err(\"Conversion error. Duration too big\");\n        }\n\n        let frequency: u128 = u32::from(arch_timer_counter_frequency()) as u128;\n        let duration: u128 = duration.as_nanos();\n\n        // This is safe, because frequency can never be greater than u32::MAX, and\n        // (Duration::MAX.as_nanos() * u32::MAX) < u128::MAX.\n        let counter_value =\n            unsafe { duration.unchecked_mul(frequency) }.div(NonZeroU128::from(NANOSEC_PER_SEC));\n\n        // Since we checked above that we are <= max_duration(), just cast to u64.\n        Ok(GenericTimerCounterValue(counter_value as u64))\n    }\n}\n\n#[inline(always)]\nfn read_cntpct() -> GenericTimerCounterValue {\n    // Prevent that the counter is read ahead of time due to out-of-order execution.\n    barrier::isb(barrier::SY);\n    let cnt = CNTPCT_EL0.get();\n\n    GenericTimerCounterValue(cnt)\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The timer's resolution.\npub fn resolution() -> Duration {\n    Duration::from(GenericTimerCounterValue(1))\n}\n\n/// The uptime since power-on of the device.\n///\n/// This includes time consumed by firmware and bootloaders.\npub fn uptime() -> Duration {\n    read_cntpct().into()\n}\n\n/// Spin for a given duration.\npub fn spin_for(duration: Duration) {\n    let curr_counter_value = read_cntpct();\n\n    let counter_value_delta: GenericTimerCounterValue = match duration.try_into() {\n        Err(msg) => {\n            warn!(\"spin_for: {}. Skipping\", msg);\n            return;\n        }\n        Ok(val) => val,\n    };\n    let counter_value_target = curr_counter_value + counter_value_delta;\n\n    // Busy wait.\n    //\n    // Read CNTPCT_EL0 directly to avoid the ISB that is part of [`read_cntpct`].\n    while GenericTimerCounterValue(CNTPCT_EL0.get()) < counter_value_target {}\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/bsp/device_driver/arm/gicv2/gicc.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GICC Driver - GIC CPU interface.\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    exception,\n    memory::{Address, Virtual},\n};\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields, register_structs,\n    registers::ReadWrite,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nregister_bitfields! {\n    u32,\n\n    /// CPU Interface Control Register\n    CTLR [\n        Enable OFFSET(0) NUMBITS(1) []\n    ],\n\n    /// Interrupt Priority Mask Register\n    PMR [\n        Priority OFFSET(0) NUMBITS(8) []\n    ],\n\n    /// Interrupt Acknowledge Register\n    IAR [\n        InterruptID OFFSET(0) NUMBITS(10) []\n    ],\n\n    /// End of Interrupt Register\n    EOIR [\n        EOIINTID OFFSET(0) NUMBITS(10) []\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    pub RegisterBlock {\n        (0x000 => CTLR: ReadWrite<u32, CTLR::Register>),\n        (0x004 => PMR: ReadWrite<u32, PMR::Register>),\n        (0x008 => _reserved1),\n        (0x00C => IAR: ReadWrite<u32, IAR::Register>),\n        (0x010 => EOIR: ReadWrite<u32, EOIR::Register>),\n        (0x014  => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the GIC CPU interface.\npub struct GICC {\n    registers: Registers,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GICC {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n        }\n    }\n\n    /// Accept interrupts of any priority.\n    ///\n    /// Quoting the GICv2 Architecture Specification:\n    ///\n    ///   \"Writing 255 to the GICC_PMR always sets it to the largest supported priority field\n    ///    value.\"\n    ///\n    /// # Safety\n    ///\n    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n    ///   of `&mut self`.\n    pub fn priority_accept_all(&self) {\n        self.registers.PMR.write(PMR::Priority.val(255)); // Comment in arch spec.\n    }\n\n    /// Enable the interface - start accepting IRQs.\n    ///\n    /// # Safety\n    ///\n    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n    ///   of `&mut self`.\n    pub fn enable(&self) {\n        self.registers.CTLR.write(CTLR::Enable::SET);\n    }\n\n    /// Extract the number of the highest-priority pending IRQ.\n    ///\n    /// Can only be called from IRQ context, which is ensured by taking an `IRQContext` token.\n    ///\n    /// # Safety\n    ///\n    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n    ///   of `&mut self`.\n    #[allow(clippy::trivially_copy_pass_by_ref)]\n    pub fn pending_irq_number<'irq_context>(\n        &self,\n        _ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) -> usize {\n        self.registers.IAR.read(IAR::InterruptID) as usize\n    }\n\n    /// Complete handling of the currently active IRQ.\n    ///\n    /// Can only be called from IRQ context, which is ensured by taking an `IRQContext` token.\n    ///\n    /// To be called after `pending_irq_number()`.\n    ///\n    /// # Safety\n    ///\n    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n    ///   of `&mut self`.\n    #[allow(clippy::trivially_copy_pass_by_ref)]\n    pub fn mark_comleted<'irq_context>(\n        &self,\n        irq_number: u32,\n        _ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) {\n        self.registers.EOIR.write(EOIR::EOIINTID.val(irq_number));\n    }\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/bsp/device_driver/arm/gicv2/gicd.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GICD Driver - GIC Distributor.\n//!\n//! # Glossary\n//!   - SPI - Shared Peripheral Interrupt.\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    memory::{Address, Virtual},\n    state, synchronization,\n    synchronization::IRQSafeNullLock,\n};\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields, register_structs,\n    registers::{ReadOnly, ReadWrite},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nregister_bitfields! {\n    u32,\n\n    /// Distributor Control Register\n    CTLR [\n        Enable OFFSET(0) NUMBITS(1) []\n    ],\n\n    /// Interrupt Controller Type Register\n    TYPER [\n        ITLinesNumber OFFSET(0)  NUMBITS(5) []\n    ],\n\n    /// Interrupt Processor Targets Registers\n    ITARGETSR [\n        Offset3 OFFSET(24) NUMBITS(8) [],\n        Offset2 OFFSET(16) NUMBITS(8) [],\n        Offset1 OFFSET(8)  NUMBITS(8) [],\n        Offset0 OFFSET(0)  NUMBITS(8) []\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    SharedRegisterBlock {\n        (0x000 => CTLR: ReadWrite<u32, CTLR::Register>),\n        (0x004 => TYPER: ReadOnly<u32, TYPER::Register>),\n        (0x008 => _reserved1),\n        (0x104 => ISENABLER: [ReadWrite<u32>; 31]),\n        (0x180 => _reserved2),\n        (0x820 => ITARGETSR: [ReadWrite<u32, ITARGETSR::Register>; 248]),\n        (0xC00 => @END),\n    }\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    BankedRegisterBlock {\n        (0x000 => _reserved1),\n        (0x100 => ISENABLER: ReadWrite<u32>),\n        (0x104 => _reserved2),\n        (0x800 => ITARGETSR: [ReadOnly<u32, ITARGETSR::Register>; 8]),\n        (0x820 => @END),\n    }\n}\n\n/// Abstraction for the non-banked parts of the associated MMIO registers.\ntype SharedRegisters = MMIODerefWrapper<SharedRegisterBlock>;\n\n/// Abstraction for the banked parts of the associated MMIO registers.\ntype BankedRegisters = MMIODerefWrapper<BankedRegisterBlock>;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the GIC Distributor.\npub struct GICD {\n    /// Access to shared registers is guarded with a lock.\n    shared_registers: IRQSafeNullLock<SharedRegisters>,\n\n    /// Access to banked registers is unguarded.\n    banked_registers: BankedRegisters,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl SharedRegisters {\n    /// Return the number of IRQs that this HW implements.\n    #[inline(always)]\n    fn num_irqs(&mut self) -> usize {\n        // Query number of implemented IRQs.\n        //\n        // Refer to GICv2 Architecture Specification, Section 4.3.2.\n        ((self.TYPER.read(TYPER::ITLinesNumber) as usize) + 1) * 32\n    }\n\n    /// Return a slice of the implemented ITARGETSR.\n    #[inline(always)]\n    fn implemented_itargets_slice(&mut self) -> &[ReadWrite<u32, ITARGETSR::Register>] {\n        assert!(self.num_irqs() >= 36);\n\n        // Calculate the max index of the shared ITARGETSR array.\n        //\n        // The first 32 IRQs are private, so not included in `shared_registers`. Each ITARGETS\n        // register has four entries, so shift right by two. Subtract one because we start\n        // counting at zero.\n        let spi_itargetsr_max_index = ((self.num_irqs() - 32) >> 2) - 1;\n\n        // Rust automatically inserts slice range sanity check, i.e. max >= min.\n        &self.ITARGETSR[0..spi_itargetsr_max_index]\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl GICD {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            shared_registers: IRQSafeNullLock::new(SharedRegisters::new(mmio_start_addr)),\n            banked_registers: BankedRegisters::new(mmio_start_addr),\n        }\n    }\n\n    /// Use a banked ITARGETSR to retrieve the executing core's GIC target mask.\n    ///\n    /// Quoting the GICv2 Architecture Specification:\n    ///\n    ///   \"GICD_ITARGETSR0 to GICD_ITARGETSR7 are read-only, and each field returns a value that\n    ///    corresponds only to the processor reading the register.\"\n    fn local_gic_target_mask(&self) -> u32 {\n        self.banked_registers.ITARGETSR[0].read(ITARGETSR::Offset0)\n    }\n\n    /// Route all SPIs to the boot core and enable the distributor.\n    pub fn boot_core_init(&self) {\n        assert!(\n            state::state_manager().is_init(),\n            \"Only allowed during kernel init phase\"\n        );\n\n        // Target all SPIs to the boot core only.\n        let mask = self.local_gic_target_mask();\n\n        self.shared_registers.lock(|regs| {\n            for i in regs.implemented_itargets_slice().iter() {\n                i.write(\n                    ITARGETSR::Offset3.val(mask)\n                        + ITARGETSR::Offset2.val(mask)\n                        + ITARGETSR::Offset1.val(mask)\n                        + ITARGETSR::Offset0.val(mask),\n                );\n            }\n\n            regs.CTLR.write(CTLR::Enable::SET);\n        });\n    }\n\n    /// Enable an interrupt.\n    pub fn enable(&self, irq_num: &super::IRQNumber) {\n        let irq_num = irq_num.get();\n\n        // Each bit in the u32 enable register corresponds to one IRQ number. Shift right by 5\n        // (division by 32) and arrive at the index for the respective ISENABLER[i].\n        let enable_reg_index = irq_num >> 5;\n        let enable_bit: u32 = 1u32 << (irq_num % 32);\n\n        // Check if we are handling a private or shared IRQ.\n        match irq_num {\n            // Private.\n            0..=31 => {\n                let enable_reg = &self.banked_registers.ISENABLER;\n                enable_reg.set(enable_reg.get() | enable_bit);\n            }\n            // Shared.\n            _ => {\n                let enable_reg_index_shared = enable_reg_index - 1;\n\n                self.shared_registers.lock(|regs| {\n                    let enable_reg = &regs.ISENABLER[enable_reg_index_shared];\n                    enable_reg.set(enable_reg.get() | enable_bit);\n                });\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/bsp/device_driver/arm/gicv2.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GICv2 Driver - ARM Generic Interrupt Controller v2.\n//!\n//! The following is a collection of excerpts with useful information from\n//!   - `Programmer's Guide for ARMv8-A`\n//!   - `ARM Generic Interrupt Controller Architecture Specification`\n//!\n//! # Programmer's Guide - 10.6.1 Configuration\n//!\n//! The GIC is accessed as a memory-mapped peripheral.\n//!\n//! All cores can access the common Distributor, but the CPU interface is banked, that is, each core\n//! uses the same address to access its own private CPU interface.\n//!\n//! It is not possible for a core to access the CPU interface of another core.\n//!\n//! # Architecture Specification - 10.6.2 Initialization\n//!\n//! Both the Distributor and the CPU interfaces are disabled at reset. The GIC must be initialized\n//! after reset before it can deliver interrupts to the core.\n//!\n//! In the Distributor, software must configure the priority, target, security and enable individual\n//! interrupts. The Distributor must subsequently be enabled through its control register\n//! (GICD_CTLR). For each CPU interface, software must program the priority mask and preemption\n//! settings.\n//!\n//! Each CPU interface block itself must be enabled through its control register (GICD_CTLR). This\n//! prepares the GIC to deliver interrupts to the core.\n//!\n//! Before interrupts are expected in the core, software prepares the core to take interrupts by\n//! setting a valid interrupt vector in the vector table, and clearing interrupt mask bits in\n//! PSTATE, and setting the routing controls.\n//!\n//! The entire interrupt mechanism in the system can be disabled by disabling the Distributor.\n//! Interrupt delivery to an individual core can be disabled by disabling its CPU interface.\n//! Individual interrupts can also be disabled (or enabled) in the distributor.\n//!\n//! For an interrupt to reach the core, the individual interrupt, Distributor and CPU interface must\n//! all be enabled. The interrupt also needs to be of sufficient priority, that is, higher than the\n//! core's priority mask.\n//!\n//! # Architecture Specification - 1.4.2 Interrupt types\n//!\n//! - Peripheral interrupt\n//!     - Private Peripheral Interrupt (PPI)\n//!         - This is a peripheral interrupt that is specific to a single processor.\n//!     - Shared Peripheral Interrupt (SPI)\n//!         - This is a peripheral interrupt that the Distributor can route to any of a specified\n//!           combination of processors.\n//!\n//! - Software-generated interrupt (SGI)\n//!     - This is an interrupt generated by software writing to a GICD_SGIR register in the GIC. The\n//!       system uses SGIs for interprocessor communication.\n//!     - An SGI has edge-triggered properties. The software triggering of the interrupt is\n//!       equivalent to the edge transition of the interrupt request signal.\n//!     - When an SGI occurs in a multiprocessor implementation, the CPUID field in the Interrupt\n//!       Acknowledge Register, GICC_IAR, or the Aliased Interrupt Acknowledge Register, GICC_AIAR,\n//!       identifies the processor that requested the interrupt.\n//!\n//! # Architecture Specification - 2.2.1 Interrupt IDs\n//!\n//! Interrupts from sources are identified using ID numbers. Each CPU interface can see up to 1020\n//! interrupts. The banking of SPIs and PPIs increases the total number of interrupts supported by\n//! the Distributor.\n//!\n//! The GIC assigns interrupt ID numbers ID0-ID1019 as follows:\n//!   - Interrupt numbers 32..1019 are used for SPIs.\n//!   - Interrupt numbers 0..31 are used for interrupts that are private to a CPU interface. These\n//!     interrupts are banked in the Distributor.\n//!       - A banked interrupt is one where the Distributor can have multiple interrupts with the\n//!         same ID. A banked interrupt is identified uniquely by its ID number and its associated\n//!         CPU interface number. Of the banked interrupt IDs:\n//!           - 00..15 SGIs\n//!           - 16..31 PPIs\n\nmod gicc;\nmod gicd;\n\nuse crate::{\n    bsp::{self, device_driver::common::BoundedUsize},\n    cpu, driver, exception,\n    memory::{Address, Virtual},\n    synchronization,\n    synchronization::InitStateLock,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\ntype HandlerTable = [Option<exception::asynchronous::IRQHandlerDescriptor<IRQNumber>>;\n    IRQNumber::MAX_INCLUSIVE + 1];\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Used for the associated type of trait [`exception::asynchronous::interface::IRQManager`].\npub type IRQNumber = BoundedUsize<{ GICv2::MAX_IRQ_NUMBER }>;\n\n/// Representation of the GIC.\npub struct GICv2 {\n    /// The Distributor.\n    gicd: gicd::GICD,\n\n    /// The CPU Interface.\n    gicc: gicc::GICC,\n\n    /// Stores registered IRQ handlers. Writable only during kernel init. RO afterwards.\n    handler_table: InitStateLock<HandlerTable>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GICv2 {\n    const MAX_IRQ_NUMBER: usize = 300; // Normally 1019, but keep it lower to save some space.\n\n    pub const COMPATIBLE: &'static str = \"GICv2 (ARM Generic Interrupt Controller v2)\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(\n        gicd_mmio_start_addr: Address<Virtual>,\n        gicc_mmio_start_addr: Address<Virtual>,\n    ) -> Self {\n        Self {\n            gicd: gicd::GICD::new(gicd_mmio_start_addr),\n            gicc: gicc::GICC::new(gicc_mmio_start_addr),\n            handler_table: InitStateLock::new([None; IRQNumber::MAX_INCLUSIVE + 1]),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::ReadWriteEx;\n\nimpl driver::interface::DeviceDriver for GICv2 {\n    type IRQNumberType = IRQNumber;\n\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n\n    unsafe fn init(&self) -> Result<(), &'static str> {\n        if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() {\n            self.gicd.boot_core_init();\n        }\n\n        self.gicc.priority_accept_all();\n        self.gicc.enable();\n\n        Ok(())\n    }\n}\n\nimpl exception::asynchronous::interface::IRQManager for GICv2 {\n    type IRQNumberType = IRQNumber;\n\n    fn register_handler(\n        &self,\n        irq_handler_descriptor: exception::asynchronous::IRQHandlerDescriptor<Self::IRQNumberType>,\n    ) -> Result<(), &'static str> {\n        self.handler_table.write(|table| {\n            let irq_number = irq_handler_descriptor.number().get();\n\n            if table[irq_number].is_some() {\n                return Err(\"IRQ handler already registered\");\n            }\n\n            table[irq_number] = Some(irq_handler_descriptor);\n\n            Ok(())\n        })\n    }\n\n    fn enable(&self, irq_number: &Self::IRQNumberType) {\n        self.gicd.enable(irq_number);\n    }\n\n    fn handle_pending_irqs<'irq_context>(\n        &'irq_context self,\n        ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) {\n        // Extract the highest priority pending IRQ number from the Interrupt Acknowledge Register\n        // (IAR).\n        let irq_number = self.gicc.pending_irq_number(ic);\n\n        // Guard against spurious interrupts.\n        if irq_number > GICv2::MAX_IRQ_NUMBER {\n            return;\n        }\n\n        // Call the IRQ handler. Panic if there is none.\n        self.handler_table.read(|table| {\n            match table[irq_number] {\n                None => panic!(\"No handler registered for IRQ {}\", irq_number),\n                Some(descriptor) => {\n                    // Call the IRQ handler. Panics on failure.\n                    descriptor.handler().handle().expect(\"Error handling IRQ\");\n                }\n            }\n        });\n\n        // Signal completion of handling.\n        self.gicc.mark_comleted(irq_number as u32, ic);\n    }\n\n    fn print_handler(&self) {\n        use crate::info;\n\n        info!(\"      Peripheral handler:\");\n\n        self.handler_table.read(|table| {\n            for (i, opt) in table.iter().skip(32).enumerate() {\n                if let Some(handler) = opt {\n                    info!(\"            {: >3}. {}\", i + 32, handler.name());\n                }\n            }\n        });\n    }\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/bsp/device_driver/arm.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! ARM driver top level.\n\npub mod gicv2;\n\npub use gicv2::*;\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GPIO Driver.\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    driver,\n    exception::asynchronous::IRQNumber,\n    memory::{Address, Virtual},\n    synchronization,\n    synchronization::IRQSafeNullLock,\n};\nuse tock_registers::{\n    interfaces::{ReadWriteable, Writeable},\n    register_bitfields, register_structs,\n    registers::ReadWrite,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// GPIO registers.\n//\n// Descriptions taken from\n// - https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf\n// - https://datasheets.raspberrypi.org/bcm2711/bcm2711-peripherals.pdf\nregister_bitfields! {\n    u32,\n\n    /// GPIO Function Select 1\n    GPFSEL1 [\n        /// Pin 15\n        FSEL15 OFFSET(15) NUMBITS(3) [\n            Input = 0b000,\n            Output = 0b001,\n            AltFunc0 = 0b100  // PL011 UART RX\n\n        ],\n\n        /// Pin 14\n        FSEL14 OFFSET(12) NUMBITS(3) [\n            Input = 0b000,\n            Output = 0b001,\n            AltFunc0 = 0b100  // PL011 UART TX\n        ]\n    ],\n\n    /// GPIO Pull-up/down Register\n    ///\n    /// BCM2837 only.\n    GPPUD [\n        /// Controls the actuation of the internal pull-up/down control line to ALL the GPIO pins.\n        PUD OFFSET(0) NUMBITS(2) [\n            Off = 0b00,\n            PullDown = 0b01,\n            PullUp = 0b10\n        ]\n    ],\n\n    /// GPIO Pull-up/down Clock Register 0\n    ///\n    /// BCM2837 only.\n    GPPUDCLK0 [\n        /// Pin 15\n        PUDCLK15 OFFSET(15) NUMBITS(1) [\n            NoEffect = 0,\n            AssertClock = 1\n        ],\n\n        /// Pin 14\n        PUDCLK14 OFFSET(14) NUMBITS(1) [\n            NoEffect = 0,\n            AssertClock = 1\n        ]\n    ],\n\n    /// GPIO Pull-up / Pull-down Register 0\n    ///\n    /// BCM2711 only.\n    GPIO_PUP_PDN_CNTRL_REG0 [\n        /// Pin 15\n        GPIO_PUP_PDN_CNTRL15 OFFSET(30) NUMBITS(2) [\n            NoResistor = 0b00,\n            PullUp = 0b01\n        ],\n\n        /// Pin 14\n        GPIO_PUP_PDN_CNTRL14 OFFSET(28) NUMBITS(2) [\n            NoResistor = 0b00,\n            PullUp = 0b01\n        ]\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    RegisterBlock {\n        (0x00 => _reserved1),\n        (0x04 => GPFSEL1: ReadWrite<u32, GPFSEL1::Register>),\n        (0x08 => _reserved2),\n        (0x94 => GPPUD: ReadWrite<u32, GPPUD::Register>),\n        (0x98 => GPPUDCLK0: ReadWrite<u32, GPPUDCLK0::Register>),\n        (0x9C => _reserved3),\n        (0xE4 => GPIO_PUP_PDN_CNTRL_REG0: ReadWrite<u32, GPIO_PUP_PDN_CNTRL_REG0::Register>),\n        (0xE8 => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\nstruct GPIOInner {\n    registers: Registers,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the GPIO HW.\npub struct GPIO {\n    inner: IRQSafeNullLock<GPIOInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GPIOInner {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n        }\n    }\n\n    /// Disable pull-up/down on pins 14 and 15.\n    #[cfg(feature = \"bsp_rpi3\")]\n    fn disable_pud_14_15_bcm2837(&mut self) {\n        use crate::time;\n        use core::time::Duration;\n\n        // The Linux 2837 GPIO driver waits 1 µs between the steps.\n        const DELAY: Duration = Duration::from_micros(1);\n\n        self.registers.GPPUD.write(GPPUD::PUD::Off);\n        time::time_manager().spin_for(DELAY);\n\n        self.registers\n            .GPPUDCLK0\n            .write(GPPUDCLK0::PUDCLK15::AssertClock + GPPUDCLK0::PUDCLK14::AssertClock);\n        time::time_manager().spin_for(DELAY);\n\n        self.registers.GPPUD.write(GPPUD::PUD::Off);\n        self.registers.GPPUDCLK0.set(0);\n    }\n\n    /// Disable pull-up/down on pins 14 and 15.\n    #[cfg(feature = \"bsp_rpi4\")]\n    fn disable_pud_14_15_bcm2711(&mut self) {\n        self.registers.GPIO_PUP_PDN_CNTRL_REG0.write(\n            GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL15::PullUp\n                + GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL14::PullUp,\n        );\n    }\n\n    /// Map PL011 UART as standard output.\n    ///\n    /// TX to pin 14\n    /// RX to pin 15\n    pub fn map_pl011_uart(&mut self) {\n        // Select the UART on pins 14 and 15.\n        self.registers\n            .GPFSEL1\n            .modify(GPFSEL1::FSEL15::AltFunc0 + GPFSEL1::FSEL14::AltFunc0);\n\n        // Disable pull-up/down on pins 14 and 15.\n        #[cfg(feature = \"bsp_rpi3\")]\n        self.disable_pud_14_15_bcm2837();\n\n        #[cfg(feature = \"bsp_rpi4\")]\n        self.disable_pud_14_15_bcm2711();\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GPIO {\n    pub const COMPATIBLE: &'static str = \"BCM GPIO\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            inner: IRQSafeNullLock::new(GPIOInner::new(mmio_start_addr)),\n        }\n    }\n\n    /// Concurrency safe version of `GPIOInner.map_pl011_uart()`\n    pub fn map_pl011_uart(&self) {\n        self.inner.lock(|inner| inner.map_pl011_uart())\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl driver::interface::DeviceDriver for GPIO {\n    type IRQNumberType = IRQNumber;\n\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Peripheral Interrupt Controller Driver.\n//!\n//! # Resources\n//!\n//! - <https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf>\n\nuse super::{PendingIRQs, PeripheralIRQ};\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    exception,\n    memory::{Address, Virtual},\n    synchronization,\n    synchronization::{IRQSafeNullLock, InitStateLock},\n};\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_structs,\n    registers::{ReadOnly, WriteOnly},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    WORegisterBlock {\n        (0x00 => _reserved1),\n        (0x10 => ENABLE_1: WriteOnly<u32>),\n        (0x14 => ENABLE_2: WriteOnly<u32>),\n        (0x18 => @END),\n    }\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    RORegisterBlock {\n        (0x00 => _reserved1),\n        (0x04 => PENDING_1: ReadOnly<u32>),\n        (0x08 => PENDING_2: ReadOnly<u32>),\n        (0x0c => @END),\n    }\n}\n\n/// Abstraction for the WriteOnly parts of the associated MMIO registers.\ntype WriteOnlyRegisters = MMIODerefWrapper<WORegisterBlock>;\n\n/// Abstraction for the ReadOnly parts of the associated MMIO registers.\ntype ReadOnlyRegisters = MMIODerefWrapper<RORegisterBlock>;\n\ntype HandlerTable = [Option<exception::asynchronous::IRQHandlerDescriptor<PeripheralIRQ>>;\n    PeripheralIRQ::MAX_INCLUSIVE + 1];\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the peripheral interrupt controller.\npub struct PeripheralIC {\n    /// Access to write registers is guarded with a lock.\n    wo_registers: IRQSafeNullLock<WriteOnlyRegisters>,\n\n    /// Register read access is unguarded.\n    ro_registers: ReadOnlyRegisters,\n\n    /// Stores registered IRQ handlers. Writable only during kernel init. RO afterwards.\n    handler_table: InitStateLock<HandlerTable>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PeripheralIC {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            wo_registers: IRQSafeNullLock::new(WriteOnlyRegisters::new(mmio_start_addr)),\n            ro_registers: ReadOnlyRegisters::new(mmio_start_addr),\n            handler_table: InitStateLock::new([None; PeripheralIRQ::MAX_INCLUSIVE + 1]),\n        }\n    }\n\n    /// Query the list of pending IRQs.\n    fn pending_irqs(&self) -> PendingIRQs {\n        let pending_mask: u64 = (u64::from(self.ro_registers.PENDING_2.get()) << 32)\n            | u64::from(self.ro_registers.PENDING_1.get());\n\n        PendingIRQs::new(pending_mask)\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::{Mutex, ReadWriteEx};\n\nimpl exception::asynchronous::interface::IRQManager for PeripheralIC {\n    type IRQNumberType = PeripheralIRQ;\n\n    fn register_handler(\n        &self,\n        irq_handler_descriptor: exception::asynchronous::IRQHandlerDescriptor<Self::IRQNumberType>,\n    ) -> Result<(), &'static str> {\n        self.handler_table.write(|table| {\n            let irq_number = irq_handler_descriptor.number().get();\n\n            if table[irq_number].is_some() {\n                return Err(\"IRQ handler already registered\");\n            }\n\n            table[irq_number] = Some(irq_handler_descriptor);\n\n            Ok(())\n        })\n    }\n\n    fn enable(&self, irq: &Self::IRQNumberType) {\n        self.wo_registers.lock(|regs| {\n            let enable_reg = if irq.get() <= 31 {\n                &regs.ENABLE_1\n            } else {\n                &regs.ENABLE_2\n            };\n\n            let enable_bit: u32 = 1 << (irq.get() % 32);\n\n            // Writing a 1 to a bit will set the corresponding IRQ enable bit. All other IRQ enable\n            // bits are unaffected. So we don't need read and OR'ing here.\n            enable_reg.set(enable_bit);\n        });\n    }\n\n    fn handle_pending_irqs<'irq_context>(\n        &'irq_context self,\n        _ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) {\n        self.handler_table.read(|table| {\n            for irq_number in self.pending_irqs() {\n                match table[irq_number] {\n                    None => panic!(\"No handler registered for IRQ {}\", irq_number),\n                    Some(descriptor) => {\n                        // Call the IRQ handler. Panics on failure.\n                        descriptor.handler().handle().expect(\"Error handling IRQ\");\n                    }\n                }\n            }\n        })\n    }\n\n    fn print_handler(&self) {\n        use crate::info;\n\n        info!(\"      Peripheral handler:\");\n\n        self.handler_table.read(|table| {\n            for (i, opt) in table.iter().enumerate() {\n                if let Some(handler) = opt {\n                    info!(\"            {: >3}. {}\", i, handler.name());\n                }\n            }\n        });\n    }\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Interrupt Controller Driver.\n\nmod peripheral_ic;\n\nuse crate::{\n    bsp::device_driver::common::BoundedUsize,\n    driver,\n    exception::{self, asynchronous::IRQHandlerDescriptor},\n    memory::{Address, Virtual},\n};\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Wrapper struct for a bitmask indicating pending IRQ numbers.\nstruct PendingIRQs {\n    bitmask: u64,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub type LocalIRQ = BoundedUsize<{ InterruptController::MAX_LOCAL_IRQ_NUMBER }>;\npub type PeripheralIRQ = BoundedUsize<{ InterruptController::MAX_PERIPHERAL_IRQ_NUMBER }>;\n\n/// Used for the associated type of trait [`exception::asynchronous::interface::IRQManager`].\n#[derive(Copy, Clone)]\n#[allow(missing_docs)]\npub enum IRQNumber {\n    Local(LocalIRQ),\n    Peripheral(PeripheralIRQ),\n}\n\n/// Representation of the Interrupt Controller.\npub struct InterruptController {\n    periph: peripheral_ic::PeripheralIC,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PendingIRQs {\n    pub fn new(bitmask: u64) -> Self {\n        Self { bitmask }\n    }\n}\n\nimpl Iterator for PendingIRQs {\n    type Item = usize;\n\n    fn next(&mut self) -> Option<Self::Item> {\n        if self.bitmask == 0 {\n            return None;\n        }\n\n        let next = self.bitmask.trailing_zeros() as usize;\n        self.bitmask &= self.bitmask.wrapping_sub(1);\n        Some(next)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl fmt::Display for IRQNumber {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        match self {\n            Self::Local(number) => write!(f, \"Local({})\", number),\n            Self::Peripheral(number) => write!(f, \"Peripheral({})\", number),\n        }\n    }\n}\n\nimpl InterruptController {\n    // Restrict to 3 for now. This makes future code for local_ic.rs more straight forward.\n    const MAX_LOCAL_IRQ_NUMBER: usize = 3;\n    const MAX_PERIPHERAL_IRQ_NUMBER: usize = 63;\n\n    pub const COMPATIBLE: &'static str = \"BCM Interrupt Controller\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(periph_mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            periph: peripheral_ic::PeripheralIC::new(periph_mmio_start_addr),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\n\nimpl driver::interface::DeviceDriver for InterruptController {\n    type IRQNumberType = IRQNumber;\n\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n}\n\nimpl exception::asynchronous::interface::IRQManager for InterruptController {\n    type IRQNumberType = IRQNumber;\n\n    fn register_handler(\n        &self,\n        irq_handler_descriptor: exception::asynchronous::IRQHandlerDescriptor<Self::IRQNumberType>,\n    ) -> Result<(), &'static str> {\n        match irq_handler_descriptor.number() {\n            IRQNumber::Local(_) => unimplemented!(\"Local IRQ controller not implemented.\"),\n            IRQNumber::Peripheral(pirq) => {\n                let periph_descriptor = IRQHandlerDescriptor::new(\n                    pirq,\n                    irq_handler_descriptor.name(),\n                    irq_handler_descriptor.handler(),\n                );\n\n                self.periph.register_handler(periph_descriptor)\n            }\n        }\n    }\n\n    fn enable(&self, irq: &Self::IRQNumberType) {\n        match irq {\n            IRQNumber::Local(_) => unimplemented!(\"Local IRQ controller not implemented.\"),\n            IRQNumber::Peripheral(pirq) => self.periph.enable(pirq),\n        }\n    }\n\n    fn handle_pending_irqs<'irq_context>(\n        &'irq_context self,\n        ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) {\n        // It can only be a peripheral IRQ pending because enable() does not support local IRQs yet.\n        self.periph.handle_pending_irqs(ic)\n    }\n\n    fn print_handler(&self) {\n        self.periph.print_handler();\n    }\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! PL011 UART driver.\n//!\n//! # Resources\n//!\n//! - <https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf>\n//! - <https://developer.arm.com/documentation/ddi0183/latest>\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    console, cpu, driver,\n    exception::{self, asynchronous::IRQNumber},\n    memory::{Address, Virtual},\n    synchronization,\n    synchronization::IRQSafeNullLock,\n};\nuse core::fmt;\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields, register_structs,\n    registers::{ReadOnly, ReadWrite, WriteOnly},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// PL011 UART registers.\n//\n// Descriptions taken from \"PrimeCell UART (PL011) Technical Reference Manual\" r1p5.\nregister_bitfields! {\n    u32,\n\n    /// Flag Register.\n    FR [\n        /// Transmit FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n        /// Line Control Register, LCR_H.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the transmit holding register is empty.\n        /// - If the FIFO is enabled, the TXFE bit is set when the transmit FIFO is empty.\n        /// - This bit does not indicate if there is data in the transmit shift register.\n        TXFE OFFSET(7) NUMBITS(1) [],\n\n        /// Transmit FIFO full. The meaning of this bit depends on the state of the FEN bit in the\n        /// LCR_H Register.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the transmit holding register is full.\n        /// - If the FIFO is enabled, the TXFF bit is set when the transmit FIFO is full.\n        TXFF OFFSET(5) NUMBITS(1) [],\n\n        /// Receive FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n        /// LCR_H Register.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the receive holding register is empty.\n        /// - If the FIFO is enabled, the RXFE bit is set when the receive FIFO is empty.\n        RXFE OFFSET(4) NUMBITS(1) [],\n\n        /// UART busy. If this bit is set to 1, the UART is busy transmitting data. This bit remains\n        /// set until the complete byte, including all the stop bits, has been sent from the shift\n        /// register.\n        ///\n        /// This bit is set as soon as the transmit FIFO becomes non-empty, regardless of whether\n        /// the UART is enabled or not.\n        BUSY OFFSET(3) NUMBITS(1) []\n    ],\n\n    /// Integer Baud Rate Divisor.\n    IBRD [\n        /// The integer baud rate divisor.\n        BAUD_DIVINT OFFSET(0) NUMBITS(16) []\n    ],\n\n    /// Fractional Baud Rate Divisor.\n    FBRD [\n        ///  The fractional baud rate divisor.\n        BAUD_DIVFRAC OFFSET(0) NUMBITS(6) []\n    ],\n\n    /// Line Control Register.\n    LCR_H [\n        /// Word length. These bits indicate the number of data bits transmitted or received in a\n        /// frame.\n        #[allow(clippy::enum_variant_names)]\n        WLEN OFFSET(5) NUMBITS(2) [\n            FiveBit = 0b00,\n            SixBit = 0b01,\n            SevenBit = 0b10,\n            EightBit = 0b11\n        ],\n\n        /// Enable FIFOs:\n        ///\n        /// 0 = FIFOs are disabled (character mode) that is, the FIFOs become 1-byte-deep holding\n        /// registers.\n        ///\n        /// 1 = Transmit and receive FIFO buffers are enabled (FIFO mode).\n        FEN  OFFSET(4) NUMBITS(1) [\n            FifosDisabled = 0,\n            FifosEnabled = 1\n        ]\n    ],\n\n    /// Control Register.\n    CR [\n        /// Receive enable. If this bit is set to 1, the receive section of the UART is enabled.\n        /// Data reception occurs for either UART signals or SIR signals depending on the setting of\n        /// the SIREN bit. When the UART is disabled in the middle of reception, it completes the\n        /// current character before stopping.\n        RXE OFFSET(9) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// Transmit enable. If this bit is set to 1, the transmit section of the UART is enabled.\n        /// Data transmission occurs for either UART signals, or SIR signals depending on the\n        /// setting of the SIREN bit. When the UART is disabled in the middle of transmission, it\n        /// completes the current character before stopping.\n        TXE OFFSET(8) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// UART enable:\n        ///\n        /// 0 = UART is disabled. If the UART is disabled in the middle of transmission or\n        /// reception, it completes the current character before stopping.\n        ///\n        /// 1 = The UART is enabled. Data transmission and reception occurs for either UART signals\n        /// or SIR signals depending on the setting of the SIREN bit\n        UARTEN OFFSET(0) NUMBITS(1) [\n            /// If the UART is disabled in the middle of transmission or reception, it completes the\n            /// current character before stopping.\n            Disabled = 0,\n            Enabled = 1\n        ]\n    ],\n\n    /// Interrupt FIFO Level Select Register.\n    IFLS [\n        /// Receive interrupt FIFO level select. The trigger points for the receive interrupt are as\n        /// follows.\n        RXIFLSEL OFFSET(3) NUMBITS(5) [\n            OneEigth = 0b000,\n            OneQuarter = 0b001,\n            OneHalf = 0b010,\n            ThreeQuarters = 0b011,\n            SevenEights = 0b100\n        ]\n    ],\n\n    /// Interrupt Mask Set/Clear Register.\n    IMSC [\n        /// Receive timeout interrupt mask. A read returns the current mask for the UARTRTINTR\n        /// interrupt.\n        ///\n        /// - On a write of 1, the mask of the UARTRTINTR interrupt is set.\n        /// - A write of 0 clears the mask.\n        RTIM OFFSET(6) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// Receive interrupt mask. A read returns the current mask for the UARTRXINTR interrupt.\n        ///\n        /// - On a write of 1, the mask of the UARTRXINTR interrupt is set.\n        /// - A write of 0 clears the mask.\n        RXIM OFFSET(4) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ]\n    ],\n\n    /// Masked Interrupt Status Register.\n    MIS [\n        /// Receive timeout masked interrupt status. Returns the masked interrupt state of the\n        /// UARTRTINTR interrupt.\n        RTMIS OFFSET(6) NUMBITS(1) [],\n\n        /// Receive masked interrupt status. Returns the masked interrupt state of the UARTRXINTR\n        /// interrupt.\n        RXMIS OFFSET(4) NUMBITS(1) []\n    ],\n\n    /// Interrupt Clear Register.\n    ICR [\n        /// Meta field for all pending interrupts.\n        ALL OFFSET(0) NUMBITS(11) []\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    pub RegisterBlock {\n        (0x00 => DR: ReadWrite<u32>),\n        (0x04 => _reserved1),\n        (0x18 => FR: ReadOnly<u32, FR::Register>),\n        (0x1c => _reserved2),\n        (0x24 => IBRD: WriteOnly<u32, IBRD::Register>),\n        (0x28 => FBRD: WriteOnly<u32, FBRD::Register>),\n        (0x2c => LCR_H: WriteOnly<u32, LCR_H::Register>),\n        (0x30 => CR: WriteOnly<u32, CR::Register>),\n        (0x34 => IFLS: ReadWrite<u32, IFLS::Register>),\n        (0x38 => IMSC: ReadWrite<u32, IMSC::Register>),\n        (0x3C => _reserved3),\n        (0x40 => MIS: ReadOnly<u32, MIS::Register>),\n        (0x44 => ICR: WriteOnly<u32, ICR::Register>),\n        (0x48 => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\n#[derive(PartialEq)]\nenum BlockingMode {\n    Blocking,\n    NonBlocking,\n}\n\nstruct PL011UartInner {\n    registers: Registers,\n    chars_written: usize,\n    chars_read: usize,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the UART.\npub struct PL011Uart {\n    inner: IRQSafeNullLock<PL011UartInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PL011UartInner {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n            chars_written: 0,\n            chars_read: 0,\n        }\n    }\n\n    /// Set up baud rate and characteristics.\n    ///\n    /// This results in 8N1 and 921_600 baud.\n    ///\n    /// The calculation for the BRD is (we set the clock to 48 MHz in config.txt):\n    /// `(48_000_000 / 16) / 921_600 = 3.2552083`.\n    ///\n    /// This means the integer part is `3` and goes into the `IBRD`.\n    /// The fractional part is `0.2552083`.\n    ///\n    /// `FBRD` calculation according to the PL011 Technical Reference Manual:\n    /// `INTEGER((0.2552083 * 64) + 0.5) = 16`.\n    ///\n    /// Therefore, the generated baud rate divider is: `3 + 16/64 = 3.25`. Which results in a\n    /// genrated baud rate of `48_000_000 / (16 * 3.25) = 923_077`.\n    ///\n    /// Error = `((923_077 - 921_600) / 921_600) * 100 = 0.16%`.\n    pub fn init(&mut self) {\n        // Execution can arrive here while there are still characters queued in the TX FIFO and\n        // actively being sent out by the UART hardware. If the UART is turned off in this case,\n        // those queued characters would be lost.\n        //\n        // For example, this can happen during runtime on a call to panic!(), because panic!()\n        // initializes its own UART instance and calls init().\n        //\n        // Hence, flush first to ensure all pending characters are transmitted.\n        self.flush();\n\n        // Turn the UART off temporarily.\n        self.registers.CR.set(0);\n\n        // Clear all pending interrupts.\n        self.registers.ICR.write(ICR::ALL::CLEAR);\n\n        // From the PL011 Technical Reference Manual:\n        //\n        // The LCR_H, IBRD, and FBRD registers form the single 30-bit wide LCR Register that is\n        // updated on a single write strobe generated by a LCR_H write. So, to internally update the\n        // contents of IBRD or FBRD, a LCR_H write must always be performed at the end.\n        //\n        // Set the baud rate, 8N1 and FIFO enabled.\n        self.registers.IBRD.write(IBRD::BAUD_DIVINT.val(3));\n        self.registers.FBRD.write(FBRD::BAUD_DIVFRAC.val(16));\n        self.registers\n            .LCR_H\n            .write(LCR_H::WLEN::EightBit + LCR_H::FEN::FifosEnabled);\n\n        // Set RX FIFO fill level at 1/8.\n        self.registers.IFLS.write(IFLS::RXIFLSEL::OneEigth);\n\n        // Enable RX IRQ + RX timeout IRQ.\n        self.registers\n            .IMSC\n            .write(IMSC::RXIM::Enabled + IMSC::RTIM::Enabled);\n\n        // Turn the UART on.\n        self.registers\n            .CR\n            .write(CR::UARTEN::Enabled + CR::TXE::Enabled + CR::RXE::Enabled);\n    }\n\n    /// Send a character.\n    fn write_char(&mut self, c: char) {\n        // Spin while TX FIFO full is set, waiting for an empty slot.\n        while self.registers.FR.matches_all(FR::TXFF::SET) {\n            cpu::nop();\n        }\n\n        // Write the character to the buffer.\n        self.registers.DR.set(c as u32);\n\n        self.chars_written += 1;\n    }\n\n    /// Block execution until the last buffered character has been physically put on the TX wire.\n    fn flush(&self) {\n        // Spin until the busy bit is cleared.\n        while self.registers.FR.matches_all(FR::BUSY::SET) {\n            cpu::nop();\n        }\n    }\n\n    /// Retrieve a character.\n    fn read_char_converting(&mut self, blocking_mode: BlockingMode) -> Option<char> {\n        // If RX FIFO is empty,\n        if self.registers.FR.matches_all(FR::RXFE::SET) {\n            // immediately return in non-blocking mode.\n            if blocking_mode == BlockingMode::NonBlocking {\n                return None;\n            }\n\n            // Otherwise, wait until a char was received.\n            while self.registers.FR.matches_all(FR::RXFE::SET) {\n                cpu::nop();\n            }\n        }\n\n        // Read one character.\n        let mut ret = self.registers.DR.get() as u8 as char;\n\n        // Convert carrige return to newline.\n        if ret == '\\r' {\n            ret = '\\n'\n        }\n\n        // Update statistics.\n        self.chars_read += 1;\n\n        Some(ret)\n    }\n}\n\n/// Implementing `core::fmt::Write` enables usage of the `format_args!` macros, which in turn are\n/// used to implement the `kernel`'s `print!` and `println!` macros. By implementing `write_str()`,\n/// we get `write_fmt()` automatically.\n///\n/// The function takes an `&mut self`, so it must be implemented for the inner struct.\n///\n/// See [`src/print.rs`].\n///\n/// [`src/print.rs`]: ../../print/index.html\nimpl fmt::Write for PL011UartInner {\n    fn write_str(&mut self, s: &str) -> fmt::Result {\n        for c in s.chars() {\n            self.write_char(c);\n        }\n\n        Ok(())\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PL011Uart {\n    pub const COMPATIBLE: &'static str = \"BCM PL011 UART\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            inner: IRQSafeNullLock::new(PL011UartInner::new(mmio_start_addr)),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl driver::interface::DeviceDriver for PL011Uart {\n    type IRQNumberType = IRQNumber;\n\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n\n    unsafe fn init(&self) -> Result<(), &'static str> {\n        self.inner.lock(|inner| inner.init());\n\n        Ok(())\n    }\n\n    fn register_and_enable_irq_handler(\n        &'static self,\n        irq_number: &Self::IRQNumberType,\n    ) -> Result<(), &'static str> {\n        use exception::asynchronous::{irq_manager, IRQHandlerDescriptor};\n\n        let descriptor = IRQHandlerDescriptor::new(*irq_number, Self::COMPATIBLE, self);\n\n        irq_manager().register_handler(descriptor)?;\n        irq_manager().enable(irq_number);\n\n        Ok(())\n    }\n}\n\nimpl console::interface::Write for PL011Uart {\n    /// Passthrough of `args` to the `core::fmt::Write` implementation, but guarded by a Mutex to\n    /// serialize access.\n    fn write_char(&self, c: char) {\n        self.inner.lock(|inner| inner.write_char(c));\n    }\n\n    fn write_fmt(&self, args: core::fmt::Arguments) -> fmt::Result {\n        // Fully qualified syntax for the call to `core::fmt::Write::write_fmt()` to increase\n        // readability.\n        self.inner.lock(|inner| fmt::Write::write_fmt(inner, args))\n    }\n\n    fn flush(&self) {\n        // Spin until TX FIFO empty is set.\n        self.inner.lock(|inner| inner.flush());\n    }\n}\n\nimpl console::interface::Read for PL011Uart {\n    fn read_char(&self) -> char {\n        self.inner\n            .lock(|inner| inner.read_char_converting(BlockingMode::Blocking).unwrap())\n    }\n\n    fn clear_rx(&self) {\n        // Read from the RX FIFO until it is indicating empty.\n        while self\n            .inner\n            .lock(|inner| inner.read_char_converting(BlockingMode::NonBlocking))\n            .is_some()\n        {}\n    }\n}\n\nimpl console::interface::Statistics for PL011Uart {\n    fn chars_written(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_written)\n    }\n\n    fn chars_read(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_read)\n    }\n}\n\nimpl console::interface::All for PL011Uart {}\n\nimpl exception::asynchronous::interface::IRQHandler for PL011Uart {\n    fn handle(&self) -> Result<(), &'static str> {\n        self.inner.lock(|inner| {\n            let pending = inner.registers.MIS.extract();\n\n            // Clear all pending IRQs.\n            inner.registers.ICR.write(ICR::ALL::CLEAR);\n\n            // Check for any kind of RX interrupt.\n            if pending.matches_any(MIS::RXMIS::SET + MIS::RTMIS::SET) {\n                // Echo any received characters.\n                while let Some(c) = inner.read_char_converting(BlockingMode::NonBlocking) {\n                    inner.write_char(c)\n                }\n            }\n        });\n\n        Ok(())\n    }\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/bsp/device_driver/bcm.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BCM driver top level.\n\nmod bcm2xxx_gpio;\n#[cfg(feature = \"bsp_rpi3\")]\nmod bcm2xxx_interrupt_controller;\nmod bcm2xxx_pl011_uart;\n\npub use bcm2xxx_gpio::*;\n#[cfg(feature = \"bsp_rpi3\")]\npub use bcm2xxx_interrupt_controller::*;\npub use bcm2xxx_pl011_uart::*;\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/bsp/device_driver/common.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Common device driver code.\n\nuse crate::memory::{Address, Virtual};\nuse core::{fmt, marker::PhantomData, ops};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct MMIODerefWrapper<T> {\n    start_addr: Address<Virtual>,\n    phantom: PhantomData<fn() -> T>,\n}\n\n/// A wrapper type for usize with integrated range bound check.\n#[derive(Copy, Clone)]\npub struct BoundedUsize<const MAX_INCLUSIVE: usize>(usize);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T> MMIODerefWrapper<T> {\n    /// Create an instance.\n    pub const unsafe fn new(start_addr: Address<Virtual>) -> Self {\n        Self {\n            start_addr,\n            phantom: PhantomData,\n        }\n    }\n}\n\nimpl<T> ops::Deref for MMIODerefWrapper<T> {\n    type Target = T;\n\n    fn deref(&self) -> &Self::Target {\n        unsafe { &*(self.start_addr.as_usize() as *const _) }\n    }\n}\n\nimpl<const MAX_INCLUSIVE: usize> BoundedUsize<{ MAX_INCLUSIVE }> {\n    pub const MAX_INCLUSIVE: usize = MAX_INCLUSIVE;\n\n    /// Creates a new instance if number <= MAX_INCLUSIVE.\n    pub const fn new(number: usize) -> Self {\n        assert!(number <= MAX_INCLUSIVE);\n\n        Self(number)\n    }\n\n    /// Return the wrapped number.\n    pub const fn get(self) -> usize {\n        self.0\n    }\n}\n\nimpl<const MAX_INCLUSIVE: usize> fmt::Display for BoundedUsize<{ MAX_INCLUSIVE }> {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        write!(f, \"{}\", self.0)\n    }\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/bsp/device_driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Device driver.\n\n#[cfg(feature = \"bsp_rpi4\")]\nmod arm;\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod bcm;\nmod common;\n\n#[cfg(feature = \"bsp_rpi4\")]\npub use arm::*;\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use bcm::*;\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/bsp/raspberrypi/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Processor code.\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Used by `arch` code to find the early boot core.\n#[no_mangle]\n#[link_section = \".text._start_arguments\"]\npub static BOOT_CORE_ID: u64 = 0;\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/bsp/raspberrypi/driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP driver support.\n\nuse super::{exception, memory::map::mmio};\nuse crate::{\n    bsp::device_driver,\n    console, driver as generic_driver,\n    exception::{self as generic_exception},\n    memory,\n    memory::mmu::MMIODescriptor,\n};\nuse core::{\n    mem::MaybeUninit,\n    sync::atomic::{AtomicBool, Ordering},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic mut PL011_UART: MaybeUninit<device_driver::PL011Uart> = MaybeUninit::uninit();\nstatic mut GPIO: MaybeUninit<device_driver::GPIO> = MaybeUninit::uninit();\n\n#[cfg(feature = \"bsp_rpi3\")]\nstatic mut INTERRUPT_CONTROLLER: MaybeUninit<device_driver::InterruptController> =\n    MaybeUninit::uninit();\n\n#[cfg(feature = \"bsp_rpi4\")]\nstatic mut INTERRUPT_CONTROLLER: MaybeUninit<device_driver::GICv2> = MaybeUninit::uninit();\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// This must be called only after successful init of the memory subsystem.\nunsafe fn instantiate_uart() -> Result<(), &'static str> {\n    let mmio_descriptor = MMIODescriptor::new(mmio::PL011_UART_START, mmio::PL011_UART_SIZE);\n    let virt_addr =\n        memory::mmu::kernel_map_mmio(device_driver::PL011Uart::COMPATIBLE, &mmio_descriptor)?;\n\n    PL011_UART.write(device_driver::PL011Uart::new(virt_addr));\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the UART driver.\nunsafe fn post_init_uart() -> Result<(), &'static str> {\n    console::register_console(PL011_UART.assume_init_ref());\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the memory subsystem.\nunsafe fn instantiate_gpio() -> Result<(), &'static str> {\n    let mmio_descriptor = MMIODescriptor::new(mmio::GPIO_START, mmio::GPIO_SIZE);\n    let virt_addr =\n        memory::mmu::kernel_map_mmio(device_driver::GPIO::COMPATIBLE, &mmio_descriptor)?;\n\n    GPIO.write(device_driver::GPIO::new(virt_addr));\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the GPIO driver.\nunsafe fn post_init_gpio() -> Result<(), &'static str> {\n    GPIO.assume_init_ref().map_pl011_uart();\n    Ok(())\n}\n\n/// This must be called only after successful init of the memory subsystem.\n#[cfg(feature = \"bsp_rpi3\")]\nunsafe fn instantiate_interrupt_controller() -> Result<(), &'static str> {\n    let periph_mmio_descriptor =\n        MMIODescriptor::new(mmio::PERIPHERAL_IC_START, mmio::PERIPHERAL_IC_SIZE);\n    let periph_virt_addr = memory::mmu::kernel_map_mmio(\n        device_driver::InterruptController::COMPATIBLE,\n        &periph_mmio_descriptor,\n    )?;\n\n    INTERRUPT_CONTROLLER.write(device_driver::InterruptController::new(periph_virt_addr));\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the memory subsystem.\n#[cfg(feature = \"bsp_rpi4\")]\nunsafe fn instantiate_interrupt_controller() -> Result<(), &'static str> {\n    let gicd_mmio_descriptor = MMIODescriptor::new(mmio::GICD_START, mmio::GICD_SIZE);\n    let gicd_virt_addr = memory::mmu::kernel_map_mmio(\"GICv2 GICD\", &gicd_mmio_descriptor)?;\n\n    let gicc_mmio_descriptor = MMIODescriptor::new(mmio::GICC_START, mmio::GICC_SIZE);\n    let gicc_virt_addr = memory::mmu::kernel_map_mmio(\"GICV2 GICC\", &gicc_mmio_descriptor)?;\n\n    INTERRUPT_CONTROLLER.write(device_driver::GICv2::new(gicd_virt_addr, gicc_virt_addr));\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the interrupt controller driver.\nunsafe fn post_init_interrupt_controller() -> Result<(), &'static str> {\n    generic_exception::asynchronous::register_irq_manager(INTERRUPT_CONTROLLER.assume_init_ref());\n\n    Ok(())\n}\n\n/// Function needs to ensure that driver registration happens only after correct instantiation.\nunsafe fn driver_uart() -> Result<(), &'static str> {\n    instantiate_uart()?;\n\n    let uart_descriptor = generic_driver::DeviceDriverDescriptor::new(\n        PL011_UART.assume_init_ref(),\n        Some(post_init_uart),\n        Some(exception::asynchronous::irq_map::PL011_UART),\n    );\n    generic_driver::driver_manager().register_driver(uart_descriptor);\n\n    Ok(())\n}\n\n/// Function needs to ensure that driver registration happens only after correct instantiation.\nunsafe fn driver_gpio() -> Result<(), &'static str> {\n    instantiate_gpio()?;\n\n    let gpio_descriptor = generic_driver::DeviceDriverDescriptor::new(\n        GPIO.assume_init_ref(),\n        Some(post_init_gpio),\n        None,\n    );\n    generic_driver::driver_manager().register_driver(gpio_descriptor);\n\n    Ok(())\n}\n\n/// Function needs to ensure that driver registration happens only after correct instantiation.\nunsafe fn driver_interrupt_controller() -> Result<(), &'static str> {\n    instantiate_interrupt_controller()?;\n\n    let interrupt_controller_descriptor = generic_driver::DeviceDriverDescriptor::new(\n        INTERRUPT_CONTROLLER.assume_init_ref(),\n        Some(post_init_interrupt_controller),\n        None,\n    );\n    generic_driver::driver_manager().register_driver(interrupt_controller_descriptor);\n\n    Ok(())\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Initialize the driver subsystem.\n///\n/// # Safety\n///\n/// See child function calls.\npub unsafe fn init() -> Result<(), &'static str> {\n    static INIT_DONE: AtomicBool = AtomicBool::new(false);\n    if INIT_DONE.load(Ordering::Relaxed) {\n        return Err(\"Init already done\");\n    }\n\n    driver_uart()?;\n    driver_gpio()?;\n    driver_interrupt_controller()?;\n\n    INIT_DONE.store(true, Ordering::Relaxed);\n    Ok(())\n}\n\n/// Minimal code needed to bring up the console in QEMU (for testing only). This is often less steps\n/// than on real hardware due to QEMU's abstractions.\n#[cfg(feature = \"test_build\")]\npub fn qemu_bring_up_console() {\n    use crate::cpu;\n\n    unsafe {\n        instantiate_uart().unwrap_or_else(|_| cpu::qemu_exit_failure());\n        console::register_console(PL011_UART.assume_init_ref());\n    };\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/bsp/raspberrypi/exception/asynchronous.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP asynchronous exception handling.\n\nuse crate::bsp;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Export for reuse in generic asynchronous.rs.\npub use bsp::device_driver::IRQNumber;\n\n#[cfg(feature = \"bsp_rpi3\")]\npub(in crate::bsp) mod irq_map {\n    use super::bsp::device_driver::{IRQNumber, PeripheralIRQ};\n\n    pub const PL011_UART: IRQNumber = IRQNumber::Peripheral(PeripheralIRQ::new(57));\n}\n\n#[cfg(feature = \"bsp_rpi4\")]\npub(in crate::bsp) mod irq_map {\n    use super::bsp::device_driver::IRQNumber;\n\n    pub const PL011_UART: IRQNumber = IRQNumber::new(153);\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/bsp/raspberrypi/exception.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP synchronous and asynchronous exception handling.\n\npub mod asynchronous;\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/bsp/raspberrypi/kernel.ld",
    "content": "/* SPDX-License-Identifier: MIT OR Apache-2.0\n *\n * Copyright (c) 2018-2022 Andre Richter <andre.o.richter@gmail.com>\n */\n\nINCLUDE kernel_virt_addr_space_size.ld;\n\nPAGE_SIZE = 64K;\nPAGE_MASK = PAGE_SIZE - 1;\n\n__rpi_phys_dram_start_addr = 0;\n\n/* The physical address at which the the kernel binary will be loaded by the Raspberry's firmware */\n__rpi_phys_binary_load_addr = 0x80000;\n\n\nENTRY(__rpi_phys_binary_load_addr)\n\n/* Flags:\n *     4 == R\n *     5 == RX\n *     6 == RW\n *\n * Segments are marked PT_LOAD below so that the ELF file provides virtual and physical addresses.\n * It doesn't mean all of them need actually be loaded.\n */\nPHDRS\n{\n    segment_boot_core_stack PT_LOAD FLAGS(6);\n    segment_code            PT_LOAD FLAGS(5);\n    segment_data            PT_LOAD FLAGS(6);\n}\n\nSECTIONS\n{\n    . =  __rpi_phys_dram_start_addr;\n\n    /***********************************************************************************************\n    * Boot Core Stack\n    ***********************************************************************************************/\n    .boot_core_stack (NOLOAD) :\n    {\n        __boot_core_stack_start = .;         /*   ^             */\n                                             /*   | stack       */\n        . += __rpi_phys_binary_load_addr;    /*   | growth      */\n                                             /*   | direction   */\n        __boot_core_stack_end_exclusive = .; /*   |             */\n    } :segment_boot_core_stack\n\n    ASSERT((. & PAGE_MASK) == 0, \"End of boot core stack is not page aligned\")\n\n    /***********************************************************************************************\n    * Code + RO Data + Global Offset Table\n    ***********************************************************************************************/\n    __code_start = .;\n    .text :\n    {\n        KEEP(*(.text._start))\n        *(.text._start_arguments) /* Constants (or statics in Rust speak) read by _start(). */\n        *(.text._start_rust)      /* The Rust entry point */\n        *(.text*)                 /* Everything else */\n    } :segment_code\n\n    .rodata : ALIGN(8) { *(.rodata*) } :segment_code\n\n    . = ALIGN(PAGE_SIZE);\n    __code_end_exclusive = .;\n\n    /***********************************************************************************************\n    * Data + BSS\n    ***********************************************************************************************/\n    __data_start = .;\n    .data : { *(.data*) } :segment_data\n\n    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */\n    .bss (NOLOAD) : ALIGN(16)\n    {\n        __bss_start = .;\n        *(.bss*);\n        . = ALIGN(16);\n        __bss_end_exclusive = .;\n    } :segment_data\n\n    . = ALIGN(PAGE_SIZE);\n    __data_end_exclusive = .;\n\n    /***********************************************************************************************\n    * MMIO Remap Reserved\n    ***********************************************************************************************/\n    __mmio_remap_start = .;\n    . += 8 * 1024 * 1024;\n    __mmio_remap_end_exclusive = .;\n\n    ASSERT((. & PAGE_MASK) == 0, \"End of boot core stack is not page aligned\")\n\n    /***********************************************************************************************\n    * Misc\n    ***********************************************************************************************/\n    .got : { *(.got*) }\n    ASSERT(SIZEOF(.got) == 0, \"Relocation support not expected\")\n\n    /DISCARD/ : { *(.comment*) }\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/bsp/raspberrypi/kernel_virt_addr_space_size.ld",
    "content": "__kernel_virt_addr_space_size = 1024 * 1024 * 1024\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/bsp/raspberrypi/memory/mmu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Memory Management Unit.\n\nuse crate::{\n    memory::{\n        mmu::{\n            self as generic_mmu, AddressSpace, AssociatedTranslationTable, AttributeFields,\n            MemoryRegion, PageAddress, TranslationGranule,\n        },\n        Physical, Virtual,\n    },\n    synchronization::InitStateLock,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\ntype KernelTranslationTable =\n    <KernelVirtAddrSpace as AssociatedTranslationTable>::TableStartFromBottom;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// The translation granule chosen by this BSP. This will be used everywhere else in the kernel to\n/// derive respective data structures and their sizes. For example, the `crate::memory::mmu::Page`.\npub type KernelGranule = TranslationGranule<{ 64 * 1024 }>;\n\n/// The kernel's virtual address space defined by this BSP.\npub type KernelVirtAddrSpace = AddressSpace<{ kernel_virt_addr_space_size() }>;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n/// The kernel translation tables.\n///\n/// It is mandatory that InitStateLock is transparent.\n///\n/// That is, `size_of(InitStateLock<KernelTranslationTable>) == size_of(KernelTranslationTable)`.\n/// There is a unit tests that checks this porperty.\n#[link_section = \".data\"]\n#[no_mangle]\nstatic KERNEL_TABLES: InitStateLock<KernelTranslationTable> =\n    InitStateLock::new(KernelTranslationTable::new_for_precompute());\n\n/// This value is needed during early boot for MMU setup.\n///\n/// This will be patched to the correct value by the \"translation table tool\" after linking. This\n/// given value here is just a dummy.\n#[link_section = \".text._start_arguments\"]\n#[no_mangle]\nstatic PHYS_KERNEL_TABLES_BASE_ADDR: u64 = 0xCCCCAAAAFFFFEEEE;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// This is a hack for retrieving the value for the kernel's virtual address space size as a\n/// constant from a common place, since it is needed as a compile-time/link-time constant in both,\n/// the linker script and the Rust sources.\n#[allow(clippy::needless_late_init)]\nconst fn kernel_virt_addr_space_size() -> usize {\n    let __kernel_virt_addr_space_size;\n\n    include!(\"../kernel_virt_addr_space_size.ld\");\n\n    __kernel_virt_addr_space_size\n}\n\n/// Helper function for calculating the number of pages the given parameter spans.\nconst fn size_to_num_pages(size: usize) -> usize {\n    assert!(size > 0);\n    assert!(size % KernelGranule::SIZE == 0);\n\n    size >> KernelGranule::SHIFT\n}\n\n/// The code pages of the kernel binary.\nfn virt_code_region() -> MemoryRegion<Virtual> {\n    let num_pages = size_to_num_pages(super::code_size());\n\n    let start_page_addr = super::virt_code_start();\n    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n\n    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n}\n\n/// The data pages of the kernel binary.\nfn virt_data_region() -> MemoryRegion<Virtual> {\n    let num_pages = size_to_num_pages(super::data_size());\n\n    let start_page_addr = super::virt_data_start();\n    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n\n    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n}\n\n/// The boot core stack pages.\nfn virt_boot_core_stack_region() -> MemoryRegion<Virtual> {\n    let num_pages = size_to_num_pages(super::boot_core_stack_size());\n\n    let start_page_addr = super::virt_boot_core_stack_start();\n    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n\n    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n}\n\n// There is no reason to expect the following conversions to fail, since they were generated offline\n// by the `translation table tool`. If it doesn't work, a panic due to the unwraps is justified.\nfn kernel_virt_to_phys_region(virt_region: MemoryRegion<Virtual>) -> MemoryRegion<Physical> {\n    let phys_start_page_addr =\n        generic_mmu::try_kernel_virt_page_addr_to_phys_page_addr(virt_region.start_page_addr())\n            .unwrap();\n\n    let phys_end_exclusive_page_addr = phys_start_page_addr\n        .checked_offset(virt_region.num_pages() as isize)\n        .unwrap();\n\n    MemoryRegion::new(phys_start_page_addr, phys_end_exclusive_page_addr)\n}\n\nfn kernel_page_attributes(virt_page_addr: PageAddress<Virtual>) -> AttributeFields {\n    generic_mmu::try_kernel_page_attributes(virt_page_addr).unwrap()\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the kernel's translation tables.\npub fn kernel_translation_tables() -> &'static InitStateLock<KernelTranslationTable> {\n    &KERNEL_TABLES\n}\n\n/// The MMIO remap pages.\npub fn virt_mmio_remap_region() -> MemoryRegion<Virtual> {\n    let num_pages = size_to_num_pages(super::mmio_remap_size());\n\n    let start_page_addr = super::virt_mmio_remap_start();\n    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n\n    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n}\n\n/// Add mapping records for the kernel binary.\n///\n/// The actual translation table entries for the kernel binary are generated using the offline\n/// `translation table tool` and patched into the kernel binary. This function just adds the mapping\n/// record entries.\npub fn kernel_add_mapping_records_for_precomputed() {\n    let virt_boot_core_stack_region = virt_boot_core_stack_region();\n    generic_mmu::kernel_add_mapping_record(\n        \"Kernel boot-core stack\",\n        &virt_boot_core_stack_region,\n        &kernel_virt_to_phys_region(virt_boot_core_stack_region),\n        &kernel_page_attributes(virt_boot_core_stack_region.start_page_addr()),\n    );\n\n    let virt_code_region = virt_code_region();\n    generic_mmu::kernel_add_mapping_record(\n        \"Kernel code and RO data\",\n        &virt_code_region,\n        &kernel_virt_to_phys_region(virt_code_region),\n        &kernel_page_attributes(virt_code_region.start_page_addr()),\n    );\n\n    let virt_data_region = virt_data_region();\n    generic_mmu::kernel_add_mapping_record(\n        \"Kernel data and bss\",\n        &virt_data_region,\n        &kernel_virt_to_phys_region(virt_data_region),\n        &kernel_page_attributes(virt_data_region.start_page_addr()),\n    );\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/bsp/raspberrypi/memory.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Memory Management.\n//!\n//! The physical memory layout.\n//!\n//! The Raspberry's firmware copies the kernel binary to 0x8_0000. The preceding region will be used\n//! as the boot core's stack.\n//!\n//! +---------------------------------------+\n//! |                                       | boot_core_stack_start @ 0x0\n//! |                                       |                                ^\n//! | Boot-core Stack                       |                                | stack\n//! |                                       |                                | growth\n//! |                                       |                                | direction\n//! +---------------------------------------+\n//! |                                       | code_start @ 0x8_0000 == boot_core_stack_end_exclusive\n//! | .text                                 |\n//! | .rodata                               |\n//! | .got                                  |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       | data_start == code_end_exclusive\n//! | .data                                 |\n//! | .bss                                  |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       | data_end_exclusive\n//! |                                       |\n//!\n//!\n//!\n//!\n//!\n//! The virtual memory layout is as follows:\n//!\n//! +---------------------------------------+\n//! |                                       | boot_core_stack_start @ 0x0\n//! |                                       |                                ^\n//! | Boot-core Stack                       |                                | stack\n//! |                                       |                                | growth\n//! |                                       |                                | direction\n//! +---------------------------------------+\n//! |                                       | code_start @ 0x8_0000 == boot_core_stack_end_exclusive\n//! | .text                                 |\n//! | .rodata                               |\n//! | .got                                  |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       | data_start == code_end_exclusive\n//! | .data                                 |\n//! | .bss                                  |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       |  mmio_remap_start == data_end_exclusive\n//! | VA region for MMIO remapping          |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       |  mmio_remap_end_exclusive\n//! |                                       |\npub mod mmu;\n\nuse crate::memory::{mmu::PageAddress, Address, Physical, Virtual};\nuse core::cell::UnsafeCell;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Symbols from the linker script.\nextern \"Rust\" {\n    static __code_start: UnsafeCell<()>;\n    static __code_end_exclusive: UnsafeCell<()>;\n\n    static __data_start: UnsafeCell<()>;\n    static __data_end_exclusive: UnsafeCell<()>;\n\n    static __mmio_remap_start: UnsafeCell<()>;\n    static __mmio_remap_end_exclusive: UnsafeCell<()>;\n\n    static __boot_core_stack_start: UnsafeCell<()>;\n    static __boot_core_stack_end_exclusive: UnsafeCell<()>;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// The board's physical memory map.\n#[rustfmt::skip]\npub(super) mod map {\n    use super::*;\n\n    /// Physical devices.\n    #[cfg(feature = \"bsp_rpi3\")]\n    pub mod mmio {\n        use super::*;\n\n        pub const PERIPHERAL_IC_START: Address<Physical> = Address::new(0x3F00_B200);\n        pub const PERIPHERAL_IC_SIZE:  usize             =              0x24;\n\n        pub const GPIO_START:          Address<Physical> = Address::new(0x3F20_0000);\n        pub const GPIO_SIZE:           usize             =              0xA0;\n\n        pub const PL011_UART_START:    Address<Physical> = Address::new(0x3F20_1000);\n        pub const PL011_UART_SIZE:     usize             =              0x48;\n\n        pub const END:                 Address<Physical> = Address::new(0x4001_0000);\n    }\n\n    /// Physical devices.\n    #[cfg(feature = \"bsp_rpi4\")]\n    pub mod mmio {\n        use super::*;\n\n        pub const GPIO_START:       Address<Physical> = Address::new(0xFE20_0000);\n        pub const GPIO_SIZE:        usize             =              0xA0;\n\n        pub const PL011_UART_START: Address<Physical> = Address::new(0xFE20_1000);\n        pub const PL011_UART_SIZE:  usize             =              0x48;\n\n        pub const GICD_START:       Address<Physical> = Address::new(0xFF84_1000);\n        pub const GICD_SIZE:        usize             =              0x824;\n\n        pub const GICC_START:       Address<Physical> = Address::new(0xFF84_2000);\n        pub const GICC_SIZE:        usize             =              0x14;\n\n        pub const END:              Address<Physical> = Address::new(0xFF85_0000);\n    }\n\n    pub const END: Address<Physical> = mmio::END;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Start page address of the code segment.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn virt_code_start() -> PageAddress<Virtual> {\n    PageAddress::from(unsafe { __code_start.get() as usize })\n}\n\n/// Size of the code segment.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn code_size() -> usize {\n    unsafe { (__code_end_exclusive.get() as usize) - (__code_start.get() as usize) }\n}\n\n/// Start page address of the data segment.\n#[inline(always)]\nfn virt_data_start() -> PageAddress<Virtual> {\n    PageAddress::from(unsafe { __data_start.get() as usize })\n}\n\n/// Size of the data segment.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn data_size() -> usize {\n    unsafe { (__data_end_exclusive.get() as usize) - (__data_start.get() as usize) }\n}\n\n/// Start page address of the MMIO remap reservation.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn virt_mmio_remap_start() -> PageAddress<Virtual> {\n    PageAddress::from(unsafe { __mmio_remap_start.get() as usize })\n}\n\n/// Size of the MMIO remap reservation.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn mmio_remap_size() -> usize {\n    unsafe { (__mmio_remap_end_exclusive.get() as usize) - (__mmio_remap_start.get() as usize) }\n}\n\n/// Start page address of the boot core's stack.\n#[inline(always)]\nfn virt_boot_core_stack_start() -> PageAddress<Virtual> {\n    PageAddress::from(unsafe { __boot_core_stack_start.get() as usize })\n}\n\n/// Size of the boot core's stack.\n#[inline(always)]\nfn boot_core_stack_size() -> usize {\n    unsafe {\n        (__boot_core_stack_end_exclusive.get() as usize) - (__boot_core_stack_start.get() as usize)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Exclusive end address of the physical address space.\n#[inline(always)]\npub fn phys_addr_space_end_exclusive_addr() -> PageAddress<Physical> {\n    PageAddress::from(map::END)\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/bsp/raspberrypi.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Top-level BSP file for the Raspberry Pi 3 and 4.\n\npub mod cpu;\npub mod driver;\npub mod exception;\npub mod memory;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Board identification.\npub fn board_name() -> &'static str {\n    #[cfg(feature = \"bsp_rpi3\")]\n    {\n        \"Raspberry Pi 3\"\n    }\n\n    #[cfg(feature = \"bsp_rpi4\")]\n    {\n        \"Raspberry Pi 4\"\n    }\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/bsp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Conditional reexporting of Board Support Packages.\n\nmod device_driver;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod raspberrypi;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use raspberrypi::*;\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/common.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! General purpose code.\n\n/// Check if a value is aligned to a given size.\n#[inline(always)]\npub const fn is_aligned(value: usize, alignment: usize) -> bool {\n    assert!(alignment.is_power_of_two());\n\n    (value & (alignment - 1)) == 0\n}\n\n/// Align down.\n#[inline(always)]\npub const fn align_down(value: usize, alignment: usize) -> usize {\n    assert!(alignment.is_power_of_two());\n\n    value & !(alignment - 1)\n}\n\n/// Align up.\n#[inline(always)]\npub const fn align_up(value: usize, alignment: usize) -> usize {\n    assert!(alignment.is_power_of_two());\n\n    (value + alignment - 1) & !(alignment - 1)\n}\n\n/// Convert a size into human readable format.\npub const fn size_human_readable_ceil(size: usize) -> (usize, &'static str) {\n    const KIB: usize = 1024;\n    const MIB: usize = 1024 * 1024;\n    const GIB: usize = 1024 * 1024 * 1024;\n\n    if (size / GIB) > 0 {\n        (size.div_ceil(GIB), \"GiB\")\n    } else if (size / MIB) > 0 {\n        (size.div_ceil(MIB), \"MiB\")\n    } else if (size / KIB) > 0 {\n        (size.div_ceil(KIB), \"KiB\")\n    } else {\n        (size, \"Byte\")\n    }\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/console/null_console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Null console.\n\nuse super::interface;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct NullConsole;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\npub static NULL_CONSOLE: NullConsole = NullConsole {};\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl interface::Write for NullConsole {\n    fn write_char(&self, _c: char) {}\n\n    fn write_fmt(&self, _args: fmt::Arguments) -> fmt::Result {\n        fmt::Result::Ok(())\n    }\n\n    fn flush(&self) {}\n}\n\nimpl interface::Read for NullConsole {\n    fn clear_rx(&self) {}\n}\n\nimpl interface::Statistics for NullConsole {}\nimpl interface::All for NullConsole {}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! System console.\n\nmod null_console;\n\nuse crate::synchronization;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Console interfaces.\npub mod interface {\n    use core::fmt;\n\n    /// Console write functions.\n    pub trait Write {\n        /// Write a single character.\n        fn write_char(&self, c: char);\n\n        /// Write a Rust format string.\n        fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result;\n\n        /// Block until the last buffered character has been physically put on the TX wire.\n        fn flush(&self);\n    }\n\n    /// Console read functions.\n    pub trait Read {\n        /// Read a single character.\n        fn read_char(&self) -> char {\n            ' '\n        }\n\n        /// Clear RX buffers, if any.\n        fn clear_rx(&self);\n    }\n\n    /// Console statistics.\n    pub trait Statistics {\n        /// Return the number of characters written.\n        fn chars_written(&self) -> usize {\n            0\n        }\n\n        /// Return the number of characters read.\n        fn chars_read(&self) -> usize {\n            0\n        }\n    }\n\n    /// Trait alias for a full-fledged console.\n    pub trait All: Write + Read + Statistics {}\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic CUR_CONSOLE: InitStateLock<&'static (dyn interface::All + Sync)> =\n    InitStateLock::new(&null_console::NULL_CONSOLE);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::{interface::ReadWriteEx, InitStateLock};\n\n/// Register a new console.\npub fn register_console(new_console: &'static (dyn interface::All + Sync)) {\n    CUR_CONSOLE.write(|con| *con = new_console);\n}\n\n/// Return a reference to the currently registered console.\n///\n/// This is the global console used by all printing macros.\npub fn console() -> &'static dyn interface::All {\n    CUR_CONSOLE.read(|con| *con)\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Boot code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/cpu/boot.rs\"]\nmod arch_boot;\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/cpu/smp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Symmetric multiprocessing.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/cpu/smp.rs\"]\nmod arch_smp;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_smp::core_id;\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Processor code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/cpu.rs\"]\nmod arch_cpu;\n\nmod boot;\n\npub mod smp;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_cpu::{nop, wait_forever};\n\n#[cfg(feature = \"test_build\")]\npub use arch_cpu::{qemu_exit_failure, qemu_exit_success};\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Driver support.\n\nuse crate::{\n    exception, info,\n    synchronization::{interface::ReadWriteEx, InitStateLock},\n};\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nconst NUM_DRIVERS: usize = 5;\n\nstruct DriverManagerInner<T>\nwhere\n    T: 'static,\n{\n    next_index: usize,\n    descriptors: [Option<DeviceDriverDescriptor<T>>; NUM_DRIVERS],\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Driver interfaces.\npub mod interface {\n    /// Device Driver functions.\n    pub trait DeviceDriver {\n        /// Different interrupt controllers might use different types for IRQ number.\n        type IRQNumberType: super::fmt::Display;\n\n        /// Return a compatibility string for identifying the driver.\n        fn compatible(&self) -> &'static str;\n\n        /// Called by the kernel to bring up the device.\n        ///\n        /// # Safety\n        ///\n        /// - During init, drivers might do stuff with system-wide impact.\n        unsafe fn init(&self) -> Result<(), &'static str> {\n            Ok(())\n        }\n\n        /// Called by the kernel to register and enable the device's IRQ handler.\n        ///\n        /// Rust's type system will prevent a call to this function unless the calling instance\n        /// itself has static lifetime.\n        fn register_and_enable_irq_handler(\n            &'static self,\n            irq_number: &Self::IRQNumberType,\n        ) -> Result<(), &'static str> {\n            panic!(\n                \"Attempt to enable IRQ {} for device {}, but driver does not support this\",\n                irq_number,\n                self.compatible()\n            )\n        }\n    }\n}\n\n/// Tpye to be used as an optional callback after a driver's init() has run.\npub type DeviceDriverPostInitCallback = unsafe fn() -> Result<(), &'static str>;\n\n/// A descriptor for device drivers.\n#[derive(Copy, Clone)]\npub struct DeviceDriverDescriptor<T>\nwhere\n    T: 'static,\n{\n    device_driver: &'static (dyn interface::DeviceDriver<IRQNumberType = T> + Sync),\n    post_init_callback: Option<DeviceDriverPostInitCallback>,\n    irq_number: Option<T>,\n}\n\n/// Provides device driver management functions.\npub struct DriverManager<T>\nwhere\n    T: 'static,\n{\n    inner: InitStateLock<DriverManagerInner<T>>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic DRIVER_MANAGER: DriverManager<exception::asynchronous::IRQNumber> = DriverManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T> DriverManagerInner<T>\nwhere\n    T: 'static + Copy,\n{\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            next_index: 0,\n            descriptors: [None; NUM_DRIVERS],\n        }\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T> DeviceDriverDescriptor<T> {\n    /// Create an instance.\n    pub fn new(\n        device_driver: &'static (dyn interface::DeviceDriver<IRQNumberType = T> + Sync),\n        post_init_callback: Option<DeviceDriverPostInitCallback>,\n        irq_number: Option<T>,\n    ) -> Self {\n        Self {\n            device_driver,\n            post_init_callback,\n            irq_number,\n        }\n    }\n}\n\n/// Return a reference to the global DriverManager.\npub fn driver_manager() -> &'static DriverManager<exception::asynchronous::IRQNumber> {\n    &DRIVER_MANAGER\n}\n\nimpl<T> DriverManager<T>\nwhere\n    T: fmt::Display + Copy,\n{\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            inner: InitStateLock::new(DriverManagerInner::new()),\n        }\n    }\n\n    /// Register a device driver with the kernel.\n    pub fn register_driver(&self, descriptor: DeviceDriverDescriptor<T>) {\n        self.inner.write(|inner| {\n            inner.descriptors[inner.next_index] = Some(descriptor);\n            inner.next_index += 1;\n        })\n    }\n\n    /// Helper for iterating over registered drivers.\n    fn for_each_descriptor<'a>(&'a self, f: impl FnMut(&'a DeviceDriverDescriptor<T>)) {\n        self.inner.read(|inner| {\n            inner\n                .descriptors\n                .iter()\n                .filter_map(|x| x.as_ref())\n                .for_each(f)\n        })\n    }\n\n    /// Fully initialize all drivers and their interrupts handlers.\n    ///\n    /// # Safety\n    ///\n    /// - During init, drivers might do stuff with system-wide impact.\n    pub unsafe fn init_drivers_and_irqs(&self) {\n        self.for_each_descriptor(|descriptor| {\n            // 1. Initialize driver.\n            if let Err(x) = descriptor.device_driver.init() {\n                panic!(\n                    \"Error initializing driver: {}: {}\",\n                    descriptor.device_driver.compatible(),\n                    x\n                );\n            }\n\n            // 2. Call corresponding post init callback.\n            if let Some(callback) = &descriptor.post_init_callback {\n                if let Err(x) = callback() {\n                    panic!(\n                        \"Error during driver post-init callback: {}: {}\",\n                        descriptor.device_driver.compatible(),\n                        x\n                    );\n                }\n            }\n        });\n\n        // 3. After all post-init callbacks were done, the interrupt controller should be\n        //    registered and functional. So let drivers register with it now.\n        self.for_each_descriptor(|descriptor| {\n            if let Some(irq_number) = &descriptor.irq_number {\n                if let Err(x) = descriptor\n                    .device_driver\n                    .register_and_enable_irq_handler(irq_number)\n                {\n                    panic!(\n                        \"Error during driver interrupt handler registration: {}: {}\",\n                        descriptor.device_driver.compatible(),\n                        x\n                    );\n                }\n            }\n        });\n    }\n\n    /// Enumerate all registered device drivers.\n    pub fn enumerate(&self) {\n        let mut i: usize = 1;\n        self.for_each_descriptor(|descriptor| {\n            info!(\"      {}. {}\", i, descriptor.device_driver.compatible());\n\n            i += 1;\n        });\n    }\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/exception/asynchronous/null_irq_manager.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Null IRQ Manager.\n\nuse super::{interface, IRQContext, IRQHandlerDescriptor};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct NullIRQManager;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\npub static NULL_IRQ_MANAGER: NullIRQManager = NullIRQManager {};\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl interface::IRQManager for NullIRQManager {\n    type IRQNumberType = super::IRQNumber;\n\n    fn register_handler(\n        &self,\n        _descriptor: IRQHandlerDescriptor<Self::IRQNumberType>,\n    ) -> Result<(), &'static str> {\n        panic!(\"No IRQ Manager registered yet\");\n    }\n\n    fn enable(&self, _irq_number: &Self::IRQNumberType) {\n        panic!(\"No IRQ Manager registered yet\");\n    }\n\n    fn handle_pending_irqs<'irq_context>(&'irq_context self, _ic: &IRQContext<'irq_context>) {\n        panic!(\"No IRQ Manager registered yet\");\n    }\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/exception/asynchronous.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Asynchronous exception handling.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/exception/asynchronous.rs\"]\nmod arch_asynchronous;\nmod null_irq_manager;\n\nuse crate::{bsp, synchronization};\nuse core::marker::PhantomData;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_asynchronous::{\n    is_local_irq_masked, local_irq_mask, local_irq_mask_save, local_irq_restore, local_irq_unmask,\n    print_state,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Interrupt number as defined by the BSP.\npub type IRQNumber = bsp::exception::asynchronous::IRQNumber;\n\n/// Interrupt descriptor.\n#[derive(Copy, Clone)]\npub struct IRQHandlerDescriptor<T>\nwhere\n    T: Copy,\n{\n    /// The IRQ number.\n    number: T,\n\n    /// Descriptive name.\n    name: &'static str,\n\n    /// Reference to handler trait object.\n    handler: &'static (dyn interface::IRQHandler + Sync),\n}\n\n/// IRQContext token.\n///\n/// An instance of this type indicates that the local core is currently executing in IRQ\n/// context, aka executing an interrupt vector or subcalls of it.\n///\n/// Concept and implementation derived from the `CriticalSection` introduced in\n/// <https://github.com/rust-embedded/bare-metal>\n#[derive(Clone, Copy)]\npub struct IRQContext<'irq_context> {\n    _0: PhantomData<&'irq_context ()>,\n}\n\n/// Asynchronous exception handling interfaces.\npub mod interface {\n\n    /// Implemented by types that handle IRQs.\n    pub trait IRQHandler {\n        /// Called when the corresponding interrupt is asserted.\n        fn handle(&self) -> Result<(), &'static str>;\n    }\n\n    /// IRQ management functions.\n    ///\n    /// The `BSP` is supposed to supply one global instance. Typically implemented by the\n    /// platform's interrupt controller.\n    pub trait IRQManager {\n        /// The IRQ number type depends on the implementation.\n        type IRQNumberType: Copy;\n\n        /// Register a handler.\n        fn register_handler(\n            &self,\n            irq_handler_descriptor: super::IRQHandlerDescriptor<Self::IRQNumberType>,\n        ) -> Result<(), &'static str>;\n\n        /// Enable an interrupt in the controller.\n        fn enable(&self, irq_number: &Self::IRQNumberType);\n\n        /// Handle pending interrupts.\n        ///\n        /// This function is called directly from the CPU's IRQ exception vector. On AArch64,\n        /// this means that the respective CPU core has disabled exception handling.\n        /// This function can therefore not be preempted and runs start to finish.\n        ///\n        /// Takes an IRQContext token to ensure it can only be called from IRQ context.\n        #[allow(clippy::trivially_copy_pass_by_ref)]\n        fn handle_pending_irqs<'irq_context>(\n            &'irq_context self,\n            ic: &super::IRQContext<'irq_context>,\n        );\n\n        /// Print list of registered handlers.\n        fn print_handler(&self) {}\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic CUR_IRQ_MANAGER: InitStateLock<\n    &'static (dyn interface::IRQManager<IRQNumberType = IRQNumber> + Sync),\n> = InitStateLock::new(&null_irq_manager::NULL_IRQ_MANAGER);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::{interface::ReadWriteEx, InitStateLock};\n\nimpl<T> IRQHandlerDescriptor<T>\nwhere\n    T: Copy,\n{\n    /// Create an instance.\n    pub const fn new(\n        number: T,\n        name: &'static str,\n        handler: &'static (dyn interface::IRQHandler + Sync),\n    ) -> Self {\n        Self {\n            number,\n            name,\n            handler,\n        }\n    }\n\n    /// Return the number.\n    pub const fn number(&self) -> T {\n        self.number\n    }\n\n    /// Return the name.\n    pub const fn name(&self) -> &'static str {\n        self.name\n    }\n\n    /// Return the handler.\n    pub const fn handler(&self) -> &'static (dyn interface::IRQHandler + Sync) {\n        self.handler\n    }\n}\n\nimpl<'irq_context> IRQContext<'irq_context> {\n    /// Creates an IRQContext token.\n    ///\n    /// # Safety\n    ///\n    /// - This must only be called when the current core is in an interrupt context and will not\n    ///   live beyond the end of it. That is, creation is allowed in interrupt vector functions. For\n    ///   example, in the ARMv8-A case, in `extern \"C\" fn current_elx_irq()`.\n    /// - Note that the lifetime `'irq_context` of the returned instance is unconstrained. User code\n    ///   must not be able to influence the lifetime picked for this type, since that might cause it\n    ///   to be inferred to `'static`.\n    #[inline(always)]\n    pub unsafe fn new() -> Self {\n        IRQContext { _0: PhantomData }\n    }\n}\n\n/// Executes the provided closure while IRQs are masked on the executing core.\n///\n/// While the function temporarily changes the HW state of the executing core, it restores it to the\n/// previous state before returning, so this is deemed safe.\n#[inline(always)]\npub fn exec_with_irq_masked<T>(f: impl FnOnce() -> T) -> T {\n    let saved = local_irq_mask_save();\n    let ret = f();\n    local_irq_restore(saved);\n\n    ret\n}\n\n/// Register a new IRQ manager.\npub fn register_irq_manager(\n    new_manager: &'static (dyn interface::IRQManager<IRQNumberType = IRQNumber> + Sync),\n) {\n    CUR_IRQ_MANAGER.write(|manager| *manager = new_manager);\n}\n\n/// Return a reference to the currently registered IRQ manager.\n///\n/// This is the IRQ manager used by the architectural interrupt handling code.\npub fn irq_manager() -> &'static dyn interface::IRQManager<IRQNumberType = IRQNumber> {\n    CUR_IRQ_MANAGER.read(|manager| *manager)\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/exception.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Synchronous and asynchronous exception handling.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/exception.rs\"]\nmod arch_exception;\n\npub mod asynchronous;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_exception::{current_privilege_level, handling_init};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Kernel privilege levels.\n#[allow(missing_docs)]\n#[derive(Eq, PartialEq)]\npub enum PrivilegeLevel {\n    User,\n    Kernel,\n    Hypervisor,\n    Unknown,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// Libkernel unit tests must execute in kernel mode.\n    #[kernel_test]\n    fn test_runner_executes_in_kernel_mode() {\n        let (level, _) = current_privilege_level();\n\n        assert!(level == PrivilegeLevel::Kernel)\n    }\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/lib.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n// Rust embedded logo for `make doc`.\n#![doc(\n    html_logo_url = \"https://raw.githubusercontent.com/rust-embedded/wg/master/assets/logo/ewg-logo-blue-white-on-transparent.png\"\n)]\n\n//! The `kernel` library.\n//!\n//! Used to compose the final kernel binary.\n//!\n//! # Code organization and architecture\n//!\n//! The code is divided into different *modules*, each representing a typical **subsystem** of the\n//! `kernel`. Top-level module files of subsystems reside directly in the `src` folder. For example,\n//! `src/memory.rs` contains code that is concerned with all things memory management.\n//!\n//! ## Visibility of processor architecture code\n//!\n//! Some of the `kernel`'s subsystems depend on low-level code that is specific to the target\n//! processor architecture. For each supported processor architecture, there exists a subfolder in\n//! `src/_arch`, for example, `src/_arch/aarch64`.\n//!\n//! The architecture folders mirror the subsystem modules laid out in `src`. For example,\n//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go\n//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in\n//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic\n//! module's name prefixed with `arch_`.\n//!\n//! For example, this is the top of `src/memory/mmu.rs`:\n//!\n//! ```\n//! #[cfg(target_arch = \"aarch64\")]\n//! #[path = \"../_arch/aarch64/memory/mmu.rs\"]\n//! mod arch_mmu;\n//! ```\n//!\n//! Often times, items from the `arch_ module` will be publicly reexported by the parent module.\n//! This way, each architecture specific module can provide its implementation of an item, while the\n//! caller must not be concerned which architecture has been conditionally compiled.\n//!\n//! ## BSP code\n//!\n//! `BSP` stands for Board Support Package. `BSP` code is organized under `src/bsp.rs` and contains\n//! target board specific definitions and functions. These are things such as the board's memory map\n//! or instances of drivers for devices that are featured on the respective board.\n//!\n//! Just like processor architecture code, the `BSP` code's module structure tries to mirror the\n//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is\n//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`.\n//!\n//! ## Kernel interfaces\n//!\n//! Both `arch` and `bsp` contain code that is conditionally compiled depending on the actual target\n//! and board for which the kernel is compiled. For example, the `interrupt controller` hardware of\n//! the `Raspberry Pi 3` and the `Raspberry Pi 4` is different, but we want the rest of the `kernel`\n//! code to play nicely with any of the two without much hassle.\n//!\n//! In order to provide a clean abstraction between `arch`, `bsp` and `generic kernel code`,\n//! `interface` traits are provided *whenever possible* and *where it makes sense*. They are defined\n//! in the respective subsystem module and help to enforce the idiom of *program to an interface,\n//! not an implementation*. For example, there will be a common IRQ handling interface which the two\n//! different interrupt controller `drivers` of both Raspberrys will implement, and only export the\n//! interface to the rest of the `kernel`.\n//!\n//! ```\n//!         +-------------------+\n//!         | Interface (Trait) |\n//!         |                   |\n//!         +--+-------------+--+\n//!            ^             ^\n//!            |             |\n//!            |             |\n//! +----------+--+       +--+----------+\n//! | kernel code |       |  bsp code   |\n//! |             |       |  arch code  |\n//! +-------------+       +-------------+\n//! ```\n//!\n//! # Summary\n//!\n//! For a logical `kernel` subsystem, corresponding code can be distributed over several physical\n//! locations. Here is an example for the **memory** subsystem:\n//!\n//! - `src/memory.rs` and `src/memory/**/*`\n//!   - Common code that is agnostic of target processor architecture and `BSP` characteristics.\n//!     - Example: A function to zero a chunk of memory.\n//!   - Interfaces for the memory subsystem that are implemented by `arch` or `BSP` code.\n//!     - Example: An `MMU` interface that defines `MMU` function prototypes.\n//! - `src/bsp/__board_name__/memory.rs` and `src/bsp/__board_name__/memory/**/*`\n//!   - `BSP` specific code.\n//!   - Example: The board's memory map (physical addresses of DRAM and MMIO devices).\n//! - `src/_arch/__arch_name__/memory.rs` and `src/_arch/__arch_name__/memory/**/*`\n//!   - Processor architecture specific code.\n//!   - Example: Implementation of the `MMU` interface for the `__arch_name__` processor\n//!     architecture.\n//!\n//! From a namespace perspective, **memory** subsystem code lives in:\n//!\n//! - `crate::memory::*`\n//! - `crate::bsp::memory::*`\n//!\n//! # Boot flow\n//!\n//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.\n//!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.\n//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.\n\n#![allow(clippy::upper_case_acronyms)]\n#![allow(incomplete_features)]\n#![feature(asm_const)]\n#![feature(const_option)]\n#![feature(core_intrinsics)]\n#![feature(format_args_nl)]\n#![feature(generic_const_exprs)]\n#![feature(int_roundings)]\n#![feature(is_sorted)]\n#![feature(linkage)]\n#![feature(nonzero_min_max)]\n#![feature(panic_info_message)]\n#![feature(step_trait)]\n#![feature(trait_alias)]\n#![feature(unchecked_math)]\n#![no_std]\n// Testing\n#![cfg_attr(test, no_main)]\n#![feature(custom_test_frameworks)]\n#![reexport_test_harness_main = \"test_main\"]\n#![test_runner(crate::test_runner)]\n\nmod panic_wait;\nmod synchronization;\n\npub mod bsp;\npub mod common;\npub mod console;\npub mod cpu;\npub mod driver;\npub mod exception;\npub mod memory;\npub mod print;\npub mod state;\npub mod time;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Version string.\npub fn version() -> &'static str {\n    concat!(\n        env!(\"CARGO_PKG_NAME\"),\n        \" version \",\n        env!(\"CARGO_PKG_VERSION\")\n    )\n}\n\n#[cfg(not(test))]\nextern \"Rust\" {\n    fn kernel_init() -> !;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n/// The default runner for unit tests.\npub fn test_runner(tests: &[&test_types::UnitTest]) {\n    // This line will be printed as the test header.\n    println!(\"Running {} tests\", tests.len());\n\n    for (i, test) in tests.iter().enumerate() {\n        print!(\"{:>3}. {:.<58}\", i + 1, test.name);\n\n        // Run the actual test.\n        (test.test_func)();\n\n        // Failed tests call panic!(). Execution reaches here only if the test has passed.\n        println!(\"[ok]\")\n    }\n}\n\n/// The `kernel_init()` for unit tests.\n#[cfg(test)]\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    test_main();\n\n    cpu::qemu_exit_success()\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/main.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n// Rust embedded logo for `make doc`.\n#![doc(\n    html_logo_url = \"https://raw.githubusercontent.com/rust-embedded/wg/master/assets/logo/ewg-logo-blue-white-on-transparent.png\"\n)]\n\n//! The `kernel` binary.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\nuse libkernel::{bsp, cpu, driver, exception, info, memory, state, time};\n\n/// Early init code.\n///\n/// When this code runs, virtual memory is already enabled.\n///\n/// # Safety\n///\n/// - Only a single core must be active and running this function.\n/// - Printing will not work until the respective driver's MMIO is remapped.\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n\n    // Initialize the BSP driver subsystem.\n    if let Err(x) = bsp::driver::init() {\n        panic!(\"Error initializing BSP driver subsystem: {}\", x);\n    }\n\n    // Initialize all device drivers.\n    driver::driver_manager().init_drivers_and_irqs();\n\n    bsp::memory::mmu::kernel_add_mapping_records_for_precomputed();\n\n    // Unmask interrupts on the boot CPU core.\n    exception::asynchronous::local_irq_unmask();\n\n    // Announce conclusion of the kernel_init() phase.\n    state::state_manager().transition_to_single_core_main();\n\n    // Transition from unsafe to safe.\n    kernel_main()\n}\n\n/// The main function running after the early init.\nfn kernel_main() -> ! {\n    info!(\"{}\", libkernel::version());\n    info!(\"Booting on: {}\", bsp::board_name());\n\n    info!(\"MMU online:\");\n    memory::mmu::kernel_print_mappings();\n\n    let (_, privilege_level) = exception::current_privilege_level();\n    info!(\"Current privilege level: {}\", privilege_level);\n\n    info!(\"Exception handling state:\");\n    exception::asynchronous::print_state();\n\n    info!(\n        \"Architectural timer resolution: {} ns\",\n        time::time_manager().resolution().as_nanos()\n    );\n\n    info!(\"Drivers loaded:\");\n    driver::driver_manager().enumerate();\n\n    info!(\"Registered IRQ handlers:\");\n    exception::asynchronous::irq_manager().print_handler();\n\n    info!(\"Echoing input now\");\n    cpu::wait_forever();\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/memory/mmu/mapping_record.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A record of mapped pages.\n\nuse super::{\n    AccessPermissions, Address, AttributeFields, MMIODescriptor, MemAttributes, MemoryRegion,\n    Physical, Virtual,\n};\nuse crate::{bsp, common, info, synchronization, synchronization::InitStateLock, warn};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Type describing a virtual memory mapping.\n#[allow(missing_docs)]\n#[derive(Copy, Clone)]\nstruct MappingRecordEntry {\n    pub users: [Option<&'static str>; 5],\n    pub phys_start_addr: Address<Physical>,\n    pub virt_start_addr: Address<Virtual>,\n    pub num_pages: usize,\n    pub attribute_fields: AttributeFields,\n}\n\nstruct MappingRecord {\n    inner: [Option<MappingRecordEntry>; 12],\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic KERNEL_MAPPING_RECORD: InitStateLock<MappingRecord> =\n    InitStateLock::new(MappingRecord::new());\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl MappingRecordEntry {\n    pub fn new(\n        name: &'static str,\n        virt_region: &MemoryRegion<Virtual>,\n        phys_region: &MemoryRegion<Physical>,\n        attr: &AttributeFields,\n    ) -> Self {\n        Self {\n            users: [Some(name), None, None, None, None],\n            phys_start_addr: phys_region.start_addr(),\n            virt_start_addr: virt_region.start_addr(),\n            num_pages: phys_region.num_pages(),\n            attribute_fields: *attr,\n        }\n    }\n\n    fn find_next_free_user(&mut self) -> Result<&mut Option<&'static str>, &'static str> {\n        if let Some(x) = self.users.iter_mut().find(|x| x.is_none()) {\n            return Ok(x);\n        };\n\n        Err(\"Storage for user info exhausted\")\n    }\n\n    pub fn add_user(&mut self, user: &'static str) -> Result<(), &'static str> {\n        let x = self.find_next_free_user()?;\n        *x = Some(user);\n        Ok(())\n    }\n}\n\nimpl MappingRecord {\n    pub const fn new() -> Self {\n        Self { inner: [None; 12] }\n    }\n\n    fn size(&self) -> usize {\n        self.inner.iter().filter(|x| x.is_some()).count()\n    }\n\n    fn sort(&mut self) {\n        let upper_bound_exclusive = self.size();\n        let entries = &mut self.inner[0..upper_bound_exclusive];\n\n        if !entries.is_sorted_by_key(|item| item.unwrap().virt_start_addr) {\n            entries.sort_unstable_by_key(|item| item.unwrap().virt_start_addr)\n        }\n    }\n\n    fn find_next_free(&mut self) -> Result<&mut Option<MappingRecordEntry>, &'static str> {\n        if let Some(x) = self.inner.iter_mut().find(|x| x.is_none()) {\n            return Ok(x);\n        }\n\n        Err(\"Storage for mapping info exhausted\")\n    }\n\n    fn find_duplicate(\n        &mut self,\n        phys_region: &MemoryRegion<Physical>,\n    ) -> Option<&mut MappingRecordEntry> {\n        self.inner\n            .iter_mut()\n            .filter_map(|x| x.as_mut())\n            .filter(|x| x.attribute_fields.mem_attributes == MemAttributes::Device)\n            .find(|x| {\n                if x.phys_start_addr != phys_region.start_addr() {\n                    return false;\n                }\n\n                if x.num_pages != phys_region.num_pages() {\n                    return false;\n                }\n\n                true\n            })\n    }\n\n    pub fn add(\n        &mut self,\n        name: &'static str,\n        virt_region: &MemoryRegion<Virtual>,\n        phys_region: &MemoryRegion<Physical>,\n        attr: &AttributeFields,\n    ) -> Result<(), &'static str> {\n        let x = self.find_next_free()?;\n\n        *x = Some(MappingRecordEntry::new(\n            name,\n            virt_region,\n            phys_region,\n            attr,\n        ));\n\n        self.sort();\n\n        Ok(())\n    }\n\n    pub fn print(&self) {\n        info!(\"      -------------------------------------------------------------------------------------------------------------------------------------------\");\n        info!(\n            \"      {:^44}     {:^30}   {:^7}   {:^9}   {:^35}\",\n            \"Virtual\", \"Physical\", \"Size\", \"Attr\", \"Entity\"\n        );\n        info!(\"      -------------------------------------------------------------------------------------------------------------------------------------------\");\n\n        for i in self.inner.iter().flatten() {\n            let size = i.num_pages * bsp::memory::mmu::KernelGranule::SIZE;\n            let virt_start = i.virt_start_addr;\n            let virt_end_inclusive = virt_start + (size - 1);\n            let phys_start = i.phys_start_addr;\n            let phys_end_inclusive = phys_start + (size - 1);\n\n            let (size, unit) = common::size_human_readable_ceil(size);\n\n            let attr = match i.attribute_fields.mem_attributes {\n                MemAttributes::CacheableDRAM => \"C\",\n                MemAttributes::Device => \"Dev\",\n            };\n\n            let acc_p = match i.attribute_fields.acc_perms {\n                AccessPermissions::ReadOnly => \"RO\",\n                AccessPermissions::ReadWrite => \"RW\",\n            };\n\n            let xn = if i.attribute_fields.execute_never {\n                \"XN\"\n            } else {\n                \"X\"\n            };\n\n            info!(\n                \"      {}..{} --> {}..{} | {:>3} {} | {:<3} {} {:<2} | {}\",\n                virt_start,\n                virt_end_inclusive,\n                phys_start,\n                phys_end_inclusive,\n                size,\n                unit,\n                attr,\n                acc_p,\n                xn,\n                i.users[0].unwrap()\n            );\n\n            for k in i.users[1..].iter() {\n                if let Some(additional_user) = *k {\n                    info!(\n                        \"                                                                                                            | {}\",\n                        additional_user\n                    );\n                }\n            }\n        }\n\n        info!(\"      -------------------------------------------------------------------------------------------------------------------------------------------\");\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::interface::ReadWriteEx;\n\n/// Add an entry to the mapping info record.\npub fn kernel_add(\n    name: &'static str,\n    virt_region: &MemoryRegion<Virtual>,\n    phys_region: &MemoryRegion<Physical>,\n    attr: &AttributeFields,\n) -> Result<(), &'static str> {\n    KERNEL_MAPPING_RECORD.write(|mr| mr.add(name, virt_region, phys_region, attr))\n}\n\npub fn kernel_find_and_insert_mmio_duplicate(\n    mmio_descriptor: &MMIODescriptor,\n    new_user: &'static str,\n) -> Option<Address<Virtual>> {\n    let phys_region: MemoryRegion<Physical> = (*mmio_descriptor).into();\n\n    KERNEL_MAPPING_RECORD.write(|mr| {\n        let dup = mr.find_duplicate(&phys_region)?;\n\n        if let Err(x) = dup.add_user(new_user) {\n            warn!(\"{}\", x);\n        }\n\n        Some(dup.virt_start_addr)\n    })\n}\n\n/// Human-readable print of all recorded kernel mappings.\npub fn kernel_print() {\n    KERNEL_MAPPING_RECORD.read(|mr| mr.print());\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/memory/mmu/page_alloc.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Page allocation.\n\nuse super::MemoryRegion;\nuse crate::{\n    memory::{AddressType, Virtual},\n    synchronization::IRQSafeNullLock,\n    warn,\n};\nuse core::num::NonZeroUsize;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// A page allocator that can be lazyily initialized.\npub struct PageAllocator<ATYPE: AddressType> {\n    pool: Option<MemoryRegion<ATYPE>>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic KERNEL_MMIO_VA_ALLOCATOR: IRQSafeNullLock<PageAllocator<Virtual>> =\n    IRQSafeNullLock::new(PageAllocator::new());\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the kernel's MMIO virtual address allocator.\npub fn kernel_mmio_va_allocator() -> &'static IRQSafeNullLock<PageAllocator<Virtual>> {\n    &KERNEL_MMIO_VA_ALLOCATOR\n}\n\nimpl<ATYPE: AddressType> PageAllocator<ATYPE> {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self { pool: None }\n    }\n\n    /// Initialize the allocator.\n    pub fn init(&mut self, pool: MemoryRegion<ATYPE>) {\n        if self.pool.is_some() {\n            warn!(\"Already initialized\");\n            return;\n        }\n\n        self.pool = Some(pool);\n    }\n\n    /// Allocate a number of pages.\n    pub fn alloc(\n        &mut self,\n        num_requested_pages: NonZeroUsize,\n    ) -> Result<MemoryRegion<ATYPE>, &'static str> {\n        if self.pool.is_none() {\n            return Err(\"Allocator not initialized\");\n        }\n\n        self.pool\n            .as_mut()\n            .unwrap()\n            .take_first_n_pages(num_requested_pages)\n    }\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/memory/mmu/translation_table.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Translation table.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../../_arch/aarch64/memory/mmu/translation_table.rs\"]\nmod arch_translation_table;\n\nuse super::{AttributeFields, MemoryRegion};\nuse crate::memory::{Address, Physical, Virtual};\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\n#[cfg(target_arch = \"aarch64\")]\npub use arch_translation_table::FixedSizeTranslationTable;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Translation table interfaces.\npub mod interface {\n    use crate::memory::mmu::PageAddress;\n\n    use super::*;\n\n    /// Translation table operations.\n    pub trait TranslationTable {\n        /// Anything that needs to run before any of the other provided functions can be used.\n        ///\n        /// # Safety\n        ///\n        /// - Implementor must ensure that this function can run only once or is harmless if invoked\n        ///   multiple times.\n        fn init(&mut self) -> Result<(), &'static str>;\n\n        /// Map the given virtual memory region to the given physical memory region.\n        ///\n        /// # Safety\n        ///\n        /// - Using wrong attributes can cause multiple issues of different nature in the system.\n        /// - It is not required that the architectural implementation prevents aliasing. That is,\n        ///   mapping to the same physical memory using multiple virtual addresses, which would\n        ///   break Rust's ownership assumptions. This should be protected against in the kernel's\n        ///   generic MMU code.\n        unsafe fn map_at(\n            &mut self,\n            virt_region: &MemoryRegion<Virtual>,\n            phys_region: &MemoryRegion<Physical>,\n            attr: &AttributeFields,\n        ) -> Result<(), &'static str>;\n\n        /// Try to translate a virtual page address to a physical page address.\n        ///\n        /// Will only succeed if there exists a valid mapping for the input page.\n        fn try_virt_page_addr_to_phys_page_addr(\n            &self,\n            virt_page_addr: PageAddress<Virtual>,\n        ) -> Result<PageAddress<Physical>, &'static str>;\n\n        /// Try to get the attributes of a page.\n        ///\n        /// Will only succeed if there exists a valid mapping for the input page.\n        fn try_page_attributes(\n            &self,\n            virt_page_addr: PageAddress<Virtual>,\n        ) -> Result<AttributeFields, &'static str>;\n\n        /// Try to translate a virtual address to a physical address.\n        ///\n        /// Will only succeed if there exists a valid mapping for the input address.\n        fn try_virt_addr_to_phys_addr(\n            &self,\n            virt_addr: Address<Virtual>,\n        ) -> Result<Address<Physical>, &'static str>;\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use crate::memory::mmu::{AccessPermissions, MemAttributes, PageAddress};\n    use arch_translation_table::MinSizeTranslationTable;\n    use interface::TranslationTable;\n    use test_macros::kernel_test;\n\n    /// Sanity checks for the TranslationTable implementation.\n    #[kernel_test]\n    fn translationtable_implementation_sanity() {\n        // This will occupy a lot of space on the stack.\n        let mut tables = MinSizeTranslationTable::new_for_runtime();\n\n        assert_eq!(tables.init(), Ok(()));\n\n        let virt_start_page_addr: PageAddress<Virtual> = PageAddress::from(0);\n        let virt_end_exclusive_page_addr: PageAddress<Virtual> =\n            virt_start_page_addr.checked_offset(5).unwrap();\n\n        let phys_start_page_addr: PageAddress<Physical> = PageAddress::from(0);\n        let phys_end_exclusive_page_addr: PageAddress<Physical> =\n            phys_start_page_addr.checked_offset(5).unwrap();\n\n        let virt_region = MemoryRegion::new(virt_start_page_addr, virt_end_exclusive_page_addr);\n        let phys_region = MemoryRegion::new(phys_start_page_addr, phys_end_exclusive_page_addr);\n\n        let attr = AttributeFields {\n            mem_attributes: MemAttributes::CacheableDRAM,\n            acc_perms: AccessPermissions::ReadWrite,\n            execute_never: true,\n        };\n\n        unsafe { assert_eq!(tables.map_at(&virt_region, &phys_region, &attr), Ok(())) };\n\n        assert_eq!(\n            tables.try_virt_page_addr_to_phys_page_addr(virt_start_page_addr),\n            Ok(phys_start_page_addr)\n        );\n\n        assert_eq!(\n            tables.try_page_attributes(virt_start_page_addr.checked_offset(6).unwrap()),\n            Err(\"Page marked invalid\")\n        );\n\n        assert_eq!(tables.try_page_attributes(virt_start_page_addr), Ok(attr));\n\n        let virt_addr = virt_start_page_addr.into_inner() + 0x100;\n        let phys_addr = phys_start_page_addr.into_inner() + 0x100;\n        assert_eq!(tables.try_virt_addr_to_phys_addr(virt_addr), Ok(phys_addr));\n    }\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/memory/mmu/types.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management Unit types.\n\nuse crate::{\n    bsp, common,\n    memory::{Address, AddressType, Physical},\n};\nuse core::{convert::From, iter::Step, num::NonZeroUsize, ops::Range};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// A wrapper type around [Address] that ensures page alignment.\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub struct PageAddress<ATYPE: AddressType> {\n    inner: Address<ATYPE>,\n}\n\n/// A type that describes a region of memory in quantities of pages.\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub struct MemoryRegion<ATYPE: AddressType> {\n    start: PageAddress<ATYPE>,\n    end_exclusive: PageAddress<ATYPE>,\n}\n\n/// Architecture agnostic memory attributes.\n#[allow(missing_docs)]\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub enum MemAttributes {\n    CacheableDRAM,\n    Device,\n}\n\n/// Architecture agnostic access permissions.\n#[allow(missing_docs)]\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub enum AccessPermissions {\n    ReadOnly,\n    ReadWrite,\n}\n\n/// Collection of memory attributes.\n#[allow(missing_docs)]\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub struct AttributeFields {\n    pub mem_attributes: MemAttributes,\n    pub acc_perms: AccessPermissions,\n    pub execute_never: bool,\n}\n\n/// An MMIO descriptor for use in device drivers.\n#[derive(Copy, Clone)]\npub struct MMIODescriptor {\n    start_addr: Address<Physical>,\n    end_addr_exclusive: Address<Physical>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n//------------------------------------------------------------------------------\n// PageAddress\n//------------------------------------------------------------------------------\nimpl<ATYPE: AddressType> PageAddress<ATYPE> {\n    /// Unwraps the value.\n    pub fn into_inner(self) -> Address<ATYPE> {\n        self.inner\n    }\n\n    /// Calculates the offset from the page address.\n    ///\n    /// `count` is in units of [PageAddress]. For example, a count of 2 means `result = self + 2 *\n    /// page_size`.\n    pub fn checked_offset(self, count: isize) -> Option<Self> {\n        if count == 0 {\n            return Some(self);\n        }\n\n        let delta = count\n            .unsigned_abs()\n            .checked_mul(bsp::memory::mmu::KernelGranule::SIZE)?;\n        let result = if count.is_positive() {\n            self.inner.as_usize().checked_add(delta)?\n        } else {\n            self.inner.as_usize().checked_sub(delta)?\n        };\n\n        Some(Self {\n            inner: Address::new(result),\n        })\n    }\n}\n\nimpl<ATYPE: AddressType> From<usize> for PageAddress<ATYPE> {\n    fn from(addr: usize) -> Self {\n        assert!(\n            common::is_aligned(addr, bsp::memory::mmu::KernelGranule::SIZE),\n            \"Input usize not page aligned\"\n        );\n\n        Self {\n            inner: Address::new(addr),\n        }\n    }\n}\n\nimpl<ATYPE: AddressType> From<Address<ATYPE>> for PageAddress<ATYPE> {\n    fn from(addr: Address<ATYPE>) -> Self {\n        assert!(addr.is_page_aligned(), \"Input Address not page aligned\");\n\n        Self { inner: addr }\n    }\n}\n\nimpl<ATYPE: AddressType> Step for PageAddress<ATYPE> {\n    fn steps_between(start: &Self, end: &Self) -> Option<usize> {\n        if start > end {\n            return None;\n        }\n\n        // Since start <= end, do unchecked arithmetic.\n        Some(\n            (end.inner.as_usize() - start.inner.as_usize())\n                >> bsp::memory::mmu::KernelGranule::SHIFT,\n        )\n    }\n\n    fn forward_checked(start: Self, count: usize) -> Option<Self> {\n        start.checked_offset(count as isize)\n    }\n\n    fn backward_checked(start: Self, count: usize) -> Option<Self> {\n        start.checked_offset(-(count as isize))\n    }\n}\n\n//------------------------------------------------------------------------------\n// MemoryRegion\n//------------------------------------------------------------------------------\nimpl<ATYPE: AddressType> MemoryRegion<ATYPE> {\n    /// Create an instance.\n    pub fn new(start: PageAddress<ATYPE>, end_exclusive: PageAddress<ATYPE>) -> Self {\n        assert!(start <= end_exclusive);\n\n        Self {\n            start,\n            end_exclusive,\n        }\n    }\n\n    fn as_range(&self) -> Range<PageAddress<ATYPE>> {\n        self.into_iter()\n    }\n\n    /// Returns the start page address.\n    pub fn start_page_addr(&self) -> PageAddress<ATYPE> {\n        self.start\n    }\n\n    /// Returns the start address.\n    pub fn start_addr(&self) -> Address<ATYPE> {\n        self.start.into_inner()\n    }\n\n    /// Returns the exclusive end page address.\n    pub fn end_exclusive_page_addr(&self) -> PageAddress<ATYPE> {\n        self.end_exclusive\n    }\n\n    /// Returns the exclusive end page address.\n    pub fn end_inclusive_page_addr(&self) -> PageAddress<ATYPE> {\n        self.end_exclusive.checked_offset(-1).unwrap()\n    }\n\n    /// Checks if self contains an address.\n    pub fn contains(&self, addr: Address<ATYPE>) -> bool {\n        let page_addr = PageAddress::from(addr.align_down_page());\n        self.as_range().contains(&page_addr)\n    }\n\n    /// Checks if there is an overlap with another memory region.\n    pub fn overlaps(&self, other_region: &Self) -> bool {\n        let self_range = self.as_range();\n\n        self_range.contains(&other_region.start_page_addr())\n            || self_range.contains(&other_region.end_inclusive_page_addr())\n    }\n\n    /// Returns the number of pages contained in this region.\n    pub fn num_pages(&self) -> usize {\n        PageAddress::steps_between(&self.start, &self.end_exclusive).unwrap()\n    }\n\n    /// Returns the size in bytes of this region.\n    pub fn size(&self) -> usize {\n        // Invariant: start <= end_exclusive, so do unchecked arithmetic.\n        let end_exclusive = self.end_exclusive.into_inner().as_usize();\n        let start = self.start.into_inner().as_usize();\n\n        end_exclusive - start\n    }\n\n    /// Splits the MemoryRegion like:\n    ///\n    /// --------------------------------------------------------------------------------\n    /// |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |\n    /// --------------------------------------------------------------------------------\n    ///   ^                               ^                                       ^\n    ///   |                               |                                       |\n    ///   left_start     left_end_exclusive                                       |\n    ///                                                                           |\n    ///                                   ^                                       |\n    ///                                   |                                       |\n    ///                                   right_start           right_end_exclusive\n    ///\n    /// Left region is returned to the caller. Right region is the new region for this struct.\n    pub fn take_first_n_pages(&mut self, num_pages: NonZeroUsize) -> Result<Self, &'static str> {\n        let count: usize = num_pages.into();\n\n        let left_end_exclusive = self.start.checked_offset(count as isize);\n        let left_end_exclusive = match left_end_exclusive {\n            None => return Err(\"Overflow while calculating left_end_exclusive\"),\n            Some(x) => x,\n        };\n\n        if left_end_exclusive > self.end_exclusive {\n            return Err(\"Not enough free pages\");\n        }\n\n        let allocation = Self {\n            start: self.start,\n            end_exclusive: left_end_exclusive,\n        };\n        self.start = left_end_exclusive;\n\n        Ok(allocation)\n    }\n}\n\nimpl<ATYPE: AddressType> IntoIterator for MemoryRegion<ATYPE> {\n    type Item = PageAddress<ATYPE>;\n    type IntoIter = Range<Self::Item>;\n\n    fn into_iter(self) -> Self::IntoIter {\n        Range {\n            start: self.start,\n            end: self.end_exclusive,\n        }\n    }\n}\n\nimpl From<MMIODescriptor> for MemoryRegion<Physical> {\n    fn from(desc: MMIODescriptor) -> Self {\n        let start = PageAddress::from(desc.start_addr.align_down_page());\n        let end_exclusive = PageAddress::from(desc.end_addr_exclusive().align_up_page());\n\n        Self {\n            start,\n            end_exclusive,\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// MMIODescriptor\n//------------------------------------------------------------------------------\n\nimpl MMIODescriptor {\n    /// Create an instance.\n    pub const fn new(start_addr: Address<Physical>, size: usize) -> Self {\n        assert!(size > 0);\n        let end_addr_exclusive = Address::new(start_addr.as_usize() + size);\n\n        Self {\n            start_addr,\n            end_addr_exclusive,\n        }\n    }\n\n    /// Return the start address.\n    pub const fn start_addr(&self) -> Address<Physical> {\n        self.start_addr\n    }\n\n    /// Return the exclusive end address.\n    pub fn end_addr_exclusive(&self) -> Address<Physical> {\n        self.end_addr_exclusive\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use crate::memory::Virtual;\n    use test_macros::kernel_test;\n\n    /// Sanity of [PageAddress] methods.\n    #[kernel_test]\n    fn pageaddress_type_method_sanity() {\n        let page_addr: PageAddress<Virtual> =\n            PageAddress::from(bsp::memory::mmu::KernelGranule::SIZE * 2);\n\n        assert_eq!(\n            page_addr.checked_offset(-2),\n            Some(PageAddress::<Virtual>::from(0))\n        );\n\n        assert_eq!(\n            page_addr.checked_offset(2),\n            Some(PageAddress::<Virtual>::from(\n                bsp::memory::mmu::KernelGranule::SIZE * 4\n            ))\n        );\n\n        assert_eq!(\n            PageAddress::<Virtual>::from(0).checked_offset(0),\n            Some(PageAddress::<Virtual>::from(0))\n        );\n        assert_eq!(PageAddress::<Virtual>::from(0).checked_offset(-1), None);\n\n        let max_page_addr = Address::<Virtual>::new(usize::MAX).align_down_page();\n        assert_eq!(\n            PageAddress::<Virtual>::from(max_page_addr).checked_offset(1),\n            None\n        );\n\n        let zero = PageAddress::<Virtual>::from(0);\n        let three = PageAddress::<Virtual>::from(bsp::memory::mmu::KernelGranule::SIZE * 3);\n        assert_eq!(PageAddress::steps_between(&zero, &three), Some(3));\n    }\n\n    /// Sanity of [MemoryRegion] methods.\n    #[kernel_test]\n    fn memoryregion_type_method_sanity() {\n        let zero = PageAddress::<Virtual>::from(0);\n        let zero_region = MemoryRegion::new(zero, zero);\n        assert_eq!(zero_region.num_pages(), 0);\n        assert_eq!(zero_region.size(), 0);\n\n        let one = PageAddress::<Virtual>::from(bsp::memory::mmu::KernelGranule::SIZE);\n        let one_region = MemoryRegion::new(zero, one);\n        assert_eq!(one_region.num_pages(), 1);\n        assert_eq!(one_region.size(), bsp::memory::mmu::KernelGranule::SIZE);\n\n        let three = PageAddress::<Virtual>::from(bsp::memory::mmu::KernelGranule::SIZE * 3);\n        let mut three_region = MemoryRegion::new(zero, three);\n        assert!(three_region.contains(zero.into_inner()));\n        assert!(!three_region.contains(three.into_inner()));\n        assert!(three_region.overlaps(&one_region));\n\n        let allocation = three_region\n            .take_first_n_pages(NonZeroUsize::new(2).unwrap())\n            .unwrap();\n        assert_eq!(allocation.num_pages(), 2);\n        assert_eq!(three_region.num_pages(), 1);\n\n        for (i, alloc) in allocation.into_iter().enumerate() {\n            assert_eq!(\n                alloc.into_inner().as_usize(),\n                i * bsp::memory::mmu::KernelGranule::SIZE\n            );\n        }\n    }\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/memory/mmu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management Unit.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/memory/mmu.rs\"]\nmod arch_mmu;\n\nmod mapping_record;\nmod page_alloc;\nmod translation_table;\nmod types;\n\nuse crate::{\n    bsp,\n    memory::{Address, Physical, Virtual},\n    synchronization::{self, interface::Mutex},\n    warn,\n};\nuse core::{fmt, num::NonZeroUsize};\n\npub use types::*;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// MMU enable errors variants.\n#[allow(missing_docs)]\n#[derive(Debug)]\npub enum MMUEnableError {\n    AlreadyEnabled,\n    Other(&'static str),\n}\n\n/// Memory Management interfaces.\npub mod interface {\n    use super::*;\n\n    /// MMU functions.\n    pub trait MMU {\n        /// Turns on the MMU for the first time and enables data and instruction caching.\n        ///\n        /// # Safety\n        ///\n        /// - Changes the HW's global state.\n        unsafe fn enable_mmu_and_caching(\n            &self,\n            phys_tables_base_addr: Address<Physical>,\n        ) -> Result<(), MMUEnableError>;\n\n        /// Returns true if the MMU is enabled, false otherwise.\n        fn is_enabled(&self) -> bool;\n    }\n}\n\n/// Describes the characteristics of a translation granule.\npub struct TranslationGranule<const GRANULE_SIZE: usize>;\n\n/// Describes properties of an address space.\npub struct AddressSpace<const AS_SIZE: usize>;\n\n/// Intended to be implemented for [`AddressSpace`].\npub trait AssociatedTranslationTable {\n    /// A translation table whose address range is:\n    ///\n    /// [AS_SIZE - 1, 0]\n    type TableStartFromBottom;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\nuse interface::MMU;\nuse synchronization::interface::ReadWriteEx;\nuse translation_table::interface::TranslationTable;\n\n/// Map a region in the kernel's translation tables.\n///\n/// No input checks done, input is passed through to the architectural implementation.\n///\n/// # Safety\n///\n/// - See `map_at()`.\n/// - Does not prevent aliasing.\nunsafe fn kernel_map_at_unchecked(\n    name: &'static str,\n    virt_region: &MemoryRegion<Virtual>,\n    phys_region: &MemoryRegion<Physical>,\n    attr: &AttributeFields,\n) -> Result<(), &'static str> {\n    bsp::memory::mmu::kernel_translation_tables()\n        .write(|tables| tables.map_at(virt_region, phys_region, attr))?;\n\n    kernel_add_mapping_record(name, virt_region, phys_region, attr);\n\n    Ok(())\n}\n\n/// Try to translate a kernel virtual address to a physical address.\n///\n/// Will only succeed if there exists a valid mapping for the input address.\nfn try_kernel_virt_addr_to_phys_addr(\n    virt_addr: Address<Virtual>,\n) -> Result<Address<Physical>, &'static str> {\n    bsp::memory::mmu::kernel_translation_tables()\n        .read(|tables| tables.try_virt_addr_to_phys_addr(virt_addr))\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl fmt::Display for MMUEnableError {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        match self {\n            MMUEnableError::AlreadyEnabled => write!(f, \"MMU is already enabled\"),\n            MMUEnableError::Other(x) => write!(f, \"{}\", x),\n        }\n    }\n}\n\nimpl<const GRANULE_SIZE: usize> TranslationGranule<GRANULE_SIZE> {\n    /// The granule's size.\n    pub const SIZE: usize = Self::size_checked();\n\n    /// The granule's mask.\n    pub const MASK: usize = Self::SIZE - 1;\n\n    /// The granule's shift, aka log2(size).\n    pub const SHIFT: usize = Self::SIZE.trailing_zeros() as usize;\n\n    const fn size_checked() -> usize {\n        assert!(GRANULE_SIZE.is_power_of_two());\n\n        GRANULE_SIZE\n    }\n}\n\nimpl<const AS_SIZE: usize> AddressSpace<AS_SIZE> {\n    /// The address space size.\n    pub const SIZE: usize = Self::size_checked();\n\n    /// The address space shift, aka log2(size).\n    pub const SIZE_SHIFT: usize = Self::SIZE.trailing_zeros() as usize;\n\n    const fn size_checked() -> usize {\n        assert!(AS_SIZE.is_power_of_two());\n\n        // Check for architectural restrictions as well.\n        Self::arch_address_space_size_sanity_checks();\n\n        AS_SIZE\n    }\n}\n\n/// Query the BSP for the reserved virtual addresses for MMIO remapping and initialize the kernel's\n/// MMIO VA allocator with it.\npub fn kernel_init_mmio_va_allocator() {\n    let region = bsp::memory::mmu::virt_mmio_remap_region();\n\n    page_alloc::kernel_mmio_va_allocator().lock(|allocator| allocator.init(region));\n}\n\n/// Add an entry to the mapping info record.\npub fn kernel_add_mapping_record(\n    name: &'static str,\n    virt_region: &MemoryRegion<Virtual>,\n    phys_region: &MemoryRegion<Physical>,\n    attr: &AttributeFields,\n) {\n    if let Err(x) = mapping_record::kernel_add(name, virt_region, phys_region, attr) {\n        warn!(\"{}\", x);\n    }\n}\n\n/// MMIO remapping in the kernel translation tables.\n///\n/// Typically used by device drivers.\n///\n/// # Safety\n///\n/// - Same as `kernel_map_at_unchecked()`, minus the aliasing part.\npub unsafe fn kernel_map_mmio(\n    name: &'static str,\n    mmio_descriptor: &MMIODescriptor,\n) -> Result<Address<Virtual>, &'static str> {\n    let phys_region = MemoryRegion::from(*mmio_descriptor);\n    let offset_into_start_page = mmio_descriptor.start_addr().offset_into_page();\n\n    // Check if an identical region has been mapped for another driver. If so, reuse it.\n    let virt_addr = if let Some(addr) =\n        mapping_record::kernel_find_and_insert_mmio_duplicate(mmio_descriptor, name)\n    {\n        addr\n    // Otherwise, allocate a new region and map it.\n    } else {\n        let num_pages = match NonZeroUsize::new(phys_region.num_pages()) {\n            None => return Err(\"Requested 0 pages\"),\n            Some(x) => x,\n        };\n\n        let virt_region =\n            page_alloc::kernel_mmio_va_allocator().lock(|allocator| allocator.alloc(num_pages))?;\n\n        kernel_map_at_unchecked(\n            name,\n            &virt_region,\n            &phys_region,\n            &AttributeFields {\n                mem_attributes: MemAttributes::Device,\n                acc_perms: AccessPermissions::ReadWrite,\n                execute_never: true,\n            },\n        )?;\n\n        virt_region.start_addr()\n    };\n\n    Ok(virt_addr + offset_into_start_page)\n}\n\n/// Try to translate a kernel virtual page address to a physical page address.\n///\n/// Will only succeed if there exists a valid mapping for the input page.\npub fn try_kernel_virt_page_addr_to_phys_page_addr(\n    virt_page_addr: PageAddress<Virtual>,\n) -> Result<PageAddress<Physical>, &'static str> {\n    bsp::memory::mmu::kernel_translation_tables()\n        .read(|tables| tables.try_virt_page_addr_to_phys_page_addr(virt_page_addr))\n}\n\n/// Try to get the attributes of a kernel page.\n///\n/// Will only succeed if there exists a valid mapping for the input page.\npub fn try_kernel_page_attributes(\n    virt_page_addr: PageAddress<Virtual>,\n) -> Result<AttributeFields, &'static str> {\n    bsp::memory::mmu::kernel_translation_tables()\n        .read(|tables| tables.try_page_attributes(virt_page_addr))\n}\n\n/// Human-readable print of all recorded kernel mappings.\npub fn kernel_print_mappings() {\n    mapping_record::kernel_print()\n}\n\n/// Enable the MMU and data + instruction caching.\n///\n/// # Safety\n///\n/// - Crucial function during kernel init. Changes the the complete memory view of the processor.\n#[inline(always)]\npub unsafe fn enable_mmu_and_caching(\n    phys_tables_base_addr: Address<Physical>,\n) -> Result<(), MMUEnableError> {\n    arch_mmu::mmu().enable_mmu_and_caching(phys_tables_base_addr)\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/memory.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management.\n\npub mod mmu;\n\nuse crate::{bsp, common};\nuse core::{\n    fmt,\n    marker::PhantomData,\n    ops::{Add, Sub},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Metadata trait for marking the type of an address.\npub trait AddressType: Copy + Clone + PartialOrd + PartialEq + Ord + Eq {}\n\n/// Zero-sized type to mark a physical address.\n#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq)]\npub enum Physical {}\n\n/// Zero-sized type to mark a virtual address.\n#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq)]\npub enum Virtual {}\n\n/// Generic address type.\n#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq)]\npub struct Address<ATYPE: AddressType> {\n    value: usize,\n    _address_type: PhantomData<fn() -> ATYPE>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl AddressType for Physical {}\nimpl AddressType for Virtual {}\n\nimpl<ATYPE: AddressType> Address<ATYPE> {\n    /// Create an instance.\n    pub const fn new(value: usize) -> Self {\n        Self {\n            value,\n            _address_type: PhantomData,\n        }\n    }\n\n    /// Convert to usize.\n    pub const fn as_usize(self) -> usize {\n        self.value\n    }\n\n    /// Align down to page size.\n    #[must_use]\n    pub const fn align_down_page(self) -> Self {\n        let aligned = common::align_down(self.value, bsp::memory::mmu::KernelGranule::SIZE);\n\n        Self::new(aligned)\n    }\n\n    /// Align up to page size.\n    #[must_use]\n    pub const fn align_up_page(self) -> Self {\n        let aligned = common::align_up(self.value, bsp::memory::mmu::KernelGranule::SIZE);\n\n        Self::new(aligned)\n    }\n\n    /// Checks if the address is page aligned.\n    pub const fn is_page_aligned(&self) -> bool {\n        common::is_aligned(self.value, bsp::memory::mmu::KernelGranule::SIZE)\n    }\n\n    /// Return the address' offset into the corresponding page.\n    pub const fn offset_into_page(&self) -> usize {\n        self.value & bsp::memory::mmu::KernelGranule::MASK\n    }\n}\n\nimpl<ATYPE: AddressType> Add<usize> for Address<ATYPE> {\n    type Output = Self;\n\n    #[inline(always)]\n    fn add(self, rhs: usize) -> Self::Output {\n        match self.value.checked_add(rhs) {\n            None => panic!(\"Overflow on Address::add\"),\n            Some(x) => Self::new(x),\n        }\n    }\n}\n\nimpl<ATYPE: AddressType> Sub<Address<ATYPE>> for Address<ATYPE> {\n    type Output = Self;\n\n    #[inline(always)]\n    fn sub(self, rhs: Address<ATYPE>) -> Self::Output {\n        match self.value.checked_sub(rhs.value) {\n            None => panic!(\"Overflow on Address::sub\"),\n            Some(x) => Self::new(x),\n        }\n    }\n}\n\nimpl fmt::Display for Address<Physical> {\n    // Don't expect to see physical addresses greater than 40 bit.\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        let q3: u8 = ((self.value >> 32) & 0xff) as u8;\n        let q2: u16 = ((self.value >> 16) & 0xffff) as u16;\n        let q1: u16 = (self.value & 0xffff) as u16;\n\n        write!(f, \"0x\")?;\n        write!(f, \"{:02x}_\", q3)?;\n        write!(f, \"{:04x}_\", q2)?;\n        write!(f, \"{:04x}\", q1)\n    }\n}\n\nimpl fmt::Display for Address<Virtual> {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        let q4: u16 = ((self.value >> 48) & 0xffff) as u16;\n        let q3: u16 = ((self.value >> 32) & 0xffff) as u16;\n        let q2: u16 = ((self.value >> 16) & 0xffff) as u16;\n        let q1: u16 = (self.value & 0xffff) as u16;\n\n        write!(f, \"0x\")?;\n        write!(f, \"{:04x}_\", q4)?;\n        write!(f, \"{:04x}_\", q3)?;\n        write!(f, \"{:04x}_\", q2)?;\n        write!(f, \"{:04x}\", q1)\n    }\n}\n\n/// Initialize the memory subsystem.\npub fn init() {\n    mmu::kernel_init_mmio_va_allocator();\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// Sanity of [Address] methods.\n    #[kernel_test]\n    fn address_type_method_sanity() {\n        let addr = Address::<Virtual>::new(bsp::memory::mmu::KernelGranule::SIZE + 100);\n\n        assert_eq!(\n            addr.align_down_page().as_usize(),\n            bsp::memory::mmu::KernelGranule::SIZE\n        );\n\n        assert_eq!(\n            addr.align_up_page().as_usize(),\n            bsp::memory::mmu::KernelGranule::SIZE * 2\n        );\n\n        assert!(!addr.is_page_aligned());\n\n        assert_eq!(addr.offset_into_page(), 100);\n    }\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/panic_wait.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A panic handler that infinitely waits.\n\nuse crate::{cpu, exception, println};\nuse core::panic::PanicInfo;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// The point of exit for `libkernel`.\n///\n/// It is linked weakly, so that the integration tests can overload its standard behavior.\n#[linkage = \"weak\"]\n#[no_mangle]\nfn _panic_exit() -> ! {\n    #[cfg(not(feature = \"test_build\"))]\n    {\n        cpu::wait_forever()\n    }\n\n    #[cfg(feature = \"test_build\")]\n    {\n        cpu::qemu_exit_failure()\n    }\n}\n\n/// Stop immediately if called a second time.\n///\n/// # Note\n///\n/// Using atomics here relieves us from needing to use `unsafe` for the static variable.\n///\n/// On `AArch64`, which is the only implemented architecture at the time of writing this,\n/// [`AtomicBool::load`] and [`AtomicBool::store`] are lowered to ordinary load and store\n/// instructions. They are therefore safe to use even with MMU + caching deactivated.\n///\n/// [`AtomicBool::load`]: core::sync::atomic::AtomicBool::load\n/// [`AtomicBool::store`]: core::sync::atomic::AtomicBool::store\nfn panic_prevent_reenter() {\n    use core::sync::atomic::{AtomicBool, Ordering};\n\n    #[cfg(not(target_arch = \"aarch64\"))]\n    compile_error!(\"Add the target_arch to above's check if the following code is safe to use\");\n\n    static PANIC_IN_PROGRESS: AtomicBool = AtomicBool::new(false);\n\n    if !PANIC_IN_PROGRESS.load(Ordering::Relaxed) {\n        PANIC_IN_PROGRESS.store(true, Ordering::Relaxed);\n\n        return;\n    }\n\n    _panic_exit()\n}\n\n#[panic_handler]\nfn panic(info: &PanicInfo) -> ! {\n    exception::asynchronous::local_irq_mask();\n\n    // Protect against panic infinite loops if any of the following code panics itself.\n    panic_prevent_reenter();\n\n    let timestamp = crate::time::time_manager().uptime();\n    let (location, line, column) = match info.location() {\n        Some(loc) => (loc.file(), loc.line(), loc.column()),\n        _ => (\"???\", 0, 0),\n    };\n\n    println!(\n        \"[  {:>3}.{:06}] Kernel panic!\\n\\n\\\n        Panic location:\\n      File '{}', line {}, column {}\\n\\n\\\n        {}\",\n        timestamp.as_secs(),\n        timestamp.subsec_micros(),\n        location,\n        line,\n        column,\n        info.message().unwrap_or(&format_args!(\"\")),\n    );\n\n    _panic_exit()\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/print.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Printing.\n\nuse crate::console;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n#[doc(hidden)]\npub fn _print(args: fmt::Arguments) {\n    console::console().write_fmt(args).unwrap();\n}\n\n/// Prints without a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! print {\n    ($($arg:tt)*) => ($crate::print::_print(format_args!($($arg)*)));\n}\n\n/// Prints with a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! println {\n    () => ($crate::print!(\"\\n\"));\n    ($($arg:tt)*) => ({\n        $crate::print::_print(format_args_nl!($($arg)*));\n    })\n}\n\n/// Prints an info, with a newline.\n#[macro_export]\nmacro_rules! info {\n    ($string:expr) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[  {:>3}.{:06}] \", $string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n        ));\n    });\n    ($format_string:expr, $($arg:tt)*) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[  {:>3}.{:06}] \", $format_string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n            $($arg)*\n        ));\n    })\n}\n\n/// Prints a warning, with a newline.\n#[macro_export]\nmacro_rules! warn {\n    ($string:expr) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[W {:>3}.{:06}] \", $string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n        ));\n    });\n    ($format_string:expr, $($arg:tt)*) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[W {:>3}.{:06}] \", $format_string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n            $($arg)*\n        ));\n    })\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/state.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! State information about the kernel itself.\n\nuse core::sync::atomic::{AtomicU8, Ordering};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Different stages in the kernel execution.\n#[derive(Copy, Clone, Eq, PartialEq)]\nenum State {\n    /// The kernel starts booting in this state.\n    Init,\n\n    /// The kernel transitions to this state when jumping to `kernel_main()` (at the end of\n    /// `kernel_init()`, after all init calls are done).\n    SingleCoreMain,\n\n    /// The kernel transitions to this state when it boots the secondary cores, aka switches\n    /// exectution mode to symmetric multiprocessing (SMP).\n    MultiCoreMain,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Maintains the kernel state and state transitions.\npub struct StateManager(AtomicU8);\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic STATE_MANAGER: StateManager = StateManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the global StateManager.\npub fn state_manager() -> &'static StateManager {\n    &STATE_MANAGER\n}\n\nimpl StateManager {\n    const INIT: u8 = 0;\n    const SINGLE_CORE_MAIN: u8 = 1;\n    const MULTI_CORE_MAIN: u8 = 2;\n\n    /// Create a new instance.\n    pub const fn new() -> Self {\n        Self(AtomicU8::new(Self::INIT))\n    }\n\n    /// Return the current state.\n    fn state(&self) -> State {\n        let state = self.0.load(Ordering::Acquire);\n\n        match state {\n            Self::INIT => State::Init,\n            Self::SINGLE_CORE_MAIN => State::SingleCoreMain,\n            Self::MULTI_CORE_MAIN => State::MultiCoreMain,\n            _ => panic!(\"Invalid KERNEL_STATE\"),\n        }\n    }\n\n    /// Return if the kernel is init state.\n    pub fn is_init(&self) -> bool {\n        self.state() == State::Init\n    }\n\n    /// Transition from Init to SingleCoreMain.\n    pub fn transition_to_single_core_main(&self) {\n        if self\n            .0\n            .compare_exchange(\n                Self::INIT,\n                Self::SINGLE_CORE_MAIN,\n                Ordering::Acquire,\n                Ordering::Relaxed,\n            )\n            .is_err()\n        {\n            panic!(\"transition_to_single_core_main() called while state != Init\");\n        }\n    }\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/synchronization.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Synchronization primitives.\n//!\n//! # Resources\n//!\n//!   - <https://doc.rust-lang.org/book/ch16-04-extensible-concurrency-sync-and-send.html>\n//!   - <https://stackoverflow.com/questions/59428096/understanding-the-send-trait>\n//!   - <https://doc.rust-lang.org/std/cell/index.html>\n\nuse core::cell::UnsafeCell;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Synchronization interfaces.\npub mod interface {\n\n    /// Any object implementing this trait guarantees exclusive access to the data wrapped within\n    /// the Mutex for the duration of the provided closure.\n    pub trait Mutex {\n        /// The type of the data that is wrapped by this mutex.\n        type Data;\n\n        /// Locks the mutex and grants the closure temporary mutable access to the wrapped data.\n        fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R;\n    }\n\n    /// A reader-writer exclusion type.\n    ///\n    /// The implementing object allows either a number of readers or at most one writer at any point\n    /// in time.\n    pub trait ReadWriteEx {\n        /// The type of encapsulated data.\n        type Data;\n\n        /// Grants temporary mutable access to the encapsulated data.\n        fn write<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R;\n\n        /// Grants temporary immutable access to the encapsulated data.\n        fn read<'a, R>(&'a self, f: impl FnOnce(&'a Self::Data) -> R) -> R;\n    }\n}\n\n/// A pseudo-lock for teaching purposes.\n///\n/// In contrast to a real Mutex implementation, does not protect against concurrent access from\n/// other cores to the contained data. This part is preserved for later lessons.\n///\n/// The lock will only be used as long as it is safe to do so, i.e. as long as the kernel is\n/// executing on a single core.\npub struct IRQSafeNullLock<T>\nwhere\n    T: ?Sized,\n{\n    data: UnsafeCell<T>,\n}\n\n/// A pseudo-lock that is RW during the single-core kernel init phase and RO afterwards.\n///\n/// Intended to encapsulate data that is populated during kernel init when no concurrency exists.\npub struct InitStateLock<T>\nwhere\n    T: ?Sized,\n{\n    data: UnsafeCell<T>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nunsafe impl<T> Send for IRQSafeNullLock<T> where T: ?Sized + Send {}\nunsafe impl<T> Sync for IRQSafeNullLock<T> where T: ?Sized + Send {}\n\nimpl<T> IRQSafeNullLock<T> {\n    /// Create an instance.\n    pub const fn new(data: T) -> Self {\n        Self {\n            data: UnsafeCell::new(data),\n        }\n    }\n}\n\nunsafe impl<T> Send for InitStateLock<T> where T: ?Sized + Send {}\nunsafe impl<T> Sync for InitStateLock<T> where T: ?Sized + Send {}\n\nimpl<T> InitStateLock<T> {\n    /// Create an instance.\n    pub const fn new(data: T) -> Self {\n        Self {\n            data: UnsafeCell::new(data),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse crate::{exception, state};\n\nimpl<T> interface::Mutex for IRQSafeNullLock<T> {\n    type Data = T;\n\n    fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R {\n        // In a real lock, there would be code encapsulating this line that ensures that this\n        // mutable reference will ever only be given out once at a time.\n        let data = unsafe { &mut *self.data.get() };\n\n        // Execute the closure while IRQs are masked.\n        exception::asynchronous::exec_with_irq_masked(|| f(data))\n    }\n}\n\nimpl<T> interface::ReadWriteEx for InitStateLock<T> {\n    type Data = T;\n\n    fn write<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R {\n        assert!(\n            state::state_manager().is_init(),\n            \"InitStateLock::write called after kernel init phase\"\n        );\n        assert!(\n            !exception::asynchronous::is_local_irq_masked(),\n            \"InitStateLock::write called with IRQs unmasked\"\n        );\n\n        let data = unsafe { &mut *self.data.get() };\n\n        f(data)\n    }\n\n    fn read<'a, R>(&'a self, f: impl FnOnce(&'a Self::Data) -> R) -> R {\n        let data = unsafe { &*self.data.get() };\n\n        f(data)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// InitStateLock must be transparent.\n    #[kernel_test]\n    fn init_state_lock_is_transparent() {\n        use core::mem::size_of;\n\n        assert_eq!(size_of::<InitStateLock<u64>>(), size_of::<u64>());\n    }\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/src/time.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Timer primitives.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/time.rs\"]\nmod arch_time;\n\nuse core::time::Duration;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Provides time management functions.\npub struct TimeManager;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic TIME_MANAGER: TimeManager = TimeManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the global TimeManager.\npub fn time_manager() -> &'static TimeManager {\n    &TIME_MANAGER\n}\n\nimpl TimeManager {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self\n    }\n\n    /// The timer's resolution.\n    pub fn resolution(&self) -> Duration {\n        arch_time::resolution()\n    }\n\n    /// The uptime since power-on of the device.\n    ///\n    /// This includes time consumed by firmware and bootloaders.\n    pub fn uptime(&self) -> Duration {\n        arch_time::uptime()\n    }\n\n    /// Spin for a given duration.\n    pub fn spin_for(&self, duration: Duration) {\n        arch_time::spin_for(duration)\n    }\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/tests/00_console_sanity.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'console_io_test'\n\n# Verify sending and receiving works as expected.\nclass TxRxHandshakeTest < SubtestBase\n    def name\n        'Transmit and Receive handshake'\n    end\n\n    def run(qemu_out, qemu_in)\n        qemu_in.write_nonblock('ABC')\n        expect_or_raise(qemu_out, 'OK1234')\n    end\nend\n\n# Check for correct TX statistics implementation. Depends on test 1 being run first.\nclass TxStatisticsTest < SubtestBase\n    def name\n        'Transmit statistics'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, '6')\n    end\nend\n\n# Check for correct RX statistics implementation. Depends on test 1 being run first.\nclass RxStatisticsTest < SubtestBase\n    def name\n        'Receive statistics'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, '3')\n    end\nend\n\n## -------------------------------------------------------------------------------------------------\n## Test registration\n## -------------------------------------------------------------------------------------------------\ndef subtest_collection\n    [TxRxHandshakeTest.new, TxStatisticsTest.new, RxStatisticsTest.new]\nend\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/tests/00_console_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Console sanity tests - RX, TX and statistics.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Console tests should time out on the I/O harness in case of panic.\nmod panic_wait_forever;\n\nuse libkernel::{bsp, console, cpu, exception, memory, print};\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    use console::console;\n\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    // Handshake\n    assert_eq!(console().read_char(), 'A');\n    assert_eq!(console().read_char(), 'B');\n    assert_eq!(console().read_char(), 'C');\n    print!(\"OK1234\");\n\n    // 6\n    print!(\"{}\", console().chars_written());\n\n    // 3\n    print!(\"{}\", console().chars_read());\n\n    // The QEMU process running this test will be closed by the I/O test harness.\n    cpu::wait_forever();\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/tests/01_timer_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Timer sanity tests.\n\n#![feature(custom_test_frameworks)]\n#![no_main]\n#![no_std]\n#![reexport_test_harness_main = \"test_main\"]\n#![test_runner(libkernel::test_runner)]\n\nuse core::time::Duration;\nuse libkernel::{bsp, cpu, exception, memory, time};\nuse test_macros::kernel_test;\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    // Depending on CPU arch, some timer bring-up code could go here. Not needed for the RPi.\n\n    test_main();\n\n    cpu::qemu_exit_success()\n}\n\n/// Simple check that the timer is running.\n#[kernel_test]\nfn timer_is_counting() {\n    assert!(time::time_manager().uptime().as_nanos() > 0)\n}\n\n/// Timer resolution must be sufficient.\n#[kernel_test]\nfn timer_resolution_is_sufficient() {\n    assert!(time::time_manager().resolution().as_nanos() > 0);\n    assert!(time::time_manager().resolution().as_nanos() < 100)\n}\n\n/// Sanity check spin_for() implementation.\n#[kernel_test]\nfn spin_accuracy_check_1_second() {\n    let t1 = time::time_manager().uptime();\n    time::time_manager().spin_for(Duration::from_secs(1));\n    let t2 = time::time_manager().uptime();\n\n    assert_eq!((t2 - t1).as_secs(), 1)\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/tests/02_exception_sync_page_fault.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Page faults must result in synchronous exceptions.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Overwrites libkernel's `panic_wait::_panic_exit()` so that it returns a \"success\" code.\n///\n/// In this test, reaching the panic is a success, because it is called from the synchronous\n/// exception handler, which is what this test wants to achieve.\n///\n/// It also means that this integration test can not use any other code that calls panic!() directly\n/// or indirectly.\nmod panic_exit_success;\n\nuse libkernel::{bsp, cpu, exception, info, memory, println};\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    // This line will be printed as the test header.\n    println!(\"Testing synchronous exception handling by causing a page fault\");\n\n    info!(\"Writing beyond mapped area to address 9 GiB...\");\n    let big_addr: u64 = 9 * 1024 * 1024 * 1024;\n    core::ptr::read_volatile(big_addr as *mut u64);\n\n    // If execution reaches here, the memory access above did not cause a page fault exception.\n    cpu::qemu_exit_failure()\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/tests/03_exception_restore_sanity.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'console_io_test'\n\n# Verify that exception restore works.\nclass ExceptionRestoreTest < SubtestBase\n    def name\n        'Exception restore'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, 'Back from system call!')\n    end\nend\n\n## -------------------------------------------------------------------------------------------------\n## Test registration\n## -------------------------------------------------------------------------------------------------\ndef subtest_collection\n    [ExceptionRestoreTest.new]\nend\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/tests/03_exception_restore_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A simple sanity test to see if exception restore code works.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Console tests should time out on the I/O harness in case of panic.\nmod panic_wait_forever;\n\nuse core::arch::asm;\nuse libkernel::{bsp, cpu, exception, info, memory, println};\n\n#[inline(never)]\nfn nested_system_call() {\n    #[cfg(target_arch = \"aarch64\")]\n    unsafe {\n        asm!(\"svc #0x1337\", options(nomem, nostack, preserves_flags));\n    }\n\n    #[cfg(not(target_arch = \"aarch64\"))]\n    {\n        info!(\"Not supported yet\");\n        cpu::wait_forever();\n    }\n}\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    // This line will be printed as the test header.\n    println!(\"Testing exception restore\");\n\n    info!(\"Making a dummy system call\");\n\n    // Calling this inside a function indirectly tests if the link register is restored properly.\n    nested_system_call();\n\n    info!(\"Back from system call!\");\n\n    // The QEMU process running this test will be closed by the I/O test harness.\n    cpu::wait_forever();\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/tests/04_exception_irq_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! IRQ handling sanity tests.\n\n#![feature(custom_test_frameworks)]\n#![no_main]\n#![no_std]\n#![reexport_test_harness_main = \"test_main\"]\n#![test_runner(libkernel::test_runner)]\n\nuse libkernel::{bsp, cpu, exception, memory};\nuse test_macros::kernel_test;\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    exception::handling_init();\n    exception::asynchronous::local_irq_unmask();\n\n    test_main();\n\n    cpu::qemu_exit_success()\n}\n\n/// Check that IRQ masking works.\n#[kernel_test]\nfn local_irq_mask_works() {\n    // Precondition: IRQs are unmasked.\n    assert!(exception::asynchronous::is_local_irq_masked());\n\n    exception::asynchronous::local_irq_mask();\n    assert!(!exception::asynchronous::is_local_irq_masked());\n\n    // Restore earlier state.\n    exception::asynchronous::local_irq_unmask();\n}\n\n/// Check that IRQ unmasking works.\n#[kernel_test]\nfn local_irq_unmask_works() {\n    // Precondition: IRQs are masked.\n    exception::asynchronous::local_irq_mask();\n    assert!(!exception::asynchronous::is_local_irq_masked());\n\n    exception::asynchronous::local_irq_unmask();\n    assert!(exception::asynchronous::is_local_irq_masked());\n}\n\n/// Check that IRQ mask save is saving \"something\".\n#[kernel_test]\nfn local_irq_mask_save_works() {\n    // Precondition: IRQs are unmasked.\n    assert!(exception::asynchronous::is_local_irq_masked());\n\n    let first = exception::asynchronous::local_irq_mask_save();\n    assert!(!exception::asynchronous::is_local_irq_masked());\n\n    let second = exception::asynchronous::local_irq_mask_save();\n    assert_ne!(first, second);\n\n    exception::asynchronous::local_irq_restore(first);\n    assert!(exception::asynchronous::is_local_irq_masked());\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/tests/boot_test_string.rb",
    "content": "# frozen_string_literal: true\n\nEXPECTED_PRINT = 'Echoing input now'\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/tests/panic_exit_success/mod.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n/// Overwrites libkernel's `panic_wait::_panic_exit()` with the QEMU-exit version.\n#[no_mangle]\nfn _panic_exit() -> ! {\n    libkernel::cpu::qemu_exit_success()\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/kernel/tests/panic_wait_forever/mod.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n/// Overwrites libkernel's `panic_wait::_panic_exit()` with wait_forever.\n#[no_mangle]\nfn _panic_exit() -> ! {\n    libkernel::cpu::wait_forever()\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/libraries/test-macros/Cargo.toml",
    "content": "[package]\nname = \"test-macros\"\nversion = \"0.1.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n\n[lib]\nproc-macro = true\n\n[dependencies]\nproc-macro2 = \"1.x\"\nquote = \"1.x\"\nsyn = { version = \"1.x\", features = [\"full\"] }\ntest-types = { path = \"../test-types\" }\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/libraries/test-macros/src/lib.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\nuse proc_macro::TokenStream;\nuse proc_macro2::Span;\nuse quote::quote;\nuse syn::{parse_macro_input, Ident, ItemFn};\n\n#[proc_macro_attribute]\npub fn kernel_test(_attr: TokenStream, input: TokenStream) -> TokenStream {\n    let f = parse_macro_input!(input as ItemFn);\n\n    let test_name = &format!(\"{}\", f.sig.ident);\n    let test_ident = Ident::new(\n        &format!(\"{}_TEST_CONTAINER\", f.sig.ident.to_string().to_uppercase()),\n        Span::call_site(),\n    );\n    let test_code_block = f.block;\n\n    quote!(\n        #[test_case]\n        const #test_ident: test_types::UnitTest = test_types::UnitTest {\n            name: #test_name,\n            test_func: || #test_code_block,\n        };\n    )\n    .into()\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/libraries/test-types/Cargo.toml",
    "content": "[package]\nname = \"test-types\"\nversion = \"0.1.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/libraries/test-types/src/lib.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Types for the `custom_test_frameworks` implementation.\n\n#![no_std]\n\n/// Unit test container.\npub struct UnitTest {\n    /// Name of the test.\n    pub name: &'static str,\n\n    /// Function pointer to the test.\n    pub test_func: fn(),\n}\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/tools/translation_table_tool/arch.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n# Bitfield manipulation.\nclass BitField\n    def initialize\n        @value = 0\n    end\n\n    def self.attr_bitfield(name, offset, num_bits)\n        define_method(\"#{name}=\") do |bits|\n            mask = (2**num_bits) - 1\n\n            raise \"Input out of range: #{name} = 0x#{bits.to_s(16)}\" if (bits & ~mask).positive?\n\n            # Clear bitfield\n            @value &= ~(mask << offset)\n\n            # Set it\n            @value |= (bits << offset)\n        end\n    end\n\n    def to_i\n        @value\n    end\n\n    def size_in_byte\n        8\n    end\nend\n\n# An array class that knows its memory location.\nclass CArray < Array\n    attr_reader :phys_start_addr\n\n    def initialize(phys_start_addr, size, &block)\n        @phys_start_addr = phys_start_addr\n\n        super(size, &block)\n    end\n\n    def size_in_byte\n        inject(0) { |sum, n| sum + n.size_in_byte }\n    end\nend\n\n#---------------------------------------------------------------------------------------------------\n# Arch::\n#---------------------------------------------------------------------------------------------------\nmodule Arch\n#---------------------------------------------------------------------------------------------------\n# Arch::ARMv8\n#---------------------------------------------------------------------------------------------------\nmodule ARMv8\n# ARMv8 Table Descriptor.\nclass Stage1TableDescriptor < BitField\n    module NextLevelTableAddr\n        OFFSET = 16\n        NUMBITS = 32\n    end\n\n    module Type\n        OFFSET = 1\n        NUMBITS = 1\n\n        BLOCK = 0\n        TABLE = 1\n    end\n\n    module Valid\n        OFFSET = 0\n        NUMBITS = 1\n\n        FALSE = 0\n        TRUE = 1\n    end\n\n    attr_bitfield(:__next_level_table_addr, NextLevelTableAddr::OFFSET, NextLevelTableAddr::NUMBITS)\n    attr_bitfield(:type, Type::OFFSET, Type::NUMBITS)\n    attr_bitfield(:valid, Valid::OFFSET, Valid::NUMBITS)\n\n    def next_level_table_addr=(addr)\n        addr >>= Granule64KiB::SHIFT\n\n        self.__next_level_table_addr = addr\n    end\n\n    private :__next_level_table_addr=\nend\n\n# ARMv8 level 3 page descriptor.\nclass Stage1PageDescriptor < BitField\n    module UXN\n        OFFSET = 54\n        NUMBITS = 1\n\n        FALSE = 0\n        TRUE = 1\n    end\n\n    module PXN\n        OFFSET = 53\n        NUMBITS = 1\n\n        FALSE = 0\n        TRUE = 1\n    end\n\n    module OutputAddr\n        OFFSET = 16\n        NUMBITS = 32\n    end\n\n    module AF\n        OFFSET = 10\n        NUMBITS = 1\n\n        FALSE = 0\n        TRUE = 1\n    end\n\n    module SH\n        OFFSET = 8\n        NUMBITS = 2\n\n        INNER_SHAREABLE = 0b11\n    end\n\n    module AP\n        OFFSET = 6\n        NUMBITS = 2\n\n        RW_EL1 = 0b00\n        RO_EL1 = 0b10\n    end\n\n    module AttrIndx\n        OFFSET = 2\n        NUMBITS = 3\n    end\n\n    module Type\n        OFFSET = 1\n        NUMBITS = 1\n\n        RESERVED_INVALID = 0\n        PAGE = 1\n    end\n\n    module Valid\n        OFFSET = 0\n        NUMBITS = 1\n\n        FALSE = 0\n        TRUE = 1\n    end\n\n    attr_bitfield(:uxn, UXN::OFFSET, UXN::NUMBITS)\n    attr_bitfield(:pxn, PXN::OFFSET, PXN::NUMBITS)\n    attr_bitfield(:__output_addr, OutputAddr::OFFSET, OutputAddr::NUMBITS)\n    attr_bitfield(:af, AF::OFFSET, AF::NUMBITS)\n    attr_bitfield(:sh, SH::OFFSET, SH::NUMBITS)\n    attr_bitfield(:ap, AP::OFFSET, AP::NUMBITS)\n    attr_bitfield(:attr_indx, AttrIndx::OFFSET, AttrIndx::NUMBITS)\n    attr_bitfield(:type, Type::OFFSET, Type::NUMBITS)\n    attr_bitfield(:valid, Valid::OFFSET, Valid::NUMBITS)\n\n    def output_addr=(addr)\n        addr >>= Granule64KiB::SHIFT\n\n        self.__output_addr = addr\n    end\n\n    private :__output_addr=\nend\n\n# Translation table representing the structure defined in translation_table.rs.\nclass TranslationTable\n    module MAIR\n        NORMAL = 1\n    end\n\n    def initialize\n        do_sanity_checks\n\n        num_lvl2_tables = BSP.kernel_virt_addr_space_size >> Granule512MiB::SHIFT\n\n        @lvl3 = new_lvl3(num_lvl2_tables, BSP.phys_addr_of_kernel_tables)\n\n        @lvl2_phys_start_addr = @lvl3.phys_start_addr + @lvl3.size_in_byte\n        @lvl2 = new_lvl2(num_lvl2_tables, @lvl2_phys_start_addr)\n\n        populate_lvl2_entries\n    end\n\n    def map_at(virt_region, phys_region, attributes)\n        return if virt_region.empty?\n\n        raise if virt_region.size != phys_region.size\n        raise if phys_region.last > BSP.phys_addr_space_end_page\n\n        virt_region.zip(phys_region).each do |virt_page, phys_page|\n            desc = page_descriptor_from(virt_page)\n            set_lvl3_entry(desc, phys_page, attributes)\n        end\n    end\n\n    def to_binary\n        data = @lvl3.flatten.map(&:to_i) + @lvl2.map(&:to_i)\n        data.pack('Q<*') # \"Q\" == uint64_t, \"<\" == little endian\n    end\n\n    def phys_tables_base_addr_binary\n        [@lvl2_phys_start_addr].pack('Q<*') # \"Q\" == uint64_t, \"<\" == little endian\n    end\n\n    def phys_tables_base_addr\n        @lvl2_phys_start_addr\n    end\n\n    private\n\n    def do_sanity_checks\n        raise unless BSP.kernel_granule::SIZE == Granule64KiB::SIZE\n        raise unless (BSP.kernel_virt_addr_space_size % Granule512MiB::SIZE).zero?\n    end\n\n    def new_lvl3(num_lvl2_tables, start_addr)\n        CArray.new(start_addr, num_lvl2_tables) do\n            temp = CArray.new(start_addr, 8192) do\n                Stage1PageDescriptor.new\n            end\n            start_addr += temp.size_in_byte\n\n            temp\n        end\n    end\n\n    def new_lvl2(num_lvl2_tables, start_addr)\n        CArray.new(start_addr, num_lvl2_tables) do\n            Stage1TableDescriptor.new\n        end\n    end\n\n    def populate_lvl2_entries\n        @lvl2.each_with_index do |descriptor, i|\n            descriptor.next_level_table_addr = @lvl3[i].phys_start_addr\n            descriptor.type = Stage1TableDescriptor::Type::TABLE\n            descriptor.valid = Stage1TableDescriptor::Valid::TRUE\n        end\n    end\n\n    def lvl2_lvl3_index_from(addr)\n        lvl2_index = addr >> Granule512MiB::SHIFT\n        lvl3_index = (addr & Granule512MiB::MASK) >> Granule64KiB::SHIFT\n\n        raise unless lvl2_index < @lvl2.size\n\n        [lvl2_index, lvl3_index]\n    end\n\n    def page_descriptor_from(virt_addr)\n        lvl2_index, lvl3_index = lvl2_lvl3_index_from(virt_addr)\n\n        @lvl3[lvl2_index][lvl3_index]\n    end\n\n    # rubocop:disable Metrics/MethodLength\n    def set_attributes(desc, attributes)\n        case attributes.mem_attributes\n        when :CacheableDRAM\n            desc.sh = Stage1PageDescriptor::SH::INNER_SHAREABLE\n            desc.attr_indx = MAIR::NORMAL\n        else\n            raise 'Invalid input'\n        end\n\n        desc.ap = case attributes.acc_perms\n                  when :ReadOnly\n                      Stage1PageDescriptor::AP::RO_EL1\n                  when :ReadWrite\n                      Stage1PageDescriptor::AP::RW_EL1\n                  else\n                      raise 'Invalid input'\n\n                  end\n\n        desc.pxn = if attributes.execute_never\n                       Stage1PageDescriptor::PXN::TRUE\n                   else\n                       Stage1PageDescriptor::PXN::FALSE\n                   end\n\n        desc.uxn = Stage1PageDescriptor::UXN::TRUE\n    end\n    # rubocop:enable Metrics/MethodLength\n\n    def set_lvl3_entry(desc, output_addr, attributes)\n        desc.output_addr = output_addr\n        desc.af = Stage1PageDescriptor::AF::TRUE\n        desc.type = Stage1PageDescriptor::Type::PAGE\n        desc.valid = Stage1PageDescriptor::Valid::TRUE\n\n        set_attributes(desc, attributes)\n    end\nend\nend\nend\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/tools/translation_table_tool/bsp.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n# Raspberry Pi 3 + 4\nclass RaspberryPi\n    attr_reader :kernel_granule, :kernel_virt_addr_space_size\n\n    MEMORY_SRC = File.read('kernel/src/bsp/raspberrypi/memory.rs').split(\"\\n\")\n\n    def initialize\n        @kernel_granule = Granule64KiB\n\n        @kernel_virt_addr_space_size = KERNEL_ELF.symbol_value('__kernel_virt_addr_space_size')\n\n        @virt_addr_of_kernel_tables = KERNEL_ELF.symbol_value('KERNEL_TABLES')\n        @virt_addr_of_phys_kernel_tables_base_addr = KERNEL_ELF.symbol_value(\n            'PHYS_KERNEL_TABLES_BASE_ADDR'\n        )\n    end\n\n    def phys_addr_of_kernel_tables\n        KERNEL_ELF.virt_to_phys(@virt_addr_of_kernel_tables)\n    end\n\n    def kernel_tables_offset_in_file\n        KERNEL_ELF.virt_addr_to_file_offset(@virt_addr_of_kernel_tables)\n    end\n\n    def phys_kernel_tables_base_addr_offset_in_file\n        KERNEL_ELF.virt_addr_to_file_offset(@virt_addr_of_phys_kernel_tables_base_addr)\n    end\n\n    def phys_addr_space_end_page\n        x = MEMORY_SRC.grep(/pub const END/)\n        x = case BSP_TYPE\n            when :rpi3\n                x[0]\n            when :rpi4\n                x[1]\n            else\n                raise\n            end\n\n        x.scan(/\\d+/).join.to_i(16)\n    end\nend\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/tools/translation_table_tool/generic.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\nmodule Granule64KiB\n    SIZE = 64 * 1024\n    SHIFT = Math.log2(SIZE).to_i\nend\n\nmodule Granule512MiB\n    SIZE = 512 * 1024 * 1024\n    SHIFT = Math.log2(SIZE).to_i\n    MASK = SIZE - 1\nend\n\n# Monkey-patch Integer with some helper functions.\nclass Integer\n    def power_of_two?\n        self[0].zero?\n    end\n\n    def aligned?(alignment)\n        raise unless alignment.power_of_two?\n\n        (self & (alignment - 1)).zero?\n    end\n\n    def align_up(alignment)\n        raise unless alignment.power_of_two?\n\n        (self + alignment - 1) & ~(alignment - 1)\n    end\n\n    def to_hex_underscore(with_leading_zeros: false)\n        fmt = with_leading_zeros ? '%016x' : '%x'\n        value = format(fmt, self).to_s.reverse.scan(/.{4}|.+/).join('_').reverse\n\n        format('0x%s', value)\n    end\nend\n\n# An array where each value is the start address of a Page.\nclass MemoryRegion < Array\n    def initialize(start_addr, size, granule_size)\n        raise unless start_addr.aligned?(granule_size)\n        raise unless size.positive?\n        raise unless (size % granule_size).zero?\n\n        num_pages = size / granule_size\n        super(num_pages) do |i|\n            (i * granule_size) + start_addr\n        end\n    end\nend\n\n# Collection of memory attributes.\nclass AttributeFields\n    attr_reader :mem_attributes, :acc_perms, :execute_never\n\n    def initialize(mem_attributes, acc_perms, execute_never)\n        @mem_attributes = mem_attributes\n        @acc_perms = acc_perms\n        @execute_never = execute_never\n    end\n\n    def to_s\n        x = case @mem_attributes\n            when :CacheableDRAM\n                'C'\n            else\n                '?'\n            end\n\n        y = case @acc_perms\n            when :ReadWrite\n                'RW'\n            when :ReadOnly\n                'RO'\n            else\n                '??'\n            end\n\n        z = @execute_never ? 'XN' : 'X '\n\n        \"#{x} #{y} #{z}\"\n    end\nend\n\n# A container that describes a virt-to-phys region mapping.\nclass MappingDescriptor\n    @max_section_name_length = 'Sections'.length\n\n    class << self\n        attr_accessor :max_section_name_length\n\n        def update_max_section_name_length(length)\n            @max_section_name_length = [@max_section_name_length, length].max\n        end\n    end\n\n    attr_reader :name, :virt_region, :phys_region, :attributes\n\n    def initialize(name, virt_region, phys_region, attributes)\n        @name = name\n        @virt_region = virt_region\n        @phys_region = phys_region\n        @attributes = attributes\n    end\n\n    def to_s\n        name = @name.ljust(self.class.max_section_name_length)\n        virt_start = @virt_region.first.to_hex_underscore(with_leading_zeros: true)\n        phys_start = @phys_region.first.to_hex_underscore(with_leading_zeros: true)\n        size = ((@virt_region.size * 65_536) / 1024).to_s.rjust(3)\n\n        \"#{name} | #{virt_start} | #{phys_start} | #{size} KiB | #{@attributes}\"\n    end\n\n    def self.print_divider\n        print '             '\n        print '-' * max_section_name_length\n        puts '--------------------------------------------------------------------'\n    end\n\n    def self.print_header\n        print_divider\n        print '             '\n        print 'Sections'.center(max_section_name_length)\n        print '   '\n        print 'Virt Start Addr'.center(21)\n        print '   '\n        print 'Phys Start Addr'.center(21)\n        print '   '\n        print 'Size'.center(7)\n        print '   '\n        print 'Attr'.center(7)\n        puts\n        print_divider\n    end\nend\n\ndef kernel_map_binary\n    mapping_descriptors = KERNEL_ELF.generate_mapping_descriptors\n\n    # Generate_mapping_descriptors updates the header being printed with this call. So it must come\n    # afterwards.\n    MappingDescriptor.print_header\n\n    mapping_descriptors.each do |i|\n        print 'Generating'.rjust(12).green.bold\n        print ' '\n        puts i\n\n        TRANSLATION_TABLES.map_at(i.virt_region, i.phys_region, i.attributes)\n    end\n\n    MappingDescriptor.print_divider\nend\n\ndef kernel_patch_tables(kernel_elf_path)\n    print 'Patching'.rjust(12).green.bold\n    print ' Kernel table struct at ELF file offset '\n    puts BSP.kernel_tables_offset_in_file.to_hex_underscore\n\n    File.binwrite(kernel_elf_path, TRANSLATION_TABLES.to_binary, BSP.kernel_tables_offset_in_file)\nend\n\ndef kernel_patch_base_addr(kernel_elf_path)\n    print 'Patching'.rjust(12).green.bold\n    print ' Kernel tables physical base address start argument to value '\n    print TRANSLATION_TABLES.phys_tables_base_addr.to_hex_underscore\n    print ' at ELF file offset '\n    puts BSP.phys_kernel_tables_base_addr_offset_in_file.to_hex_underscore\n\n    File.binwrite(kernel_elf_path, TRANSLATION_TABLES.phys_tables_base_addr_binary,\n                  BSP.phys_kernel_tables_base_addr_offset_in_file)\nend\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/tools/translation_table_tool/kernel_elf.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n# KernelELF\nclass KernelELF\n    SECTION_FLAG_ALLOC = 2\n\n    def initialize(kernel_elf_path)\n        @elf = ELFTools::ELFFile.new(File.open(kernel_elf_path))\n        @symtab_section = @elf.section_by_name('.symtab')\n    end\n\n    def machine\n        @elf.machine.to_sym\n    end\n\n    def symbol_value(symbol_name)\n        @symtab_section.symbol_by_name(symbol_name).header.st_value\n    end\n\n    def segment_containing_virt_addr(virt_addr)\n        @elf.each_segments do |segment|\n            return segment if segment.vma_in?(virt_addr)\n        end\n    end\n\n    def virt_to_phys(virt_addr)\n        segment = segment_containing_virt_addr(virt_addr)\n        translation_offset = segment.header.p_vaddr - segment.header.p_paddr\n\n        virt_addr - translation_offset\n    end\n\n    def virt_addr_to_file_offset(virt_addr)\n        segment = segment_containing_virt_addr(virt_addr)\n        segment.vma_to_offset(virt_addr)\n    end\n\n    def sections_in_segment(segment)\n        head = segment.mem_head\n        tail = segment.mem_tail\n\n        sections = @elf.each_sections.select do |section|\n            file_offset = section.header.sh_addr\n            flags = section.header.sh_flags\n\n            file_offset >= head && file_offset < tail && (flags & SECTION_FLAG_ALLOC != 0)\n        end\n\n        sections.map(&:name).join(' ')\n    end\n\n    def select_load_segments\n        @elf.each_segments.select do |segment|\n            segment.instance_of?(ELFTools::Segments::LoadSegment)\n        end\n    end\n\n    def segment_get_acc_perms(segment)\n        if segment.readable? && segment.writable?\n            :ReadWrite\n        elsif segment.readable?\n            :ReadOnly\n        else\n            :Invalid\n        end\n    end\n\n    def update_max_section_name_length(descriptors)\n        MappingDescriptor.update_max_section_name_length(descriptors.map { |i| i.name.size }.max)\n    end\n\n    def generate_mapping_descriptors\n        descriptors = select_load_segments.map do |segment|\n            # Assume each segment is page aligned.\n            size = segment.mem_size.align_up(BSP.kernel_granule::SIZE)\n            virt_start_addr = segment.header.p_vaddr\n            phys_start_addr = segment.header.p_paddr\n            acc_perms = segment_get_acc_perms(segment)\n            execute_never = !segment.executable?\n            section_names = sections_in_segment(segment)\n\n            virt_region = MemoryRegion.new(virt_start_addr, size, BSP.kernel_granule::SIZE)\n            phys_region = MemoryRegion.new(phys_start_addr, size, BSP.kernel_granule::SIZE)\n            attributes = AttributeFields.new(:CacheableDRAM, acc_perms, execute_never)\n\n            MappingDescriptor.new(section_names, virt_region, phys_region, attributes)\n        end\n\n        update_max_section_name_length(descriptors)\n        descriptors\n    end\nend\n"
  },
  {
    "path": "15_virtual_mem_part3_precomputed_tables/tools/translation_table_tool/main.rb",
    "content": "#!/usr/bin/env ruby\n# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'rubygems'\nrequire 'bundler/setup'\nrequire 'colorize'\nrequire 'elftools'\n\nrequire_relative 'generic'\nrequire_relative 'kernel_elf'\nrequire_relative 'bsp'\nrequire_relative 'arch'\n\nBSP_TYPE = ARGV[0].to_sym\nkernel_elf_path = ARGV[1]\n\nstart = Time.now\n\nKERNEL_ELF = KernelELF.new(kernel_elf_path)\n\nBSP = case BSP_TYPE\n      when :rpi3, :rpi4\n          RaspberryPi.new\n      else\n          raise\n      end\n\nTRANSLATION_TABLES = case KERNEL_ELF.machine\n                     when :AArch64\n                         Arch::ARMv8::TranslationTable.new\n                     else\n                         raise\n                     end\n\nkernel_map_binary\nkernel_patch_tables(kernel_elf_path)\nkernel_patch_base_addr(kernel_elf_path)\n\nelapsed = Time.now - start\n\nprint 'Finished'.rjust(12).green.bold\nputs \" in #{elapsed.round(2)}s\"\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/.cargo/config.toml",
    "content": "[target.'cfg(target_os = \"none\")']\nrunner = \"target/kernel_test_runner.sh\"\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/.vscode/settings.json",
    "content": "{\n    \"editor.formatOnSave\": true,\n    \"editor.rulers\": [100],\n    \"rust-analyzer.cargo.target\": \"aarch64-unknown-none-softfloat\",\n    \"rust-analyzer.cargo.features\": [\"bsp_rpi3\"],\n    \"rust-analyzer.checkOnSave.allTargets\": false,\n    \"rust-analyzer.checkOnSave.extraArgs\": [\"--lib\", \"--bins\"],\n    \"rust-analyzer.lens.debug\": false,\n    \"rust-analyzer.lens.run\": false\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/Cargo.toml",
    "content": "[workspace]\n\nmembers = [\n        \"libraries/*\",\n        \"kernel\"\n]\n\n[profile.release]\nlto = true\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/Makefile",
    "content": "## SPDX-License-Identifier: MIT OR Apache-2.0\n##\n## Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\ninclude ../common/docker.mk\ninclude ../common/format.mk\ninclude ../common/operating_system.mk\n\n##--------------------------------------------------------------------------------------------------\n## Optional, user-provided configuration values\n##--------------------------------------------------------------------------------------------------\n\n# Default to the RPi3.\nBSP ?= rpi3\n\n# Default to a serial device name that is common in Linux.\nDEV_SERIAL ?= /dev/ttyUSB0\n\n# Optional integration test name.\nifdef TEST\n    TEST_ARG = --test $(TEST)\nelse\n    TEST_ARG = --test '*'\nendif\n\n\n\n##--------------------------------------------------------------------------------------------------\n## BSP-specific configuration values\n##--------------------------------------------------------------------------------------------------\nQEMU_MISSING_STRING = \"This board is not yet supported for QEMU.\"\n\nifeq ($(BSP),rpi3)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE = raspi3\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    QEMU_TEST_ARGS    = $(QEMU_RELEASE_ARGS) -semihosting\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    OPENOCD_ARG       = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi3.cfg\n    JTAG_BOOT_IMAGE   = ../X1_JTAG_boot/jtag_boot_rpi3.img\n    LD_SCRIPT_PATH    = $(shell pwd)/kernel/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a53\nelse ifeq ($(BSP),rpi4)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE =\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    QEMU_TEST_ARGS    = $(QEMU_RELEASE_ARGS) -semihosting\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    OPENOCD_ARG       = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi4.cfg\n    JTAG_BOOT_IMAGE   = ../X1_JTAG_boot/jtag_boot_rpi4.img\n    LD_SCRIPT_PATH    = $(shell pwd)/kernel/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a72\nendif\n\n# Export for build.rs.\nexport LD_SCRIPT_PATH\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets and Prerequisites\n##--------------------------------------------------------------------------------------------------\nKERNEL_MANIFEST      = kernel/Cargo.toml\nKERNEL_LINKER_SCRIPT = kernel.ld\nLAST_BUILD_CONFIG    = target/$(BSP).build_config\n\nKERNEL_ELF_RAW      = target/$(TARGET)/release/kernel\n# This parses cargo's dep-info file.\n# https://doc.rust-lang.org/cargo/guide/build-cache.html#dep-info-files\nKERNEL_ELF_RAW_DEPS = $(filter-out %: ,$(file < $(KERNEL_ELF_RAW).d)) $(KERNEL_MANIFEST) $(LAST_BUILD_CONFIG)\n\n##------------------------------------------------------------------------------\n## Translation tables\n##------------------------------------------------------------------------------\nTT_TOOL_PATH = tools/translation_table_tool\n\nKERNEL_ELF_TTABLES      = target/$(TARGET)/release/kernel+ttables\nKERNEL_ELF_TTABLES_DEPS = $(KERNEL_ELF_RAW) $(wildcard $(TT_TOOL_PATH)/*)\n\nKERNEL_ELF = $(KERNEL_ELF_TTABLES)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Command building blocks\n##--------------------------------------------------------------------------------------------------\nRUSTFLAGS = $(RUSTC_MISC_ARGS)                   \\\n    -C link-arg=--library-path=$(LD_SCRIPT_PATH) \\\n    -C link-arg=--script=$(KERNEL_LINKER_SCRIPT)\n\nRUSTFLAGS_PEDANTIC = $(RUSTFLAGS) \\\n    -D warnings                   \\\n    -D missing_docs\n\nFEATURES      = --features bsp_$(BSP)\nCOMPILER_ARGS = --target=$(TARGET) \\\n    $(FEATURES)                    \\\n    --release\n\nRUSTC_CMD   = cargo rustc $(COMPILER_ARGS) --manifest-path $(KERNEL_MANIFEST)\nDOC_CMD     = cargo doc $(COMPILER_ARGS)\nCLIPPY_CMD  = cargo clippy $(COMPILER_ARGS)\nTEST_CMD    = cargo test $(COMPILER_ARGS) --manifest-path $(KERNEL_MANIFEST)\nOBJCOPY_CMD = rust-objcopy \\\n    --strip-all            \\\n    -O binary\n\nEXEC_QEMU          = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE)\nEXEC_TT_TOOL       = ruby $(TT_TOOL_PATH)/main.rb\nEXEC_TEST_DISPATCH = ruby ../common/tests/dispatch.rb\nEXEC_MINIPUSH      = ruby ../common/serial/minipush.rb\n\n##------------------------------------------------------------------------------\n## Dockerization\n##------------------------------------------------------------------------------\nDOCKER_CMD            = docker run -t --rm -v $(shell pwd):/work/tutorial -w /work/tutorial\nDOCKER_CMD_INTERACT   = $(DOCKER_CMD) -i\nDOCKER_ARG_DIR_COMMON = -v $(shell pwd)/../common:/work/common\nDOCKER_ARG_DIR_JTAG   = -v $(shell pwd)/../X1_JTAG_boot:/work/X1_JTAG_boot\nDOCKER_ARG_DEV        = --privileged -v /dev:/dev\nDOCKER_ARG_NET        = --network host\n\n# DOCKER_IMAGE defined in include file (see top of this file).\nDOCKER_QEMU  = $(DOCKER_CMD_INTERACT) $(DOCKER_IMAGE)\nDOCKER_TOOLS = $(DOCKER_CMD) $(DOCKER_IMAGE)\nDOCKER_TEST  = $(DOCKER_CMD) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\nDOCKER_GDB   = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_NET) $(DOCKER_IMAGE)\n\n# Dockerize commands, which require USB device passthrough, only on Linux.\nifeq ($(shell uname -s),Linux)\n    DOCKER_CMD_DEV = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_DEV)\n\n    DOCKER_CHAINBOOT = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\n    DOCKER_JTAGBOOT  = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_ARG_DIR_JTAG) $(DOCKER_IMAGE)\n    DOCKER_OPENOCD   = $(DOCKER_CMD_DEV) $(DOCKER_ARG_NET) $(DOCKER_IMAGE)\nelse\n    DOCKER_OPENOCD   = echo \"Not yet supported on non-Linux systems.\"; \\#\nendif\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: all doc qemu chainboot clippy clean readelf objdump nm check\n\nall: $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Save the configuration as a file, so make understands if it changed.\n##------------------------------------------------------------------------------\n$(LAST_BUILD_CONFIG):\n\t@rm -f target/*.build_config\n\t@mkdir -p target\n\t@touch $(LAST_BUILD_CONFIG)\n\n##------------------------------------------------------------------------------\n## Compile the kernel ELF\n##------------------------------------------------------------------------------\n$(KERNEL_ELF_RAW): $(KERNEL_ELF_RAW_DEPS)\n\t$(call color_header, \"Compiling kernel ELF - $(BSP)\")\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(RUSTC_CMD)\n\n##------------------------------------------------------------------------------\n## Precompute the kernel translation tables and patch them into the kernel ELF\n##------------------------------------------------------------------------------\n$(KERNEL_ELF_TTABLES): $(KERNEL_ELF_TTABLES_DEPS)\n\t$(call color_header, \"Precomputing kernel translation tables and patching kernel ELF\")\n\t@cp $(KERNEL_ELF_RAW) $(KERNEL_ELF_TTABLES)\n\t@$(DOCKER_TOOLS) $(EXEC_TT_TOOL) $(BSP) $(KERNEL_ELF_TTABLES)\n\n##------------------------------------------------------------------------------\n## Generate the stripped kernel binary\n##------------------------------------------------------------------------------\n$(KERNEL_BIN): $(KERNEL_ELF_TTABLES)\n\t$(call color_header, \"Generating stripped binary\")\n\t@$(OBJCOPY_CMD) $(KERNEL_ELF_TTABLES) $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Name\")\n\t@echo $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Size\")\n\t$(call disk_usage_KiB, $(KERNEL_BIN))\n\n##------------------------------------------------------------------------------\n## Generate the documentation\n##------------------------------------------------------------------------------\ndoc:\n\t$(call color_header, \"Generating docs\")\n\t@$(DOC_CMD) --document-private-items --open\n\n##------------------------------------------------------------------------------\n## Run the kernel in QEMU\n##------------------------------------------------------------------------------\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\nqemu:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\nqemu: $(KERNEL_BIN)\n\t$(call color_header, \"Launching QEMU\")\n\t@$(DOCKER_QEMU) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\nendif\n\n##------------------------------------------------------------------------------\n## Push the kernel to the real HW target\n##------------------------------------------------------------------------------\nchainboot: $(KERNEL_BIN)\n\t@$(DOCKER_CHAINBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run clippy\n##------------------------------------------------------------------------------\nclippy:\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(CLIPPY_CMD)\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(CLIPPY_CMD) --features test_build --tests \\\n                --manifest-path $(KERNEL_MANIFEST)\n\n##------------------------------------------------------------------------------\n## Clean\n##------------------------------------------------------------------------------\nclean:\n\trm -rf target $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run readelf\n##------------------------------------------------------------------------------\nreadelf: $(KERNEL_ELF)\n\t$(call color_header, \"Launching readelf\")\n\t@$(DOCKER_TOOLS) $(READELF_BINARY) --headers $(KERNEL_ELF)\n\n##------------------------------------------------------------------------------\n## Run objdump\n##------------------------------------------------------------------------------\nobjdump: $(KERNEL_ELF)\n\t$(call color_header, \"Launching objdump\")\n\t@$(DOCKER_TOOLS) $(OBJDUMP_BINARY) --disassemble --demangle \\\n                --section .text   \\\n                --section .rodata \\\n                $(KERNEL_ELF) | rustfilt\n\n##------------------------------------------------------------------------------\n## Run nm\n##------------------------------------------------------------------------------\nnm: $(KERNEL_ELF)\n\t$(call color_header, \"Launching nm\")\n\t@$(DOCKER_TOOLS) $(NM_BINARY) --demangle --print-size $(KERNEL_ELF) | sort | rustfilt\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Debugging targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: jtagboot openocd gdb gdb-opt0\n\n##------------------------------------------------------------------------------\n## Push the JTAG boot image to the real HW target\n##------------------------------------------------------------------------------\njtagboot:\n\t@$(DOCKER_JTAGBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(JTAG_BOOT_IMAGE)\n\n##------------------------------------------------------------------------------\n## Start OpenOCD session\n##------------------------------------------------------------------------------\nopenocd:\n\t$(call color_header, \"Launching OpenOCD\")\n\t@$(DOCKER_OPENOCD) openocd $(OPENOCD_ARG)\n\n##------------------------------------------------------------------------------\n## Start GDB session\n##------------------------------------------------------------------------------\ngdb: RUSTC_MISC_ARGS += -C debuginfo=2\ngdb-opt0: RUSTC_MISC_ARGS += -C debuginfo=2 -C opt-level=0\ngdb gdb-opt0: $(KERNEL_ELF)\n\t$(call color_header, \"Launching GDB\")\n\t@$(DOCKER_GDB) gdb-multiarch -q $(KERNEL_ELF)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Testing targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: test test_boot test_unit test_integration\n\ntest_unit test_integration: FEATURES += --features test_build\n\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\ntest_boot test_unit test_integration test:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\n##------------------------------------------------------------------------------\n## Run boot test\n##------------------------------------------------------------------------------\ntest_boot: $(KERNEL_BIN)\n\t$(call color_header, \"Boot test - $(BSP)\")\n\t@$(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Helpers for unit and integration test targets\n##------------------------------------------------------------------------------\ndefine KERNEL_TEST_RUNNER\n#!/usr/bin/env bash\n\n    # The cargo test runner seems to change into the crate under test's directory. Therefore, ensure\n    # this script executes from the root.\n    cd $(shell pwd)\n\n    TEST_ELF=$$(echo $$1 | sed -e 's/.*target/target/g')\n    TEST_BINARY=$$(echo $$1.img | sed -e 's/.*target/target/g')\n\n    $(DOCKER_TOOLS) $(EXEC_TT_TOOL) $(BSP) $$TEST_ELF > /dev/null\n    $(OBJCOPY_CMD) $$TEST_ELF $$TEST_BINARY\n    $(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_TEST_ARGS) -kernel $$TEST_BINARY\nendef\n\nexport KERNEL_TEST_RUNNER\n\ndefine test_prepare\n    @mkdir -p target\n    @echo \"$$KERNEL_TEST_RUNNER\" > target/kernel_test_runner.sh\n    @chmod +x target/kernel_test_runner.sh\nendef\n\n##------------------------------------------------------------------------------\n## Run unit test(s)\n##------------------------------------------------------------------------------\ntest_unit:\n\t$(call color_header, \"Compiling unit test(s) - $(BSP)\")\n\t$(call test_prepare)\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(TEST_CMD) --lib\n\n##------------------------------------------------------------------------------\n## Run integration test(s)\n##------------------------------------------------------------------------------\ntest_integration:\n\t$(call color_header, \"Compiling integration test(s) - $(BSP)\")\n\t$(call test_prepare)\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(TEST_CMD) $(TEST_ARG)\n\ntest: test_boot test_unit test_integration\n\nendif\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/README.md",
    "content": "# Tutorial 16 - Virtual Memory Part 4: Higher-Half Kernel\n\n## tl;dr\n\n- The time has come: We map and run the kernel from the top of the 64 bit virtual address space! 🥳\n\n## Table of Contents\n\n- [Introduction](#introduction)\n- [Implementation](#implementation)\n  - [Linking Changes](#linking-changes)\n  - [Position-Independent Boot Code](#position-independent-boot-code)\n- [Test it](#test-it)\n- [Diff to previous](#diff-to-previous)\n\n## Introduction\n\nA long time in the making, in this tutorial we finally map the kernel to the most significant area\n(alternatively: higher-half) of the 64 bit virtual address space. This makes room for future\napplications to use the whole of the least significant area of the virtual memory space.\n\nAs has been teased since `tutorial 14`, we will make use of the `AArch64`'s `TTBR1`. Since the\nkernel's virtual address space size currently is `1 GiB` (defined in\n`bsp/__board_name__/memory/mmu.rs`), `TTBR1` will cover the range from `0xffff_ffff_ffff_ffff` down\nto `ffff_ffff_c000_0000` (both inclusive).\n\n## Implementation\n\nIn `src/memory/mmu.rs`, we extend the `AssociatedTranslationTable` trait with a `TableStartFromTop`\nassociated type:\n\n```rust\n/// Intended to be implemented for [`AddressSpace`].\npub trait AssociatedTranslationTable {\n    /// A translation table whose address range is:\n    ///\n    /// [u64::MAX, (u64::MAX - AS_SIZE) + 1]\n    type TableStartFromTop;\n\n    /// A translation table whose address range is:\n    ///\n    /// [AS_SIZE - 1, 0]\n    type TableStartFromBottom;\n}\n```\n\nArchitecture specific code in `_arch/aarch64/memory/mmu/translation_table.rs` populates both types\nnow by making use of a new generic that is added to `FixedSizeTranslationTable`, which defines\nwhether the covered address space starts from the top or the bottom:\n\n```rust\npub struct FixedSizeTranslationTable<const NUM_TABLES: usize, const START_FROM_TOP: bool> {\n    ...\n```\n\n```rust\nimpl<const AS_SIZE: usize> memory::mmu::AssociatedTranslationTable\n    for memory::mmu::AddressSpace<AS_SIZE>\nwhere\n    [u8; Self::SIZE >> Granule512MiB::SHIFT]: Sized,\n{\n    type TableStartFromTop =\n        FixedSizeTranslationTable<{ Self::SIZE >> Granule512MiB::SHIFT }, true>;\n\n    type TableStartFromBottom =\n        FixedSizeTranslationTable<{ Self::SIZE >> Granule512MiB::SHIFT }, false>;\n}\n```\n\nThanks to this infrastructure, `BSP` Rust code in `bsp/__board_name__/memory/mmu.rs` only needs to\nchange to this newly introduced type in order to switch from lower half to higher half translation\ntables for the kernel:\n\n```rust\ntype KernelTranslationTable =\n    <KernelVirtAddrSpace as AssociatedTranslationTable>::TableStartFromTop;\n```\n\n### Linking Changes\n\nIn the `kernel.ld` linker script, we define a new symbol `__kernel_virt_start_addr` now, which is\nthe start address of the kernel's virtual address space, calculated as `(u64::MAX -\n__kernel_virt_addr_space_size) + 1`. Before the first section definition, we set the linker script's\nlocation counter to this address:\n\n```ld.s\nSECTIONS\n{\n    . =  __kernel_virt_start_addr;\n\n    ASSERT((. & PAGE_MASK) == 0, \"Start of address space is not page aligned\")\n\n    /***********************************************************************************************\n    * Code + RO Data + Global Offset Table\n    ***********************************************************************************************/\n```\n\nSince we are not identity mapping anymore, we start to make use of the `AT` keyword in the output\nsection specification:\n\n```ld.s\n/* The physical address at which the the kernel binary will be loaded by the Raspberry's firmware */\n__rpi_phys_binary_load_addr = 0x80000;\n\n/* omitted */\n\nSECTIONS\n{\n    . =  __kernel_virt_start_addr;\n\n    /* omitted */\n\n    __code_start = .;\n    .text : AT(__rpi_phys_binary_load_addr)\n```\n\nThis will manifest in the kernel ELF `segment` attributes, as can be inspected using the `make\nreadelf` command:\n\n```console\n$ make readelf\n\nProgram Headers:\n  Type           Offset             VirtAddr           PhysAddr\n                 FileSiz            MemSiz              Flags  Align\n  LOAD           0x0000000000010000 0xffffffffc0000000 0x0000000000080000\n                 0x000000000000cb08 0x000000000000cb08  R E    0x10000\n  LOAD           0x0000000000020000 0xffffffffc0010000 0x0000000000090000\n                 0x0000000000030dc0 0x0000000000030de0  RW     0x10000\n  LOAD           0x0000000000060000 0xffffffffc0860000 0x0000000000000000\n                 0x0000000000000000 0x0000000000080000  RW     0x10000\n\n Section to Segment mapping:\n  Segment Sections...\n   00     .text .rodata\n   01     .data .bss\n   02     .boot_core_stack\n\n```\n\nAs you can see, `VirtAddr` and `PhysAddr` are different now, as compared to all the previous\ntutorials where they were identical. This information from the `ELF` file will eventually be parsed\nby the `translation table tool` and incorporated when compiling the precomputed translation tables.\n\nYou might have noticed that `.text .rodata` and `.boot_core_stack` exchanged places as compared to\nprevious tutorials. The reason this was done is that with a remapped kernel, this is trivial to do\nwithout affecting the physical layout. This allows us to place an unmapped `guard page` between the\n`boot core stack` and the `mmio remap region` in the VA space, which nicely protects the kernel from\nstack overflows now:\n\n```ld.s\n/***********************************************************************************************\n* MMIO Remap Reserved\n***********************************************************************************************/\n__mmio_remap_start = .;\n. += 8 * 1024 * 1024;\n__mmio_remap_end_exclusive = .;\n\nASSERT((. & PAGE_MASK) == 0, \"MMIO remap reservation is not page aligned\")\n\n/***********************************************************************************************\n* Guard Page\n***********************************************************************************************/\n. += PAGE_SIZE;\n\n/***********************************************************************************************\n* Boot Core Stack\n***********************************************************************************************/\n.boot_core_stack (NOLOAD) : AT(__rpi_phys_dram_start_addr)\n{\n    __boot_core_stack_start = .;         /*   ^             */\n                                         /*   | stack       */\n    . += __rpi_phys_binary_load_addr;    /*   | growth      */\n                                         /*   | direction   */\n    __boot_core_stack_end_exclusive = .; /*   |             */\n} :segment_boot_core_stack\n\nASSERT((. & PAGE_MASK) == 0, \"End of boot core stack is not page aligned\")\n```\n\nChanges in the `_arch` `MMU` driver are minimal, and mostly concerned with configuring `TCR_EL1` for\nuse with `TTBR1_EL1` now. And of course, setting `TTBR1_EL1` in `fn enable_mmu_and_caching(...)`.\n\n### Position-Independent Boot Code\n\nRemember all the fuss that we made about `position-independent code` that will be needed until the\n`MMU` is enabled. Let's quickly check what it means for us in reality now:\n\nIn `_arch/aarch64/cpu/boot.rs`, we turn on the `MMU` just before returning from `EL2` to `EL1`. So\nby the time the CPU enters `EL1`, virtual memory will be active, and the CPU must therefore use the\nnew higher-half `virtual addresses` for everything it does.\n\nSpecifically, this means the address from which the CPU should execute upon entering `EL1` (function\n`kernel_init()`) must be a valid _virtual address_, same as the stack pointer's address. Both of\nthem are programmed in function `fn prepare_el2_to_el1_transition(...)`, so we must ensure now that\n_link-time_ addresses are used here. For this reason, retrieval of these addresses happens in\n`assembly` in `boot.s`, where we can explicitly enforce generation of **absolute** addresses:\n\n```asm\n// Load the _absolute_ addresses of the following symbols. Since the kernel is linked at\n// the top of the 64 bit address space, these are effectively virtual addresses.\nADR_ABS\tx1, __boot_core_stack_end_exclusive\nADR_ABS\tx2, kernel_init\n```\n\nBoth values are forwarded to the Rust entry point function `_start_rust()`, which in turn forwards\nthem to `fn prepare_el2_to_el1_transition(...)`.\n\nOne more thing to consider is that we keep on programming the boot core's stack address for `EL2`\nusing an address that is calculated `PC-relative`, because all the `EL2` code will still run while\nvirtual memory _is disabled_. As such, we need the \"physical\" address of the stack, so to speak.\n\nThe previous tutorial also explained that it is not easily possible to compile select files using\n`-fpic` in `Rust`. Still, we are doing some function calls in `Rust` before virtual memory is\nenabled, so _theoretically_, there is room for failure. However, branches to local code in `AArch64`\nare usually generated PC-relative. So it is a small risk worth taking. Should it still fail someday,\nat least our automated CI pipeline would give notice when the tests start to fail.\n\n## Test it\n\nThat's it! We are ready for a higher-half kernel now. Power up your Raspberrys and marvel at those\nbeautiful (virtual) addresses:\n\nRaspberry Pi 3:\n\n```console\n$ make chainboot\n[...]\n\nPrecomputing kernel translation tables and patching kernel ELF\n             ------------------------------------------------------------------------------------\n                 Sections          Virt Start Addr         Phys Start Addr       Size      Attr\n             ------------------------------------------------------------------------------------\n  Generating .text .rodata    | 0xffff_ffff_c000_0000 | 0x0000_0000_0008_0000 |  64 KiB | C RO X\n  Generating .data .bss       | 0xffff_ffff_c001_0000 | 0x0000_0000_0009_0000 | 256 KiB | C RW XN\n  Generating .boot_core_stack | 0xffff_ffff_c086_0000 | 0x0000_0000_0000_0000 | 512 KiB | C RW XN\n             ------------------------------------------------------------------------------------\n    Patching Kernel table struct at ELF file offset 0x2_0000\n    Patching Kernel tables physical base address start argument to value 0xb_0000 at ELF file offset 0x1_0088\n    Finished in 0.14s\n\nMinipush 1.0\n\n[MP] ⏳ Waiting for /dev/ttyUSB0\n[MP] ✅ Serial connected\n[MP] 🔌 Please power the target now\n\n __  __ _      _ _                 _\n|  \\/  (_)_ _ (_) |   ___  __ _ __| |\n| |\\/| | | ' \\| | |__/ _ \\/ _` / _` |\n|_|  |_|_|_||_|_|____\\___/\\__,_\\__,_|\n\n           Raspberry Pi 3\n\n[ML] Requesting binary\n[MP] ⏩ Pushing 257 KiB ======================================🦀 100% 128 KiB/s Time: 00:00:02\n[ML] Loaded! Executing the payload now\n\n[    2.870248] mingo version 0.16.0\n[    2.870456] Booting on: Raspberry Pi 3\n[    2.870911] MMU online:\n[    2.871203]       -------------------------------------------------------------------------------------------------------------------------------------------\n[    2.872947]                         Virtual                                   Physical               Size       Attr                    Entity\n[    2.874691]       -------------------------------------------------------------------------------------------------------------------------------------------\n[    2.876436]       0xffff_ffff_c000_0000..0xffff_ffff_c000_ffff --> 0x00_0008_0000..0x00_0008_ffff |  64 KiB | C   RO X  | Kernel code and RO data\n[    2.878050]       0xffff_ffff_c001_0000..0xffff_ffff_c004_ffff --> 0x00_0009_0000..0x00_000c_ffff | 256 KiB | C   RW XN | Kernel data and bss\n[    2.879621]       0xffff_ffff_c005_0000..0xffff_ffff_c005_ffff --> 0x00_3f20_0000..0x00_3f20_ffff |  64 KiB | Dev RW XN | BCM PL011 UART\n[    2.881137]                                                                                                             | BCM GPIO\n[    2.882589]       0xffff_ffff_c006_0000..0xffff_ffff_c006_ffff --> 0x00_3f00_0000..0x00_3f00_ffff |  64 KiB | Dev RW XN | BCM Interrupt Controller\n[    2.884214]       0xffff_ffff_c086_0000..0xffff_ffff_c08d_ffff --> 0x00_0000_0000..0x00_0007_ffff | 512 KiB | C   RW XN | Kernel boot-core stack\n[    2.885818]       -------------------------------------------------------------------------------------------------------------------------------------------\n```\n\nRaspberry Pi 4:\n\n```console\n$ BSP=rpi4 make chainboot\n[...]\n\nPrecomputing kernel translation tables and patching kernel ELF\n             ------------------------------------------------------------------------------------\n                 Sections          Virt Start Addr         Phys Start Addr       Size      Attr\n             ------------------------------------------------------------------------------------\n  Generating .text .rodata    | 0xffff_ffff_c000_0000 | 0x0000_0000_0008_0000 |  64 KiB | C RO X\n  Generating .data .bss       | 0xffff_ffff_c001_0000 | 0x0000_0000_0009_0000 | 256 KiB | C RW XN\n  Generating .boot_core_stack | 0xffff_ffff_c086_0000 | 0x0000_0000_0000_0000 | 512 KiB | C RW XN\n             ------------------------------------------------------------------------------------\n    Patching Kernel table struct at ELF file offset 0x2_0000\n    Patching Kernel tables physical base address start argument to value 0xb_0000 at ELF file offset 0x1_0080\n    Finished in 0.13s\n\nMinipush 1.0\n\n[MP] ⏳ Waiting for /dev/ttyUSB0\n[MP] ✅ Serial connected\n[MP] 🔌 Please power the target now\n\n __  __ _      _ _                 _\n|  \\/  (_)_ _ (_) |   ___  __ _ __| |\n| |\\/| | | ' \\| | |__/ _ \\/ _` / _` |\n|_|  |_|_|_||_|_|____\\___/\\__,_\\__,_|\n\n           Raspberry Pi 4\n\n[ML] Requesting binary\n[MP] ⏩ Pushing 257 KiB ======================================🦀 100% 128 KiB/s Time: 00:00:02\n[ML] Loaded! Executing the payload now\n\n[    2.871960] mingo version 0.16.0\n[    2.871994] Booting on: Raspberry Pi 4\n[    2.872449] MMU online:\n[    2.872742]       -------------------------------------------------------------------------------------------------------------------------------------------\n[    2.874486]                         Virtual                                   Physical               Size       Attr                    Entity\n[    2.876230]       -------------------------------------------------------------------------------------------------------------------------------------------\n[    2.877975]       0xffff_ffff_c000_0000..0xffff_ffff_c000_ffff --> 0x00_0008_0000..0x00_0008_ffff |  64 KiB | C   RO X  | Kernel code and RO data\n[    2.879589]       0xffff_ffff_c001_0000..0xffff_ffff_c004_ffff --> 0x00_0009_0000..0x00_000c_ffff | 256 KiB | C   RW XN | Kernel data and bss\n[    2.881159]       0xffff_ffff_c005_0000..0xffff_ffff_c005_ffff --> 0x00_fe20_0000..0x00_fe20_ffff |  64 KiB | Dev RW XN | BCM PL011 UART\n[    2.882676]                                                                                                             | BCM GPIO\n[    2.884128]       0xffff_ffff_c006_0000..0xffff_ffff_c006_ffff --> 0x00_ff84_0000..0x00_ff84_ffff |  64 KiB | Dev RW XN | GICv2 GICD\n[    2.885601]                                                                                                             | GICV2 GICC\n[    2.887074]       0xffff_ffff_c086_0000..0xffff_ffff_c08d_ffff --> 0x00_0000_0000..0x00_0007_ffff | 512 KiB | C   RW XN | Kernel boot-core stack\n[    2.888678]       -------------------------------------------------------------------------------------------------------------------------------------------\n```\n\n## Diff to previous\n```diff\n\ndiff -uNr 15_virtual_mem_part3_precomputed_tables/kernel/Cargo.toml 16_virtual_mem_part4_higher_half_kernel/kernel/Cargo.toml\n--- 15_virtual_mem_part3_precomputed_tables/kernel/Cargo.toml\n+++ 16_virtual_mem_part4_higher_half_kernel/kernel/Cargo.toml\n@@ -1,6 +1,6 @@\n [package]\n name = \"mingo\"\n-version = \"0.15.0\"\n+version = \"0.16.0\"\n authors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\n edition = \"2021\"\n\n\ndiff -uNr 15_virtual_mem_part3_precomputed_tables/kernel/src/_arch/aarch64/cpu/boot.rs 16_virtual_mem_part4_higher_half_kernel/kernel/src/_arch/aarch64/cpu/boot.rs\n--- 15_virtual_mem_part3_precomputed_tables/kernel/src/_arch/aarch64/cpu/boot.rs\n+++ 16_virtual_mem_part4_higher_half_kernel/kernel/src/_arch/aarch64/cpu/boot.rs\n@@ -34,7 +34,10 @@\n /// - The `bss` section is not initialized yet. The code must not use or reference it in any way.\n /// - The HW state of EL1 must be prepared in a sound way.\n #[inline(always)]\n-unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: u64) {\n+unsafe fn prepare_el2_to_el1_transition(\n+    virt_boot_core_stack_end_exclusive_addr: u64,\n+    virt_kernel_init_addr: u64,\n+) {\n     // Enable timer counter registers for EL1.\n     CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET);\n\n@@ -57,11 +60,11 @@\n     );\n\n     // Second, let the link register point to kernel_init().\n-    ELR_EL2.set(crate::kernel_init as *const () as u64);\n+    ELR_EL2.set(virt_kernel_init_addr);\n\n     // Set up SP_EL1 (stack pointer), which will be used by EL1 once we \"return\" to it. Since there\n     // are no plans to ever return to EL2, just re-use the same stack.\n-    SP_EL1.set(phys_boot_core_stack_end_exclusive_addr);\n+    SP_EL1.set(virt_boot_core_stack_end_exclusive_addr);\n }\n\n //--------------------------------------------------------------------------------------------------\n@@ -78,14 +81,19 @@\n #[no_mangle]\n pub unsafe extern \"C\" fn _start_rust(\n     phys_kernel_tables_base_addr: u64,\n-    phys_boot_core_stack_end_exclusive_addr: u64,\n+    virt_boot_core_stack_end_exclusive_addr: u64,\n+    virt_kernel_init_addr: u64,\n ) -> ! {\n-    prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr);\n+    prepare_el2_to_el1_transition(\n+        virt_boot_core_stack_end_exclusive_addr,\n+        virt_kernel_init_addr,\n+    );\n\n     // Turn on the MMU for EL1.\n     let addr = Address::new(phys_kernel_tables_base_addr as usize);\n     memory::mmu::enable_mmu_and_caching(addr).unwrap();\n\n-    // Use `eret` to \"return\" to EL1. This results in execution of kernel_init() in EL1.\n+    // Use `eret` to \"return\" to EL1. Since virtual memory will already be enabled, this results in\n+    // execution of kernel_init() in EL1 from its _virtual address_.\n     asm::eret()\n }\n\ndiff -uNr 15_virtual_mem_part3_precomputed_tables/kernel/src/_arch/aarch64/cpu/boot.s 16_virtual_mem_part4_higher_half_kernel/kernel/src/_arch/aarch64/cpu/boot.s\n--- 15_virtual_mem_part3_precomputed_tables/kernel/src/_arch/aarch64/cpu/boot.s\n+++ 16_virtual_mem_part4_higher_half_kernel/kernel/src/_arch/aarch64/cpu/boot.s\n@@ -18,6 +18,18 @@\n \tadd\t\\register, \\register, #:lo12:\\symbol\n .endm\n\n+// Load the address of a symbol into a register, absolute.\n+//\n+// # Resources\n+//\n+// - https://sourceware.org/binutils/docs-2.36/as/AArch64_002dRelocations.html\n+.macro ADR_ABS register, symbol\n+\tmovz\t\\register, #:abs_g3:\\symbol\n+\tmovk\t\\register, #:abs_g2_nc:\\symbol\n+\tmovk\t\\register, #:abs_g1_nc:\\symbol\n+\tmovk\t\\register, #:abs_g0_nc:\\symbol\n+.endm\n+\n //--------------------------------------------------------------------------------------------------\n // Public Code\n //--------------------------------------------------------------------------------------------------\n@@ -56,19 +68,31 @@\n \t// Load the base address of the kernel's translation tables.\n \tldr\tx0, PHYS_KERNEL_TABLES_BASE_ADDR // provided by bsp/__board_name__/memory/mmu.rs\n\n-\t// Set the stack pointer. This ensures that any code in EL2 that needs the stack will work.\n-\tADR_REL\tx1, __boot_core_stack_end_exclusive\n-\tmov\tsp, x1\n+\t// Load the _absolute_ addresses of the following symbols. Since the kernel is linked at\n+\t// the top of the 64 bit address space, these are effectively virtual addresses.\n+\tADR_ABS\tx1, __boot_core_stack_end_exclusive\n+\tADR_ABS\tx2, kernel_init\n+\n+\t// Load the PC-relative address of the stack and set the stack pointer.\n+\t//\n+\t// Since _start() is the first function that runs after the firmware has loaded the kernel\n+\t// into memory, retrieving this symbol PC-relative returns the \"physical\" address.\n+\t//\n+\t// Setting the stack pointer to this value ensures that anything that still runs in EL2,\n+\t// until the kernel returns to EL1 with the MMU enabled, works as well. After the return to\n+\t// EL1, the virtual address of the stack retrieved above will be used.\n+\tADR_REL\tx3, __boot_core_stack_end_exclusive\n+\tmov\tsp, x3\n\n \t// Read the CPU's timer counter frequency and store it in ARCH_TIMER_COUNTER_FREQUENCY.\n \t// Abort if the frequency read back as 0.\n-\tADR_REL\tx2, ARCH_TIMER_COUNTER_FREQUENCY // provided by aarch64/time.rs\n-\tmrs\tx3, CNTFRQ_EL0\n-\tcmp\tx3, xzr\n+\tADR_REL\tx4, ARCH_TIMER_COUNTER_FREQUENCY // provided by aarch64/time.rs\n+\tmrs\tx5, CNTFRQ_EL0\n+\tcmp\tx5, xzr\n \tb.eq\t.L_parking_loop\n-\tstr\tw3, [x2]\n+\tstr\tw5, [x4]\n\n-\t// Jump to Rust code. x0 and x1 hold the function arguments provided to _start_rust().\n+\t// Jump to Rust code. x0, x1 and x2 hold the function arguments provided to _start_rust().\n \tb\t_start_rust\n\n \t// Infinitely wait for events (aka \"park the core\").\n\ndiff -uNr 15_virtual_mem_part3_precomputed_tables/kernel/src/_arch/aarch64/memory/mmu/translation_table.rs 16_virtual_mem_part4_higher_half_kernel/kernel/src/_arch/aarch64/memory/mmu/translation_table.rs\n--- 15_virtual_mem_part3_precomputed_tables/kernel/src/_arch/aarch64/memory/mmu/translation_table.rs\n+++ 16_virtual_mem_part4_higher_half_kernel/kernel/src/_arch/aarch64/memory/mmu/translation_table.rs\n@@ -136,7 +136,7 @@\n /// aligned, so the lvl3 is put first.\n #[repr(C)]\n #[repr(align(65536))]\n-pub struct FixedSizeTranslationTable<const NUM_TABLES: usize> {\n+pub struct FixedSizeTranslationTable<const NUM_TABLES: usize, const START_FROM_TOP: bool> {\n     /// Page descriptors, covering 64 KiB windows per entry.\n     lvl3: [[PageDescriptor; 8192]; NUM_TABLES],\n\n@@ -302,10 +302,19 @@\n where\n     [u8; Self::SIZE >> Granule512MiB::SHIFT]: Sized,\n {\n-    type TableStartFromBottom = FixedSizeTranslationTable<{ Self::SIZE >> Granule512MiB::SHIFT }>;\n+    type TableStartFromTop =\n+        FixedSizeTranslationTable<{ Self::SIZE >> Granule512MiB::SHIFT }, true>;\n+\n+    type TableStartFromBottom =\n+        FixedSizeTranslationTable<{ Self::SIZE >> Granule512MiB::SHIFT }, false>;\n }\n\n-impl<const NUM_TABLES: usize> FixedSizeTranslationTable<NUM_TABLES> {\n+impl<const NUM_TABLES: usize, const START_FROM_TOP: bool>\n+    FixedSizeTranslationTable<NUM_TABLES, START_FROM_TOP>\n+{\n+    const START_FROM_TOP_OFFSET: Address<Virtual> =\n+        Address::new((usize::MAX - (Granule512MiB::SIZE * NUM_TABLES)) + 1);\n+\n     /// Create an instance.\n     #[allow(clippy::assertions_on_constants)]\n     const fn _new(for_precompute: bool) -> Self {\n@@ -336,9 +345,14 @@\n         &self,\n         virt_page_addr: PageAddress<Virtual>,\n     ) -> Result<(usize, usize), &'static str> {\n-        let addr = virt_page_addr.into_inner().as_usize();\n-        let lvl2_index = addr >> Granule512MiB::SHIFT;\n-        let lvl3_index = (addr & Granule512MiB::MASK) >> Granule64KiB::SHIFT;\n+        let mut addr = virt_page_addr.into_inner();\n+\n+        if START_FROM_TOP {\n+            addr = addr - Self::START_FROM_TOP_OFFSET;\n+        }\n+\n+        let lvl2_index = addr.as_usize() >> Granule512MiB::SHIFT;\n+        let lvl3_index = (addr.as_usize() & Granule512MiB::MASK) >> Granule64KiB::SHIFT;\n\n         if lvl2_index > (NUM_TABLES - 1) {\n             return Err(\"Virtual page is out of bounds of translation table\");\n@@ -384,8 +398,9 @@\n // OS Interface Code\n //------------------------------------------------------------------------------\n\n-impl<const NUM_TABLES: usize> memory::mmu::translation_table::interface::TranslationTable\n-    for FixedSizeTranslationTable<NUM_TABLES>\n+impl<const NUM_TABLES: usize, const START_FROM_TOP: bool>\n+    memory::mmu::translation_table::interface::TranslationTable\n+    for FixedSizeTranslationTable<NUM_TABLES, START_FROM_TOP>\n {\n     fn init(&mut self) -> Result<(), &'static str> {\n         if self.initialized {\n@@ -479,7 +494,7 @@\n //--------------------------------------------------------------------------------------------------\n\n #[cfg(test)]\n-pub type MinSizeTranslationTable = FixedSizeTranslationTable<1>;\n+pub type MinSizeTranslationTable = FixedSizeTranslationTable<1, true>;\n\n #[cfg(test)]\n mod tests {\n\ndiff -uNr 15_virtual_mem_part3_precomputed_tables/kernel/src/_arch/aarch64/memory/mmu.rs 16_virtual_mem_part4_higher_half_kernel/kernel/src/_arch/aarch64/memory/mmu.rs\n--- 15_virtual_mem_part3_precomputed_tables/kernel/src/_arch/aarch64/memory/mmu.rs\n+++ 16_virtual_mem_part4_higher_half_kernel/kernel/src/_arch/aarch64/memory/mmu.rs\n@@ -66,6 +66,7 @@\n\n impl MemoryManagementUnit {\n     /// Setup function for the MAIR_EL1 register.\n+    #[inline(always)]\n     fn set_up_mair(&self) {\n         // Define the memory types being mapped.\n         MAIR_EL1.write(\n@@ -79,20 +80,21 @@\n     }\n\n     /// Configure various settings of stage 1 of the EL1 translation regime.\n+    #[inline(always)]\n     fn configure_translation_control(&self) {\n-        let t0sz = (64 - bsp::memory::mmu::KernelVirtAddrSpace::SIZE_SHIFT) as u64;\n+        let t1sz = (64 - bsp::memory::mmu::KernelVirtAddrSpace::SIZE_SHIFT) as u64;\n\n         TCR_EL1.write(\n-            TCR_EL1::TBI0::Used\n+            TCR_EL1::TBI1::Used\n                 + TCR_EL1::IPS::Bits_40\n-                + TCR_EL1::TG0::KiB_64\n-                + TCR_EL1::SH0::Inner\n-                + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable\n-                + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable\n-                + TCR_EL1::EPD0::EnableTTBR0Walks\n-                + TCR_EL1::A1::TTBR0\n-                + TCR_EL1::T0SZ.val(t0sz)\n-                + TCR_EL1::EPD1::DisableTTBR1Walks,\n+                + TCR_EL1::TG1::KiB_64\n+                + TCR_EL1::SH1::Inner\n+                + TCR_EL1::ORGN1::WriteBack_ReadAlloc_WriteAlloc_Cacheable\n+                + TCR_EL1::IRGN1::WriteBack_ReadAlloc_WriteAlloc_Cacheable\n+                + TCR_EL1::EPD1::EnableTTBR1Walks\n+                + TCR_EL1::A1::TTBR1\n+                + TCR_EL1::T1SZ.val(t1sz)\n+                + TCR_EL1::EPD0::DisableTTBR0Walks,\n         );\n     }\n }\n@@ -131,7 +133,7 @@\n         self.set_up_mair();\n\n         // Set the \"Translation Table Base Register\".\n-        TTBR0_EL1.set_baddr(phys_tables_base_addr.as_usize() as u64);\n+        TTBR1_EL1.set_baddr(phys_tables_base_addr.as_usize() as u64);\n\n         self.configure_translation_control();\n\n\ndiff -uNr 15_virtual_mem_part3_precomputed_tables/kernel/src/bsp/raspberrypi/kernel.ld 16_virtual_mem_part4_higher_half_kernel/kernel/src/bsp/raspberrypi/kernel.ld\n--- 15_virtual_mem_part3_precomputed_tables/kernel/src/bsp/raspberrypi/kernel.ld\n+++ 16_virtual_mem_part4_higher_half_kernel/kernel/src/bsp/raspberrypi/kernel.ld\n@@ -8,6 +8,13 @@\n PAGE_SIZE = 64K;\n PAGE_MASK = PAGE_SIZE - 1;\n\n+/* The kernel's virtual address range will be:\n+ *\n+ * [END_ADDRESS_INCLUSIVE, START_ADDRESS]\n+ * [u64::MAX             , (u64::MAX - __kernel_virt_addr_space_size) + 1]\n+ */\n+__kernel_virt_start_addr = ((0xffffffffffffffff - __kernel_virt_addr_space_size) + 1);\n+\n __rpi_phys_dram_start_addr = 0;\n\n /* The physical address at which the the kernel binary will be loaded by the Raspberry's firmware */\n@@ -26,34 +33,22 @@\n  */\n PHDRS\n {\n-    segment_boot_core_stack PT_LOAD FLAGS(6);\n     segment_code            PT_LOAD FLAGS(5);\n     segment_data            PT_LOAD FLAGS(6);\n+    segment_boot_core_stack PT_LOAD FLAGS(6);\n }\n\n SECTIONS\n {\n-    . =  __rpi_phys_dram_start_addr;\n-\n-    /***********************************************************************************************\n-    * Boot Core Stack\n-    ***********************************************************************************************/\n-    .boot_core_stack (NOLOAD) :\n-    {\n-        __boot_core_stack_start = .;         /*   ^             */\n-                                             /*   | stack       */\n-        . += __rpi_phys_binary_load_addr;    /*   | growth      */\n-                                             /*   | direction   */\n-        __boot_core_stack_end_exclusive = .; /*   |             */\n-    } :segment_boot_core_stack\n+    . =  __kernel_virt_start_addr;\n\n-    ASSERT((. & PAGE_MASK) == 0, \"End of boot core stack is not page aligned\")\n+    ASSERT((. & PAGE_MASK) == 0, \"Start of address space is not page aligned\")\n\n     /***********************************************************************************************\n     * Code + RO Data + Global Offset Table\n     ***********************************************************************************************/\n     __code_start = .;\n-    .text :\n+    .text : AT(__rpi_phys_binary_load_addr)\n     {\n         KEEP(*(.text._start))\n         *(.text._start_arguments) /* Constants (or statics in Rust speak) read by _start(). */\n@@ -91,6 +86,25 @@\n     . += 8 * 1024 * 1024;\n     __mmio_remap_end_exclusive = .;\n\n+    ASSERT((. & PAGE_MASK) == 0, \"MMIO remap reservation is not page aligned\")\n+\n+    /***********************************************************************************************\n+    * Guard Page\n+    ***********************************************************************************************/\n+    . += PAGE_SIZE;\n+\n+    /***********************************************************************************************\n+    * Boot Core Stack\n+    ***********************************************************************************************/\n+    .boot_core_stack (NOLOAD) : AT(__rpi_phys_dram_start_addr)\n+    {\n+        __boot_core_stack_start = .;         /*   ^             */\n+                                             /*   | stack       */\n+        . += __rpi_phys_binary_load_addr;    /*   | growth      */\n+                                             /*   | direction   */\n+        __boot_core_stack_end_exclusive = .; /*   |             */\n+    } :segment_boot_core_stack\n+\n     ASSERT((. & PAGE_MASK) == 0, \"End of boot core stack is not page aligned\")\n\n     /***********************************************************************************************\n\ndiff -uNr 15_virtual_mem_part3_precomputed_tables/kernel/src/bsp/raspberrypi/memory/mmu.rs 16_virtual_mem_part4_higher_half_kernel/kernel/src/bsp/raspberrypi/memory/mmu.rs\n--- 15_virtual_mem_part3_precomputed_tables/kernel/src/bsp/raspberrypi/memory/mmu.rs\n+++ 16_virtual_mem_part4_higher_half_kernel/kernel/src/bsp/raspberrypi/memory/mmu.rs\n@@ -20,7 +20,7 @@\n //--------------------------------------------------------------------------------------------------\n\n type KernelTranslationTable =\n-    <KernelVirtAddrSpace as AssociatedTranslationTable>::TableStartFromBottom;\n+    <KernelVirtAddrSpace as AssociatedTranslationTable>::TableStartFromTop;\n\n //--------------------------------------------------------------------------------------------------\n // Public Definitions\n@@ -153,14 +153,6 @@\n /// `translation table tool` and patched into the kernel binary. This function just adds the mapping\n /// record entries.\n pub fn kernel_add_mapping_records_for_precomputed() {\n-    let virt_boot_core_stack_region = virt_boot_core_stack_region();\n-    generic_mmu::kernel_add_mapping_record(\n-        \"Kernel boot-core stack\",\n-        &virt_boot_core_stack_region,\n-        &kernel_virt_to_phys_region(virt_boot_core_stack_region),\n-        &kernel_page_attributes(virt_boot_core_stack_region.start_page_addr()),\n-    );\n-\n     let virt_code_region = virt_code_region();\n     generic_mmu::kernel_add_mapping_record(\n         \"Kernel code and RO data\",\n@@ -176,4 +168,12 @@\n         &kernel_virt_to_phys_region(virt_data_region),\n         &kernel_page_attributes(virt_data_region.start_page_addr()),\n     );\n+\n+    let virt_boot_core_stack_region = virt_boot_core_stack_region();\n+    generic_mmu::kernel_add_mapping_record(\n+        \"Kernel boot-core stack\",\n+        &virt_boot_core_stack_region,\n+        &kernel_virt_to_phys_region(virt_boot_core_stack_region),\n+        &kernel_page_attributes(virt_boot_core_stack_region.start_page_addr()),\n+    );\n }\n\ndiff -uNr 15_virtual_mem_part3_precomputed_tables/kernel/src/bsp/raspberrypi/memory.rs 16_virtual_mem_part4_higher_half_kernel/kernel/src/bsp/raspberrypi/memory.rs\n--- 15_virtual_mem_part3_precomputed_tables/kernel/src/bsp/raspberrypi/memory.rs\n+++ 16_virtual_mem_part4_higher_half_kernel/kernel/src/bsp/raspberrypi/memory.rs\n@@ -37,13 +37,7 @@\n //! The virtual memory layout is as follows:\n //!\n //! +---------------------------------------+\n-//! |                                       | boot_core_stack_start @ 0x0\n-//! |                                       |                                ^\n-//! | Boot-core Stack                       |                                | stack\n-//! |                                       |                                | growth\n-//! |                                       |                                | direction\n-//! +---------------------------------------+\n-//! |                                       | code_start @ 0x8_0000 == boot_core_stack_end_exclusive\n+//! |                                       | code_start @ __kernel_virt_start_addr\n //! | .text                                 |\n //! | .rodata                               |\n //! | .got                                  |\n@@ -59,6 +53,16 @@\n //! |                                       |\n //! +---------------------------------------+\n //! |                                       |  mmio_remap_end_exclusive\n+//! | Unmapped guard page                   |\n+//! |                                       |\n+//! +---------------------------------------+\n+//! |                                       | boot_core_stack_start\n+//! |                                       |                                ^\n+//! | Boot-core Stack                       |                                | stack\n+//! |                                       |                                | growth\n+//! |                                       |                                | direction\n+//! +---------------------------------------+\n+//! |                                       | boot_core_stack_end_exclusive\n //! |                                       |\n pub mod mmu;\n\n\ndiff -uNr 15_virtual_mem_part3_precomputed_tables/kernel/src/lib.rs 16_virtual_mem_part4_higher_half_kernel/kernel/src/lib.rs\n--- 15_virtual_mem_part3_precomputed_tables/kernel/src/lib.rs\n+++ 16_virtual_mem_part4_higher_half_kernel/kernel/src/lib.rs\n@@ -157,11 +157,6 @@\n     )\n }\n\n-#[cfg(not(test))]\n-extern \"Rust\" {\n-    fn kernel_init() -> !;\n-}\n-\n //--------------------------------------------------------------------------------------------------\n // Testing\n //--------------------------------------------------------------------------------------------------\n\ndiff -uNr 15_virtual_mem_part3_precomputed_tables/kernel/src/memory/mmu/translation_table.rs 16_virtual_mem_part4_higher_half_kernel/kernel/src/memory/mmu/translation_table.rs\n--- 15_virtual_mem_part3_precomputed_tables/kernel/src/memory/mmu/translation_table.rs\n+++ 16_virtual_mem_part4_higher_half_kernel/kernel/src/memory/mmu/translation_table.rs\n@@ -99,9 +99,9 @@\n\n         assert_eq!(tables.init(), Ok(()));\n\n-        let virt_start_page_addr: PageAddress<Virtual> = PageAddress::from(0);\n-        let virt_end_exclusive_page_addr: PageAddress<Virtual> =\n-            virt_start_page_addr.checked_offset(5).unwrap();\n+        let virt_end_exclusive_page_addr: PageAddress<Virtual> = PageAddress::MAX;\n+        let virt_start_page_addr: PageAddress<Virtual> =\n+            virt_end_exclusive_page_addr.checked_offset(-5).unwrap();\n\n         let phys_start_page_addr: PageAddress<Physical> = PageAddress::from(0);\n         let phys_end_exclusive_page_addr: PageAddress<Physical> =\n@@ -124,7 +124,7 @@\n         );\n\n         assert_eq!(\n-            tables.try_page_attributes(virt_start_page_addr.checked_offset(6).unwrap()),\n+            tables.try_page_attributes(virt_start_page_addr.checked_offset(-1).unwrap()),\n             Err(\"Page marked invalid\")\n         );\n\n\ndiff -uNr 15_virtual_mem_part3_precomputed_tables/kernel/src/memory/mmu/types.rs 16_virtual_mem_part4_higher_half_kernel/kernel/src/memory/mmu/types.rs\n--- 15_virtual_mem_part3_precomputed_tables/kernel/src/memory/mmu/types.rs\n+++ 16_virtual_mem_part4_higher_half_kernel/kernel/src/memory/mmu/types.rs\n@@ -67,6 +67,11 @@\n // PageAddress\n //------------------------------------------------------------------------------\n impl<ATYPE: AddressType> PageAddress<ATYPE> {\n+    /// The largest value that can be represented by this type.\n+    pub const MAX: Self = PageAddress {\n+        inner: Address::new(usize::MAX).align_down_page(),\n+    };\n+\n     /// Unwraps the value.\n     pub fn into_inner(self) -> Address<ATYPE> {\n         self.inner\n\ndiff -uNr 15_virtual_mem_part3_precomputed_tables/kernel/src/memory/mmu.rs 16_virtual_mem_part4_higher_half_kernel/kernel/src/memory/mmu.rs\n--- 15_virtual_mem_part3_precomputed_tables/kernel/src/memory/mmu.rs\n+++ 16_virtual_mem_part4_higher_half_kernel/kernel/src/memory/mmu.rs\n@@ -66,6 +66,11 @@\n pub trait AssociatedTranslationTable {\n     /// A translation table whose address range is:\n     ///\n+    /// [u64::MAX, (u64::MAX - AS_SIZE) + 1]\n+    type TableStartFromTop;\n+\n+    /// A translation table whose address range is:\n+    ///\n     /// [AS_SIZE - 1, 0]\n     type TableStartFromBottom;\n }\n\ndiff -uNr 15_virtual_mem_part3_precomputed_tables/kernel/tests/02_exception_sync_page_fault.rs 16_virtual_mem_part4_higher_half_kernel/kernel/tests/02_exception_sync_page_fault.rs\n--- 15_virtual_mem_part3_precomputed_tables/kernel/tests/02_exception_sync_page_fault.rs\n+++ 16_virtual_mem_part4_higher_half_kernel/kernel/tests/02_exception_sync_page_fault.rs\n@@ -28,8 +28,8 @@\n     // This line will be printed as the test header.\n     println!(\"Testing synchronous exception handling by causing a page fault\");\n\n-    info!(\"Writing beyond mapped area to address 9 GiB...\");\n-    let big_addr: u64 = 9 * 1024 * 1024 * 1024;\n+    info!(\"Writing to bottom of address space to address 1 GiB...\");\n+    let big_addr: u64 = 1024 * 1024 * 1024;\n     core::ptr::read_volatile(big_addr as *mut u64);\n\n     // If execution reaches here, the memory access above did not cause a page fault exception.\n\ndiff -uNr 15_virtual_mem_part3_precomputed_tables/tools/translation_table_tool/arch.rb 16_virtual_mem_part4_higher_half_kernel/tools/translation_table_tool/arch.rb\n--- 15_virtual_mem_part3_precomputed_tables/tools/translation_table_tool/arch.rb\n+++ 16_virtual_mem_part4_higher_half_kernel/tools/translation_table_tool/arch.rb\n@@ -255,6 +255,8 @@\n     end\n\n     def lvl2_lvl3_index_from(addr)\n+        addr -= BSP.kernel_virt_start_addr\n+\n         lvl2_index = addr >> Granule512MiB::SHIFT\n         lvl3_index = (addr & Granule512MiB::MASK) >> Granule64KiB::SHIFT\n\n\ndiff -uNr 15_virtual_mem_part3_precomputed_tables/tools/translation_table_tool/bsp.rb 16_virtual_mem_part4_higher_half_kernel/tools/translation_table_tool/bsp.rb\n--- 15_virtual_mem_part3_precomputed_tables/tools/translation_table_tool/bsp.rb\n+++ 16_virtual_mem_part4_higher_half_kernel/tools/translation_table_tool/bsp.rb\n@@ -6,7 +6,7 @@\n\n # Raspberry Pi 3 + 4\n class RaspberryPi\n-    attr_reader :kernel_granule, :kernel_virt_addr_space_size\n+    attr_reader :kernel_granule, :kernel_virt_addr_space_size, :kernel_virt_start_addr\n\n     MEMORY_SRC = File.read('kernel/src/bsp/raspberrypi/memory.rs').split(\"\\n\")\n\n@@ -14,6 +14,7 @@\n         @kernel_granule = Granule64KiB\n\n         @kernel_virt_addr_space_size = KERNEL_ELF.symbol_value('__kernel_virt_addr_space_size')\n+        @kernel_virt_start_addr = KERNEL_ELF.symbol_value('__kernel_virt_start_addr')\n\n         @virt_addr_of_kernel_tables = KERNEL_ELF.symbol_value('KERNEL_TABLES')\n         @virt_addr_of_phys_kernel_tables_base_addr = KERNEL_ELF.symbol_value(\n\n```\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/Cargo.toml",
    "content": "[package]\nname = \"mingo\"\nversion = \"0.16.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n\n[features]\ndefault = []\nbsp_rpi3 = [\"tock-registers\"]\nbsp_rpi4 = [\"tock-registers\"]\ntest_build = [\"qemu-exit\"]\n\n##--------------------------------------------------------------------------------------------------\n## Dependencies\n##--------------------------------------------------------------------------------------------------\n\n[dependencies]\ntest-types = { path = \"../libraries/test-types\" }\n\n# Optional dependencies\ntock-registers = { version = \"0.8.x\", default-features = false, features = [\"register_types\"], optional = true }\nqemu-exit = { version = \"3.x.x\", optional = true }\n\n# Platform specific dependencies\n[target.'cfg(target_arch = \"aarch64\")'.dependencies]\naarch64-cpu = { version = \"9.x.x\" }\n\n##--------------------------------------------------------------------------------------------------\n## Testing\n##--------------------------------------------------------------------------------------------------\n\n[dev-dependencies]\ntest-macros = { path = \"../libraries/test-macros\" }\n\n# Unit tests are done in the library part of the kernel.\n[lib]\nname = \"libkernel\"\ntest = true\n\n# Disable unit tests for the kernel binary.\n[[bin]]\nname = \"kernel\"\npath = \"src/main.rs\"\ntest = false\n\n# List of tests without harness.\n[[test]]\nname = \"00_console_sanity\"\nharness = false\n\n[[test]]\nname = \"02_exception_sync_page_fault\"\nharness = false\n\n[[test]]\nname = \"03_exception_restore_sanity\"\nharness = false\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/build.rs",
    "content": "use std::{env, fs, process};\n\nfn main() {\n    let ld_script_path = match env::var(\"LD_SCRIPT_PATH\") {\n        Ok(var) => var,\n        _ => process::exit(0),\n    };\n\n    let files = fs::read_dir(ld_script_path).unwrap();\n    files\n        .filter_map(Result::ok)\n        .filter(|d| {\n            if let Some(e) = d.path().extension() {\n                e == \"ld\"\n            } else {\n                false\n            }\n        })\n        .for_each(|f| println!(\"cargo:rerun-if-changed={}\", f.path().display()));\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/_arch/aarch64/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural boot code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::boot::arch_boot\n\nuse crate::{memory, memory::Address};\nuse aarch64_cpu::{asm, registers::*};\nuse core::arch::global_asm;\nuse tock_registers::interfaces::Writeable;\n\n// Assembly counterpart to this file.\nglobal_asm!(\n    include_str!(\"boot.s\"),\n    CONST_CURRENTEL_EL2 = const 0x8,\n    CONST_CORE_ID_MASK = const 0b11\n);\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Prepares the transition from EL2 to EL1.\n///\n/// # Safety\n///\n/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.\n/// - The HW state of EL1 must be prepared in a sound way.\n#[inline(always)]\nunsafe fn prepare_el2_to_el1_transition(\n    virt_boot_core_stack_end_exclusive_addr: u64,\n    virt_kernel_init_addr: u64,\n) {\n    // Enable timer counter registers for EL1.\n    CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET);\n\n    // No offset for reading the counters.\n    CNTVOFF_EL2.set(0);\n\n    // Set EL1 execution state to AArch64.\n    HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64);\n\n    // Set up a simulated exception return.\n    //\n    // First, fake a saved program status where all interrupts were masked and SP_EL1 was used as a\n    // stack pointer.\n    SPSR_EL2.write(\n        SPSR_EL2::D::Masked\n            + SPSR_EL2::A::Masked\n            + SPSR_EL2::I::Masked\n            + SPSR_EL2::F::Masked\n            + SPSR_EL2::M::EL1h,\n    );\n\n    // Second, let the link register point to kernel_init().\n    ELR_EL2.set(virt_kernel_init_addr);\n\n    // Set up SP_EL1 (stack pointer), which will be used by EL1 once we \"return\" to it. Since there\n    // are no plans to ever return to EL2, just re-use the same stack.\n    SP_EL1.set(virt_boot_core_stack_end_exclusive_addr);\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The Rust entry of the `kernel` binary.\n///\n/// The function is called from the assembly `_start` function.\n///\n/// # Safety\n///\n/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`.\n#[no_mangle]\npub unsafe extern \"C\" fn _start_rust(\n    phys_kernel_tables_base_addr: u64,\n    virt_boot_core_stack_end_exclusive_addr: u64,\n    virt_kernel_init_addr: u64,\n) -> ! {\n    prepare_el2_to_el1_transition(\n        virt_boot_core_stack_end_exclusive_addr,\n        virt_kernel_init_addr,\n    );\n\n    // Turn on the MMU for EL1.\n    let addr = Address::new(phys_kernel_tables_base_addr as usize);\n    memory::mmu::enable_mmu_and_caching(addr).unwrap();\n\n    // Use `eret` to \"return\" to EL1. Since virtual memory will already be enabled, this results in\n    // execution of kernel_init() in EL1 from its _virtual address_.\n    asm::eret()\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/_arch/aarch64/cpu/boot.s",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2022 Andre Richter <andre.o.richter@gmail.com>\n\n//--------------------------------------------------------------------------------------------------\n// Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Load the address of a symbol into a register, PC-relative.\n//\n// The symbol must lie within +/- 4 GiB of the Program Counter.\n//\n// # Resources\n//\n// - https://sourceware.org/binutils/docs-2.36/as/AArch64_002dRelocations.html\n.macro ADR_REL register, symbol\n\tadrp\t\\register, \\symbol\n\tadd\t\\register, \\register, #:lo12:\\symbol\n.endm\n\n// Load the address of a symbol into a register, absolute.\n//\n// # Resources\n//\n// - https://sourceware.org/binutils/docs-2.36/as/AArch64_002dRelocations.html\n.macro ADR_ABS register, symbol\n\tmovz\t\\register, #:abs_g3:\\symbol\n\tmovk\t\\register, #:abs_g2_nc:\\symbol\n\tmovk\t\\register, #:abs_g1_nc:\\symbol\n\tmovk\t\\register, #:abs_g0_nc:\\symbol\n.endm\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n.section .text._start\n\n//------------------------------------------------------------------------------\n// fn _start()\n//------------------------------------------------------------------------------\n_start:\n\t// Only proceed if the core executes in EL2. Park it otherwise.\n\tmrs\tx0, CurrentEL\n\tcmp\tx0, {CONST_CURRENTEL_EL2}\n\tb.ne\t.L_parking_loop\n\n\t// Only proceed on the boot core. Park it otherwise.\n\tmrs\tx1, MPIDR_EL1\n\tand\tx1, x1, {CONST_CORE_ID_MASK}\n\tldr\tx2, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs\n\tcmp\tx1, x2\n\tb.ne\t.L_parking_loop\n\n\t// If execution reaches here, it is the boot core.\n\n\t// Initialize DRAM.\n\tADR_REL\tx0, __bss_start\n\tADR_REL x1, __bss_end_exclusive\n\n.L_bss_init_loop:\n\tcmp\tx0, x1\n\tb.eq\t.L_prepare_rust\n\tstp\txzr, xzr, [x0], #16\n\tb\t.L_bss_init_loop\n\n\t// Prepare the jump to Rust code.\n.L_prepare_rust:\n\t// Load the base address of the kernel's translation tables.\n\tldr\tx0, PHYS_KERNEL_TABLES_BASE_ADDR // provided by bsp/__board_name__/memory/mmu.rs\n\n\t// Load the _absolute_ addresses of the following symbols. Since the kernel is linked at\n\t// the top of the 64 bit address space, these are effectively virtual addresses.\n\tADR_ABS\tx1, __boot_core_stack_end_exclusive\n\tADR_ABS\tx2, kernel_init\n\n\t// Load the PC-relative address of the stack and set the stack pointer.\n\t//\n\t// Since _start() is the first function that runs after the firmware has loaded the kernel\n\t// into memory, retrieving this symbol PC-relative returns the \"physical\" address.\n\t//\n\t// Setting the stack pointer to this value ensures that anything that still runs in EL2,\n\t// until the kernel returns to EL1 with the MMU enabled, works as well. After the return to\n\t// EL1, the virtual address of the stack retrieved above will be used.\n\tADR_REL\tx3, __boot_core_stack_end_exclusive\n\tmov\tsp, x3\n\n\t// Read the CPU's timer counter frequency and store it in ARCH_TIMER_COUNTER_FREQUENCY.\n\t// Abort if the frequency read back as 0.\n\tADR_REL\tx4, ARCH_TIMER_COUNTER_FREQUENCY // provided by aarch64/time.rs\n\tmrs\tx5, CNTFRQ_EL0\n\tcmp\tx5, xzr\n\tb.eq\t.L_parking_loop\n\tstr\tw5, [x4]\n\n\t// Jump to Rust code. x0, x1 and x2 hold the function arguments provided to _start_rust().\n\tb\t_start_rust\n\n\t// Infinitely wait for events (aka \"park the core\").\n.L_parking_loop:\n\twfe\n\tb\t.L_parking_loop\n\n.size\t_start, . - _start\n.type\t_start, function\n.global\t_start\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/_arch/aarch64/cpu/smp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural symmetric multiprocessing.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::smp::arch_smp\n\nuse aarch64_cpu::registers::*;\nuse tock_registers::interfaces::Readable;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return the executing core's id.\n#[inline(always)]\npub fn core_id<T>() -> T\nwhere\n    T: From<u8>,\n{\n    const CORE_MASK: u64 = 0b11;\n\n    T::from((MPIDR_EL1.get() & CORE_MASK) as u8)\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/_arch/aarch64/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural processor code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::arch_cpu\n\nuse aarch64_cpu::asm;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\npub use asm::nop;\n\n/// Pause execution on the core.\n#[inline(always)]\npub fn wait_forever() -> ! {\n    loop {\n        asm::wfe()\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n#[cfg(feature = \"test_build\")]\nuse qemu_exit::QEMUExit;\n\n#[cfg(feature = \"test_build\")]\nconst QEMU_EXIT_HANDLE: qemu_exit::AArch64 = qemu_exit::AArch64::new();\n\n/// Make the host QEMU binary execute `exit(1)`.\n#[cfg(feature = \"test_build\")]\npub fn qemu_exit_failure() -> ! {\n    QEMU_EXIT_HANDLE.exit_failure()\n}\n\n/// Make the host QEMU binary execute `exit(0)`.\n#[cfg(feature = \"test_build\")]\npub fn qemu_exit_success() -> ! {\n    QEMU_EXIT_HANDLE.exit_success()\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/_arch/aarch64/exception/asynchronous.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural asynchronous exception handling.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::exception::asynchronous::arch_asynchronous\n\nuse aarch64_cpu::registers::*;\nuse core::arch::asm;\nuse tock_registers::interfaces::{Readable, Writeable};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nmod daif_bits {\n    pub const IRQ: u8 = 0b0010;\n}\n\ntrait DaifField {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register>;\n}\n\nstruct Debug;\nstruct SError;\nstruct IRQ;\nstruct FIQ;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl DaifField for Debug {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::D\n    }\n}\n\nimpl DaifField for SError {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::A\n    }\n}\n\nimpl DaifField for IRQ {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::I\n    }\n}\n\nimpl DaifField for FIQ {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::F\n    }\n}\n\nfn is_masked<T>() -> bool\nwhere\n    T: DaifField,\n{\n    DAIF.is_set(T::daif_field())\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Returns whether IRQs are masked on the executing core.\npub fn is_local_irq_masked() -> bool {\n    !is_masked::<IRQ>()\n}\n\n/// Unmask IRQs on the executing core.\n///\n/// It is not needed to place an explicit instruction synchronization barrier after the `msr`.\n/// Quoting the Architecture Reference Manual for ARMv8-A, section C5.1.3:\n///\n/// \"Writes to PSTATE.{PAN, D, A, I, F} occur in program order without the need for additional\n/// synchronization.\"\n#[inline(always)]\npub fn local_irq_unmask() {\n    unsafe {\n        asm!(\n            \"msr DAIFClr, {arg}\",\n            arg = const daif_bits::IRQ,\n            options(nomem, nostack, preserves_flags)\n        );\n    }\n}\n\n/// Mask IRQs on the executing core.\n#[inline(always)]\npub fn local_irq_mask() {\n    unsafe {\n        asm!(\n            \"msr DAIFSet, {arg}\",\n            arg = const daif_bits::IRQ,\n            options(nomem, nostack, preserves_flags)\n        );\n    }\n}\n\n/// Mask IRQs on the executing core and return the previously saved interrupt mask bits (DAIF).\n#[inline(always)]\npub fn local_irq_mask_save() -> u64 {\n    let saved = DAIF.get();\n    local_irq_mask();\n\n    saved\n}\n\n/// Restore the interrupt mask bits (DAIF) using the callee's argument.\n///\n/// # Invariant\n///\n/// - No sanity checks on the input.\n#[inline(always)]\npub fn local_irq_restore(saved: u64) {\n    DAIF.set(saved);\n}\n\n/// Print the AArch64 exceptions status.\n#[rustfmt::skip]\npub fn print_state() {\n    use crate::info;\n\n    let to_mask_str = |x| -> _ {\n        if x { \"Masked\" } else { \"Unmasked\" }\n    };\n\n    info!(\"      Debug:  {}\", to_mask_str(is_masked::<Debug>()));\n    info!(\"      SError: {}\", to_mask_str(is_masked::<SError>()));\n    info!(\"      IRQ:    {}\", to_mask_str(is_masked::<IRQ>()));\n    info!(\"      FIQ:    {}\", to_mask_str(is_masked::<FIQ>()));\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/_arch/aarch64/exception.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural synchronous and asynchronous exception handling.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::exception::arch_exception\n\nuse crate::exception;\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::{arch::global_asm, cell::UnsafeCell, fmt};\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    registers::InMemoryRegister,\n};\n\n// Assembly counterpart to this file.\nglobal_asm!(include_str!(\"exception.s\"));\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Wrapper structs for memory copies of registers.\n#[repr(transparent)]\nstruct SpsrEL1(InMemoryRegister<u64, SPSR_EL1::Register>);\nstruct EsrEL1(InMemoryRegister<u64, ESR_EL1::Register>);\n\n/// The exception context as it is stored on the stack on exception entry.\n#[repr(C)]\nstruct ExceptionContext {\n    /// General Purpose Registers.\n    gpr: [u64; 30],\n\n    /// The link register, aka x30.\n    lr: u64,\n\n    /// Exception link register. The program counter at the time the exception happened.\n    elr_el1: u64,\n\n    /// Saved program status.\n    spsr_el1: SpsrEL1,\n\n    /// Exception syndrome register.\n    esr_el1: EsrEL1,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Prints verbose information about the exception and then panics.\nfn default_exception_handler(exc: &ExceptionContext) {\n    panic!(\n        \"CPU Exception!\\n\\n\\\n        {}\",\n        exc\n    );\n}\n\n//------------------------------------------------------------------------------\n// Current, EL0\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn current_el0_synchronous(_e: &mut ExceptionContext) {\n    panic!(\"Should not be here. Use of SP_EL0 in EL1 is not supported.\")\n}\n\n#[no_mangle]\nextern \"C\" fn current_el0_irq(_e: &mut ExceptionContext) {\n    panic!(\"Should not be here. Use of SP_EL0 in EL1 is not supported.\")\n}\n\n#[no_mangle]\nextern \"C\" fn current_el0_serror(_e: &mut ExceptionContext) {\n    panic!(\"Should not be here. Use of SP_EL0 in EL1 is not supported.\")\n}\n\n//------------------------------------------------------------------------------\n// Current, ELx\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn current_elx_synchronous(e: &mut ExceptionContext) {\n    #[cfg(feature = \"test_build\")]\n    {\n        const TEST_SVC_ID: u64 = 0x1337;\n\n        if let Some(ESR_EL1::EC::Value::SVC64) = e.esr_el1.exception_class() {\n            if e.esr_el1.iss() == TEST_SVC_ID {\n                return;\n            }\n        }\n    }\n\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn current_elx_irq(_e: &mut ExceptionContext) {\n    let token = unsafe { &exception::asynchronous::IRQContext::new() };\n    exception::asynchronous::irq_manager().handle_pending_irqs(token);\n}\n\n#[no_mangle]\nextern \"C\" fn current_elx_serror(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n//------------------------------------------------------------------------------\n// Lower, AArch64\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn lower_aarch64_synchronous(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch64_irq(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch64_serror(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n//------------------------------------------------------------------------------\n// Lower, AArch32\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn lower_aarch32_synchronous(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch32_irq(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch32_serror(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n//------------------------------------------------------------------------------\n// Misc\n//------------------------------------------------------------------------------\n\n/// Human readable SPSR_EL1.\n#[rustfmt::skip]\nimpl fmt::Display for SpsrEL1 {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        // Raw value.\n        writeln!(f, \"SPSR_EL1: {:#010x}\", self.0.get())?;\n\n        let to_flag_str = |x| -> _ {\n            if x { \"Set\" } else { \"Not set\" }\n         };\n\n        writeln!(f, \"      Flags:\")?;\n        writeln!(f, \"            Negative (N): {}\", to_flag_str(self.0.is_set(SPSR_EL1::N)))?;\n        writeln!(f, \"            Zero     (Z): {}\", to_flag_str(self.0.is_set(SPSR_EL1::Z)))?;\n        writeln!(f, \"            Carry    (C): {}\", to_flag_str(self.0.is_set(SPSR_EL1::C)))?;\n        writeln!(f, \"            Overflow (V): {}\", to_flag_str(self.0.is_set(SPSR_EL1::V)))?;\n\n        let to_mask_str = |x| -> _ {\n            if x { \"Masked\" } else { \"Unmasked\" }\n        };\n\n        writeln!(f, \"      Exception handling state:\")?;\n        writeln!(f, \"            Debug  (D): {}\", to_mask_str(self.0.is_set(SPSR_EL1::D)))?;\n        writeln!(f, \"            SError (A): {}\", to_mask_str(self.0.is_set(SPSR_EL1::A)))?;\n        writeln!(f, \"            IRQ    (I): {}\", to_mask_str(self.0.is_set(SPSR_EL1::I)))?;\n        writeln!(f, \"            FIQ    (F): {}\", to_mask_str(self.0.is_set(SPSR_EL1::F)))?;\n\n        write!(f, \"      Illegal Execution State (IL): {}\",\n            to_flag_str(self.0.is_set(SPSR_EL1::IL))\n        )\n    }\n}\n\nimpl EsrEL1 {\n    #[inline(always)]\n    fn exception_class(&self) -> Option<ESR_EL1::EC::Value> {\n        self.0.read_as_enum(ESR_EL1::EC)\n    }\n\n    #[cfg(feature = \"test_build\")]\n    #[inline(always)]\n    fn iss(&self) -> u64 {\n        self.0.read(ESR_EL1::ISS)\n    }\n}\n\n/// Human readable ESR_EL1.\n#[rustfmt::skip]\nimpl fmt::Display for EsrEL1 {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        // Raw print of whole register.\n        writeln!(f, \"ESR_EL1: {:#010x}\", self.0.get())?;\n\n        // Raw print of exception class.\n        write!(f, \"      Exception Class         (EC) : {:#x}\", self.0.read(ESR_EL1::EC))?;\n\n        // Exception class.\n        let ec_translation = match self.exception_class() {\n            Some(ESR_EL1::EC::Value::DataAbortCurrentEL) => \"Data Abort, current EL\",\n            _ => \"N/A\",\n        };\n        writeln!(f, \" - {}\", ec_translation)?;\n\n        // Raw print of instruction specific syndrome.\n        write!(f, \"      Instr Specific Syndrome (ISS): {:#x}\", self.0.read(ESR_EL1::ISS))\n    }\n}\n\nimpl ExceptionContext {\n    #[inline(always)]\n    fn exception_class(&self) -> Option<ESR_EL1::EC::Value> {\n        self.esr_el1.exception_class()\n    }\n\n    #[inline(always)]\n    fn fault_address_valid(&self) -> bool {\n        use ESR_EL1::EC::Value::*;\n\n        match self.exception_class() {\n            None => false,\n            Some(ec) => matches!(\n                ec,\n                InstrAbortLowerEL\n                    | InstrAbortCurrentEL\n                    | PCAlignmentFault\n                    | DataAbortLowerEL\n                    | DataAbortCurrentEL\n                    | WatchpointLowerEL\n                    | WatchpointCurrentEL\n            ),\n        }\n    }\n}\n\n/// Human readable print of the exception context.\nimpl fmt::Display for ExceptionContext {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        writeln!(f, \"{}\", self.esr_el1)?;\n\n        if self.fault_address_valid() {\n            writeln!(f, \"FAR_EL1: {:#018x}\", FAR_EL1.get() as usize)?;\n        }\n\n        writeln!(f, \"{}\", self.spsr_el1)?;\n        writeln!(f, \"ELR_EL1: {:#018x}\", self.elr_el1)?;\n        writeln!(f)?;\n        writeln!(f, \"General purpose register:\")?;\n\n        #[rustfmt::skip]\n        let alternating = |x| -> _ {\n            if x % 2 == 0 { \"   \" } else { \"\\n\" }\n        };\n\n        // Print two registers per line.\n        for (i, reg) in self.gpr.iter().enumerate() {\n            write!(f, \"      x{: <2}: {: >#018x}{}\", i, reg, alternating(i))?;\n        }\n        write!(f, \"      lr : {:#018x}\", self.lr)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse crate::exception::PrivilegeLevel;\n\n/// The processing element's current privilege level.\npub fn current_privilege_level() -> (PrivilegeLevel, &'static str) {\n    let el = CurrentEL.read_as_enum(CurrentEL::EL);\n    match el {\n        Some(CurrentEL::EL::Value::EL2) => (PrivilegeLevel::Hypervisor, \"EL2\"),\n        Some(CurrentEL::EL::Value::EL1) => (PrivilegeLevel::Kernel, \"EL1\"),\n        Some(CurrentEL::EL::Value::EL0) => (PrivilegeLevel::User, \"EL0\"),\n        _ => (PrivilegeLevel::Unknown, \"Unknown\"),\n    }\n}\n\n/// Init exception handling by setting the exception vector base address register.\n///\n/// # Safety\n///\n/// - Changes the HW state of the executing core.\n/// - The vector table and the symbol `__exception_vector_table_start` from the linker script must\n///   adhere to the alignment and size constraints demanded by the ARMv8-A Architecture Reference\n///   Manual.\npub unsafe fn handling_init() {\n    // Provided by exception.S.\n    extern \"Rust\" {\n        static __exception_vector_start: UnsafeCell<()>;\n    }\n\n    VBAR_EL1.set(__exception_vector_start.get() as u64);\n\n    // Force VBAR update to complete before next instruction.\n    barrier::isb(barrier::SY);\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/_arch/aarch64/exception.s",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2022 Andre Richter <andre.o.richter@gmail.com>\n\n//--------------------------------------------------------------------------------------------------\n// Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Call the function provided by parameter `\\handler` after saving the exception context. Provide\n/// the context as the first parameter to '\\handler'.\n.macro CALL_WITH_CONTEXT handler\n__vector_\\handler:\n\t// Make room on the stack for the exception context.\n\tsub\tsp,  sp,  #16 * 17\n\n\t// Store all general purpose registers on the stack.\n\tstp\tx0,  x1,  [sp, #16 * 0]\n\tstp\tx2,  x3,  [sp, #16 * 1]\n\tstp\tx4,  x5,  [sp, #16 * 2]\n\tstp\tx6,  x7,  [sp, #16 * 3]\n\tstp\tx8,  x9,  [sp, #16 * 4]\n\tstp\tx10, x11, [sp, #16 * 5]\n\tstp\tx12, x13, [sp, #16 * 6]\n\tstp\tx14, x15, [sp, #16 * 7]\n\tstp\tx16, x17, [sp, #16 * 8]\n\tstp\tx18, x19, [sp, #16 * 9]\n\tstp\tx20, x21, [sp, #16 * 10]\n\tstp\tx22, x23, [sp, #16 * 11]\n\tstp\tx24, x25, [sp, #16 * 12]\n\tstp\tx26, x27, [sp, #16 * 13]\n\tstp\tx28, x29, [sp, #16 * 14]\n\n\t// Add the exception link register (ELR_EL1), saved program status (SPSR_EL1) and exception\n\t// syndrome register (ESR_EL1).\n\tmrs\tx1,  ELR_EL1\n\tmrs\tx2,  SPSR_EL1\n\tmrs\tx3,  ESR_EL1\n\n\tstp\tlr,  x1,  [sp, #16 * 15]\n\tstp\tx2,  x3,  [sp, #16 * 16]\n\n\t// x0 is the first argument for the function called through `\\handler`.\n\tmov\tx0,  sp\n\n\t// Call `\\handler`.\n\tbl\t\\handler\n\n\t// After returning from exception handling code, replay the saved context and return via\n\t// `eret`.\n\tb\t__exception_restore_context\n\n.size\t__vector_\\handler, . - __vector_\\handler\n.type\t__vector_\\handler, function\n.endm\n\n.macro FIQ_SUSPEND\n1:\twfe\n\tb\t1b\n.endm\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n.section .text\n\n//------------------------------------------------------------------------------\n// The exception vector table.\n//------------------------------------------------------------------------------\n\n// Align by 2^11 bytes, as demanded by ARMv8-A. Same as ALIGN(2048) in an ld script.\n.align 11\n\n// Export a symbol for the Rust code to use.\n__exception_vector_start:\n\n// Current exception level with SP_EL0.\n//\n// .org sets the offset relative to section start.\n//\n// # Safety\n//\n// - It must be ensured that `CALL_WITH_CONTEXT` <= 0x80 bytes.\n.org 0x000\n\tCALL_WITH_CONTEXT current_el0_synchronous\n.org 0x080\n\tCALL_WITH_CONTEXT current_el0_irq\n.org 0x100\n\tFIQ_SUSPEND\n.org 0x180\n\tCALL_WITH_CONTEXT current_el0_serror\n\n// Current exception level with SP_ELx, x > 0.\n.org 0x200\n\tCALL_WITH_CONTEXT current_elx_synchronous\n.org 0x280\n\tCALL_WITH_CONTEXT current_elx_irq\n.org 0x300\n\tFIQ_SUSPEND\n.org 0x380\n\tCALL_WITH_CONTEXT current_elx_serror\n\n// Lower exception level, AArch64\n.org 0x400\n\tCALL_WITH_CONTEXT lower_aarch64_synchronous\n.org 0x480\n\tCALL_WITH_CONTEXT lower_aarch64_irq\n.org 0x500\n\tFIQ_SUSPEND\n.org 0x580\n\tCALL_WITH_CONTEXT lower_aarch64_serror\n\n// Lower exception level, AArch32\n.org 0x600\n\tCALL_WITH_CONTEXT lower_aarch32_synchronous\n.org 0x680\n\tCALL_WITH_CONTEXT lower_aarch32_irq\n.org 0x700\n\tFIQ_SUSPEND\n.org 0x780\n\tCALL_WITH_CONTEXT lower_aarch32_serror\n.org 0x800\n\n//------------------------------------------------------------------------------\n// fn __exception_restore_context()\n//------------------------------------------------------------------------------\n__exception_restore_context:\n\tldr\tw19,      [sp, #16 * 16]\n\tldp\tlr,  x20, [sp, #16 * 15]\n\n\tmsr\tSPSR_EL1, x19\n\tmsr\tELR_EL1,  x20\n\n\tldp\tx0,  x1,  [sp, #16 * 0]\n\tldp\tx2,  x3,  [sp, #16 * 1]\n\tldp\tx4,  x5,  [sp, #16 * 2]\n\tldp\tx6,  x7,  [sp, #16 * 3]\n\tldp\tx8,  x9,  [sp, #16 * 4]\n\tldp\tx10, x11, [sp, #16 * 5]\n\tldp\tx12, x13, [sp, #16 * 6]\n\tldp\tx14, x15, [sp, #16 * 7]\n\tldp\tx16, x17, [sp, #16 * 8]\n\tldp\tx18, x19, [sp, #16 * 9]\n\tldp\tx20, x21, [sp, #16 * 10]\n\tldp\tx22, x23, [sp, #16 * 11]\n\tldp\tx24, x25, [sp, #16 * 12]\n\tldp\tx26, x27, [sp, #16 * 13]\n\tldp\tx28, x29, [sp, #16 * 14]\n\n\tadd\tsp,  sp,  #16 * 17\n\n\teret\n\n.size\t__exception_restore_context, . - __exception_restore_context\n.type\t__exception_restore_context, function\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/_arch/aarch64/memory/mmu/translation_table.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural translation table.\n//!\n//! Only 64 KiB granule is supported.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::memory::mmu::translation_table::arch_translation_table\n\nuse crate::{\n    bsp,\n    memory::{\n        self,\n        mmu::{\n            arch_mmu::{Granule512MiB, Granule64KiB},\n            AccessPermissions, AttributeFields, MemAttributes, MemoryRegion, PageAddress,\n        },\n        Address, Physical, Virtual,\n    },\n};\nuse core::convert;\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields,\n    registers::InMemoryRegister,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// A table descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-15.\nregister_bitfields! {u64,\n    STAGE1_TABLE_DESCRIPTOR [\n        /// Physical address of the next descriptor.\n        NEXT_LEVEL_TABLE_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16]\n\n        TYPE  OFFSET(1) NUMBITS(1) [\n            Block = 0,\n            Table = 1\n        ],\n\n        VALID OFFSET(0) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ]\n    ]\n}\n\n// A level 3 page descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-17.\nregister_bitfields! {u64,\n    STAGE1_PAGE_DESCRIPTOR [\n        /// Unprivileged execute-never.\n        UXN      OFFSET(54) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ],\n\n        /// Privileged execute-never.\n        PXN      OFFSET(53) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ],\n\n        /// Physical address of the next table descriptor (lvl2) or the page descriptor (lvl3).\n        OUTPUT_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16]\n\n        /// Access flag.\n        AF       OFFSET(10) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ],\n\n        /// Shareability field.\n        SH       OFFSET(8) NUMBITS(2) [\n            OuterShareable = 0b10,\n            InnerShareable = 0b11\n        ],\n\n        /// Access Permissions.\n        AP       OFFSET(6) NUMBITS(2) [\n            RW_EL1 = 0b00,\n            RW_EL1_EL0 = 0b01,\n            RO_EL1 = 0b10,\n            RO_EL1_EL0 = 0b11\n        ],\n\n        /// Memory attributes index into the MAIR_EL1 register.\n        AttrIndx OFFSET(2) NUMBITS(3) [],\n\n        TYPE     OFFSET(1) NUMBITS(1) [\n            Reserved_Invalid = 0,\n            Page = 1\n        ],\n\n        VALID    OFFSET(0) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ]\n    ]\n}\n\n/// A table descriptor for 64 KiB aperture.\n///\n/// The output points to the next table.\n#[derive(Copy, Clone)]\n#[repr(C)]\nstruct TableDescriptor {\n    value: u64,\n}\n\n/// A page descriptor with 64 KiB aperture.\n///\n/// The output points to physical memory.\n#[derive(Copy, Clone)]\n#[repr(C)]\nstruct PageDescriptor {\n    value: u64,\n}\n\ntrait StartAddr {\n    fn virt_start_addr(&self) -> Address<Virtual>;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB\n/// aligned, so the lvl3 is put first.\n#[repr(C)]\n#[repr(align(65536))]\npub struct FixedSizeTranslationTable<const NUM_TABLES: usize, const START_FROM_TOP: bool> {\n    /// Page descriptors, covering 64 KiB windows per entry.\n    lvl3: [[PageDescriptor; 8192]; NUM_TABLES],\n\n    /// Table descriptors, covering 512 MiB windows.\n    lvl2: [TableDescriptor; NUM_TABLES],\n\n    /// Have the tables been initialized?\n    initialized: bool,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T, const N: usize> StartAddr for [T; N] {\n    fn virt_start_addr(&self) -> Address<Virtual> {\n        Address::new(self as *const _ as usize)\n    }\n}\n\nimpl TableDescriptor {\n    /// Create an instance.\n    ///\n    /// Descriptor is invalid by default.\n    pub const fn new_zeroed() -> Self {\n        Self { value: 0 }\n    }\n\n    /// Create an instance pointing to the supplied address.\n    pub fn from_next_lvl_table_addr(phys_next_lvl_table_addr: Address<Physical>) -> Self {\n        let val = InMemoryRegister::<u64, STAGE1_TABLE_DESCRIPTOR::Register>::new(0);\n\n        let shifted = phys_next_lvl_table_addr.as_usize() >> Granule64KiB::SHIFT;\n        val.write(\n            STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64)\n                + STAGE1_TABLE_DESCRIPTOR::TYPE::Table\n                + STAGE1_TABLE_DESCRIPTOR::VALID::True,\n        );\n\n        TableDescriptor { value: val.get() }\n    }\n}\n\n/// Convert the kernel's generic memory attributes to HW-specific attributes of the MMU.\nimpl convert::From<AttributeFields>\n    for tock_registers::fields::FieldValue<u64, STAGE1_PAGE_DESCRIPTOR::Register>\n{\n    fn from(attribute_fields: AttributeFields) -> Self {\n        // Memory attributes.\n        let mut desc = match attribute_fields.mem_attributes {\n            MemAttributes::CacheableDRAM => {\n                STAGE1_PAGE_DESCRIPTOR::SH::InnerShareable\n                    + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::NORMAL)\n            }\n            MemAttributes::Device => {\n                STAGE1_PAGE_DESCRIPTOR::SH::OuterShareable\n                    + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::DEVICE)\n            }\n        };\n\n        // Access Permissions.\n        desc += match attribute_fields.acc_perms {\n            AccessPermissions::ReadOnly => STAGE1_PAGE_DESCRIPTOR::AP::RO_EL1,\n            AccessPermissions::ReadWrite => STAGE1_PAGE_DESCRIPTOR::AP::RW_EL1,\n        };\n\n        // The execute-never attribute is mapped to PXN in AArch64.\n        desc += if attribute_fields.execute_never {\n            STAGE1_PAGE_DESCRIPTOR::PXN::True\n        } else {\n            STAGE1_PAGE_DESCRIPTOR::PXN::False\n        };\n\n        // Always set unprivileged exectue-never as long as userspace is not implemented yet.\n        desc += STAGE1_PAGE_DESCRIPTOR::UXN::True;\n\n        desc\n    }\n}\n\n/// Convert the HW-specific attributes of the MMU to kernel's generic memory attributes.\nimpl convert::TryFrom<InMemoryRegister<u64, STAGE1_PAGE_DESCRIPTOR::Register>> for AttributeFields {\n    type Error = &'static str;\n\n    fn try_from(\n        desc: InMemoryRegister<u64, STAGE1_PAGE_DESCRIPTOR::Register>,\n    ) -> Result<AttributeFields, Self::Error> {\n        let mem_attributes = match desc.read(STAGE1_PAGE_DESCRIPTOR::AttrIndx) {\n            memory::mmu::arch_mmu::mair::NORMAL => MemAttributes::CacheableDRAM,\n            memory::mmu::arch_mmu::mair::DEVICE => MemAttributes::Device,\n            _ => return Err(\"Unexpected memory attribute\"),\n        };\n\n        let acc_perms = match desc.read_as_enum(STAGE1_PAGE_DESCRIPTOR::AP) {\n            Some(STAGE1_PAGE_DESCRIPTOR::AP::Value::RO_EL1) => AccessPermissions::ReadOnly,\n            Some(STAGE1_PAGE_DESCRIPTOR::AP::Value::RW_EL1) => AccessPermissions::ReadWrite,\n            _ => return Err(\"Unexpected access permission\"),\n        };\n\n        let execute_never = desc.read(STAGE1_PAGE_DESCRIPTOR::PXN) > 0;\n\n        Ok(AttributeFields {\n            mem_attributes,\n            acc_perms,\n            execute_never,\n        })\n    }\n}\n\nimpl PageDescriptor {\n    /// Create an instance.\n    ///\n    /// Descriptor is invalid by default.\n    pub const fn new_zeroed() -> Self {\n        Self { value: 0 }\n    }\n\n    /// Create an instance.\n    pub fn from_output_page_addr(\n        phys_output_page_addr: PageAddress<Physical>,\n        attribute_fields: &AttributeFields,\n    ) -> Self {\n        let val = InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(0);\n\n        let shifted = phys_output_page_addr.into_inner().as_usize() >> Granule64KiB::SHIFT;\n        val.write(\n            STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted as u64)\n                + STAGE1_PAGE_DESCRIPTOR::AF::True\n                + STAGE1_PAGE_DESCRIPTOR::TYPE::Page\n                + STAGE1_PAGE_DESCRIPTOR::VALID::True\n                + (*attribute_fields).into(),\n        );\n\n        Self { value: val.get() }\n    }\n\n    /// Returns the valid bit.\n    fn is_valid(&self) -> bool {\n        InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(self.value)\n            .is_set(STAGE1_PAGE_DESCRIPTOR::VALID)\n    }\n\n    /// Returns the output page.\n    fn output_page_addr(&self) -> PageAddress<Physical> {\n        let shifted = InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(self.value)\n            .read(STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB) as usize;\n\n        PageAddress::from(shifted << Granule64KiB::SHIFT)\n    }\n\n    /// Returns the attributes.\n    fn try_attributes(&self) -> Result<AttributeFields, &'static str> {\n        InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(self.value).try_into()\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<const AS_SIZE: usize> memory::mmu::AssociatedTranslationTable\n    for memory::mmu::AddressSpace<AS_SIZE>\nwhere\n    [u8; Self::SIZE >> Granule512MiB::SHIFT]: Sized,\n{\n    type TableStartFromTop =\n        FixedSizeTranslationTable<{ Self::SIZE >> Granule512MiB::SHIFT }, true>;\n\n    type TableStartFromBottom =\n        FixedSizeTranslationTable<{ Self::SIZE >> Granule512MiB::SHIFT }, false>;\n}\n\nimpl<const NUM_TABLES: usize, const START_FROM_TOP: bool>\n    FixedSizeTranslationTable<NUM_TABLES, START_FROM_TOP>\n{\n    const START_FROM_TOP_OFFSET: Address<Virtual> =\n        Address::new((usize::MAX - (Granule512MiB::SIZE * NUM_TABLES)) + 1);\n\n    /// Create an instance.\n    #[allow(clippy::assertions_on_constants)]\n    const fn _new(for_precompute: bool) -> Self {\n        assert!(bsp::memory::mmu::KernelGranule::SIZE == Granule64KiB::SIZE);\n\n        // Can't have a zero-sized address space.\n        assert!(NUM_TABLES > 0);\n\n        Self {\n            lvl3: [[PageDescriptor::new_zeroed(); 8192]; NUM_TABLES],\n            lvl2: [TableDescriptor::new_zeroed(); NUM_TABLES],\n            initialized: for_precompute,\n        }\n    }\n\n    pub const fn new_for_precompute() -> Self {\n        Self::_new(true)\n    }\n\n    #[cfg(test)]\n    pub fn new_for_runtime() -> Self {\n        Self::_new(false)\n    }\n\n    /// Helper to calculate the lvl2 and lvl3 indices from an address.\n    #[inline(always)]\n    fn lvl2_lvl3_index_from_page_addr(\n        &self,\n        virt_page_addr: PageAddress<Virtual>,\n    ) -> Result<(usize, usize), &'static str> {\n        let mut addr = virt_page_addr.into_inner();\n\n        if START_FROM_TOP {\n            addr = addr - Self::START_FROM_TOP_OFFSET;\n        }\n\n        let lvl2_index = addr.as_usize() >> Granule512MiB::SHIFT;\n        let lvl3_index = (addr.as_usize() & Granule512MiB::MASK) >> Granule64KiB::SHIFT;\n\n        if lvl2_index > (NUM_TABLES - 1) {\n            return Err(\"Virtual page is out of bounds of translation table\");\n        }\n\n        Ok((lvl2_index, lvl3_index))\n    }\n\n    /// Returns the PageDescriptor corresponding to the supplied page address.\n    #[inline(always)]\n    fn page_descriptor_from_page_addr(\n        &self,\n        virt_page_addr: PageAddress<Virtual>,\n    ) -> Result<&PageDescriptor, &'static str> {\n        let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from_page_addr(virt_page_addr)?;\n        let desc = &self.lvl3[lvl2_index][lvl3_index];\n\n        Ok(desc)\n    }\n\n    /// Sets the PageDescriptor corresponding to the supplied page address.\n    ///\n    /// Doesn't allow overriding an already valid page.\n    #[inline(always)]\n    fn set_page_descriptor_from_page_addr(\n        &mut self,\n        virt_page_addr: PageAddress<Virtual>,\n        new_desc: &PageDescriptor,\n    ) -> Result<(), &'static str> {\n        let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from_page_addr(virt_page_addr)?;\n        let desc = &mut self.lvl3[lvl2_index][lvl3_index];\n\n        if desc.is_valid() {\n            return Err(\"Virtual page is already mapped\");\n        }\n\n        *desc = *new_desc;\n        Ok(())\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\n\nimpl<const NUM_TABLES: usize, const START_FROM_TOP: bool>\n    memory::mmu::translation_table::interface::TranslationTable\n    for FixedSizeTranslationTable<NUM_TABLES, START_FROM_TOP>\n{\n    fn init(&mut self) -> Result<(), &'static str> {\n        if self.initialized {\n            return Ok(());\n        }\n\n        // Populate the l2 entries.\n        for (lvl2_nr, lvl2_entry) in self.lvl2.iter_mut().enumerate() {\n            let virt_table_addr = self.lvl3[lvl2_nr].virt_start_addr();\n            let phys_table_addr = memory::mmu::try_kernel_virt_addr_to_phys_addr(virt_table_addr)?;\n\n            let new_desc = TableDescriptor::from_next_lvl_table_addr(phys_table_addr);\n            *lvl2_entry = new_desc;\n        }\n\n        self.initialized = true;\n\n        Ok(())\n    }\n\n    unsafe fn map_at(\n        &mut self,\n        virt_region: &MemoryRegion<Virtual>,\n        phys_region: &MemoryRegion<Physical>,\n        attr: &AttributeFields,\n    ) -> Result<(), &'static str> {\n        assert!(self.initialized, \"Translation tables not initialized\");\n\n        if virt_region.size() != phys_region.size() {\n            return Err(\"Tried to map memory regions with unequal sizes\");\n        }\n\n        if phys_region.end_exclusive_page_addr() > bsp::memory::phys_addr_space_end_exclusive_addr()\n        {\n            return Err(\"Tried to map outside of physical address space\");\n        }\n\n        let iter = phys_region.into_iter().zip(virt_region.into_iter());\n        for (phys_page_addr, virt_page_addr) in iter {\n            let new_desc = PageDescriptor::from_output_page_addr(phys_page_addr, attr);\n            let virt_page = virt_page_addr;\n\n            self.set_page_descriptor_from_page_addr(virt_page, &new_desc)?;\n        }\n\n        Ok(())\n    }\n\n    fn try_virt_page_addr_to_phys_page_addr(\n        &self,\n        virt_page_addr: PageAddress<Virtual>,\n    ) -> Result<PageAddress<Physical>, &'static str> {\n        let page_desc = self.page_descriptor_from_page_addr(virt_page_addr)?;\n\n        if !page_desc.is_valid() {\n            return Err(\"Page marked invalid\");\n        }\n\n        Ok(page_desc.output_page_addr())\n    }\n\n    fn try_page_attributes(\n        &self,\n        virt_page_addr: PageAddress<Virtual>,\n    ) -> Result<AttributeFields, &'static str> {\n        let page_desc = self.page_descriptor_from_page_addr(virt_page_addr)?;\n\n        if !page_desc.is_valid() {\n            return Err(\"Page marked invalid\");\n        }\n\n        page_desc.try_attributes()\n    }\n\n    /// Try to translate a virtual address to a physical address.\n    ///\n    /// Will only succeed if there exists a valid mapping for the input address.\n    fn try_virt_addr_to_phys_addr(\n        &self,\n        virt_addr: Address<Virtual>,\n    ) -> Result<Address<Physical>, &'static str> {\n        let virt_page = PageAddress::from(virt_addr.align_down_page());\n        let phys_page = self.try_virt_page_addr_to_phys_page_addr(virt_page)?;\n\n        Ok(phys_page.into_inner() + virt_addr.offset_into_page())\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\npub type MinSizeTranslationTable = FixedSizeTranslationTable<1, true>;\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// Check if the size of `struct TableDescriptor` is as expected.\n    #[kernel_test]\n    fn size_of_tabledescriptor_equals_64_bit() {\n        assert_eq!(\n            core::mem::size_of::<TableDescriptor>(),\n            core::mem::size_of::<u64>()\n        );\n    }\n\n    /// Check if the size of `struct PageDescriptor` is as expected.\n    #[kernel_test]\n    fn size_of_pagedescriptor_equals_64_bit() {\n        assert_eq!(\n            core::mem::size_of::<PageDescriptor>(),\n            core::mem::size_of::<u64>()\n        );\n    }\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/_arch/aarch64/memory/mmu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management Unit Driver.\n//!\n//! Only 64 KiB granule is supported.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::memory::mmu::arch_mmu\n\nuse crate::{\n    bsp, memory,\n    memory::{mmu::TranslationGranule, Address, Physical},\n};\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::intrinsics::unlikely;\nuse tock_registers::interfaces::{ReadWriteable, Readable, Writeable};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Memory Management Unit type.\nstruct MemoryManagementUnit;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub type Granule512MiB = TranslationGranule<{ 512 * 1024 * 1024 }>;\npub type Granule64KiB = TranslationGranule<{ 64 * 1024 }>;\n\n/// Constants for indexing the MAIR_EL1.\n#[allow(dead_code)]\npub mod mair {\n    pub const DEVICE: u64 = 0;\n    pub const NORMAL: u64 = 1;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic MMU: MemoryManagementUnit = MemoryManagementUnit;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<const AS_SIZE: usize> memory::mmu::AddressSpace<AS_SIZE> {\n    /// Checks for architectural restrictions.\n    pub const fn arch_address_space_size_sanity_checks() {\n        // Size must be at least one full 512 MiB table.\n        assert!((AS_SIZE % Granule512MiB::SIZE) == 0);\n\n        // Check for 48 bit virtual address size as maximum, which is supported by any ARMv8\n        // version.\n        assert!(AS_SIZE <= (1 << 48));\n    }\n}\n\nimpl MemoryManagementUnit {\n    /// Setup function for the MAIR_EL1 register.\n    #[inline(always)]\n    fn set_up_mair(&self) {\n        // Define the memory types being mapped.\n        MAIR_EL1.write(\n            // Attribute 1 - Cacheable normal DRAM.\n            MAIR_EL1::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc +\n        MAIR_EL1::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc +\n\n        // Attribute 0 - Device.\n        MAIR_EL1::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck,\n        );\n    }\n\n    /// Configure various settings of stage 1 of the EL1 translation regime.\n    #[inline(always)]\n    fn configure_translation_control(&self) {\n        let t1sz = (64 - bsp::memory::mmu::KernelVirtAddrSpace::SIZE_SHIFT) as u64;\n\n        TCR_EL1.write(\n            TCR_EL1::TBI1::Used\n                + TCR_EL1::IPS::Bits_40\n                + TCR_EL1::TG1::KiB_64\n                + TCR_EL1::SH1::Inner\n                + TCR_EL1::ORGN1::WriteBack_ReadAlloc_WriteAlloc_Cacheable\n                + TCR_EL1::IRGN1::WriteBack_ReadAlloc_WriteAlloc_Cacheable\n                + TCR_EL1::EPD1::EnableTTBR1Walks\n                + TCR_EL1::A1::TTBR1\n                + TCR_EL1::T1SZ.val(t1sz)\n                + TCR_EL1::EPD0::DisableTTBR0Walks,\n        );\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the MMU instance.\npub fn mmu() -> &'static impl memory::mmu::interface::MMU {\n    &MMU\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse memory::mmu::MMUEnableError;\n\nimpl memory::mmu::interface::MMU for MemoryManagementUnit {\n    unsafe fn enable_mmu_and_caching(\n        &self,\n        phys_tables_base_addr: Address<Physical>,\n    ) -> Result<(), MMUEnableError> {\n        if unlikely(self.is_enabled()) {\n            return Err(MMUEnableError::AlreadyEnabled);\n        }\n\n        // Fail early if translation granule is not supported.\n        if unlikely(!ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran64::Supported)) {\n            return Err(MMUEnableError::Other(\n                \"Translation granule not supported in HW\",\n            ));\n        }\n\n        // Prepare the memory attribute indirection register.\n        self.set_up_mair();\n\n        // Set the \"Translation Table Base Register\".\n        TTBR1_EL1.set_baddr(phys_tables_base_addr.as_usize() as u64);\n\n        self.configure_translation_control();\n\n        // Switch the MMU on.\n        //\n        // First, force all previous changes to be seen before the MMU is enabled.\n        barrier::isb(barrier::SY);\n\n        // Enable the MMU and turn on data and instruction caching.\n        SCTLR_EL1.modify(SCTLR_EL1::M::Enable + SCTLR_EL1::C::Cacheable + SCTLR_EL1::I::Cacheable);\n\n        // Force MMU init to complete before next instruction.\n        barrier::isb(barrier::SY);\n\n        Ok(())\n    }\n\n    #[inline(always)]\n    fn is_enabled(&self) -> bool {\n        SCTLR_EL1.matches_all(SCTLR_EL1::M::Enable)\n    }\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/_arch/aarch64/time.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural timer primitives.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::time::arch_time\n\nuse crate::warn;\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::{\n    num::{NonZeroU128, NonZeroU32, NonZeroU64},\n    ops::{Add, Div},\n    time::Duration,\n};\nuse tock_registers::interfaces::Readable;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nconst NANOSEC_PER_SEC: NonZeroU64 = NonZeroU64::new(1_000_000_000).unwrap();\n\n#[derive(Copy, Clone, PartialOrd, PartialEq)]\nstruct GenericTimerCounterValue(u64);\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n/// Boot assembly code overwrites this value with the value of CNTFRQ_EL0 before any Rust code is\n/// executed. This given value here is just a (safe) dummy.\n#[no_mangle]\nstatic ARCH_TIMER_COUNTER_FREQUENCY: NonZeroU32 = NonZeroU32::MIN;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nfn arch_timer_counter_frequency() -> NonZeroU32 {\n    // Read volatile is needed here to prevent the compiler from optimizing\n    // ARCH_TIMER_COUNTER_FREQUENCY away.\n    //\n    // This is safe, because all the safety requirements as stated in read_volatile()'s\n    // documentation are fulfilled.\n    unsafe { core::ptr::read_volatile(&ARCH_TIMER_COUNTER_FREQUENCY) }\n}\n\nimpl GenericTimerCounterValue {\n    pub const MAX: Self = GenericTimerCounterValue(u64::MAX);\n}\n\nimpl Add for GenericTimerCounterValue {\n    type Output = Self;\n\n    fn add(self, other: Self) -> Self {\n        GenericTimerCounterValue(self.0.wrapping_add(other.0))\n    }\n}\n\nimpl From<GenericTimerCounterValue> for Duration {\n    fn from(counter_value: GenericTimerCounterValue) -> Self {\n        if counter_value.0 == 0 {\n            return Duration::ZERO;\n        }\n\n        let frequency: NonZeroU64 = arch_timer_counter_frequency().into();\n\n        // Div<NonZeroU64> implementation for u64 cannot panic.\n        let secs = counter_value.0.div(frequency);\n\n        // This is safe, because frequency can never be greater than u32::MAX, which means the\n        // largest theoretical value for sub_second_counter_value is (u32::MAX - 1). Therefore,\n        // (sub_second_counter_value * NANOSEC_PER_SEC) cannot overflow an u64.\n        //\n        // The subsequent division ensures the result fits into u32, since the max result is smaller\n        // than NANOSEC_PER_SEC. Therefore, just cast it to u32 using `as`.\n        let sub_second_counter_value = counter_value.0 % frequency;\n        let nanos = unsafe { sub_second_counter_value.unchecked_mul(u64::from(NANOSEC_PER_SEC)) }\n            .div(frequency) as u32;\n\n        Duration::new(secs, nanos)\n    }\n}\n\nfn max_duration() -> Duration {\n    Duration::from(GenericTimerCounterValue::MAX)\n}\n\nimpl TryFrom<Duration> for GenericTimerCounterValue {\n    type Error = &'static str;\n\n    fn try_from(duration: Duration) -> Result<Self, Self::Error> {\n        if duration < resolution() {\n            return Ok(GenericTimerCounterValue(0));\n        }\n\n        if duration > max_duration() {\n            return Err(\"Conversion error. Duration too big\");\n        }\n\n        let frequency: u128 = u32::from(arch_timer_counter_frequency()) as u128;\n        let duration: u128 = duration.as_nanos();\n\n        // This is safe, because frequency can never be greater than u32::MAX, and\n        // (Duration::MAX.as_nanos() * u32::MAX) < u128::MAX.\n        let counter_value =\n            unsafe { duration.unchecked_mul(frequency) }.div(NonZeroU128::from(NANOSEC_PER_SEC));\n\n        // Since we checked above that we are <= max_duration(), just cast to u64.\n        Ok(GenericTimerCounterValue(counter_value as u64))\n    }\n}\n\n#[inline(always)]\nfn read_cntpct() -> GenericTimerCounterValue {\n    // Prevent that the counter is read ahead of time due to out-of-order execution.\n    barrier::isb(barrier::SY);\n    let cnt = CNTPCT_EL0.get();\n\n    GenericTimerCounterValue(cnt)\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The timer's resolution.\npub fn resolution() -> Duration {\n    Duration::from(GenericTimerCounterValue(1))\n}\n\n/// The uptime since power-on of the device.\n///\n/// This includes time consumed by firmware and bootloaders.\npub fn uptime() -> Duration {\n    read_cntpct().into()\n}\n\n/// Spin for a given duration.\npub fn spin_for(duration: Duration) {\n    let curr_counter_value = read_cntpct();\n\n    let counter_value_delta: GenericTimerCounterValue = match duration.try_into() {\n        Err(msg) => {\n            warn!(\"spin_for: {}. Skipping\", msg);\n            return;\n        }\n        Ok(val) => val,\n    };\n    let counter_value_target = curr_counter_value + counter_value_delta;\n\n    // Busy wait.\n    //\n    // Read CNTPCT_EL0 directly to avoid the ISB that is part of [`read_cntpct`].\n    while GenericTimerCounterValue(CNTPCT_EL0.get()) < counter_value_target {}\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/bsp/device_driver/arm/gicv2/gicc.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GICC Driver - GIC CPU interface.\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    exception,\n    memory::{Address, Virtual},\n};\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields, register_structs,\n    registers::ReadWrite,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nregister_bitfields! {\n    u32,\n\n    /// CPU Interface Control Register\n    CTLR [\n        Enable OFFSET(0) NUMBITS(1) []\n    ],\n\n    /// Interrupt Priority Mask Register\n    PMR [\n        Priority OFFSET(0) NUMBITS(8) []\n    ],\n\n    /// Interrupt Acknowledge Register\n    IAR [\n        InterruptID OFFSET(0) NUMBITS(10) []\n    ],\n\n    /// End of Interrupt Register\n    EOIR [\n        EOIINTID OFFSET(0) NUMBITS(10) []\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    pub RegisterBlock {\n        (0x000 => CTLR: ReadWrite<u32, CTLR::Register>),\n        (0x004 => PMR: ReadWrite<u32, PMR::Register>),\n        (0x008 => _reserved1),\n        (0x00C => IAR: ReadWrite<u32, IAR::Register>),\n        (0x010 => EOIR: ReadWrite<u32, EOIR::Register>),\n        (0x014  => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the GIC CPU interface.\npub struct GICC {\n    registers: Registers,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GICC {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n        }\n    }\n\n    /// Accept interrupts of any priority.\n    ///\n    /// Quoting the GICv2 Architecture Specification:\n    ///\n    ///   \"Writing 255 to the GICC_PMR always sets it to the largest supported priority field\n    ///    value.\"\n    ///\n    /// # Safety\n    ///\n    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n    ///   of `&mut self`.\n    pub fn priority_accept_all(&self) {\n        self.registers.PMR.write(PMR::Priority.val(255)); // Comment in arch spec.\n    }\n\n    /// Enable the interface - start accepting IRQs.\n    ///\n    /// # Safety\n    ///\n    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n    ///   of `&mut self`.\n    pub fn enable(&self) {\n        self.registers.CTLR.write(CTLR::Enable::SET);\n    }\n\n    /// Extract the number of the highest-priority pending IRQ.\n    ///\n    /// Can only be called from IRQ context, which is ensured by taking an `IRQContext` token.\n    ///\n    /// # Safety\n    ///\n    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n    ///   of `&mut self`.\n    #[allow(clippy::trivially_copy_pass_by_ref)]\n    pub fn pending_irq_number<'irq_context>(\n        &self,\n        _ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) -> usize {\n        self.registers.IAR.read(IAR::InterruptID) as usize\n    }\n\n    /// Complete handling of the currently active IRQ.\n    ///\n    /// Can only be called from IRQ context, which is ensured by taking an `IRQContext` token.\n    ///\n    /// To be called after `pending_irq_number()`.\n    ///\n    /// # Safety\n    ///\n    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n    ///   of `&mut self`.\n    #[allow(clippy::trivially_copy_pass_by_ref)]\n    pub fn mark_comleted<'irq_context>(\n        &self,\n        irq_number: u32,\n        _ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) {\n        self.registers.EOIR.write(EOIR::EOIINTID.val(irq_number));\n    }\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/bsp/device_driver/arm/gicv2/gicd.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GICD Driver - GIC Distributor.\n//!\n//! # Glossary\n//!   - SPI - Shared Peripheral Interrupt.\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    memory::{Address, Virtual},\n    state, synchronization,\n    synchronization::IRQSafeNullLock,\n};\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields, register_structs,\n    registers::{ReadOnly, ReadWrite},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nregister_bitfields! {\n    u32,\n\n    /// Distributor Control Register\n    CTLR [\n        Enable OFFSET(0) NUMBITS(1) []\n    ],\n\n    /// Interrupt Controller Type Register\n    TYPER [\n        ITLinesNumber OFFSET(0)  NUMBITS(5) []\n    ],\n\n    /// Interrupt Processor Targets Registers\n    ITARGETSR [\n        Offset3 OFFSET(24) NUMBITS(8) [],\n        Offset2 OFFSET(16) NUMBITS(8) [],\n        Offset1 OFFSET(8)  NUMBITS(8) [],\n        Offset0 OFFSET(0)  NUMBITS(8) []\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    SharedRegisterBlock {\n        (0x000 => CTLR: ReadWrite<u32, CTLR::Register>),\n        (0x004 => TYPER: ReadOnly<u32, TYPER::Register>),\n        (0x008 => _reserved1),\n        (0x104 => ISENABLER: [ReadWrite<u32>; 31]),\n        (0x180 => _reserved2),\n        (0x820 => ITARGETSR: [ReadWrite<u32, ITARGETSR::Register>; 248]),\n        (0xC00 => @END),\n    }\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    BankedRegisterBlock {\n        (0x000 => _reserved1),\n        (0x100 => ISENABLER: ReadWrite<u32>),\n        (0x104 => _reserved2),\n        (0x800 => ITARGETSR: [ReadOnly<u32, ITARGETSR::Register>; 8]),\n        (0x820 => @END),\n    }\n}\n\n/// Abstraction for the non-banked parts of the associated MMIO registers.\ntype SharedRegisters = MMIODerefWrapper<SharedRegisterBlock>;\n\n/// Abstraction for the banked parts of the associated MMIO registers.\ntype BankedRegisters = MMIODerefWrapper<BankedRegisterBlock>;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the GIC Distributor.\npub struct GICD {\n    /// Access to shared registers is guarded with a lock.\n    shared_registers: IRQSafeNullLock<SharedRegisters>,\n\n    /// Access to banked registers is unguarded.\n    banked_registers: BankedRegisters,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl SharedRegisters {\n    /// Return the number of IRQs that this HW implements.\n    #[inline(always)]\n    fn num_irqs(&mut self) -> usize {\n        // Query number of implemented IRQs.\n        //\n        // Refer to GICv2 Architecture Specification, Section 4.3.2.\n        ((self.TYPER.read(TYPER::ITLinesNumber) as usize) + 1) * 32\n    }\n\n    /// Return a slice of the implemented ITARGETSR.\n    #[inline(always)]\n    fn implemented_itargets_slice(&mut self) -> &[ReadWrite<u32, ITARGETSR::Register>] {\n        assert!(self.num_irqs() >= 36);\n\n        // Calculate the max index of the shared ITARGETSR array.\n        //\n        // The first 32 IRQs are private, so not included in `shared_registers`. Each ITARGETS\n        // register has four entries, so shift right by two. Subtract one because we start\n        // counting at zero.\n        let spi_itargetsr_max_index = ((self.num_irqs() - 32) >> 2) - 1;\n\n        // Rust automatically inserts slice range sanity check, i.e. max >= min.\n        &self.ITARGETSR[0..spi_itargetsr_max_index]\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl GICD {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            shared_registers: IRQSafeNullLock::new(SharedRegisters::new(mmio_start_addr)),\n            banked_registers: BankedRegisters::new(mmio_start_addr),\n        }\n    }\n\n    /// Use a banked ITARGETSR to retrieve the executing core's GIC target mask.\n    ///\n    /// Quoting the GICv2 Architecture Specification:\n    ///\n    ///   \"GICD_ITARGETSR0 to GICD_ITARGETSR7 are read-only, and each field returns a value that\n    ///    corresponds only to the processor reading the register.\"\n    fn local_gic_target_mask(&self) -> u32 {\n        self.banked_registers.ITARGETSR[0].read(ITARGETSR::Offset0)\n    }\n\n    /// Route all SPIs to the boot core and enable the distributor.\n    pub fn boot_core_init(&self) {\n        assert!(\n            state::state_manager().is_init(),\n            \"Only allowed during kernel init phase\"\n        );\n\n        // Target all SPIs to the boot core only.\n        let mask = self.local_gic_target_mask();\n\n        self.shared_registers.lock(|regs| {\n            for i in regs.implemented_itargets_slice().iter() {\n                i.write(\n                    ITARGETSR::Offset3.val(mask)\n                        + ITARGETSR::Offset2.val(mask)\n                        + ITARGETSR::Offset1.val(mask)\n                        + ITARGETSR::Offset0.val(mask),\n                );\n            }\n\n            regs.CTLR.write(CTLR::Enable::SET);\n        });\n    }\n\n    /// Enable an interrupt.\n    pub fn enable(&self, irq_num: &super::IRQNumber) {\n        let irq_num = irq_num.get();\n\n        // Each bit in the u32 enable register corresponds to one IRQ number. Shift right by 5\n        // (division by 32) and arrive at the index for the respective ISENABLER[i].\n        let enable_reg_index = irq_num >> 5;\n        let enable_bit: u32 = 1u32 << (irq_num % 32);\n\n        // Check if we are handling a private or shared IRQ.\n        match irq_num {\n            // Private.\n            0..=31 => {\n                let enable_reg = &self.banked_registers.ISENABLER;\n                enable_reg.set(enable_reg.get() | enable_bit);\n            }\n            // Shared.\n            _ => {\n                let enable_reg_index_shared = enable_reg_index - 1;\n\n                self.shared_registers.lock(|regs| {\n                    let enable_reg = &regs.ISENABLER[enable_reg_index_shared];\n                    enable_reg.set(enable_reg.get() | enable_bit);\n                });\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/bsp/device_driver/arm/gicv2.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GICv2 Driver - ARM Generic Interrupt Controller v2.\n//!\n//! The following is a collection of excerpts with useful information from\n//!   - `Programmer's Guide for ARMv8-A`\n//!   - `ARM Generic Interrupt Controller Architecture Specification`\n//!\n//! # Programmer's Guide - 10.6.1 Configuration\n//!\n//! The GIC is accessed as a memory-mapped peripheral.\n//!\n//! All cores can access the common Distributor, but the CPU interface is banked, that is, each core\n//! uses the same address to access its own private CPU interface.\n//!\n//! It is not possible for a core to access the CPU interface of another core.\n//!\n//! # Architecture Specification - 10.6.2 Initialization\n//!\n//! Both the Distributor and the CPU interfaces are disabled at reset. The GIC must be initialized\n//! after reset before it can deliver interrupts to the core.\n//!\n//! In the Distributor, software must configure the priority, target, security and enable individual\n//! interrupts. The Distributor must subsequently be enabled through its control register\n//! (GICD_CTLR). For each CPU interface, software must program the priority mask and preemption\n//! settings.\n//!\n//! Each CPU interface block itself must be enabled through its control register (GICD_CTLR). This\n//! prepares the GIC to deliver interrupts to the core.\n//!\n//! Before interrupts are expected in the core, software prepares the core to take interrupts by\n//! setting a valid interrupt vector in the vector table, and clearing interrupt mask bits in\n//! PSTATE, and setting the routing controls.\n//!\n//! The entire interrupt mechanism in the system can be disabled by disabling the Distributor.\n//! Interrupt delivery to an individual core can be disabled by disabling its CPU interface.\n//! Individual interrupts can also be disabled (or enabled) in the distributor.\n//!\n//! For an interrupt to reach the core, the individual interrupt, Distributor and CPU interface must\n//! all be enabled. The interrupt also needs to be of sufficient priority, that is, higher than the\n//! core's priority mask.\n//!\n//! # Architecture Specification - 1.4.2 Interrupt types\n//!\n//! - Peripheral interrupt\n//!     - Private Peripheral Interrupt (PPI)\n//!         - This is a peripheral interrupt that is specific to a single processor.\n//!     - Shared Peripheral Interrupt (SPI)\n//!         - This is a peripheral interrupt that the Distributor can route to any of a specified\n//!           combination of processors.\n//!\n//! - Software-generated interrupt (SGI)\n//!     - This is an interrupt generated by software writing to a GICD_SGIR register in the GIC. The\n//!       system uses SGIs for interprocessor communication.\n//!     - An SGI has edge-triggered properties. The software triggering of the interrupt is\n//!       equivalent to the edge transition of the interrupt request signal.\n//!     - When an SGI occurs in a multiprocessor implementation, the CPUID field in the Interrupt\n//!       Acknowledge Register, GICC_IAR, or the Aliased Interrupt Acknowledge Register, GICC_AIAR,\n//!       identifies the processor that requested the interrupt.\n//!\n//! # Architecture Specification - 2.2.1 Interrupt IDs\n//!\n//! Interrupts from sources are identified using ID numbers. Each CPU interface can see up to 1020\n//! interrupts. The banking of SPIs and PPIs increases the total number of interrupts supported by\n//! the Distributor.\n//!\n//! The GIC assigns interrupt ID numbers ID0-ID1019 as follows:\n//!   - Interrupt numbers 32..1019 are used for SPIs.\n//!   - Interrupt numbers 0..31 are used for interrupts that are private to a CPU interface. These\n//!     interrupts are banked in the Distributor.\n//!       - A banked interrupt is one where the Distributor can have multiple interrupts with the\n//!         same ID. A banked interrupt is identified uniquely by its ID number and its associated\n//!         CPU interface number. Of the banked interrupt IDs:\n//!           - 00..15 SGIs\n//!           - 16..31 PPIs\n\nmod gicc;\nmod gicd;\n\nuse crate::{\n    bsp::{self, device_driver::common::BoundedUsize},\n    cpu, driver, exception,\n    memory::{Address, Virtual},\n    synchronization,\n    synchronization::InitStateLock,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\ntype HandlerTable = [Option<exception::asynchronous::IRQHandlerDescriptor<IRQNumber>>;\n    IRQNumber::MAX_INCLUSIVE + 1];\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Used for the associated type of trait [`exception::asynchronous::interface::IRQManager`].\npub type IRQNumber = BoundedUsize<{ GICv2::MAX_IRQ_NUMBER }>;\n\n/// Representation of the GIC.\npub struct GICv2 {\n    /// The Distributor.\n    gicd: gicd::GICD,\n\n    /// The CPU Interface.\n    gicc: gicc::GICC,\n\n    /// Stores registered IRQ handlers. Writable only during kernel init. RO afterwards.\n    handler_table: InitStateLock<HandlerTable>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GICv2 {\n    const MAX_IRQ_NUMBER: usize = 300; // Normally 1019, but keep it lower to save some space.\n\n    pub const COMPATIBLE: &'static str = \"GICv2 (ARM Generic Interrupt Controller v2)\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(\n        gicd_mmio_start_addr: Address<Virtual>,\n        gicc_mmio_start_addr: Address<Virtual>,\n    ) -> Self {\n        Self {\n            gicd: gicd::GICD::new(gicd_mmio_start_addr),\n            gicc: gicc::GICC::new(gicc_mmio_start_addr),\n            handler_table: InitStateLock::new([None; IRQNumber::MAX_INCLUSIVE + 1]),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::ReadWriteEx;\n\nimpl driver::interface::DeviceDriver for GICv2 {\n    type IRQNumberType = IRQNumber;\n\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n\n    unsafe fn init(&self) -> Result<(), &'static str> {\n        if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() {\n            self.gicd.boot_core_init();\n        }\n\n        self.gicc.priority_accept_all();\n        self.gicc.enable();\n\n        Ok(())\n    }\n}\n\nimpl exception::asynchronous::interface::IRQManager for GICv2 {\n    type IRQNumberType = IRQNumber;\n\n    fn register_handler(\n        &self,\n        irq_handler_descriptor: exception::asynchronous::IRQHandlerDescriptor<Self::IRQNumberType>,\n    ) -> Result<(), &'static str> {\n        self.handler_table.write(|table| {\n            let irq_number = irq_handler_descriptor.number().get();\n\n            if table[irq_number].is_some() {\n                return Err(\"IRQ handler already registered\");\n            }\n\n            table[irq_number] = Some(irq_handler_descriptor);\n\n            Ok(())\n        })\n    }\n\n    fn enable(&self, irq_number: &Self::IRQNumberType) {\n        self.gicd.enable(irq_number);\n    }\n\n    fn handle_pending_irqs<'irq_context>(\n        &'irq_context self,\n        ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) {\n        // Extract the highest priority pending IRQ number from the Interrupt Acknowledge Register\n        // (IAR).\n        let irq_number = self.gicc.pending_irq_number(ic);\n\n        // Guard against spurious interrupts.\n        if irq_number > GICv2::MAX_IRQ_NUMBER {\n            return;\n        }\n\n        // Call the IRQ handler. Panic if there is none.\n        self.handler_table.read(|table| {\n            match table[irq_number] {\n                None => panic!(\"No handler registered for IRQ {}\", irq_number),\n                Some(descriptor) => {\n                    // Call the IRQ handler. Panics on failure.\n                    descriptor.handler().handle().expect(\"Error handling IRQ\");\n                }\n            }\n        });\n\n        // Signal completion of handling.\n        self.gicc.mark_comleted(irq_number as u32, ic);\n    }\n\n    fn print_handler(&self) {\n        use crate::info;\n\n        info!(\"      Peripheral handler:\");\n\n        self.handler_table.read(|table| {\n            for (i, opt) in table.iter().skip(32).enumerate() {\n                if let Some(handler) = opt {\n                    info!(\"            {: >3}. {}\", i + 32, handler.name());\n                }\n            }\n        });\n    }\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/bsp/device_driver/arm.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! ARM driver top level.\n\npub mod gicv2;\n\npub use gicv2::*;\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GPIO Driver.\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    driver,\n    exception::asynchronous::IRQNumber,\n    memory::{Address, Virtual},\n    synchronization,\n    synchronization::IRQSafeNullLock,\n};\nuse tock_registers::{\n    interfaces::{ReadWriteable, Writeable},\n    register_bitfields, register_structs,\n    registers::ReadWrite,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// GPIO registers.\n//\n// Descriptions taken from\n// - https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf\n// - https://datasheets.raspberrypi.org/bcm2711/bcm2711-peripherals.pdf\nregister_bitfields! {\n    u32,\n\n    /// GPIO Function Select 1\n    GPFSEL1 [\n        /// Pin 15\n        FSEL15 OFFSET(15) NUMBITS(3) [\n            Input = 0b000,\n            Output = 0b001,\n            AltFunc0 = 0b100  // PL011 UART RX\n\n        ],\n\n        /// Pin 14\n        FSEL14 OFFSET(12) NUMBITS(3) [\n            Input = 0b000,\n            Output = 0b001,\n            AltFunc0 = 0b100  // PL011 UART TX\n        ]\n    ],\n\n    /// GPIO Pull-up/down Register\n    ///\n    /// BCM2837 only.\n    GPPUD [\n        /// Controls the actuation of the internal pull-up/down control line to ALL the GPIO pins.\n        PUD OFFSET(0) NUMBITS(2) [\n            Off = 0b00,\n            PullDown = 0b01,\n            PullUp = 0b10\n        ]\n    ],\n\n    /// GPIO Pull-up/down Clock Register 0\n    ///\n    /// BCM2837 only.\n    GPPUDCLK0 [\n        /// Pin 15\n        PUDCLK15 OFFSET(15) NUMBITS(1) [\n            NoEffect = 0,\n            AssertClock = 1\n        ],\n\n        /// Pin 14\n        PUDCLK14 OFFSET(14) NUMBITS(1) [\n            NoEffect = 0,\n            AssertClock = 1\n        ]\n    ],\n\n    /// GPIO Pull-up / Pull-down Register 0\n    ///\n    /// BCM2711 only.\n    GPIO_PUP_PDN_CNTRL_REG0 [\n        /// Pin 15\n        GPIO_PUP_PDN_CNTRL15 OFFSET(30) NUMBITS(2) [\n            NoResistor = 0b00,\n            PullUp = 0b01\n        ],\n\n        /// Pin 14\n        GPIO_PUP_PDN_CNTRL14 OFFSET(28) NUMBITS(2) [\n            NoResistor = 0b00,\n            PullUp = 0b01\n        ]\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    RegisterBlock {\n        (0x00 => _reserved1),\n        (0x04 => GPFSEL1: ReadWrite<u32, GPFSEL1::Register>),\n        (0x08 => _reserved2),\n        (0x94 => GPPUD: ReadWrite<u32, GPPUD::Register>),\n        (0x98 => GPPUDCLK0: ReadWrite<u32, GPPUDCLK0::Register>),\n        (0x9C => _reserved3),\n        (0xE4 => GPIO_PUP_PDN_CNTRL_REG0: ReadWrite<u32, GPIO_PUP_PDN_CNTRL_REG0::Register>),\n        (0xE8 => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\nstruct GPIOInner {\n    registers: Registers,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the GPIO HW.\npub struct GPIO {\n    inner: IRQSafeNullLock<GPIOInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GPIOInner {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n        }\n    }\n\n    /// Disable pull-up/down on pins 14 and 15.\n    #[cfg(feature = \"bsp_rpi3\")]\n    fn disable_pud_14_15_bcm2837(&mut self) {\n        use crate::time;\n        use core::time::Duration;\n\n        // The Linux 2837 GPIO driver waits 1 µs between the steps.\n        const DELAY: Duration = Duration::from_micros(1);\n\n        self.registers.GPPUD.write(GPPUD::PUD::Off);\n        time::time_manager().spin_for(DELAY);\n\n        self.registers\n            .GPPUDCLK0\n            .write(GPPUDCLK0::PUDCLK15::AssertClock + GPPUDCLK0::PUDCLK14::AssertClock);\n        time::time_manager().spin_for(DELAY);\n\n        self.registers.GPPUD.write(GPPUD::PUD::Off);\n        self.registers.GPPUDCLK0.set(0);\n    }\n\n    /// Disable pull-up/down on pins 14 and 15.\n    #[cfg(feature = \"bsp_rpi4\")]\n    fn disable_pud_14_15_bcm2711(&mut self) {\n        self.registers.GPIO_PUP_PDN_CNTRL_REG0.write(\n            GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL15::PullUp\n                + GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL14::PullUp,\n        );\n    }\n\n    /// Map PL011 UART as standard output.\n    ///\n    /// TX to pin 14\n    /// RX to pin 15\n    pub fn map_pl011_uart(&mut self) {\n        // Select the UART on pins 14 and 15.\n        self.registers\n            .GPFSEL1\n            .modify(GPFSEL1::FSEL15::AltFunc0 + GPFSEL1::FSEL14::AltFunc0);\n\n        // Disable pull-up/down on pins 14 and 15.\n        #[cfg(feature = \"bsp_rpi3\")]\n        self.disable_pud_14_15_bcm2837();\n\n        #[cfg(feature = \"bsp_rpi4\")]\n        self.disable_pud_14_15_bcm2711();\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GPIO {\n    pub const COMPATIBLE: &'static str = \"BCM GPIO\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            inner: IRQSafeNullLock::new(GPIOInner::new(mmio_start_addr)),\n        }\n    }\n\n    /// Concurrency safe version of `GPIOInner.map_pl011_uart()`\n    pub fn map_pl011_uart(&self) {\n        self.inner.lock(|inner| inner.map_pl011_uart())\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl driver::interface::DeviceDriver for GPIO {\n    type IRQNumberType = IRQNumber;\n\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Peripheral Interrupt Controller Driver.\n//!\n//! # Resources\n//!\n//! - <https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf>\n\nuse super::{PendingIRQs, PeripheralIRQ};\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    exception,\n    memory::{Address, Virtual},\n    synchronization,\n    synchronization::{IRQSafeNullLock, InitStateLock},\n};\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_structs,\n    registers::{ReadOnly, WriteOnly},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    WORegisterBlock {\n        (0x00 => _reserved1),\n        (0x10 => ENABLE_1: WriteOnly<u32>),\n        (0x14 => ENABLE_2: WriteOnly<u32>),\n        (0x18 => @END),\n    }\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    RORegisterBlock {\n        (0x00 => _reserved1),\n        (0x04 => PENDING_1: ReadOnly<u32>),\n        (0x08 => PENDING_2: ReadOnly<u32>),\n        (0x0c => @END),\n    }\n}\n\n/// Abstraction for the WriteOnly parts of the associated MMIO registers.\ntype WriteOnlyRegisters = MMIODerefWrapper<WORegisterBlock>;\n\n/// Abstraction for the ReadOnly parts of the associated MMIO registers.\ntype ReadOnlyRegisters = MMIODerefWrapper<RORegisterBlock>;\n\ntype HandlerTable = [Option<exception::asynchronous::IRQHandlerDescriptor<PeripheralIRQ>>;\n    PeripheralIRQ::MAX_INCLUSIVE + 1];\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the peripheral interrupt controller.\npub struct PeripheralIC {\n    /// Access to write registers is guarded with a lock.\n    wo_registers: IRQSafeNullLock<WriteOnlyRegisters>,\n\n    /// Register read access is unguarded.\n    ro_registers: ReadOnlyRegisters,\n\n    /// Stores registered IRQ handlers. Writable only during kernel init. RO afterwards.\n    handler_table: InitStateLock<HandlerTable>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PeripheralIC {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            wo_registers: IRQSafeNullLock::new(WriteOnlyRegisters::new(mmio_start_addr)),\n            ro_registers: ReadOnlyRegisters::new(mmio_start_addr),\n            handler_table: InitStateLock::new([None; PeripheralIRQ::MAX_INCLUSIVE + 1]),\n        }\n    }\n\n    /// Query the list of pending IRQs.\n    fn pending_irqs(&self) -> PendingIRQs {\n        let pending_mask: u64 = (u64::from(self.ro_registers.PENDING_2.get()) << 32)\n            | u64::from(self.ro_registers.PENDING_1.get());\n\n        PendingIRQs::new(pending_mask)\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::{Mutex, ReadWriteEx};\n\nimpl exception::asynchronous::interface::IRQManager for PeripheralIC {\n    type IRQNumberType = PeripheralIRQ;\n\n    fn register_handler(\n        &self,\n        irq_handler_descriptor: exception::asynchronous::IRQHandlerDescriptor<Self::IRQNumberType>,\n    ) -> Result<(), &'static str> {\n        self.handler_table.write(|table| {\n            let irq_number = irq_handler_descriptor.number().get();\n\n            if table[irq_number].is_some() {\n                return Err(\"IRQ handler already registered\");\n            }\n\n            table[irq_number] = Some(irq_handler_descriptor);\n\n            Ok(())\n        })\n    }\n\n    fn enable(&self, irq: &Self::IRQNumberType) {\n        self.wo_registers.lock(|regs| {\n            let enable_reg = if irq.get() <= 31 {\n                &regs.ENABLE_1\n            } else {\n                &regs.ENABLE_2\n            };\n\n            let enable_bit: u32 = 1 << (irq.get() % 32);\n\n            // Writing a 1 to a bit will set the corresponding IRQ enable bit. All other IRQ enable\n            // bits are unaffected. So we don't need read and OR'ing here.\n            enable_reg.set(enable_bit);\n        });\n    }\n\n    fn handle_pending_irqs<'irq_context>(\n        &'irq_context self,\n        _ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) {\n        self.handler_table.read(|table| {\n            for irq_number in self.pending_irqs() {\n                match table[irq_number] {\n                    None => panic!(\"No handler registered for IRQ {}\", irq_number),\n                    Some(descriptor) => {\n                        // Call the IRQ handler. Panics on failure.\n                        descriptor.handler().handle().expect(\"Error handling IRQ\");\n                    }\n                }\n            }\n        })\n    }\n\n    fn print_handler(&self) {\n        use crate::info;\n\n        info!(\"      Peripheral handler:\");\n\n        self.handler_table.read(|table| {\n            for (i, opt) in table.iter().enumerate() {\n                if let Some(handler) = opt {\n                    info!(\"            {: >3}. {}\", i, handler.name());\n                }\n            }\n        });\n    }\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Interrupt Controller Driver.\n\nmod peripheral_ic;\n\nuse crate::{\n    bsp::device_driver::common::BoundedUsize,\n    driver,\n    exception::{self, asynchronous::IRQHandlerDescriptor},\n    memory::{Address, Virtual},\n};\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Wrapper struct for a bitmask indicating pending IRQ numbers.\nstruct PendingIRQs {\n    bitmask: u64,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub type LocalIRQ = BoundedUsize<{ InterruptController::MAX_LOCAL_IRQ_NUMBER }>;\npub type PeripheralIRQ = BoundedUsize<{ InterruptController::MAX_PERIPHERAL_IRQ_NUMBER }>;\n\n/// Used for the associated type of trait [`exception::asynchronous::interface::IRQManager`].\n#[derive(Copy, Clone)]\n#[allow(missing_docs)]\npub enum IRQNumber {\n    Local(LocalIRQ),\n    Peripheral(PeripheralIRQ),\n}\n\n/// Representation of the Interrupt Controller.\npub struct InterruptController {\n    periph: peripheral_ic::PeripheralIC,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PendingIRQs {\n    pub fn new(bitmask: u64) -> Self {\n        Self { bitmask }\n    }\n}\n\nimpl Iterator for PendingIRQs {\n    type Item = usize;\n\n    fn next(&mut self) -> Option<Self::Item> {\n        if self.bitmask == 0 {\n            return None;\n        }\n\n        let next = self.bitmask.trailing_zeros() as usize;\n        self.bitmask &= self.bitmask.wrapping_sub(1);\n        Some(next)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl fmt::Display for IRQNumber {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        match self {\n            Self::Local(number) => write!(f, \"Local({})\", number),\n            Self::Peripheral(number) => write!(f, \"Peripheral({})\", number),\n        }\n    }\n}\n\nimpl InterruptController {\n    // Restrict to 3 for now. This makes future code for local_ic.rs more straight forward.\n    const MAX_LOCAL_IRQ_NUMBER: usize = 3;\n    const MAX_PERIPHERAL_IRQ_NUMBER: usize = 63;\n\n    pub const COMPATIBLE: &'static str = \"BCM Interrupt Controller\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(periph_mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            periph: peripheral_ic::PeripheralIC::new(periph_mmio_start_addr),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\n\nimpl driver::interface::DeviceDriver for InterruptController {\n    type IRQNumberType = IRQNumber;\n\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n}\n\nimpl exception::asynchronous::interface::IRQManager for InterruptController {\n    type IRQNumberType = IRQNumber;\n\n    fn register_handler(\n        &self,\n        irq_handler_descriptor: exception::asynchronous::IRQHandlerDescriptor<Self::IRQNumberType>,\n    ) -> Result<(), &'static str> {\n        match irq_handler_descriptor.number() {\n            IRQNumber::Local(_) => unimplemented!(\"Local IRQ controller not implemented.\"),\n            IRQNumber::Peripheral(pirq) => {\n                let periph_descriptor = IRQHandlerDescriptor::new(\n                    pirq,\n                    irq_handler_descriptor.name(),\n                    irq_handler_descriptor.handler(),\n                );\n\n                self.periph.register_handler(periph_descriptor)\n            }\n        }\n    }\n\n    fn enable(&self, irq: &Self::IRQNumberType) {\n        match irq {\n            IRQNumber::Local(_) => unimplemented!(\"Local IRQ controller not implemented.\"),\n            IRQNumber::Peripheral(pirq) => self.periph.enable(pirq),\n        }\n    }\n\n    fn handle_pending_irqs<'irq_context>(\n        &'irq_context self,\n        ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) {\n        // It can only be a peripheral IRQ pending because enable() does not support local IRQs yet.\n        self.periph.handle_pending_irqs(ic)\n    }\n\n    fn print_handler(&self) {\n        self.periph.print_handler();\n    }\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! PL011 UART driver.\n//!\n//! # Resources\n//!\n//! - <https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf>\n//! - <https://developer.arm.com/documentation/ddi0183/latest>\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    console, cpu, driver,\n    exception::{self, asynchronous::IRQNumber},\n    memory::{Address, Virtual},\n    synchronization,\n    synchronization::IRQSafeNullLock,\n};\nuse core::fmt;\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields, register_structs,\n    registers::{ReadOnly, ReadWrite, WriteOnly},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// PL011 UART registers.\n//\n// Descriptions taken from \"PrimeCell UART (PL011) Technical Reference Manual\" r1p5.\nregister_bitfields! {\n    u32,\n\n    /// Flag Register.\n    FR [\n        /// Transmit FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n        /// Line Control Register, LCR_H.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the transmit holding register is empty.\n        /// - If the FIFO is enabled, the TXFE bit is set when the transmit FIFO is empty.\n        /// - This bit does not indicate if there is data in the transmit shift register.\n        TXFE OFFSET(7) NUMBITS(1) [],\n\n        /// Transmit FIFO full. The meaning of this bit depends on the state of the FEN bit in the\n        /// LCR_H Register.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the transmit holding register is full.\n        /// - If the FIFO is enabled, the TXFF bit is set when the transmit FIFO is full.\n        TXFF OFFSET(5) NUMBITS(1) [],\n\n        /// Receive FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n        /// LCR_H Register.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the receive holding register is empty.\n        /// - If the FIFO is enabled, the RXFE bit is set when the receive FIFO is empty.\n        RXFE OFFSET(4) NUMBITS(1) [],\n\n        /// UART busy. If this bit is set to 1, the UART is busy transmitting data. This bit remains\n        /// set until the complete byte, including all the stop bits, has been sent from the shift\n        /// register.\n        ///\n        /// This bit is set as soon as the transmit FIFO becomes non-empty, regardless of whether\n        /// the UART is enabled or not.\n        BUSY OFFSET(3) NUMBITS(1) []\n    ],\n\n    /// Integer Baud Rate Divisor.\n    IBRD [\n        /// The integer baud rate divisor.\n        BAUD_DIVINT OFFSET(0) NUMBITS(16) []\n    ],\n\n    /// Fractional Baud Rate Divisor.\n    FBRD [\n        ///  The fractional baud rate divisor.\n        BAUD_DIVFRAC OFFSET(0) NUMBITS(6) []\n    ],\n\n    /// Line Control Register.\n    LCR_H [\n        /// Word length. These bits indicate the number of data bits transmitted or received in a\n        /// frame.\n        #[allow(clippy::enum_variant_names)]\n        WLEN OFFSET(5) NUMBITS(2) [\n            FiveBit = 0b00,\n            SixBit = 0b01,\n            SevenBit = 0b10,\n            EightBit = 0b11\n        ],\n\n        /// Enable FIFOs:\n        ///\n        /// 0 = FIFOs are disabled (character mode) that is, the FIFOs become 1-byte-deep holding\n        /// registers.\n        ///\n        /// 1 = Transmit and receive FIFO buffers are enabled (FIFO mode).\n        FEN  OFFSET(4) NUMBITS(1) [\n            FifosDisabled = 0,\n            FifosEnabled = 1\n        ]\n    ],\n\n    /// Control Register.\n    CR [\n        /// Receive enable. If this bit is set to 1, the receive section of the UART is enabled.\n        /// Data reception occurs for either UART signals or SIR signals depending on the setting of\n        /// the SIREN bit. When the UART is disabled in the middle of reception, it completes the\n        /// current character before stopping.\n        RXE OFFSET(9) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// Transmit enable. If this bit is set to 1, the transmit section of the UART is enabled.\n        /// Data transmission occurs for either UART signals, or SIR signals depending on the\n        /// setting of the SIREN bit. When the UART is disabled in the middle of transmission, it\n        /// completes the current character before stopping.\n        TXE OFFSET(8) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// UART enable:\n        ///\n        /// 0 = UART is disabled. If the UART is disabled in the middle of transmission or\n        /// reception, it completes the current character before stopping.\n        ///\n        /// 1 = The UART is enabled. Data transmission and reception occurs for either UART signals\n        /// or SIR signals depending on the setting of the SIREN bit\n        UARTEN OFFSET(0) NUMBITS(1) [\n            /// If the UART is disabled in the middle of transmission or reception, it completes the\n            /// current character before stopping.\n            Disabled = 0,\n            Enabled = 1\n        ]\n    ],\n\n    /// Interrupt FIFO Level Select Register.\n    IFLS [\n        /// Receive interrupt FIFO level select. The trigger points for the receive interrupt are as\n        /// follows.\n        RXIFLSEL OFFSET(3) NUMBITS(5) [\n            OneEigth = 0b000,\n            OneQuarter = 0b001,\n            OneHalf = 0b010,\n            ThreeQuarters = 0b011,\n            SevenEights = 0b100\n        ]\n    ],\n\n    /// Interrupt Mask Set/Clear Register.\n    IMSC [\n        /// Receive timeout interrupt mask. A read returns the current mask for the UARTRTINTR\n        /// interrupt.\n        ///\n        /// - On a write of 1, the mask of the UARTRTINTR interrupt is set.\n        /// - A write of 0 clears the mask.\n        RTIM OFFSET(6) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// Receive interrupt mask. A read returns the current mask for the UARTRXINTR interrupt.\n        ///\n        /// - On a write of 1, the mask of the UARTRXINTR interrupt is set.\n        /// - A write of 0 clears the mask.\n        RXIM OFFSET(4) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ]\n    ],\n\n    /// Masked Interrupt Status Register.\n    MIS [\n        /// Receive timeout masked interrupt status. Returns the masked interrupt state of the\n        /// UARTRTINTR interrupt.\n        RTMIS OFFSET(6) NUMBITS(1) [],\n\n        /// Receive masked interrupt status. Returns the masked interrupt state of the UARTRXINTR\n        /// interrupt.\n        RXMIS OFFSET(4) NUMBITS(1) []\n    ],\n\n    /// Interrupt Clear Register.\n    ICR [\n        /// Meta field for all pending interrupts.\n        ALL OFFSET(0) NUMBITS(11) []\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    pub RegisterBlock {\n        (0x00 => DR: ReadWrite<u32>),\n        (0x04 => _reserved1),\n        (0x18 => FR: ReadOnly<u32, FR::Register>),\n        (0x1c => _reserved2),\n        (0x24 => IBRD: WriteOnly<u32, IBRD::Register>),\n        (0x28 => FBRD: WriteOnly<u32, FBRD::Register>),\n        (0x2c => LCR_H: WriteOnly<u32, LCR_H::Register>),\n        (0x30 => CR: WriteOnly<u32, CR::Register>),\n        (0x34 => IFLS: ReadWrite<u32, IFLS::Register>),\n        (0x38 => IMSC: ReadWrite<u32, IMSC::Register>),\n        (0x3C => _reserved3),\n        (0x40 => MIS: ReadOnly<u32, MIS::Register>),\n        (0x44 => ICR: WriteOnly<u32, ICR::Register>),\n        (0x48 => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\n#[derive(PartialEq)]\nenum BlockingMode {\n    Blocking,\n    NonBlocking,\n}\n\nstruct PL011UartInner {\n    registers: Registers,\n    chars_written: usize,\n    chars_read: usize,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the UART.\npub struct PL011Uart {\n    inner: IRQSafeNullLock<PL011UartInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PL011UartInner {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n            chars_written: 0,\n            chars_read: 0,\n        }\n    }\n\n    /// Set up baud rate and characteristics.\n    ///\n    /// This results in 8N1 and 921_600 baud.\n    ///\n    /// The calculation for the BRD is (we set the clock to 48 MHz in config.txt):\n    /// `(48_000_000 / 16) / 921_600 = 3.2552083`.\n    ///\n    /// This means the integer part is `3` and goes into the `IBRD`.\n    /// The fractional part is `0.2552083`.\n    ///\n    /// `FBRD` calculation according to the PL011 Technical Reference Manual:\n    /// `INTEGER((0.2552083 * 64) + 0.5) = 16`.\n    ///\n    /// Therefore, the generated baud rate divider is: `3 + 16/64 = 3.25`. Which results in a\n    /// genrated baud rate of `48_000_000 / (16 * 3.25) = 923_077`.\n    ///\n    /// Error = `((923_077 - 921_600) / 921_600) * 100 = 0.16%`.\n    pub fn init(&mut self) {\n        // Execution can arrive here while there are still characters queued in the TX FIFO and\n        // actively being sent out by the UART hardware. If the UART is turned off in this case,\n        // those queued characters would be lost.\n        //\n        // For example, this can happen during runtime on a call to panic!(), because panic!()\n        // initializes its own UART instance and calls init().\n        //\n        // Hence, flush first to ensure all pending characters are transmitted.\n        self.flush();\n\n        // Turn the UART off temporarily.\n        self.registers.CR.set(0);\n\n        // Clear all pending interrupts.\n        self.registers.ICR.write(ICR::ALL::CLEAR);\n\n        // From the PL011 Technical Reference Manual:\n        //\n        // The LCR_H, IBRD, and FBRD registers form the single 30-bit wide LCR Register that is\n        // updated on a single write strobe generated by a LCR_H write. So, to internally update the\n        // contents of IBRD or FBRD, a LCR_H write must always be performed at the end.\n        //\n        // Set the baud rate, 8N1 and FIFO enabled.\n        self.registers.IBRD.write(IBRD::BAUD_DIVINT.val(3));\n        self.registers.FBRD.write(FBRD::BAUD_DIVFRAC.val(16));\n        self.registers\n            .LCR_H\n            .write(LCR_H::WLEN::EightBit + LCR_H::FEN::FifosEnabled);\n\n        // Set RX FIFO fill level at 1/8.\n        self.registers.IFLS.write(IFLS::RXIFLSEL::OneEigth);\n\n        // Enable RX IRQ + RX timeout IRQ.\n        self.registers\n            .IMSC\n            .write(IMSC::RXIM::Enabled + IMSC::RTIM::Enabled);\n\n        // Turn the UART on.\n        self.registers\n            .CR\n            .write(CR::UARTEN::Enabled + CR::TXE::Enabled + CR::RXE::Enabled);\n    }\n\n    /// Send a character.\n    fn write_char(&mut self, c: char) {\n        // Spin while TX FIFO full is set, waiting for an empty slot.\n        while self.registers.FR.matches_all(FR::TXFF::SET) {\n            cpu::nop();\n        }\n\n        // Write the character to the buffer.\n        self.registers.DR.set(c as u32);\n\n        self.chars_written += 1;\n    }\n\n    /// Block execution until the last buffered character has been physically put on the TX wire.\n    fn flush(&self) {\n        // Spin until the busy bit is cleared.\n        while self.registers.FR.matches_all(FR::BUSY::SET) {\n            cpu::nop();\n        }\n    }\n\n    /// Retrieve a character.\n    fn read_char_converting(&mut self, blocking_mode: BlockingMode) -> Option<char> {\n        // If RX FIFO is empty,\n        if self.registers.FR.matches_all(FR::RXFE::SET) {\n            // immediately return in non-blocking mode.\n            if blocking_mode == BlockingMode::NonBlocking {\n                return None;\n            }\n\n            // Otherwise, wait until a char was received.\n            while self.registers.FR.matches_all(FR::RXFE::SET) {\n                cpu::nop();\n            }\n        }\n\n        // Read one character.\n        let mut ret = self.registers.DR.get() as u8 as char;\n\n        // Convert carrige return to newline.\n        if ret == '\\r' {\n            ret = '\\n'\n        }\n\n        // Update statistics.\n        self.chars_read += 1;\n\n        Some(ret)\n    }\n}\n\n/// Implementing `core::fmt::Write` enables usage of the `format_args!` macros, which in turn are\n/// used to implement the `kernel`'s `print!` and `println!` macros. By implementing `write_str()`,\n/// we get `write_fmt()` automatically.\n///\n/// The function takes an `&mut self`, so it must be implemented for the inner struct.\n///\n/// See [`src/print.rs`].\n///\n/// [`src/print.rs`]: ../../print/index.html\nimpl fmt::Write for PL011UartInner {\n    fn write_str(&mut self, s: &str) -> fmt::Result {\n        for c in s.chars() {\n            self.write_char(c);\n        }\n\n        Ok(())\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PL011Uart {\n    pub const COMPATIBLE: &'static str = \"BCM PL011 UART\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            inner: IRQSafeNullLock::new(PL011UartInner::new(mmio_start_addr)),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl driver::interface::DeviceDriver for PL011Uart {\n    type IRQNumberType = IRQNumber;\n\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n\n    unsafe fn init(&self) -> Result<(), &'static str> {\n        self.inner.lock(|inner| inner.init());\n\n        Ok(())\n    }\n\n    fn register_and_enable_irq_handler(\n        &'static self,\n        irq_number: &Self::IRQNumberType,\n    ) -> Result<(), &'static str> {\n        use exception::asynchronous::{irq_manager, IRQHandlerDescriptor};\n\n        let descriptor = IRQHandlerDescriptor::new(*irq_number, Self::COMPATIBLE, self);\n\n        irq_manager().register_handler(descriptor)?;\n        irq_manager().enable(irq_number);\n\n        Ok(())\n    }\n}\n\nimpl console::interface::Write for PL011Uart {\n    /// Passthrough of `args` to the `core::fmt::Write` implementation, but guarded by a Mutex to\n    /// serialize access.\n    fn write_char(&self, c: char) {\n        self.inner.lock(|inner| inner.write_char(c));\n    }\n\n    fn write_fmt(&self, args: core::fmt::Arguments) -> fmt::Result {\n        // Fully qualified syntax for the call to `core::fmt::Write::write_fmt()` to increase\n        // readability.\n        self.inner.lock(|inner| fmt::Write::write_fmt(inner, args))\n    }\n\n    fn flush(&self) {\n        // Spin until TX FIFO empty is set.\n        self.inner.lock(|inner| inner.flush());\n    }\n}\n\nimpl console::interface::Read for PL011Uart {\n    fn read_char(&self) -> char {\n        self.inner\n            .lock(|inner| inner.read_char_converting(BlockingMode::Blocking).unwrap())\n    }\n\n    fn clear_rx(&self) {\n        // Read from the RX FIFO until it is indicating empty.\n        while self\n            .inner\n            .lock(|inner| inner.read_char_converting(BlockingMode::NonBlocking))\n            .is_some()\n        {}\n    }\n}\n\nimpl console::interface::Statistics for PL011Uart {\n    fn chars_written(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_written)\n    }\n\n    fn chars_read(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_read)\n    }\n}\n\nimpl console::interface::All for PL011Uart {}\n\nimpl exception::asynchronous::interface::IRQHandler for PL011Uart {\n    fn handle(&self) -> Result<(), &'static str> {\n        self.inner.lock(|inner| {\n            let pending = inner.registers.MIS.extract();\n\n            // Clear all pending IRQs.\n            inner.registers.ICR.write(ICR::ALL::CLEAR);\n\n            // Check for any kind of RX interrupt.\n            if pending.matches_any(MIS::RXMIS::SET + MIS::RTMIS::SET) {\n                // Echo any received characters.\n                while let Some(c) = inner.read_char_converting(BlockingMode::NonBlocking) {\n                    inner.write_char(c)\n                }\n            }\n        });\n\n        Ok(())\n    }\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/bsp/device_driver/bcm.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BCM driver top level.\n\nmod bcm2xxx_gpio;\n#[cfg(feature = \"bsp_rpi3\")]\nmod bcm2xxx_interrupt_controller;\nmod bcm2xxx_pl011_uart;\n\npub use bcm2xxx_gpio::*;\n#[cfg(feature = \"bsp_rpi3\")]\npub use bcm2xxx_interrupt_controller::*;\npub use bcm2xxx_pl011_uart::*;\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/bsp/device_driver/common.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Common device driver code.\n\nuse crate::memory::{Address, Virtual};\nuse core::{fmt, marker::PhantomData, ops};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct MMIODerefWrapper<T> {\n    start_addr: Address<Virtual>,\n    phantom: PhantomData<fn() -> T>,\n}\n\n/// A wrapper type for usize with integrated range bound check.\n#[derive(Copy, Clone)]\npub struct BoundedUsize<const MAX_INCLUSIVE: usize>(usize);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T> MMIODerefWrapper<T> {\n    /// Create an instance.\n    pub const unsafe fn new(start_addr: Address<Virtual>) -> Self {\n        Self {\n            start_addr,\n            phantom: PhantomData,\n        }\n    }\n}\n\nimpl<T> ops::Deref for MMIODerefWrapper<T> {\n    type Target = T;\n\n    fn deref(&self) -> &Self::Target {\n        unsafe { &*(self.start_addr.as_usize() as *const _) }\n    }\n}\n\nimpl<const MAX_INCLUSIVE: usize> BoundedUsize<{ MAX_INCLUSIVE }> {\n    pub const MAX_INCLUSIVE: usize = MAX_INCLUSIVE;\n\n    /// Creates a new instance if number <= MAX_INCLUSIVE.\n    pub const fn new(number: usize) -> Self {\n        assert!(number <= MAX_INCLUSIVE);\n\n        Self(number)\n    }\n\n    /// Return the wrapped number.\n    pub const fn get(self) -> usize {\n        self.0\n    }\n}\n\nimpl<const MAX_INCLUSIVE: usize> fmt::Display for BoundedUsize<{ MAX_INCLUSIVE }> {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        write!(f, \"{}\", self.0)\n    }\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/bsp/device_driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Device driver.\n\n#[cfg(feature = \"bsp_rpi4\")]\nmod arm;\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod bcm;\nmod common;\n\n#[cfg(feature = \"bsp_rpi4\")]\npub use arm::*;\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use bcm::*;\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/bsp/raspberrypi/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Processor code.\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Used by `arch` code to find the early boot core.\n#[no_mangle]\n#[link_section = \".text._start_arguments\"]\npub static BOOT_CORE_ID: u64 = 0;\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/bsp/raspberrypi/driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP driver support.\n\nuse super::{exception, memory::map::mmio};\nuse crate::{\n    bsp::device_driver,\n    console, driver as generic_driver,\n    exception::{self as generic_exception},\n    memory,\n    memory::mmu::MMIODescriptor,\n};\nuse core::{\n    mem::MaybeUninit,\n    sync::atomic::{AtomicBool, Ordering},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic mut PL011_UART: MaybeUninit<device_driver::PL011Uart> = MaybeUninit::uninit();\nstatic mut GPIO: MaybeUninit<device_driver::GPIO> = MaybeUninit::uninit();\n\n#[cfg(feature = \"bsp_rpi3\")]\nstatic mut INTERRUPT_CONTROLLER: MaybeUninit<device_driver::InterruptController> =\n    MaybeUninit::uninit();\n\n#[cfg(feature = \"bsp_rpi4\")]\nstatic mut INTERRUPT_CONTROLLER: MaybeUninit<device_driver::GICv2> = MaybeUninit::uninit();\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// This must be called only after successful init of the memory subsystem.\nunsafe fn instantiate_uart() -> Result<(), &'static str> {\n    let mmio_descriptor = MMIODescriptor::new(mmio::PL011_UART_START, mmio::PL011_UART_SIZE);\n    let virt_addr =\n        memory::mmu::kernel_map_mmio(device_driver::PL011Uart::COMPATIBLE, &mmio_descriptor)?;\n\n    PL011_UART.write(device_driver::PL011Uart::new(virt_addr));\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the UART driver.\nunsafe fn post_init_uart() -> Result<(), &'static str> {\n    console::register_console(PL011_UART.assume_init_ref());\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the memory subsystem.\nunsafe fn instantiate_gpio() -> Result<(), &'static str> {\n    let mmio_descriptor = MMIODescriptor::new(mmio::GPIO_START, mmio::GPIO_SIZE);\n    let virt_addr =\n        memory::mmu::kernel_map_mmio(device_driver::GPIO::COMPATIBLE, &mmio_descriptor)?;\n\n    GPIO.write(device_driver::GPIO::new(virt_addr));\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the GPIO driver.\nunsafe fn post_init_gpio() -> Result<(), &'static str> {\n    GPIO.assume_init_ref().map_pl011_uart();\n    Ok(())\n}\n\n/// This must be called only after successful init of the memory subsystem.\n#[cfg(feature = \"bsp_rpi3\")]\nunsafe fn instantiate_interrupt_controller() -> Result<(), &'static str> {\n    let periph_mmio_descriptor =\n        MMIODescriptor::new(mmio::PERIPHERAL_IC_START, mmio::PERIPHERAL_IC_SIZE);\n    let periph_virt_addr = memory::mmu::kernel_map_mmio(\n        device_driver::InterruptController::COMPATIBLE,\n        &periph_mmio_descriptor,\n    )?;\n\n    INTERRUPT_CONTROLLER.write(device_driver::InterruptController::new(periph_virt_addr));\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the memory subsystem.\n#[cfg(feature = \"bsp_rpi4\")]\nunsafe fn instantiate_interrupt_controller() -> Result<(), &'static str> {\n    let gicd_mmio_descriptor = MMIODescriptor::new(mmio::GICD_START, mmio::GICD_SIZE);\n    let gicd_virt_addr = memory::mmu::kernel_map_mmio(\"GICv2 GICD\", &gicd_mmio_descriptor)?;\n\n    let gicc_mmio_descriptor = MMIODescriptor::new(mmio::GICC_START, mmio::GICC_SIZE);\n    let gicc_virt_addr = memory::mmu::kernel_map_mmio(\"GICV2 GICC\", &gicc_mmio_descriptor)?;\n\n    INTERRUPT_CONTROLLER.write(device_driver::GICv2::new(gicd_virt_addr, gicc_virt_addr));\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the interrupt controller driver.\nunsafe fn post_init_interrupt_controller() -> Result<(), &'static str> {\n    generic_exception::asynchronous::register_irq_manager(INTERRUPT_CONTROLLER.assume_init_ref());\n\n    Ok(())\n}\n\n/// Function needs to ensure that driver registration happens only after correct instantiation.\nunsafe fn driver_uart() -> Result<(), &'static str> {\n    instantiate_uart()?;\n\n    let uart_descriptor = generic_driver::DeviceDriverDescriptor::new(\n        PL011_UART.assume_init_ref(),\n        Some(post_init_uart),\n        Some(exception::asynchronous::irq_map::PL011_UART),\n    );\n    generic_driver::driver_manager().register_driver(uart_descriptor);\n\n    Ok(())\n}\n\n/// Function needs to ensure that driver registration happens only after correct instantiation.\nunsafe fn driver_gpio() -> Result<(), &'static str> {\n    instantiate_gpio()?;\n\n    let gpio_descriptor = generic_driver::DeviceDriverDescriptor::new(\n        GPIO.assume_init_ref(),\n        Some(post_init_gpio),\n        None,\n    );\n    generic_driver::driver_manager().register_driver(gpio_descriptor);\n\n    Ok(())\n}\n\n/// Function needs to ensure that driver registration happens only after correct instantiation.\nunsafe fn driver_interrupt_controller() -> Result<(), &'static str> {\n    instantiate_interrupt_controller()?;\n\n    let interrupt_controller_descriptor = generic_driver::DeviceDriverDescriptor::new(\n        INTERRUPT_CONTROLLER.assume_init_ref(),\n        Some(post_init_interrupt_controller),\n        None,\n    );\n    generic_driver::driver_manager().register_driver(interrupt_controller_descriptor);\n\n    Ok(())\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Initialize the driver subsystem.\n///\n/// # Safety\n///\n/// See child function calls.\npub unsafe fn init() -> Result<(), &'static str> {\n    static INIT_DONE: AtomicBool = AtomicBool::new(false);\n    if INIT_DONE.load(Ordering::Relaxed) {\n        return Err(\"Init already done\");\n    }\n\n    driver_uart()?;\n    driver_gpio()?;\n    driver_interrupt_controller()?;\n\n    INIT_DONE.store(true, Ordering::Relaxed);\n    Ok(())\n}\n\n/// Minimal code needed to bring up the console in QEMU (for testing only). This is often less steps\n/// than on real hardware due to QEMU's abstractions.\n#[cfg(feature = \"test_build\")]\npub fn qemu_bring_up_console() {\n    use crate::cpu;\n\n    unsafe {\n        instantiate_uart().unwrap_or_else(|_| cpu::qemu_exit_failure());\n        console::register_console(PL011_UART.assume_init_ref());\n    };\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/bsp/raspberrypi/exception/asynchronous.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP asynchronous exception handling.\n\nuse crate::bsp;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Export for reuse in generic asynchronous.rs.\npub use bsp::device_driver::IRQNumber;\n\n#[cfg(feature = \"bsp_rpi3\")]\npub(in crate::bsp) mod irq_map {\n    use super::bsp::device_driver::{IRQNumber, PeripheralIRQ};\n\n    pub const PL011_UART: IRQNumber = IRQNumber::Peripheral(PeripheralIRQ::new(57));\n}\n\n#[cfg(feature = \"bsp_rpi4\")]\npub(in crate::bsp) mod irq_map {\n    use super::bsp::device_driver::IRQNumber;\n\n    pub const PL011_UART: IRQNumber = IRQNumber::new(153);\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/bsp/raspberrypi/exception.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP synchronous and asynchronous exception handling.\n\npub mod asynchronous;\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/bsp/raspberrypi/kernel.ld",
    "content": "/* SPDX-License-Identifier: MIT OR Apache-2.0\n *\n * Copyright (c) 2018-2022 Andre Richter <andre.o.richter@gmail.com>\n */\n\nINCLUDE kernel_virt_addr_space_size.ld;\n\nPAGE_SIZE = 64K;\nPAGE_MASK = PAGE_SIZE - 1;\n\n/* The kernel's virtual address range will be:\n *\n * [END_ADDRESS_INCLUSIVE, START_ADDRESS]\n * [u64::MAX             , (u64::MAX - __kernel_virt_addr_space_size) + 1]\n */\n__kernel_virt_start_addr = ((0xffffffffffffffff - __kernel_virt_addr_space_size) + 1);\n\n__rpi_phys_dram_start_addr = 0;\n\n/* The physical address at which the the kernel binary will be loaded by the Raspberry's firmware */\n__rpi_phys_binary_load_addr = 0x80000;\n\n\nENTRY(__rpi_phys_binary_load_addr)\n\n/* Flags:\n *     4 == R\n *     5 == RX\n *     6 == RW\n *\n * Segments are marked PT_LOAD below so that the ELF file provides virtual and physical addresses.\n * It doesn't mean all of them need actually be loaded.\n */\nPHDRS\n{\n    segment_code            PT_LOAD FLAGS(5);\n    segment_data            PT_LOAD FLAGS(6);\n    segment_boot_core_stack PT_LOAD FLAGS(6);\n}\n\nSECTIONS\n{\n    . =  __kernel_virt_start_addr;\n\n    ASSERT((. & PAGE_MASK) == 0, \"Start of address space is not page aligned\")\n\n    /***********************************************************************************************\n    * Code + RO Data + Global Offset Table\n    ***********************************************************************************************/\n    __code_start = .;\n    .text : AT(__rpi_phys_binary_load_addr)\n    {\n        KEEP(*(.text._start))\n        *(.text._start_arguments) /* Constants (or statics in Rust speak) read by _start(). */\n        *(.text._start_rust)      /* The Rust entry point */\n        *(.text*)                 /* Everything else */\n    } :segment_code\n\n    .rodata : ALIGN(8) { *(.rodata*) } :segment_code\n\n    . = ALIGN(PAGE_SIZE);\n    __code_end_exclusive = .;\n\n    /***********************************************************************************************\n    * Data + BSS\n    ***********************************************************************************************/\n    __data_start = .;\n    .data : { *(.data*) } :segment_data\n\n    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */\n    .bss (NOLOAD) : ALIGN(16)\n    {\n        __bss_start = .;\n        *(.bss*);\n        . = ALIGN(16);\n        __bss_end_exclusive = .;\n    } :segment_data\n\n    . = ALIGN(PAGE_SIZE);\n    __data_end_exclusive = .;\n\n    /***********************************************************************************************\n    * MMIO Remap Reserved\n    ***********************************************************************************************/\n    __mmio_remap_start = .;\n    . += 8 * 1024 * 1024;\n    __mmio_remap_end_exclusive = .;\n\n    ASSERT((. & PAGE_MASK) == 0, \"MMIO remap reservation is not page aligned\")\n\n    /***********************************************************************************************\n    * Guard Page\n    ***********************************************************************************************/\n    . += PAGE_SIZE;\n\n    /***********************************************************************************************\n    * Boot Core Stack\n    ***********************************************************************************************/\n    .boot_core_stack (NOLOAD) : AT(__rpi_phys_dram_start_addr)\n    {\n        __boot_core_stack_start = .;         /*   ^             */\n                                             /*   | stack       */\n        . += __rpi_phys_binary_load_addr;    /*   | growth      */\n                                             /*   | direction   */\n        __boot_core_stack_end_exclusive = .; /*   |             */\n    } :segment_boot_core_stack\n\n    ASSERT((. & PAGE_MASK) == 0, \"End of boot core stack is not page aligned\")\n\n    /***********************************************************************************************\n    * Misc\n    ***********************************************************************************************/\n    .got : { *(.got*) }\n    ASSERT(SIZEOF(.got) == 0, \"Relocation support not expected\")\n\n    /DISCARD/ : { *(.comment*) }\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/bsp/raspberrypi/kernel_virt_addr_space_size.ld",
    "content": "__kernel_virt_addr_space_size = 1024 * 1024 * 1024\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/bsp/raspberrypi/memory/mmu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Memory Management Unit.\n\nuse crate::{\n    memory::{\n        mmu::{\n            self as generic_mmu, AddressSpace, AssociatedTranslationTable, AttributeFields,\n            MemoryRegion, PageAddress, TranslationGranule,\n        },\n        Physical, Virtual,\n    },\n    synchronization::InitStateLock,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\ntype KernelTranslationTable =\n    <KernelVirtAddrSpace as AssociatedTranslationTable>::TableStartFromTop;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// The translation granule chosen by this BSP. This will be used everywhere else in the kernel to\n/// derive respective data structures and their sizes. For example, the `crate::memory::mmu::Page`.\npub type KernelGranule = TranslationGranule<{ 64 * 1024 }>;\n\n/// The kernel's virtual address space defined by this BSP.\npub type KernelVirtAddrSpace = AddressSpace<{ kernel_virt_addr_space_size() }>;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n/// The kernel translation tables.\n///\n/// It is mandatory that InitStateLock is transparent.\n///\n/// That is, `size_of(InitStateLock<KernelTranslationTable>) == size_of(KernelTranslationTable)`.\n/// There is a unit tests that checks this porperty.\n#[link_section = \".data\"]\n#[no_mangle]\nstatic KERNEL_TABLES: InitStateLock<KernelTranslationTable> =\n    InitStateLock::new(KernelTranslationTable::new_for_precompute());\n\n/// This value is needed during early boot for MMU setup.\n///\n/// This will be patched to the correct value by the \"translation table tool\" after linking. This\n/// given value here is just a dummy.\n#[link_section = \".text._start_arguments\"]\n#[no_mangle]\nstatic PHYS_KERNEL_TABLES_BASE_ADDR: u64 = 0xCCCCAAAAFFFFEEEE;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// This is a hack for retrieving the value for the kernel's virtual address space size as a\n/// constant from a common place, since it is needed as a compile-time/link-time constant in both,\n/// the linker script and the Rust sources.\n#[allow(clippy::needless_late_init)]\nconst fn kernel_virt_addr_space_size() -> usize {\n    let __kernel_virt_addr_space_size;\n\n    include!(\"../kernel_virt_addr_space_size.ld\");\n\n    __kernel_virt_addr_space_size\n}\n\n/// Helper function for calculating the number of pages the given parameter spans.\nconst fn size_to_num_pages(size: usize) -> usize {\n    assert!(size > 0);\n    assert!(size % KernelGranule::SIZE == 0);\n\n    size >> KernelGranule::SHIFT\n}\n\n/// The code pages of the kernel binary.\nfn virt_code_region() -> MemoryRegion<Virtual> {\n    let num_pages = size_to_num_pages(super::code_size());\n\n    let start_page_addr = super::virt_code_start();\n    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n\n    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n}\n\n/// The data pages of the kernel binary.\nfn virt_data_region() -> MemoryRegion<Virtual> {\n    let num_pages = size_to_num_pages(super::data_size());\n\n    let start_page_addr = super::virt_data_start();\n    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n\n    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n}\n\n/// The boot core stack pages.\nfn virt_boot_core_stack_region() -> MemoryRegion<Virtual> {\n    let num_pages = size_to_num_pages(super::boot_core_stack_size());\n\n    let start_page_addr = super::virt_boot_core_stack_start();\n    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n\n    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n}\n\n// There is no reason to expect the following conversions to fail, since they were generated offline\n// by the `translation table tool`. If it doesn't work, a panic due to the unwraps is justified.\nfn kernel_virt_to_phys_region(virt_region: MemoryRegion<Virtual>) -> MemoryRegion<Physical> {\n    let phys_start_page_addr =\n        generic_mmu::try_kernel_virt_page_addr_to_phys_page_addr(virt_region.start_page_addr())\n            .unwrap();\n\n    let phys_end_exclusive_page_addr = phys_start_page_addr\n        .checked_offset(virt_region.num_pages() as isize)\n        .unwrap();\n\n    MemoryRegion::new(phys_start_page_addr, phys_end_exclusive_page_addr)\n}\n\nfn kernel_page_attributes(virt_page_addr: PageAddress<Virtual>) -> AttributeFields {\n    generic_mmu::try_kernel_page_attributes(virt_page_addr).unwrap()\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the kernel's translation tables.\npub fn kernel_translation_tables() -> &'static InitStateLock<KernelTranslationTable> {\n    &KERNEL_TABLES\n}\n\n/// The MMIO remap pages.\npub fn virt_mmio_remap_region() -> MemoryRegion<Virtual> {\n    let num_pages = size_to_num_pages(super::mmio_remap_size());\n\n    let start_page_addr = super::virt_mmio_remap_start();\n    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n\n    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n}\n\n/// Add mapping records for the kernel binary.\n///\n/// The actual translation table entries for the kernel binary are generated using the offline\n/// `translation table tool` and patched into the kernel binary. This function just adds the mapping\n/// record entries.\npub fn kernel_add_mapping_records_for_precomputed() {\n    let virt_code_region = virt_code_region();\n    generic_mmu::kernel_add_mapping_record(\n        \"Kernel code and RO data\",\n        &virt_code_region,\n        &kernel_virt_to_phys_region(virt_code_region),\n        &kernel_page_attributes(virt_code_region.start_page_addr()),\n    );\n\n    let virt_data_region = virt_data_region();\n    generic_mmu::kernel_add_mapping_record(\n        \"Kernel data and bss\",\n        &virt_data_region,\n        &kernel_virt_to_phys_region(virt_data_region),\n        &kernel_page_attributes(virt_data_region.start_page_addr()),\n    );\n\n    let virt_boot_core_stack_region = virt_boot_core_stack_region();\n    generic_mmu::kernel_add_mapping_record(\n        \"Kernel boot-core stack\",\n        &virt_boot_core_stack_region,\n        &kernel_virt_to_phys_region(virt_boot_core_stack_region),\n        &kernel_page_attributes(virt_boot_core_stack_region.start_page_addr()),\n    );\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/bsp/raspberrypi/memory.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Memory Management.\n//!\n//! The physical memory layout.\n//!\n//! The Raspberry's firmware copies the kernel binary to 0x8_0000. The preceding region will be used\n//! as the boot core's stack.\n//!\n//! +---------------------------------------+\n//! |                                       | boot_core_stack_start @ 0x0\n//! |                                       |                                ^\n//! | Boot-core Stack                       |                                | stack\n//! |                                       |                                | growth\n//! |                                       |                                | direction\n//! +---------------------------------------+\n//! |                                       | code_start @ 0x8_0000 == boot_core_stack_end_exclusive\n//! | .text                                 |\n//! | .rodata                               |\n//! | .got                                  |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       | data_start == code_end_exclusive\n//! | .data                                 |\n//! | .bss                                  |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       | data_end_exclusive\n//! |                                       |\n//!\n//!\n//!\n//!\n//!\n//! The virtual memory layout is as follows:\n//!\n//! +---------------------------------------+\n//! |                                       | code_start @ __kernel_virt_start_addr\n//! | .text                                 |\n//! | .rodata                               |\n//! | .got                                  |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       | data_start == code_end_exclusive\n//! | .data                                 |\n//! | .bss                                  |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       |  mmio_remap_start == data_end_exclusive\n//! | VA region for MMIO remapping          |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       |  mmio_remap_end_exclusive\n//! | Unmapped guard page                   |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       | boot_core_stack_start\n//! |                                       |                                ^\n//! | Boot-core Stack                       |                                | stack\n//! |                                       |                                | growth\n//! |                                       |                                | direction\n//! +---------------------------------------+\n//! |                                       | boot_core_stack_end_exclusive\n//! |                                       |\npub mod mmu;\n\nuse crate::memory::{mmu::PageAddress, Address, Physical, Virtual};\nuse core::cell::UnsafeCell;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Symbols from the linker script.\nextern \"Rust\" {\n    static __code_start: UnsafeCell<()>;\n    static __code_end_exclusive: UnsafeCell<()>;\n\n    static __data_start: UnsafeCell<()>;\n    static __data_end_exclusive: UnsafeCell<()>;\n\n    static __mmio_remap_start: UnsafeCell<()>;\n    static __mmio_remap_end_exclusive: UnsafeCell<()>;\n\n    static __boot_core_stack_start: UnsafeCell<()>;\n    static __boot_core_stack_end_exclusive: UnsafeCell<()>;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// The board's physical memory map.\n#[rustfmt::skip]\npub(super) mod map {\n    use super::*;\n\n    /// Physical devices.\n    #[cfg(feature = \"bsp_rpi3\")]\n    pub mod mmio {\n        use super::*;\n\n        pub const PERIPHERAL_IC_START: Address<Physical> = Address::new(0x3F00_B200);\n        pub const PERIPHERAL_IC_SIZE:  usize             =              0x24;\n\n        pub const GPIO_START:          Address<Physical> = Address::new(0x3F20_0000);\n        pub const GPIO_SIZE:           usize             =              0xA0;\n\n        pub const PL011_UART_START:    Address<Physical> = Address::new(0x3F20_1000);\n        pub const PL011_UART_SIZE:     usize             =              0x48;\n\n        pub const END:                 Address<Physical> = Address::new(0x4001_0000);\n    }\n\n    /// Physical devices.\n    #[cfg(feature = \"bsp_rpi4\")]\n    pub mod mmio {\n        use super::*;\n\n        pub const GPIO_START:       Address<Physical> = Address::new(0xFE20_0000);\n        pub const GPIO_SIZE:        usize             =              0xA0;\n\n        pub const PL011_UART_START: Address<Physical> = Address::new(0xFE20_1000);\n        pub const PL011_UART_SIZE:  usize             =              0x48;\n\n        pub const GICD_START:       Address<Physical> = Address::new(0xFF84_1000);\n        pub const GICD_SIZE:        usize             =              0x824;\n\n        pub const GICC_START:       Address<Physical> = Address::new(0xFF84_2000);\n        pub const GICC_SIZE:        usize             =              0x14;\n\n        pub const END:              Address<Physical> = Address::new(0xFF85_0000);\n    }\n\n    pub const END: Address<Physical> = mmio::END;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Start page address of the code segment.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn virt_code_start() -> PageAddress<Virtual> {\n    PageAddress::from(unsafe { __code_start.get() as usize })\n}\n\n/// Size of the code segment.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn code_size() -> usize {\n    unsafe { (__code_end_exclusive.get() as usize) - (__code_start.get() as usize) }\n}\n\n/// Start page address of the data segment.\n#[inline(always)]\nfn virt_data_start() -> PageAddress<Virtual> {\n    PageAddress::from(unsafe { __data_start.get() as usize })\n}\n\n/// Size of the data segment.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn data_size() -> usize {\n    unsafe { (__data_end_exclusive.get() as usize) - (__data_start.get() as usize) }\n}\n\n/// Start page address of the MMIO remap reservation.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn virt_mmio_remap_start() -> PageAddress<Virtual> {\n    PageAddress::from(unsafe { __mmio_remap_start.get() as usize })\n}\n\n/// Size of the MMIO remap reservation.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn mmio_remap_size() -> usize {\n    unsafe { (__mmio_remap_end_exclusive.get() as usize) - (__mmio_remap_start.get() as usize) }\n}\n\n/// Start page address of the boot core's stack.\n#[inline(always)]\nfn virt_boot_core_stack_start() -> PageAddress<Virtual> {\n    PageAddress::from(unsafe { __boot_core_stack_start.get() as usize })\n}\n\n/// Size of the boot core's stack.\n#[inline(always)]\nfn boot_core_stack_size() -> usize {\n    unsafe {\n        (__boot_core_stack_end_exclusive.get() as usize) - (__boot_core_stack_start.get() as usize)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Exclusive end address of the physical address space.\n#[inline(always)]\npub fn phys_addr_space_end_exclusive_addr() -> PageAddress<Physical> {\n    PageAddress::from(map::END)\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/bsp/raspberrypi.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Top-level BSP file for the Raspberry Pi 3 and 4.\n\npub mod cpu;\npub mod driver;\npub mod exception;\npub mod memory;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Board identification.\npub fn board_name() -> &'static str {\n    #[cfg(feature = \"bsp_rpi3\")]\n    {\n        \"Raspberry Pi 3\"\n    }\n\n    #[cfg(feature = \"bsp_rpi4\")]\n    {\n        \"Raspberry Pi 4\"\n    }\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/bsp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Conditional reexporting of Board Support Packages.\n\nmod device_driver;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod raspberrypi;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use raspberrypi::*;\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/common.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! General purpose code.\n\n/// Check if a value is aligned to a given size.\n#[inline(always)]\npub const fn is_aligned(value: usize, alignment: usize) -> bool {\n    assert!(alignment.is_power_of_two());\n\n    (value & (alignment - 1)) == 0\n}\n\n/// Align down.\n#[inline(always)]\npub const fn align_down(value: usize, alignment: usize) -> usize {\n    assert!(alignment.is_power_of_two());\n\n    value & !(alignment - 1)\n}\n\n/// Align up.\n#[inline(always)]\npub const fn align_up(value: usize, alignment: usize) -> usize {\n    assert!(alignment.is_power_of_two());\n\n    (value + alignment - 1) & !(alignment - 1)\n}\n\n/// Convert a size into human readable format.\npub const fn size_human_readable_ceil(size: usize) -> (usize, &'static str) {\n    const KIB: usize = 1024;\n    const MIB: usize = 1024 * 1024;\n    const GIB: usize = 1024 * 1024 * 1024;\n\n    if (size / GIB) > 0 {\n        (size.div_ceil(GIB), \"GiB\")\n    } else if (size / MIB) > 0 {\n        (size.div_ceil(MIB), \"MiB\")\n    } else if (size / KIB) > 0 {\n        (size.div_ceil(KIB), \"KiB\")\n    } else {\n        (size, \"Byte\")\n    }\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/console/null_console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Null console.\n\nuse super::interface;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct NullConsole;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\npub static NULL_CONSOLE: NullConsole = NullConsole {};\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl interface::Write for NullConsole {\n    fn write_char(&self, _c: char) {}\n\n    fn write_fmt(&self, _args: fmt::Arguments) -> fmt::Result {\n        fmt::Result::Ok(())\n    }\n\n    fn flush(&self) {}\n}\n\nimpl interface::Read for NullConsole {\n    fn clear_rx(&self) {}\n}\n\nimpl interface::Statistics for NullConsole {}\nimpl interface::All for NullConsole {}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! System console.\n\nmod null_console;\n\nuse crate::synchronization;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Console interfaces.\npub mod interface {\n    use core::fmt;\n\n    /// Console write functions.\n    pub trait Write {\n        /// Write a single character.\n        fn write_char(&self, c: char);\n\n        /// Write a Rust format string.\n        fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result;\n\n        /// Block until the last buffered character has been physically put on the TX wire.\n        fn flush(&self);\n    }\n\n    /// Console read functions.\n    pub trait Read {\n        /// Read a single character.\n        fn read_char(&self) -> char {\n            ' '\n        }\n\n        /// Clear RX buffers, if any.\n        fn clear_rx(&self);\n    }\n\n    /// Console statistics.\n    pub trait Statistics {\n        /// Return the number of characters written.\n        fn chars_written(&self) -> usize {\n            0\n        }\n\n        /// Return the number of characters read.\n        fn chars_read(&self) -> usize {\n            0\n        }\n    }\n\n    /// Trait alias for a full-fledged console.\n    pub trait All: Write + Read + Statistics {}\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic CUR_CONSOLE: InitStateLock<&'static (dyn interface::All + Sync)> =\n    InitStateLock::new(&null_console::NULL_CONSOLE);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::{interface::ReadWriteEx, InitStateLock};\n\n/// Register a new console.\npub fn register_console(new_console: &'static (dyn interface::All + Sync)) {\n    CUR_CONSOLE.write(|con| *con = new_console);\n}\n\n/// Return a reference to the currently registered console.\n///\n/// This is the global console used by all printing macros.\npub fn console() -> &'static dyn interface::All {\n    CUR_CONSOLE.read(|con| *con)\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Boot code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/cpu/boot.rs\"]\nmod arch_boot;\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/cpu/smp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Symmetric multiprocessing.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/cpu/smp.rs\"]\nmod arch_smp;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_smp::core_id;\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Processor code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/cpu.rs\"]\nmod arch_cpu;\n\nmod boot;\n\npub mod smp;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_cpu::{nop, wait_forever};\n\n#[cfg(feature = \"test_build\")]\npub use arch_cpu::{qemu_exit_failure, qemu_exit_success};\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Driver support.\n\nuse crate::{\n    exception, info,\n    synchronization::{interface::ReadWriteEx, InitStateLock},\n};\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nconst NUM_DRIVERS: usize = 5;\n\nstruct DriverManagerInner<T>\nwhere\n    T: 'static,\n{\n    next_index: usize,\n    descriptors: [Option<DeviceDriverDescriptor<T>>; NUM_DRIVERS],\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Driver interfaces.\npub mod interface {\n    /// Device Driver functions.\n    pub trait DeviceDriver {\n        /// Different interrupt controllers might use different types for IRQ number.\n        type IRQNumberType: super::fmt::Display;\n\n        /// Return a compatibility string for identifying the driver.\n        fn compatible(&self) -> &'static str;\n\n        /// Called by the kernel to bring up the device.\n        ///\n        /// # Safety\n        ///\n        /// - During init, drivers might do stuff with system-wide impact.\n        unsafe fn init(&self) -> Result<(), &'static str> {\n            Ok(())\n        }\n\n        /// Called by the kernel to register and enable the device's IRQ handler.\n        ///\n        /// Rust's type system will prevent a call to this function unless the calling instance\n        /// itself has static lifetime.\n        fn register_and_enable_irq_handler(\n            &'static self,\n            irq_number: &Self::IRQNumberType,\n        ) -> Result<(), &'static str> {\n            panic!(\n                \"Attempt to enable IRQ {} for device {}, but driver does not support this\",\n                irq_number,\n                self.compatible()\n            )\n        }\n    }\n}\n\n/// Tpye to be used as an optional callback after a driver's init() has run.\npub type DeviceDriverPostInitCallback = unsafe fn() -> Result<(), &'static str>;\n\n/// A descriptor for device drivers.\n#[derive(Copy, Clone)]\npub struct DeviceDriverDescriptor<T>\nwhere\n    T: 'static,\n{\n    device_driver: &'static (dyn interface::DeviceDriver<IRQNumberType = T> + Sync),\n    post_init_callback: Option<DeviceDriverPostInitCallback>,\n    irq_number: Option<T>,\n}\n\n/// Provides device driver management functions.\npub struct DriverManager<T>\nwhere\n    T: 'static,\n{\n    inner: InitStateLock<DriverManagerInner<T>>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic DRIVER_MANAGER: DriverManager<exception::asynchronous::IRQNumber> = DriverManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T> DriverManagerInner<T>\nwhere\n    T: 'static + Copy,\n{\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            next_index: 0,\n            descriptors: [None; NUM_DRIVERS],\n        }\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T> DeviceDriverDescriptor<T> {\n    /// Create an instance.\n    pub fn new(\n        device_driver: &'static (dyn interface::DeviceDriver<IRQNumberType = T> + Sync),\n        post_init_callback: Option<DeviceDriverPostInitCallback>,\n        irq_number: Option<T>,\n    ) -> Self {\n        Self {\n            device_driver,\n            post_init_callback,\n            irq_number,\n        }\n    }\n}\n\n/// Return a reference to the global DriverManager.\npub fn driver_manager() -> &'static DriverManager<exception::asynchronous::IRQNumber> {\n    &DRIVER_MANAGER\n}\n\nimpl<T> DriverManager<T>\nwhere\n    T: fmt::Display + Copy,\n{\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            inner: InitStateLock::new(DriverManagerInner::new()),\n        }\n    }\n\n    /// Register a device driver with the kernel.\n    pub fn register_driver(&self, descriptor: DeviceDriverDescriptor<T>) {\n        self.inner.write(|inner| {\n            inner.descriptors[inner.next_index] = Some(descriptor);\n            inner.next_index += 1;\n        })\n    }\n\n    /// Helper for iterating over registered drivers.\n    fn for_each_descriptor<'a>(&'a self, f: impl FnMut(&'a DeviceDriverDescriptor<T>)) {\n        self.inner.read(|inner| {\n            inner\n                .descriptors\n                .iter()\n                .filter_map(|x| x.as_ref())\n                .for_each(f)\n        })\n    }\n\n    /// Fully initialize all drivers and their interrupts handlers.\n    ///\n    /// # Safety\n    ///\n    /// - During init, drivers might do stuff with system-wide impact.\n    pub unsafe fn init_drivers_and_irqs(&self) {\n        self.for_each_descriptor(|descriptor| {\n            // 1. Initialize driver.\n            if let Err(x) = descriptor.device_driver.init() {\n                panic!(\n                    \"Error initializing driver: {}: {}\",\n                    descriptor.device_driver.compatible(),\n                    x\n                );\n            }\n\n            // 2. Call corresponding post init callback.\n            if let Some(callback) = &descriptor.post_init_callback {\n                if let Err(x) = callback() {\n                    panic!(\n                        \"Error during driver post-init callback: {}: {}\",\n                        descriptor.device_driver.compatible(),\n                        x\n                    );\n                }\n            }\n        });\n\n        // 3. After all post-init callbacks were done, the interrupt controller should be\n        //    registered and functional. So let drivers register with it now.\n        self.for_each_descriptor(|descriptor| {\n            if let Some(irq_number) = &descriptor.irq_number {\n                if let Err(x) = descriptor\n                    .device_driver\n                    .register_and_enable_irq_handler(irq_number)\n                {\n                    panic!(\n                        \"Error during driver interrupt handler registration: {}: {}\",\n                        descriptor.device_driver.compatible(),\n                        x\n                    );\n                }\n            }\n        });\n    }\n\n    /// Enumerate all registered device drivers.\n    pub fn enumerate(&self) {\n        let mut i: usize = 1;\n        self.for_each_descriptor(|descriptor| {\n            info!(\"      {}. {}\", i, descriptor.device_driver.compatible());\n\n            i += 1;\n        });\n    }\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/exception/asynchronous/null_irq_manager.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Null IRQ Manager.\n\nuse super::{interface, IRQContext, IRQHandlerDescriptor};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct NullIRQManager;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\npub static NULL_IRQ_MANAGER: NullIRQManager = NullIRQManager {};\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl interface::IRQManager for NullIRQManager {\n    type IRQNumberType = super::IRQNumber;\n\n    fn register_handler(\n        &self,\n        _descriptor: IRQHandlerDescriptor<Self::IRQNumberType>,\n    ) -> Result<(), &'static str> {\n        panic!(\"No IRQ Manager registered yet\");\n    }\n\n    fn enable(&self, _irq_number: &Self::IRQNumberType) {\n        panic!(\"No IRQ Manager registered yet\");\n    }\n\n    fn handle_pending_irqs<'irq_context>(&'irq_context self, _ic: &IRQContext<'irq_context>) {\n        panic!(\"No IRQ Manager registered yet\");\n    }\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/exception/asynchronous.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Asynchronous exception handling.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/exception/asynchronous.rs\"]\nmod arch_asynchronous;\nmod null_irq_manager;\n\nuse crate::{bsp, synchronization};\nuse core::marker::PhantomData;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_asynchronous::{\n    is_local_irq_masked, local_irq_mask, local_irq_mask_save, local_irq_restore, local_irq_unmask,\n    print_state,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Interrupt number as defined by the BSP.\npub type IRQNumber = bsp::exception::asynchronous::IRQNumber;\n\n/// Interrupt descriptor.\n#[derive(Copy, Clone)]\npub struct IRQHandlerDescriptor<T>\nwhere\n    T: Copy,\n{\n    /// The IRQ number.\n    number: T,\n\n    /// Descriptive name.\n    name: &'static str,\n\n    /// Reference to handler trait object.\n    handler: &'static (dyn interface::IRQHandler + Sync),\n}\n\n/// IRQContext token.\n///\n/// An instance of this type indicates that the local core is currently executing in IRQ\n/// context, aka executing an interrupt vector or subcalls of it.\n///\n/// Concept and implementation derived from the `CriticalSection` introduced in\n/// <https://github.com/rust-embedded/bare-metal>\n#[derive(Clone, Copy)]\npub struct IRQContext<'irq_context> {\n    _0: PhantomData<&'irq_context ()>,\n}\n\n/// Asynchronous exception handling interfaces.\npub mod interface {\n\n    /// Implemented by types that handle IRQs.\n    pub trait IRQHandler {\n        /// Called when the corresponding interrupt is asserted.\n        fn handle(&self) -> Result<(), &'static str>;\n    }\n\n    /// IRQ management functions.\n    ///\n    /// The `BSP` is supposed to supply one global instance. Typically implemented by the\n    /// platform's interrupt controller.\n    pub trait IRQManager {\n        /// The IRQ number type depends on the implementation.\n        type IRQNumberType: Copy;\n\n        /// Register a handler.\n        fn register_handler(\n            &self,\n            irq_handler_descriptor: super::IRQHandlerDescriptor<Self::IRQNumberType>,\n        ) -> Result<(), &'static str>;\n\n        /// Enable an interrupt in the controller.\n        fn enable(&self, irq_number: &Self::IRQNumberType);\n\n        /// Handle pending interrupts.\n        ///\n        /// This function is called directly from the CPU's IRQ exception vector. On AArch64,\n        /// this means that the respective CPU core has disabled exception handling.\n        /// This function can therefore not be preempted and runs start to finish.\n        ///\n        /// Takes an IRQContext token to ensure it can only be called from IRQ context.\n        #[allow(clippy::trivially_copy_pass_by_ref)]\n        fn handle_pending_irqs<'irq_context>(\n            &'irq_context self,\n            ic: &super::IRQContext<'irq_context>,\n        );\n\n        /// Print list of registered handlers.\n        fn print_handler(&self) {}\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic CUR_IRQ_MANAGER: InitStateLock<\n    &'static (dyn interface::IRQManager<IRQNumberType = IRQNumber> + Sync),\n> = InitStateLock::new(&null_irq_manager::NULL_IRQ_MANAGER);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::{interface::ReadWriteEx, InitStateLock};\n\nimpl<T> IRQHandlerDescriptor<T>\nwhere\n    T: Copy,\n{\n    /// Create an instance.\n    pub const fn new(\n        number: T,\n        name: &'static str,\n        handler: &'static (dyn interface::IRQHandler + Sync),\n    ) -> Self {\n        Self {\n            number,\n            name,\n            handler,\n        }\n    }\n\n    /// Return the number.\n    pub const fn number(&self) -> T {\n        self.number\n    }\n\n    /// Return the name.\n    pub const fn name(&self) -> &'static str {\n        self.name\n    }\n\n    /// Return the handler.\n    pub const fn handler(&self) -> &'static (dyn interface::IRQHandler + Sync) {\n        self.handler\n    }\n}\n\nimpl<'irq_context> IRQContext<'irq_context> {\n    /// Creates an IRQContext token.\n    ///\n    /// # Safety\n    ///\n    /// - This must only be called when the current core is in an interrupt context and will not\n    ///   live beyond the end of it. That is, creation is allowed in interrupt vector functions. For\n    ///   example, in the ARMv8-A case, in `extern \"C\" fn current_elx_irq()`.\n    /// - Note that the lifetime `'irq_context` of the returned instance is unconstrained. User code\n    ///   must not be able to influence the lifetime picked for this type, since that might cause it\n    ///   to be inferred to `'static`.\n    #[inline(always)]\n    pub unsafe fn new() -> Self {\n        IRQContext { _0: PhantomData }\n    }\n}\n\n/// Executes the provided closure while IRQs are masked on the executing core.\n///\n/// While the function temporarily changes the HW state of the executing core, it restores it to the\n/// previous state before returning, so this is deemed safe.\n#[inline(always)]\npub fn exec_with_irq_masked<T>(f: impl FnOnce() -> T) -> T {\n    let saved = local_irq_mask_save();\n    let ret = f();\n    local_irq_restore(saved);\n\n    ret\n}\n\n/// Register a new IRQ manager.\npub fn register_irq_manager(\n    new_manager: &'static (dyn interface::IRQManager<IRQNumberType = IRQNumber> + Sync),\n) {\n    CUR_IRQ_MANAGER.write(|manager| *manager = new_manager);\n}\n\n/// Return a reference to the currently registered IRQ manager.\n///\n/// This is the IRQ manager used by the architectural interrupt handling code.\npub fn irq_manager() -> &'static dyn interface::IRQManager<IRQNumberType = IRQNumber> {\n    CUR_IRQ_MANAGER.read(|manager| *manager)\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/exception.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Synchronous and asynchronous exception handling.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/exception.rs\"]\nmod arch_exception;\n\npub mod asynchronous;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_exception::{current_privilege_level, handling_init};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Kernel privilege levels.\n#[allow(missing_docs)]\n#[derive(Eq, PartialEq)]\npub enum PrivilegeLevel {\n    User,\n    Kernel,\n    Hypervisor,\n    Unknown,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// Libkernel unit tests must execute in kernel mode.\n    #[kernel_test]\n    fn test_runner_executes_in_kernel_mode() {\n        let (level, _) = current_privilege_level();\n\n        assert!(level == PrivilegeLevel::Kernel)\n    }\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/lib.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n// Rust embedded logo for `make doc`.\n#![doc(\n    html_logo_url = \"https://raw.githubusercontent.com/rust-embedded/wg/master/assets/logo/ewg-logo-blue-white-on-transparent.png\"\n)]\n\n//! The `kernel` library.\n//!\n//! Used to compose the final kernel binary.\n//!\n//! # Code organization and architecture\n//!\n//! The code is divided into different *modules*, each representing a typical **subsystem** of the\n//! `kernel`. Top-level module files of subsystems reside directly in the `src` folder. For example,\n//! `src/memory.rs` contains code that is concerned with all things memory management.\n//!\n//! ## Visibility of processor architecture code\n//!\n//! Some of the `kernel`'s subsystems depend on low-level code that is specific to the target\n//! processor architecture. For each supported processor architecture, there exists a subfolder in\n//! `src/_arch`, for example, `src/_arch/aarch64`.\n//!\n//! The architecture folders mirror the subsystem modules laid out in `src`. For example,\n//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go\n//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in\n//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic\n//! module's name prefixed with `arch_`.\n//!\n//! For example, this is the top of `src/memory/mmu.rs`:\n//!\n//! ```\n//! #[cfg(target_arch = \"aarch64\")]\n//! #[path = \"../_arch/aarch64/memory/mmu.rs\"]\n//! mod arch_mmu;\n//! ```\n//!\n//! Often times, items from the `arch_ module` will be publicly reexported by the parent module.\n//! This way, each architecture specific module can provide its implementation of an item, while the\n//! caller must not be concerned which architecture has been conditionally compiled.\n//!\n//! ## BSP code\n//!\n//! `BSP` stands for Board Support Package. `BSP` code is organized under `src/bsp.rs` and contains\n//! target board specific definitions and functions. These are things such as the board's memory map\n//! or instances of drivers for devices that are featured on the respective board.\n//!\n//! Just like processor architecture code, the `BSP` code's module structure tries to mirror the\n//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is\n//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`.\n//!\n//! ## Kernel interfaces\n//!\n//! Both `arch` and `bsp` contain code that is conditionally compiled depending on the actual target\n//! and board for which the kernel is compiled. For example, the `interrupt controller` hardware of\n//! the `Raspberry Pi 3` and the `Raspberry Pi 4` is different, but we want the rest of the `kernel`\n//! code to play nicely with any of the two without much hassle.\n//!\n//! In order to provide a clean abstraction between `arch`, `bsp` and `generic kernel code`,\n//! `interface` traits are provided *whenever possible* and *where it makes sense*. They are defined\n//! in the respective subsystem module and help to enforce the idiom of *program to an interface,\n//! not an implementation*. For example, there will be a common IRQ handling interface which the two\n//! different interrupt controller `drivers` of both Raspberrys will implement, and only export the\n//! interface to the rest of the `kernel`.\n//!\n//! ```\n//!         +-------------------+\n//!         | Interface (Trait) |\n//!         |                   |\n//!         +--+-------------+--+\n//!            ^             ^\n//!            |             |\n//!            |             |\n//! +----------+--+       +--+----------+\n//! | kernel code |       |  bsp code   |\n//! |             |       |  arch code  |\n//! +-------------+       +-------------+\n//! ```\n//!\n//! # Summary\n//!\n//! For a logical `kernel` subsystem, corresponding code can be distributed over several physical\n//! locations. Here is an example for the **memory** subsystem:\n//!\n//! - `src/memory.rs` and `src/memory/**/*`\n//!   - Common code that is agnostic of target processor architecture and `BSP` characteristics.\n//!     - Example: A function to zero a chunk of memory.\n//!   - Interfaces for the memory subsystem that are implemented by `arch` or `BSP` code.\n//!     - Example: An `MMU` interface that defines `MMU` function prototypes.\n//! - `src/bsp/__board_name__/memory.rs` and `src/bsp/__board_name__/memory/**/*`\n//!   - `BSP` specific code.\n//!   - Example: The board's memory map (physical addresses of DRAM and MMIO devices).\n//! - `src/_arch/__arch_name__/memory.rs` and `src/_arch/__arch_name__/memory/**/*`\n//!   - Processor architecture specific code.\n//!   - Example: Implementation of the `MMU` interface for the `__arch_name__` processor\n//!     architecture.\n//!\n//! From a namespace perspective, **memory** subsystem code lives in:\n//!\n//! - `crate::memory::*`\n//! - `crate::bsp::memory::*`\n//!\n//! # Boot flow\n//!\n//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.\n//!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.\n//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.\n\n#![allow(clippy::upper_case_acronyms)]\n#![allow(incomplete_features)]\n#![feature(asm_const)]\n#![feature(const_option)]\n#![feature(core_intrinsics)]\n#![feature(format_args_nl)]\n#![feature(generic_const_exprs)]\n#![feature(int_roundings)]\n#![feature(is_sorted)]\n#![feature(linkage)]\n#![feature(nonzero_min_max)]\n#![feature(panic_info_message)]\n#![feature(step_trait)]\n#![feature(trait_alias)]\n#![feature(unchecked_math)]\n#![no_std]\n// Testing\n#![cfg_attr(test, no_main)]\n#![feature(custom_test_frameworks)]\n#![reexport_test_harness_main = \"test_main\"]\n#![test_runner(crate::test_runner)]\n\nmod panic_wait;\nmod synchronization;\n\npub mod bsp;\npub mod common;\npub mod console;\npub mod cpu;\npub mod driver;\npub mod exception;\npub mod memory;\npub mod print;\npub mod state;\npub mod time;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Version string.\npub fn version() -> &'static str {\n    concat!(\n        env!(\"CARGO_PKG_NAME\"),\n        \" version \",\n        env!(\"CARGO_PKG_VERSION\")\n    )\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n/// The default runner for unit tests.\npub fn test_runner(tests: &[&test_types::UnitTest]) {\n    // This line will be printed as the test header.\n    println!(\"Running {} tests\", tests.len());\n\n    for (i, test) in tests.iter().enumerate() {\n        print!(\"{:>3}. {:.<58}\", i + 1, test.name);\n\n        // Run the actual test.\n        (test.test_func)();\n\n        // Failed tests call panic!(). Execution reaches here only if the test has passed.\n        println!(\"[ok]\")\n    }\n}\n\n/// The `kernel_init()` for unit tests.\n#[cfg(test)]\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    test_main();\n\n    cpu::qemu_exit_success()\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/main.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n// Rust embedded logo for `make doc`.\n#![doc(\n    html_logo_url = \"https://raw.githubusercontent.com/rust-embedded/wg/master/assets/logo/ewg-logo-blue-white-on-transparent.png\"\n)]\n\n//! The `kernel` binary.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\nuse libkernel::{bsp, cpu, driver, exception, info, memory, state, time};\n\n/// Early init code.\n///\n/// When this code runs, virtual memory is already enabled.\n///\n/// # Safety\n///\n/// - Only a single core must be active and running this function.\n/// - Printing will not work until the respective driver's MMIO is remapped.\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n\n    // Initialize the BSP driver subsystem.\n    if let Err(x) = bsp::driver::init() {\n        panic!(\"Error initializing BSP driver subsystem: {}\", x);\n    }\n\n    // Initialize all device drivers.\n    driver::driver_manager().init_drivers_and_irqs();\n\n    bsp::memory::mmu::kernel_add_mapping_records_for_precomputed();\n\n    // Unmask interrupts on the boot CPU core.\n    exception::asynchronous::local_irq_unmask();\n\n    // Announce conclusion of the kernel_init() phase.\n    state::state_manager().transition_to_single_core_main();\n\n    // Transition from unsafe to safe.\n    kernel_main()\n}\n\n/// The main function running after the early init.\nfn kernel_main() -> ! {\n    info!(\"{}\", libkernel::version());\n    info!(\"Booting on: {}\", bsp::board_name());\n\n    info!(\"MMU online:\");\n    memory::mmu::kernel_print_mappings();\n\n    let (_, privilege_level) = exception::current_privilege_level();\n    info!(\"Current privilege level: {}\", privilege_level);\n\n    info!(\"Exception handling state:\");\n    exception::asynchronous::print_state();\n\n    info!(\n        \"Architectural timer resolution: {} ns\",\n        time::time_manager().resolution().as_nanos()\n    );\n\n    info!(\"Drivers loaded:\");\n    driver::driver_manager().enumerate();\n\n    info!(\"Registered IRQ handlers:\");\n    exception::asynchronous::irq_manager().print_handler();\n\n    info!(\"Echoing input now\");\n    cpu::wait_forever();\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/memory/mmu/mapping_record.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A record of mapped pages.\n\nuse super::{\n    AccessPermissions, Address, AttributeFields, MMIODescriptor, MemAttributes, MemoryRegion,\n    Physical, Virtual,\n};\nuse crate::{bsp, common, info, synchronization, synchronization::InitStateLock, warn};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Type describing a virtual memory mapping.\n#[allow(missing_docs)]\n#[derive(Copy, Clone)]\nstruct MappingRecordEntry {\n    pub users: [Option<&'static str>; 5],\n    pub phys_start_addr: Address<Physical>,\n    pub virt_start_addr: Address<Virtual>,\n    pub num_pages: usize,\n    pub attribute_fields: AttributeFields,\n}\n\nstruct MappingRecord {\n    inner: [Option<MappingRecordEntry>; 12],\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic KERNEL_MAPPING_RECORD: InitStateLock<MappingRecord> =\n    InitStateLock::new(MappingRecord::new());\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl MappingRecordEntry {\n    pub fn new(\n        name: &'static str,\n        virt_region: &MemoryRegion<Virtual>,\n        phys_region: &MemoryRegion<Physical>,\n        attr: &AttributeFields,\n    ) -> Self {\n        Self {\n            users: [Some(name), None, None, None, None],\n            phys_start_addr: phys_region.start_addr(),\n            virt_start_addr: virt_region.start_addr(),\n            num_pages: phys_region.num_pages(),\n            attribute_fields: *attr,\n        }\n    }\n\n    fn find_next_free_user(&mut self) -> Result<&mut Option<&'static str>, &'static str> {\n        if let Some(x) = self.users.iter_mut().find(|x| x.is_none()) {\n            return Ok(x);\n        };\n\n        Err(\"Storage for user info exhausted\")\n    }\n\n    pub fn add_user(&mut self, user: &'static str) -> Result<(), &'static str> {\n        let x = self.find_next_free_user()?;\n        *x = Some(user);\n        Ok(())\n    }\n}\n\nimpl MappingRecord {\n    pub const fn new() -> Self {\n        Self { inner: [None; 12] }\n    }\n\n    fn size(&self) -> usize {\n        self.inner.iter().filter(|x| x.is_some()).count()\n    }\n\n    fn sort(&mut self) {\n        let upper_bound_exclusive = self.size();\n        let entries = &mut self.inner[0..upper_bound_exclusive];\n\n        if !entries.is_sorted_by_key(|item| item.unwrap().virt_start_addr) {\n            entries.sort_unstable_by_key(|item| item.unwrap().virt_start_addr)\n        }\n    }\n\n    fn find_next_free(&mut self) -> Result<&mut Option<MappingRecordEntry>, &'static str> {\n        if let Some(x) = self.inner.iter_mut().find(|x| x.is_none()) {\n            return Ok(x);\n        }\n\n        Err(\"Storage for mapping info exhausted\")\n    }\n\n    fn find_duplicate(\n        &mut self,\n        phys_region: &MemoryRegion<Physical>,\n    ) -> Option<&mut MappingRecordEntry> {\n        self.inner\n            .iter_mut()\n            .filter_map(|x| x.as_mut())\n            .filter(|x| x.attribute_fields.mem_attributes == MemAttributes::Device)\n            .find(|x| {\n                if x.phys_start_addr != phys_region.start_addr() {\n                    return false;\n                }\n\n                if x.num_pages != phys_region.num_pages() {\n                    return false;\n                }\n\n                true\n            })\n    }\n\n    pub fn add(\n        &mut self,\n        name: &'static str,\n        virt_region: &MemoryRegion<Virtual>,\n        phys_region: &MemoryRegion<Physical>,\n        attr: &AttributeFields,\n    ) -> Result<(), &'static str> {\n        let x = self.find_next_free()?;\n\n        *x = Some(MappingRecordEntry::new(\n            name,\n            virt_region,\n            phys_region,\n            attr,\n        ));\n\n        self.sort();\n\n        Ok(())\n    }\n\n    pub fn print(&self) {\n        info!(\"      -------------------------------------------------------------------------------------------------------------------------------------------\");\n        info!(\n            \"      {:^44}     {:^30}   {:^7}   {:^9}   {:^35}\",\n            \"Virtual\", \"Physical\", \"Size\", \"Attr\", \"Entity\"\n        );\n        info!(\"      -------------------------------------------------------------------------------------------------------------------------------------------\");\n\n        for i in self.inner.iter().flatten() {\n            let size = i.num_pages * bsp::memory::mmu::KernelGranule::SIZE;\n            let virt_start = i.virt_start_addr;\n            let virt_end_inclusive = virt_start + (size - 1);\n            let phys_start = i.phys_start_addr;\n            let phys_end_inclusive = phys_start + (size - 1);\n\n            let (size, unit) = common::size_human_readable_ceil(size);\n\n            let attr = match i.attribute_fields.mem_attributes {\n                MemAttributes::CacheableDRAM => \"C\",\n                MemAttributes::Device => \"Dev\",\n            };\n\n            let acc_p = match i.attribute_fields.acc_perms {\n                AccessPermissions::ReadOnly => \"RO\",\n                AccessPermissions::ReadWrite => \"RW\",\n            };\n\n            let xn = if i.attribute_fields.execute_never {\n                \"XN\"\n            } else {\n                \"X\"\n            };\n\n            info!(\n                \"      {}..{} --> {}..{} | {:>3} {} | {:<3} {} {:<2} | {}\",\n                virt_start,\n                virt_end_inclusive,\n                phys_start,\n                phys_end_inclusive,\n                size,\n                unit,\n                attr,\n                acc_p,\n                xn,\n                i.users[0].unwrap()\n            );\n\n            for k in i.users[1..].iter() {\n                if let Some(additional_user) = *k {\n                    info!(\n                        \"                                                                                                            | {}\",\n                        additional_user\n                    );\n                }\n            }\n        }\n\n        info!(\"      -------------------------------------------------------------------------------------------------------------------------------------------\");\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::interface::ReadWriteEx;\n\n/// Add an entry to the mapping info record.\npub fn kernel_add(\n    name: &'static str,\n    virt_region: &MemoryRegion<Virtual>,\n    phys_region: &MemoryRegion<Physical>,\n    attr: &AttributeFields,\n) -> Result<(), &'static str> {\n    KERNEL_MAPPING_RECORD.write(|mr| mr.add(name, virt_region, phys_region, attr))\n}\n\npub fn kernel_find_and_insert_mmio_duplicate(\n    mmio_descriptor: &MMIODescriptor,\n    new_user: &'static str,\n) -> Option<Address<Virtual>> {\n    let phys_region: MemoryRegion<Physical> = (*mmio_descriptor).into();\n\n    KERNEL_MAPPING_RECORD.write(|mr| {\n        let dup = mr.find_duplicate(&phys_region)?;\n\n        if let Err(x) = dup.add_user(new_user) {\n            warn!(\"{}\", x);\n        }\n\n        Some(dup.virt_start_addr)\n    })\n}\n\n/// Human-readable print of all recorded kernel mappings.\npub fn kernel_print() {\n    KERNEL_MAPPING_RECORD.read(|mr| mr.print());\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/memory/mmu/page_alloc.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Page allocation.\n\nuse super::MemoryRegion;\nuse crate::{\n    memory::{AddressType, Virtual},\n    synchronization::IRQSafeNullLock,\n    warn,\n};\nuse core::num::NonZeroUsize;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// A page allocator that can be lazyily initialized.\npub struct PageAllocator<ATYPE: AddressType> {\n    pool: Option<MemoryRegion<ATYPE>>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic KERNEL_MMIO_VA_ALLOCATOR: IRQSafeNullLock<PageAllocator<Virtual>> =\n    IRQSafeNullLock::new(PageAllocator::new());\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the kernel's MMIO virtual address allocator.\npub fn kernel_mmio_va_allocator() -> &'static IRQSafeNullLock<PageAllocator<Virtual>> {\n    &KERNEL_MMIO_VA_ALLOCATOR\n}\n\nimpl<ATYPE: AddressType> PageAllocator<ATYPE> {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self { pool: None }\n    }\n\n    /// Initialize the allocator.\n    pub fn init(&mut self, pool: MemoryRegion<ATYPE>) {\n        if self.pool.is_some() {\n            warn!(\"Already initialized\");\n            return;\n        }\n\n        self.pool = Some(pool);\n    }\n\n    /// Allocate a number of pages.\n    pub fn alloc(\n        &mut self,\n        num_requested_pages: NonZeroUsize,\n    ) -> Result<MemoryRegion<ATYPE>, &'static str> {\n        if self.pool.is_none() {\n            return Err(\"Allocator not initialized\");\n        }\n\n        self.pool\n            .as_mut()\n            .unwrap()\n            .take_first_n_pages(num_requested_pages)\n    }\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/memory/mmu/translation_table.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Translation table.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../../_arch/aarch64/memory/mmu/translation_table.rs\"]\nmod arch_translation_table;\n\nuse super::{AttributeFields, MemoryRegion};\nuse crate::memory::{Address, Physical, Virtual};\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\n#[cfg(target_arch = \"aarch64\")]\npub use arch_translation_table::FixedSizeTranslationTable;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Translation table interfaces.\npub mod interface {\n    use crate::memory::mmu::PageAddress;\n\n    use super::*;\n\n    /// Translation table operations.\n    pub trait TranslationTable {\n        /// Anything that needs to run before any of the other provided functions can be used.\n        ///\n        /// # Safety\n        ///\n        /// - Implementor must ensure that this function can run only once or is harmless if invoked\n        ///   multiple times.\n        fn init(&mut self) -> Result<(), &'static str>;\n\n        /// Map the given virtual memory region to the given physical memory region.\n        ///\n        /// # Safety\n        ///\n        /// - Using wrong attributes can cause multiple issues of different nature in the system.\n        /// - It is not required that the architectural implementation prevents aliasing. That is,\n        ///   mapping to the same physical memory using multiple virtual addresses, which would\n        ///   break Rust's ownership assumptions. This should be protected against in the kernel's\n        ///   generic MMU code.\n        unsafe fn map_at(\n            &mut self,\n            virt_region: &MemoryRegion<Virtual>,\n            phys_region: &MemoryRegion<Physical>,\n            attr: &AttributeFields,\n        ) -> Result<(), &'static str>;\n\n        /// Try to translate a virtual page address to a physical page address.\n        ///\n        /// Will only succeed if there exists a valid mapping for the input page.\n        fn try_virt_page_addr_to_phys_page_addr(\n            &self,\n            virt_page_addr: PageAddress<Virtual>,\n        ) -> Result<PageAddress<Physical>, &'static str>;\n\n        /// Try to get the attributes of a page.\n        ///\n        /// Will only succeed if there exists a valid mapping for the input page.\n        fn try_page_attributes(\n            &self,\n            virt_page_addr: PageAddress<Virtual>,\n        ) -> Result<AttributeFields, &'static str>;\n\n        /// Try to translate a virtual address to a physical address.\n        ///\n        /// Will only succeed if there exists a valid mapping for the input address.\n        fn try_virt_addr_to_phys_addr(\n            &self,\n            virt_addr: Address<Virtual>,\n        ) -> Result<Address<Physical>, &'static str>;\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use crate::memory::mmu::{AccessPermissions, MemAttributes, PageAddress};\n    use arch_translation_table::MinSizeTranslationTable;\n    use interface::TranslationTable;\n    use test_macros::kernel_test;\n\n    /// Sanity checks for the TranslationTable implementation.\n    #[kernel_test]\n    fn translationtable_implementation_sanity() {\n        // This will occupy a lot of space on the stack.\n        let mut tables = MinSizeTranslationTable::new_for_runtime();\n\n        assert_eq!(tables.init(), Ok(()));\n\n        let virt_end_exclusive_page_addr: PageAddress<Virtual> = PageAddress::MAX;\n        let virt_start_page_addr: PageAddress<Virtual> =\n            virt_end_exclusive_page_addr.checked_offset(-5).unwrap();\n\n        let phys_start_page_addr: PageAddress<Physical> = PageAddress::from(0);\n        let phys_end_exclusive_page_addr: PageAddress<Physical> =\n            phys_start_page_addr.checked_offset(5).unwrap();\n\n        let virt_region = MemoryRegion::new(virt_start_page_addr, virt_end_exclusive_page_addr);\n        let phys_region = MemoryRegion::new(phys_start_page_addr, phys_end_exclusive_page_addr);\n\n        let attr = AttributeFields {\n            mem_attributes: MemAttributes::CacheableDRAM,\n            acc_perms: AccessPermissions::ReadWrite,\n            execute_never: true,\n        };\n\n        unsafe { assert_eq!(tables.map_at(&virt_region, &phys_region, &attr), Ok(())) };\n\n        assert_eq!(\n            tables.try_virt_page_addr_to_phys_page_addr(virt_start_page_addr),\n            Ok(phys_start_page_addr)\n        );\n\n        assert_eq!(\n            tables.try_page_attributes(virt_start_page_addr.checked_offset(-1).unwrap()),\n            Err(\"Page marked invalid\")\n        );\n\n        assert_eq!(tables.try_page_attributes(virt_start_page_addr), Ok(attr));\n\n        let virt_addr = virt_start_page_addr.into_inner() + 0x100;\n        let phys_addr = phys_start_page_addr.into_inner() + 0x100;\n        assert_eq!(tables.try_virt_addr_to_phys_addr(virt_addr), Ok(phys_addr));\n    }\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/memory/mmu/types.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management Unit types.\n\nuse crate::{\n    bsp, common,\n    memory::{Address, AddressType, Physical},\n};\nuse core::{convert::From, iter::Step, num::NonZeroUsize, ops::Range};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// A wrapper type around [Address] that ensures page alignment.\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub struct PageAddress<ATYPE: AddressType> {\n    inner: Address<ATYPE>,\n}\n\n/// A type that describes a region of memory in quantities of pages.\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub struct MemoryRegion<ATYPE: AddressType> {\n    start: PageAddress<ATYPE>,\n    end_exclusive: PageAddress<ATYPE>,\n}\n\n/// Architecture agnostic memory attributes.\n#[allow(missing_docs)]\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub enum MemAttributes {\n    CacheableDRAM,\n    Device,\n}\n\n/// Architecture agnostic access permissions.\n#[allow(missing_docs)]\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub enum AccessPermissions {\n    ReadOnly,\n    ReadWrite,\n}\n\n/// Collection of memory attributes.\n#[allow(missing_docs)]\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub struct AttributeFields {\n    pub mem_attributes: MemAttributes,\n    pub acc_perms: AccessPermissions,\n    pub execute_never: bool,\n}\n\n/// An MMIO descriptor for use in device drivers.\n#[derive(Copy, Clone)]\npub struct MMIODescriptor {\n    start_addr: Address<Physical>,\n    end_addr_exclusive: Address<Physical>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n//------------------------------------------------------------------------------\n// PageAddress\n//------------------------------------------------------------------------------\nimpl<ATYPE: AddressType> PageAddress<ATYPE> {\n    /// The largest value that can be represented by this type.\n    pub const MAX: Self = PageAddress {\n        inner: Address::new(usize::MAX).align_down_page(),\n    };\n\n    /// Unwraps the value.\n    pub fn into_inner(self) -> Address<ATYPE> {\n        self.inner\n    }\n\n    /// Calculates the offset from the page address.\n    ///\n    /// `count` is in units of [PageAddress]. For example, a count of 2 means `result = self + 2 *\n    /// page_size`.\n    pub fn checked_offset(self, count: isize) -> Option<Self> {\n        if count == 0 {\n            return Some(self);\n        }\n\n        let delta = count\n            .unsigned_abs()\n            .checked_mul(bsp::memory::mmu::KernelGranule::SIZE)?;\n        let result = if count.is_positive() {\n            self.inner.as_usize().checked_add(delta)?\n        } else {\n            self.inner.as_usize().checked_sub(delta)?\n        };\n\n        Some(Self {\n            inner: Address::new(result),\n        })\n    }\n}\n\nimpl<ATYPE: AddressType> From<usize> for PageAddress<ATYPE> {\n    fn from(addr: usize) -> Self {\n        assert!(\n            common::is_aligned(addr, bsp::memory::mmu::KernelGranule::SIZE),\n            \"Input usize not page aligned\"\n        );\n\n        Self {\n            inner: Address::new(addr),\n        }\n    }\n}\n\nimpl<ATYPE: AddressType> From<Address<ATYPE>> for PageAddress<ATYPE> {\n    fn from(addr: Address<ATYPE>) -> Self {\n        assert!(addr.is_page_aligned(), \"Input Address not page aligned\");\n\n        Self { inner: addr }\n    }\n}\n\nimpl<ATYPE: AddressType> Step for PageAddress<ATYPE> {\n    fn steps_between(start: &Self, end: &Self) -> Option<usize> {\n        if start > end {\n            return None;\n        }\n\n        // Since start <= end, do unchecked arithmetic.\n        Some(\n            (end.inner.as_usize() - start.inner.as_usize())\n                >> bsp::memory::mmu::KernelGranule::SHIFT,\n        )\n    }\n\n    fn forward_checked(start: Self, count: usize) -> Option<Self> {\n        start.checked_offset(count as isize)\n    }\n\n    fn backward_checked(start: Self, count: usize) -> Option<Self> {\n        start.checked_offset(-(count as isize))\n    }\n}\n\n//------------------------------------------------------------------------------\n// MemoryRegion\n//------------------------------------------------------------------------------\nimpl<ATYPE: AddressType> MemoryRegion<ATYPE> {\n    /// Create an instance.\n    pub fn new(start: PageAddress<ATYPE>, end_exclusive: PageAddress<ATYPE>) -> Self {\n        assert!(start <= end_exclusive);\n\n        Self {\n            start,\n            end_exclusive,\n        }\n    }\n\n    fn as_range(&self) -> Range<PageAddress<ATYPE>> {\n        self.into_iter()\n    }\n\n    /// Returns the start page address.\n    pub fn start_page_addr(&self) -> PageAddress<ATYPE> {\n        self.start\n    }\n\n    /// Returns the start address.\n    pub fn start_addr(&self) -> Address<ATYPE> {\n        self.start.into_inner()\n    }\n\n    /// Returns the exclusive end page address.\n    pub fn end_exclusive_page_addr(&self) -> PageAddress<ATYPE> {\n        self.end_exclusive\n    }\n\n    /// Returns the exclusive end page address.\n    pub fn end_inclusive_page_addr(&self) -> PageAddress<ATYPE> {\n        self.end_exclusive.checked_offset(-1).unwrap()\n    }\n\n    /// Checks if self contains an address.\n    pub fn contains(&self, addr: Address<ATYPE>) -> bool {\n        let page_addr = PageAddress::from(addr.align_down_page());\n        self.as_range().contains(&page_addr)\n    }\n\n    /// Checks if there is an overlap with another memory region.\n    pub fn overlaps(&self, other_region: &Self) -> bool {\n        let self_range = self.as_range();\n\n        self_range.contains(&other_region.start_page_addr())\n            || self_range.contains(&other_region.end_inclusive_page_addr())\n    }\n\n    /// Returns the number of pages contained in this region.\n    pub fn num_pages(&self) -> usize {\n        PageAddress::steps_between(&self.start, &self.end_exclusive).unwrap()\n    }\n\n    /// Returns the size in bytes of this region.\n    pub fn size(&self) -> usize {\n        // Invariant: start <= end_exclusive, so do unchecked arithmetic.\n        let end_exclusive = self.end_exclusive.into_inner().as_usize();\n        let start = self.start.into_inner().as_usize();\n\n        end_exclusive - start\n    }\n\n    /// Splits the MemoryRegion like:\n    ///\n    /// --------------------------------------------------------------------------------\n    /// |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |\n    /// --------------------------------------------------------------------------------\n    ///   ^                               ^                                       ^\n    ///   |                               |                                       |\n    ///   left_start     left_end_exclusive                                       |\n    ///                                                                           |\n    ///                                   ^                                       |\n    ///                                   |                                       |\n    ///                                   right_start           right_end_exclusive\n    ///\n    /// Left region is returned to the caller. Right region is the new region for this struct.\n    pub fn take_first_n_pages(&mut self, num_pages: NonZeroUsize) -> Result<Self, &'static str> {\n        let count: usize = num_pages.into();\n\n        let left_end_exclusive = self.start.checked_offset(count as isize);\n        let left_end_exclusive = match left_end_exclusive {\n            None => return Err(\"Overflow while calculating left_end_exclusive\"),\n            Some(x) => x,\n        };\n\n        if left_end_exclusive > self.end_exclusive {\n            return Err(\"Not enough free pages\");\n        }\n\n        let allocation = Self {\n            start: self.start,\n            end_exclusive: left_end_exclusive,\n        };\n        self.start = left_end_exclusive;\n\n        Ok(allocation)\n    }\n}\n\nimpl<ATYPE: AddressType> IntoIterator for MemoryRegion<ATYPE> {\n    type Item = PageAddress<ATYPE>;\n    type IntoIter = Range<Self::Item>;\n\n    fn into_iter(self) -> Self::IntoIter {\n        Range {\n            start: self.start,\n            end: self.end_exclusive,\n        }\n    }\n}\n\nimpl From<MMIODescriptor> for MemoryRegion<Physical> {\n    fn from(desc: MMIODescriptor) -> Self {\n        let start = PageAddress::from(desc.start_addr.align_down_page());\n        let end_exclusive = PageAddress::from(desc.end_addr_exclusive().align_up_page());\n\n        Self {\n            start,\n            end_exclusive,\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// MMIODescriptor\n//------------------------------------------------------------------------------\n\nimpl MMIODescriptor {\n    /// Create an instance.\n    pub const fn new(start_addr: Address<Physical>, size: usize) -> Self {\n        assert!(size > 0);\n        let end_addr_exclusive = Address::new(start_addr.as_usize() + size);\n\n        Self {\n            start_addr,\n            end_addr_exclusive,\n        }\n    }\n\n    /// Return the start address.\n    pub const fn start_addr(&self) -> Address<Physical> {\n        self.start_addr\n    }\n\n    /// Return the exclusive end address.\n    pub fn end_addr_exclusive(&self) -> Address<Physical> {\n        self.end_addr_exclusive\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use crate::memory::Virtual;\n    use test_macros::kernel_test;\n\n    /// Sanity of [PageAddress] methods.\n    #[kernel_test]\n    fn pageaddress_type_method_sanity() {\n        let page_addr: PageAddress<Virtual> =\n            PageAddress::from(bsp::memory::mmu::KernelGranule::SIZE * 2);\n\n        assert_eq!(\n            page_addr.checked_offset(-2),\n            Some(PageAddress::<Virtual>::from(0))\n        );\n\n        assert_eq!(\n            page_addr.checked_offset(2),\n            Some(PageAddress::<Virtual>::from(\n                bsp::memory::mmu::KernelGranule::SIZE * 4\n            ))\n        );\n\n        assert_eq!(\n            PageAddress::<Virtual>::from(0).checked_offset(0),\n            Some(PageAddress::<Virtual>::from(0))\n        );\n        assert_eq!(PageAddress::<Virtual>::from(0).checked_offset(-1), None);\n\n        let max_page_addr = Address::<Virtual>::new(usize::MAX).align_down_page();\n        assert_eq!(\n            PageAddress::<Virtual>::from(max_page_addr).checked_offset(1),\n            None\n        );\n\n        let zero = PageAddress::<Virtual>::from(0);\n        let three = PageAddress::<Virtual>::from(bsp::memory::mmu::KernelGranule::SIZE * 3);\n        assert_eq!(PageAddress::steps_between(&zero, &three), Some(3));\n    }\n\n    /// Sanity of [MemoryRegion] methods.\n    #[kernel_test]\n    fn memoryregion_type_method_sanity() {\n        let zero = PageAddress::<Virtual>::from(0);\n        let zero_region = MemoryRegion::new(zero, zero);\n        assert_eq!(zero_region.num_pages(), 0);\n        assert_eq!(zero_region.size(), 0);\n\n        let one = PageAddress::<Virtual>::from(bsp::memory::mmu::KernelGranule::SIZE);\n        let one_region = MemoryRegion::new(zero, one);\n        assert_eq!(one_region.num_pages(), 1);\n        assert_eq!(one_region.size(), bsp::memory::mmu::KernelGranule::SIZE);\n\n        let three = PageAddress::<Virtual>::from(bsp::memory::mmu::KernelGranule::SIZE * 3);\n        let mut three_region = MemoryRegion::new(zero, three);\n        assert!(three_region.contains(zero.into_inner()));\n        assert!(!three_region.contains(three.into_inner()));\n        assert!(three_region.overlaps(&one_region));\n\n        let allocation = three_region\n            .take_first_n_pages(NonZeroUsize::new(2).unwrap())\n            .unwrap();\n        assert_eq!(allocation.num_pages(), 2);\n        assert_eq!(three_region.num_pages(), 1);\n\n        for (i, alloc) in allocation.into_iter().enumerate() {\n            assert_eq!(\n                alloc.into_inner().as_usize(),\n                i * bsp::memory::mmu::KernelGranule::SIZE\n            );\n        }\n    }\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/memory/mmu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management Unit.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/memory/mmu.rs\"]\nmod arch_mmu;\n\nmod mapping_record;\nmod page_alloc;\nmod translation_table;\nmod types;\n\nuse crate::{\n    bsp,\n    memory::{Address, Physical, Virtual},\n    synchronization::{self, interface::Mutex},\n    warn,\n};\nuse core::{fmt, num::NonZeroUsize};\n\npub use types::*;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// MMU enable errors variants.\n#[allow(missing_docs)]\n#[derive(Debug)]\npub enum MMUEnableError {\n    AlreadyEnabled,\n    Other(&'static str),\n}\n\n/// Memory Management interfaces.\npub mod interface {\n    use super::*;\n\n    /// MMU functions.\n    pub trait MMU {\n        /// Turns on the MMU for the first time and enables data and instruction caching.\n        ///\n        /// # Safety\n        ///\n        /// - Changes the HW's global state.\n        unsafe fn enable_mmu_and_caching(\n            &self,\n            phys_tables_base_addr: Address<Physical>,\n        ) -> Result<(), MMUEnableError>;\n\n        /// Returns true if the MMU is enabled, false otherwise.\n        fn is_enabled(&self) -> bool;\n    }\n}\n\n/// Describes the characteristics of a translation granule.\npub struct TranslationGranule<const GRANULE_SIZE: usize>;\n\n/// Describes properties of an address space.\npub struct AddressSpace<const AS_SIZE: usize>;\n\n/// Intended to be implemented for [`AddressSpace`].\npub trait AssociatedTranslationTable {\n    /// A translation table whose address range is:\n    ///\n    /// [u64::MAX, (u64::MAX - AS_SIZE) + 1]\n    type TableStartFromTop;\n\n    /// A translation table whose address range is:\n    ///\n    /// [AS_SIZE - 1, 0]\n    type TableStartFromBottom;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\nuse interface::MMU;\nuse synchronization::interface::ReadWriteEx;\nuse translation_table::interface::TranslationTable;\n\n/// Map a region in the kernel's translation tables.\n///\n/// No input checks done, input is passed through to the architectural implementation.\n///\n/// # Safety\n///\n/// - See `map_at()`.\n/// - Does not prevent aliasing.\nunsafe fn kernel_map_at_unchecked(\n    name: &'static str,\n    virt_region: &MemoryRegion<Virtual>,\n    phys_region: &MemoryRegion<Physical>,\n    attr: &AttributeFields,\n) -> Result<(), &'static str> {\n    bsp::memory::mmu::kernel_translation_tables()\n        .write(|tables| tables.map_at(virt_region, phys_region, attr))?;\n\n    kernel_add_mapping_record(name, virt_region, phys_region, attr);\n\n    Ok(())\n}\n\n/// Try to translate a kernel virtual address to a physical address.\n///\n/// Will only succeed if there exists a valid mapping for the input address.\nfn try_kernel_virt_addr_to_phys_addr(\n    virt_addr: Address<Virtual>,\n) -> Result<Address<Physical>, &'static str> {\n    bsp::memory::mmu::kernel_translation_tables()\n        .read(|tables| tables.try_virt_addr_to_phys_addr(virt_addr))\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl fmt::Display for MMUEnableError {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        match self {\n            MMUEnableError::AlreadyEnabled => write!(f, \"MMU is already enabled\"),\n            MMUEnableError::Other(x) => write!(f, \"{}\", x),\n        }\n    }\n}\n\nimpl<const GRANULE_SIZE: usize> TranslationGranule<GRANULE_SIZE> {\n    /// The granule's size.\n    pub const SIZE: usize = Self::size_checked();\n\n    /// The granule's mask.\n    pub const MASK: usize = Self::SIZE - 1;\n\n    /// The granule's shift, aka log2(size).\n    pub const SHIFT: usize = Self::SIZE.trailing_zeros() as usize;\n\n    const fn size_checked() -> usize {\n        assert!(GRANULE_SIZE.is_power_of_two());\n\n        GRANULE_SIZE\n    }\n}\n\nimpl<const AS_SIZE: usize> AddressSpace<AS_SIZE> {\n    /// The address space size.\n    pub const SIZE: usize = Self::size_checked();\n\n    /// The address space shift, aka log2(size).\n    pub const SIZE_SHIFT: usize = Self::SIZE.trailing_zeros() as usize;\n\n    const fn size_checked() -> usize {\n        assert!(AS_SIZE.is_power_of_two());\n\n        // Check for architectural restrictions as well.\n        Self::arch_address_space_size_sanity_checks();\n\n        AS_SIZE\n    }\n}\n\n/// Query the BSP for the reserved virtual addresses for MMIO remapping and initialize the kernel's\n/// MMIO VA allocator with it.\npub fn kernel_init_mmio_va_allocator() {\n    let region = bsp::memory::mmu::virt_mmio_remap_region();\n\n    page_alloc::kernel_mmio_va_allocator().lock(|allocator| allocator.init(region));\n}\n\n/// Add an entry to the mapping info record.\npub fn kernel_add_mapping_record(\n    name: &'static str,\n    virt_region: &MemoryRegion<Virtual>,\n    phys_region: &MemoryRegion<Physical>,\n    attr: &AttributeFields,\n) {\n    if let Err(x) = mapping_record::kernel_add(name, virt_region, phys_region, attr) {\n        warn!(\"{}\", x);\n    }\n}\n\n/// MMIO remapping in the kernel translation tables.\n///\n/// Typically used by device drivers.\n///\n/// # Safety\n///\n/// - Same as `kernel_map_at_unchecked()`, minus the aliasing part.\npub unsafe fn kernel_map_mmio(\n    name: &'static str,\n    mmio_descriptor: &MMIODescriptor,\n) -> Result<Address<Virtual>, &'static str> {\n    let phys_region = MemoryRegion::from(*mmio_descriptor);\n    let offset_into_start_page = mmio_descriptor.start_addr().offset_into_page();\n\n    // Check if an identical region has been mapped for another driver. If so, reuse it.\n    let virt_addr = if let Some(addr) =\n        mapping_record::kernel_find_and_insert_mmio_duplicate(mmio_descriptor, name)\n    {\n        addr\n    // Otherwise, allocate a new region and map it.\n    } else {\n        let num_pages = match NonZeroUsize::new(phys_region.num_pages()) {\n            None => return Err(\"Requested 0 pages\"),\n            Some(x) => x,\n        };\n\n        let virt_region =\n            page_alloc::kernel_mmio_va_allocator().lock(|allocator| allocator.alloc(num_pages))?;\n\n        kernel_map_at_unchecked(\n            name,\n            &virt_region,\n            &phys_region,\n            &AttributeFields {\n                mem_attributes: MemAttributes::Device,\n                acc_perms: AccessPermissions::ReadWrite,\n                execute_never: true,\n            },\n        )?;\n\n        virt_region.start_addr()\n    };\n\n    Ok(virt_addr + offset_into_start_page)\n}\n\n/// Try to translate a kernel virtual page address to a physical page address.\n///\n/// Will only succeed if there exists a valid mapping for the input page.\npub fn try_kernel_virt_page_addr_to_phys_page_addr(\n    virt_page_addr: PageAddress<Virtual>,\n) -> Result<PageAddress<Physical>, &'static str> {\n    bsp::memory::mmu::kernel_translation_tables()\n        .read(|tables| tables.try_virt_page_addr_to_phys_page_addr(virt_page_addr))\n}\n\n/// Try to get the attributes of a kernel page.\n///\n/// Will only succeed if there exists a valid mapping for the input page.\npub fn try_kernel_page_attributes(\n    virt_page_addr: PageAddress<Virtual>,\n) -> Result<AttributeFields, &'static str> {\n    bsp::memory::mmu::kernel_translation_tables()\n        .read(|tables| tables.try_page_attributes(virt_page_addr))\n}\n\n/// Human-readable print of all recorded kernel mappings.\npub fn kernel_print_mappings() {\n    mapping_record::kernel_print()\n}\n\n/// Enable the MMU and data + instruction caching.\n///\n/// # Safety\n///\n/// - Crucial function during kernel init. Changes the the complete memory view of the processor.\n#[inline(always)]\npub unsafe fn enable_mmu_and_caching(\n    phys_tables_base_addr: Address<Physical>,\n) -> Result<(), MMUEnableError> {\n    arch_mmu::mmu().enable_mmu_and_caching(phys_tables_base_addr)\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/memory.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management.\n\npub mod mmu;\n\nuse crate::{bsp, common};\nuse core::{\n    fmt,\n    marker::PhantomData,\n    ops::{Add, Sub},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Metadata trait for marking the type of an address.\npub trait AddressType: Copy + Clone + PartialOrd + PartialEq + Ord + Eq {}\n\n/// Zero-sized type to mark a physical address.\n#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq)]\npub enum Physical {}\n\n/// Zero-sized type to mark a virtual address.\n#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq)]\npub enum Virtual {}\n\n/// Generic address type.\n#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq)]\npub struct Address<ATYPE: AddressType> {\n    value: usize,\n    _address_type: PhantomData<fn() -> ATYPE>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl AddressType for Physical {}\nimpl AddressType for Virtual {}\n\nimpl<ATYPE: AddressType> Address<ATYPE> {\n    /// Create an instance.\n    pub const fn new(value: usize) -> Self {\n        Self {\n            value,\n            _address_type: PhantomData,\n        }\n    }\n\n    /// Convert to usize.\n    pub const fn as_usize(self) -> usize {\n        self.value\n    }\n\n    /// Align down to page size.\n    #[must_use]\n    pub const fn align_down_page(self) -> Self {\n        let aligned = common::align_down(self.value, bsp::memory::mmu::KernelGranule::SIZE);\n\n        Self::new(aligned)\n    }\n\n    /// Align up to page size.\n    #[must_use]\n    pub const fn align_up_page(self) -> Self {\n        let aligned = common::align_up(self.value, bsp::memory::mmu::KernelGranule::SIZE);\n\n        Self::new(aligned)\n    }\n\n    /// Checks if the address is page aligned.\n    pub const fn is_page_aligned(&self) -> bool {\n        common::is_aligned(self.value, bsp::memory::mmu::KernelGranule::SIZE)\n    }\n\n    /// Return the address' offset into the corresponding page.\n    pub const fn offset_into_page(&self) -> usize {\n        self.value & bsp::memory::mmu::KernelGranule::MASK\n    }\n}\n\nimpl<ATYPE: AddressType> Add<usize> for Address<ATYPE> {\n    type Output = Self;\n\n    #[inline(always)]\n    fn add(self, rhs: usize) -> Self::Output {\n        match self.value.checked_add(rhs) {\n            None => panic!(\"Overflow on Address::add\"),\n            Some(x) => Self::new(x),\n        }\n    }\n}\n\nimpl<ATYPE: AddressType> Sub<Address<ATYPE>> for Address<ATYPE> {\n    type Output = Self;\n\n    #[inline(always)]\n    fn sub(self, rhs: Address<ATYPE>) -> Self::Output {\n        match self.value.checked_sub(rhs.value) {\n            None => panic!(\"Overflow on Address::sub\"),\n            Some(x) => Self::new(x),\n        }\n    }\n}\n\nimpl fmt::Display for Address<Physical> {\n    // Don't expect to see physical addresses greater than 40 bit.\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        let q3: u8 = ((self.value >> 32) & 0xff) as u8;\n        let q2: u16 = ((self.value >> 16) & 0xffff) as u16;\n        let q1: u16 = (self.value & 0xffff) as u16;\n\n        write!(f, \"0x\")?;\n        write!(f, \"{:02x}_\", q3)?;\n        write!(f, \"{:04x}_\", q2)?;\n        write!(f, \"{:04x}\", q1)\n    }\n}\n\nimpl fmt::Display for Address<Virtual> {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        let q4: u16 = ((self.value >> 48) & 0xffff) as u16;\n        let q3: u16 = ((self.value >> 32) & 0xffff) as u16;\n        let q2: u16 = ((self.value >> 16) & 0xffff) as u16;\n        let q1: u16 = (self.value & 0xffff) as u16;\n\n        write!(f, \"0x\")?;\n        write!(f, \"{:04x}_\", q4)?;\n        write!(f, \"{:04x}_\", q3)?;\n        write!(f, \"{:04x}_\", q2)?;\n        write!(f, \"{:04x}\", q1)\n    }\n}\n\n/// Initialize the memory subsystem.\npub fn init() {\n    mmu::kernel_init_mmio_va_allocator();\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// Sanity of [Address] methods.\n    #[kernel_test]\n    fn address_type_method_sanity() {\n        let addr = Address::<Virtual>::new(bsp::memory::mmu::KernelGranule::SIZE + 100);\n\n        assert_eq!(\n            addr.align_down_page().as_usize(),\n            bsp::memory::mmu::KernelGranule::SIZE\n        );\n\n        assert_eq!(\n            addr.align_up_page().as_usize(),\n            bsp::memory::mmu::KernelGranule::SIZE * 2\n        );\n\n        assert!(!addr.is_page_aligned());\n\n        assert_eq!(addr.offset_into_page(), 100);\n    }\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/panic_wait.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A panic handler that infinitely waits.\n\nuse crate::{cpu, exception, println};\nuse core::panic::PanicInfo;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// The point of exit for `libkernel`.\n///\n/// It is linked weakly, so that the integration tests can overload its standard behavior.\n#[linkage = \"weak\"]\n#[no_mangle]\nfn _panic_exit() -> ! {\n    #[cfg(not(feature = \"test_build\"))]\n    {\n        cpu::wait_forever()\n    }\n\n    #[cfg(feature = \"test_build\")]\n    {\n        cpu::qemu_exit_failure()\n    }\n}\n\n/// Stop immediately if called a second time.\n///\n/// # Note\n///\n/// Using atomics here relieves us from needing to use `unsafe` for the static variable.\n///\n/// On `AArch64`, which is the only implemented architecture at the time of writing this,\n/// [`AtomicBool::load`] and [`AtomicBool::store`] are lowered to ordinary load and store\n/// instructions. They are therefore safe to use even with MMU + caching deactivated.\n///\n/// [`AtomicBool::load`]: core::sync::atomic::AtomicBool::load\n/// [`AtomicBool::store`]: core::sync::atomic::AtomicBool::store\nfn panic_prevent_reenter() {\n    use core::sync::atomic::{AtomicBool, Ordering};\n\n    #[cfg(not(target_arch = \"aarch64\"))]\n    compile_error!(\"Add the target_arch to above's check if the following code is safe to use\");\n\n    static PANIC_IN_PROGRESS: AtomicBool = AtomicBool::new(false);\n\n    if !PANIC_IN_PROGRESS.load(Ordering::Relaxed) {\n        PANIC_IN_PROGRESS.store(true, Ordering::Relaxed);\n\n        return;\n    }\n\n    _panic_exit()\n}\n\n#[panic_handler]\nfn panic(info: &PanicInfo) -> ! {\n    exception::asynchronous::local_irq_mask();\n\n    // Protect against panic infinite loops if any of the following code panics itself.\n    panic_prevent_reenter();\n\n    let timestamp = crate::time::time_manager().uptime();\n    let (location, line, column) = match info.location() {\n        Some(loc) => (loc.file(), loc.line(), loc.column()),\n        _ => (\"???\", 0, 0),\n    };\n\n    println!(\n        \"[  {:>3}.{:06}] Kernel panic!\\n\\n\\\n        Panic location:\\n      File '{}', line {}, column {}\\n\\n\\\n        {}\",\n        timestamp.as_secs(),\n        timestamp.subsec_micros(),\n        location,\n        line,\n        column,\n        info.message().unwrap_or(&format_args!(\"\")),\n    );\n\n    _panic_exit()\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/print.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Printing.\n\nuse crate::console;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n#[doc(hidden)]\npub fn _print(args: fmt::Arguments) {\n    console::console().write_fmt(args).unwrap();\n}\n\n/// Prints without a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! print {\n    ($($arg:tt)*) => ($crate::print::_print(format_args!($($arg)*)));\n}\n\n/// Prints with a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! println {\n    () => ($crate::print!(\"\\n\"));\n    ($($arg:tt)*) => ({\n        $crate::print::_print(format_args_nl!($($arg)*));\n    })\n}\n\n/// Prints an info, with a newline.\n#[macro_export]\nmacro_rules! info {\n    ($string:expr) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[  {:>3}.{:06}] \", $string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n        ));\n    });\n    ($format_string:expr, $($arg:tt)*) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[  {:>3}.{:06}] \", $format_string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n            $($arg)*\n        ));\n    })\n}\n\n/// Prints a warning, with a newline.\n#[macro_export]\nmacro_rules! warn {\n    ($string:expr) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[W {:>3}.{:06}] \", $string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n        ));\n    });\n    ($format_string:expr, $($arg:tt)*) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[W {:>3}.{:06}] \", $format_string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n            $($arg)*\n        ));\n    })\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/state.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! State information about the kernel itself.\n\nuse core::sync::atomic::{AtomicU8, Ordering};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Different stages in the kernel execution.\n#[derive(Copy, Clone, Eq, PartialEq)]\nenum State {\n    /// The kernel starts booting in this state.\n    Init,\n\n    /// The kernel transitions to this state when jumping to `kernel_main()` (at the end of\n    /// `kernel_init()`, after all init calls are done).\n    SingleCoreMain,\n\n    /// The kernel transitions to this state when it boots the secondary cores, aka switches\n    /// exectution mode to symmetric multiprocessing (SMP).\n    MultiCoreMain,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Maintains the kernel state and state transitions.\npub struct StateManager(AtomicU8);\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic STATE_MANAGER: StateManager = StateManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the global StateManager.\npub fn state_manager() -> &'static StateManager {\n    &STATE_MANAGER\n}\n\nimpl StateManager {\n    const INIT: u8 = 0;\n    const SINGLE_CORE_MAIN: u8 = 1;\n    const MULTI_CORE_MAIN: u8 = 2;\n\n    /// Create a new instance.\n    pub const fn new() -> Self {\n        Self(AtomicU8::new(Self::INIT))\n    }\n\n    /// Return the current state.\n    fn state(&self) -> State {\n        let state = self.0.load(Ordering::Acquire);\n\n        match state {\n            Self::INIT => State::Init,\n            Self::SINGLE_CORE_MAIN => State::SingleCoreMain,\n            Self::MULTI_CORE_MAIN => State::MultiCoreMain,\n            _ => panic!(\"Invalid KERNEL_STATE\"),\n        }\n    }\n\n    /// Return if the kernel is init state.\n    pub fn is_init(&self) -> bool {\n        self.state() == State::Init\n    }\n\n    /// Transition from Init to SingleCoreMain.\n    pub fn transition_to_single_core_main(&self) {\n        if self\n            .0\n            .compare_exchange(\n                Self::INIT,\n                Self::SINGLE_CORE_MAIN,\n                Ordering::Acquire,\n                Ordering::Relaxed,\n            )\n            .is_err()\n        {\n            panic!(\"transition_to_single_core_main() called while state != Init\");\n        }\n    }\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/synchronization.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Synchronization primitives.\n//!\n//! # Resources\n//!\n//!   - <https://doc.rust-lang.org/book/ch16-04-extensible-concurrency-sync-and-send.html>\n//!   - <https://stackoverflow.com/questions/59428096/understanding-the-send-trait>\n//!   - <https://doc.rust-lang.org/std/cell/index.html>\n\nuse core::cell::UnsafeCell;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Synchronization interfaces.\npub mod interface {\n\n    /// Any object implementing this trait guarantees exclusive access to the data wrapped within\n    /// the Mutex for the duration of the provided closure.\n    pub trait Mutex {\n        /// The type of the data that is wrapped by this mutex.\n        type Data;\n\n        /// Locks the mutex and grants the closure temporary mutable access to the wrapped data.\n        fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R;\n    }\n\n    /// A reader-writer exclusion type.\n    ///\n    /// The implementing object allows either a number of readers or at most one writer at any point\n    /// in time.\n    pub trait ReadWriteEx {\n        /// The type of encapsulated data.\n        type Data;\n\n        /// Grants temporary mutable access to the encapsulated data.\n        fn write<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R;\n\n        /// Grants temporary immutable access to the encapsulated data.\n        fn read<'a, R>(&'a self, f: impl FnOnce(&'a Self::Data) -> R) -> R;\n    }\n}\n\n/// A pseudo-lock for teaching purposes.\n///\n/// In contrast to a real Mutex implementation, does not protect against concurrent access from\n/// other cores to the contained data. This part is preserved for later lessons.\n///\n/// The lock will only be used as long as it is safe to do so, i.e. as long as the kernel is\n/// executing on a single core.\npub struct IRQSafeNullLock<T>\nwhere\n    T: ?Sized,\n{\n    data: UnsafeCell<T>,\n}\n\n/// A pseudo-lock that is RW during the single-core kernel init phase and RO afterwards.\n///\n/// Intended to encapsulate data that is populated during kernel init when no concurrency exists.\npub struct InitStateLock<T>\nwhere\n    T: ?Sized,\n{\n    data: UnsafeCell<T>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nunsafe impl<T> Send for IRQSafeNullLock<T> where T: ?Sized + Send {}\nunsafe impl<T> Sync for IRQSafeNullLock<T> where T: ?Sized + Send {}\n\nimpl<T> IRQSafeNullLock<T> {\n    /// Create an instance.\n    pub const fn new(data: T) -> Self {\n        Self {\n            data: UnsafeCell::new(data),\n        }\n    }\n}\n\nunsafe impl<T> Send for InitStateLock<T> where T: ?Sized + Send {}\nunsafe impl<T> Sync for InitStateLock<T> where T: ?Sized + Send {}\n\nimpl<T> InitStateLock<T> {\n    /// Create an instance.\n    pub const fn new(data: T) -> Self {\n        Self {\n            data: UnsafeCell::new(data),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse crate::{exception, state};\n\nimpl<T> interface::Mutex for IRQSafeNullLock<T> {\n    type Data = T;\n\n    fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R {\n        // In a real lock, there would be code encapsulating this line that ensures that this\n        // mutable reference will ever only be given out once at a time.\n        let data = unsafe { &mut *self.data.get() };\n\n        // Execute the closure while IRQs are masked.\n        exception::asynchronous::exec_with_irq_masked(|| f(data))\n    }\n}\n\nimpl<T> interface::ReadWriteEx for InitStateLock<T> {\n    type Data = T;\n\n    fn write<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R {\n        assert!(\n            state::state_manager().is_init(),\n            \"InitStateLock::write called after kernel init phase\"\n        );\n        assert!(\n            !exception::asynchronous::is_local_irq_masked(),\n            \"InitStateLock::write called with IRQs unmasked\"\n        );\n\n        let data = unsafe { &mut *self.data.get() };\n\n        f(data)\n    }\n\n    fn read<'a, R>(&'a self, f: impl FnOnce(&'a Self::Data) -> R) -> R {\n        let data = unsafe { &*self.data.get() };\n\n        f(data)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// InitStateLock must be transparent.\n    #[kernel_test]\n    fn init_state_lock_is_transparent() {\n        use core::mem::size_of;\n\n        assert_eq!(size_of::<InitStateLock<u64>>(), size_of::<u64>());\n    }\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/src/time.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Timer primitives.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/time.rs\"]\nmod arch_time;\n\nuse core::time::Duration;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Provides time management functions.\npub struct TimeManager;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic TIME_MANAGER: TimeManager = TimeManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the global TimeManager.\npub fn time_manager() -> &'static TimeManager {\n    &TIME_MANAGER\n}\n\nimpl TimeManager {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self\n    }\n\n    /// The timer's resolution.\n    pub fn resolution(&self) -> Duration {\n        arch_time::resolution()\n    }\n\n    /// The uptime since power-on of the device.\n    ///\n    /// This includes time consumed by firmware and bootloaders.\n    pub fn uptime(&self) -> Duration {\n        arch_time::uptime()\n    }\n\n    /// Spin for a given duration.\n    pub fn spin_for(&self, duration: Duration) {\n        arch_time::spin_for(duration)\n    }\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/tests/00_console_sanity.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'console_io_test'\n\n# Verify sending and receiving works as expected.\nclass TxRxHandshakeTest < SubtestBase\n    def name\n        'Transmit and Receive handshake'\n    end\n\n    def run(qemu_out, qemu_in)\n        qemu_in.write_nonblock('ABC')\n        expect_or_raise(qemu_out, 'OK1234')\n    end\nend\n\n# Check for correct TX statistics implementation. Depends on test 1 being run first.\nclass TxStatisticsTest < SubtestBase\n    def name\n        'Transmit statistics'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, '6')\n    end\nend\n\n# Check for correct RX statistics implementation. Depends on test 1 being run first.\nclass RxStatisticsTest < SubtestBase\n    def name\n        'Receive statistics'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, '3')\n    end\nend\n\n## -------------------------------------------------------------------------------------------------\n## Test registration\n## -------------------------------------------------------------------------------------------------\ndef subtest_collection\n    [TxRxHandshakeTest.new, TxStatisticsTest.new, RxStatisticsTest.new]\nend\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/tests/00_console_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Console sanity tests - RX, TX and statistics.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Console tests should time out on the I/O harness in case of panic.\nmod panic_wait_forever;\n\nuse libkernel::{bsp, console, cpu, exception, memory, print};\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    use console::console;\n\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    // Handshake\n    assert_eq!(console().read_char(), 'A');\n    assert_eq!(console().read_char(), 'B');\n    assert_eq!(console().read_char(), 'C');\n    print!(\"OK1234\");\n\n    // 6\n    print!(\"{}\", console().chars_written());\n\n    // 3\n    print!(\"{}\", console().chars_read());\n\n    // The QEMU process running this test will be closed by the I/O test harness.\n    cpu::wait_forever();\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/tests/01_timer_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Timer sanity tests.\n\n#![feature(custom_test_frameworks)]\n#![no_main]\n#![no_std]\n#![reexport_test_harness_main = \"test_main\"]\n#![test_runner(libkernel::test_runner)]\n\nuse core::time::Duration;\nuse libkernel::{bsp, cpu, exception, memory, time};\nuse test_macros::kernel_test;\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    // Depending on CPU arch, some timer bring-up code could go here. Not needed for the RPi.\n\n    test_main();\n\n    cpu::qemu_exit_success()\n}\n\n/// Simple check that the timer is running.\n#[kernel_test]\nfn timer_is_counting() {\n    assert!(time::time_manager().uptime().as_nanos() > 0)\n}\n\n/// Timer resolution must be sufficient.\n#[kernel_test]\nfn timer_resolution_is_sufficient() {\n    assert!(time::time_manager().resolution().as_nanos() > 0);\n    assert!(time::time_manager().resolution().as_nanos() < 100)\n}\n\n/// Sanity check spin_for() implementation.\n#[kernel_test]\nfn spin_accuracy_check_1_second() {\n    let t1 = time::time_manager().uptime();\n    time::time_manager().spin_for(Duration::from_secs(1));\n    let t2 = time::time_manager().uptime();\n\n    assert_eq!((t2 - t1).as_secs(), 1)\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/tests/02_exception_sync_page_fault.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Page faults must result in synchronous exceptions.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Overwrites libkernel's `panic_wait::_panic_exit()` so that it returns a \"success\" code.\n///\n/// In this test, reaching the panic is a success, because it is called from the synchronous\n/// exception handler, which is what this test wants to achieve.\n///\n/// It also means that this integration test can not use any other code that calls panic!() directly\n/// or indirectly.\nmod panic_exit_success;\n\nuse libkernel::{bsp, cpu, exception, info, memory, println};\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    // This line will be printed as the test header.\n    println!(\"Testing synchronous exception handling by causing a page fault\");\n\n    info!(\"Writing to bottom of address space to address 1 GiB...\");\n    let big_addr: u64 = 1024 * 1024 * 1024;\n    core::ptr::read_volatile(big_addr as *mut u64);\n\n    // If execution reaches here, the memory access above did not cause a page fault exception.\n    cpu::qemu_exit_failure()\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/tests/03_exception_restore_sanity.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'console_io_test'\n\n# Verify that exception restore works.\nclass ExceptionRestoreTest < SubtestBase\n    def name\n        'Exception restore'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, 'Back from system call!')\n    end\nend\n\n## -------------------------------------------------------------------------------------------------\n## Test registration\n## -------------------------------------------------------------------------------------------------\ndef subtest_collection\n    [ExceptionRestoreTest.new]\nend\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/tests/03_exception_restore_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A simple sanity test to see if exception restore code works.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Console tests should time out on the I/O harness in case of panic.\nmod panic_wait_forever;\n\nuse core::arch::asm;\nuse libkernel::{bsp, cpu, exception, info, memory, println};\n\n#[inline(never)]\nfn nested_system_call() {\n    #[cfg(target_arch = \"aarch64\")]\n    unsafe {\n        asm!(\"svc #0x1337\", options(nomem, nostack, preserves_flags));\n    }\n\n    #[cfg(not(target_arch = \"aarch64\"))]\n    {\n        info!(\"Not supported yet\");\n        cpu::wait_forever();\n    }\n}\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    // This line will be printed as the test header.\n    println!(\"Testing exception restore\");\n\n    info!(\"Making a dummy system call\");\n\n    // Calling this inside a function indirectly tests if the link register is restored properly.\n    nested_system_call();\n\n    info!(\"Back from system call!\");\n\n    // The QEMU process running this test will be closed by the I/O test harness.\n    cpu::wait_forever();\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/tests/04_exception_irq_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! IRQ handling sanity tests.\n\n#![feature(custom_test_frameworks)]\n#![no_main]\n#![no_std]\n#![reexport_test_harness_main = \"test_main\"]\n#![test_runner(libkernel::test_runner)]\n\nuse libkernel::{bsp, cpu, exception, memory};\nuse test_macros::kernel_test;\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    exception::handling_init();\n    exception::asynchronous::local_irq_unmask();\n\n    test_main();\n\n    cpu::qemu_exit_success()\n}\n\n/// Check that IRQ masking works.\n#[kernel_test]\nfn local_irq_mask_works() {\n    // Precondition: IRQs are unmasked.\n    assert!(exception::asynchronous::is_local_irq_masked());\n\n    exception::asynchronous::local_irq_mask();\n    assert!(!exception::asynchronous::is_local_irq_masked());\n\n    // Restore earlier state.\n    exception::asynchronous::local_irq_unmask();\n}\n\n/// Check that IRQ unmasking works.\n#[kernel_test]\nfn local_irq_unmask_works() {\n    // Precondition: IRQs are masked.\n    exception::asynchronous::local_irq_mask();\n    assert!(!exception::asynchronous::is_local_irq_masked());\n\n    exception::asynchronous::local_irq_unmask();\n    assert!(exception::asynchronous::is_local_irq_masked());\n}\n\n/// Check that IRQ mask save is saving \"something\".\n#[kernel_test]\nfn local_irq_mask_save_works() {\n    // Precondition: IRQs are unmasked.\n    assert!(exception::asynchronous::is_local_irq_masked());\n\n    let first = exception::asynchronous::local_irq_mask_save();\n    assert!(!exception::asynchronous::is_local_irq_masked());\n\n    let second = exception::asynchronous::local_irq_mask_save();\n    assert_ne!(first, second);\n\n    exception::asynchronous::local_irq_restore(first);\n    assert!(exception::asynchronous::is_local_irq_masked());\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/tests/boot_test_string.rb",
    "content": "# frozen_string_literal: true\n\nEXPECTED_PRINT = 'Echoing input now'\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/tests/panic_exit_success/mod.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n/// Overwrites libkernel's `panic_wait::_panic_exit()` with the QEMU-exit version.\n#[no_mangle]\nfn _panic_exit() -> ! {\n    libkernel::cpu::qemu_exit_success()\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/kernel/tests/panic_wait_forever/mod.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n/// Overwrites libkernel's `panic_wait::_panic_exit()` with wait_forever.\n#[no_mangle]\nfn _panic_exit() -> ! {\n    libkernel::cpu::wait_forever()\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/libraries/test-macros/Cargo.toml",
    "content": "[package]\nname = \"test-macros\"\nversion = \"0.1.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n\n[lib]\nproc-macro = true\n\n[dependencies]\nproc-macro2 = \"1.x\"\nquote = \"1.x\"\nsyn = { version = \"1.x\", features = [\"full\"] }\ntest-types = { path = \"../test-types\" }\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/libraries/test-macros/src/lib.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\nuse proc_macro::TokenStream;\nuse proc_macro2::Span;\nuse quote::quote;\nuse syn::{parse_macro_input, Ident, ItemFn};\n\n#[proc_macro_attribute]\npub fn kernel_test(_attr: TokenStream, input: TokenStream) -> TokenStream {\n    let f = parse_macro_input!(input as ItemFn);\n\n    let test_name = &format!(\"{}\", f.sig.ident);\n    let test_ident = Ident::new(\n        &format!(\"{}_TEST_CONTAINER\", f.sig.ident.to_string().to_uppercase()),\n        Span::call_site(),\n    );\n    let test_code_block = f.block;\n\n    quote!(\n        #[test_case]\n        const #test_ident: test_types::UnitTest = test_types::UnitTest {\n            name: #test_name,\n            test_func: || #test_code_block,\n        };\n    )\n    .into()\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/libraries/test-types/Cargo.toml",
    "content": "[package]\nname = \"test-types\"\nversion = \"0.1.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/libraries/test-types/src/lib.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Types for the `custom_test_frameworks` implementation.\n\n#![no_std]\n\n/// Unit test container.\npub struct UnitTest {\n    /// Name of the test.\n    pub name: &'static str,\n\n    /// Function pointer to the test.\n    pub test_func: fn(),\n}\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/tools/translation_table_tool/arch.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n# Bitfield manipulation.\nclass BitField\n    def initialize\n        @value = 0\n    end\n\n    def self.attr_bitfield(name, offset, num_bits)\n        define_method(\"#{name}=\") do |bits|\n            mask = (2**num_bits) - 1\n\n            raise \"Input out of range: #{name} = 0x#{bits.to_s(16)}\" if (bits & ~mask).positive?\n\n            # Clear bitfield\n            @value &= ~(mask << offset)\n\n            # Set it\n            @value |= (bits << offset)\n        end\n    end\n\n    def to_i\n        @value\n    end\n\n    def size_in_byte\n        8\n    end\nend\n\n# An array class that knows its memory location.\nclass CArray < Array\n    attr_reader :phys_start_addr\n\n    def initialize(phys_start_addr, size, &block)\n        @phys_start_addr = phys_start_addr\n\n        super(size, &block)\n    end\n\n    def size_in_byte\n        inject(0) { |sum, n| sum + n.size_in_byte }\n    end\nend\n\n#---------------------------------------------------------------------------------------------------\n# Arch::\n#---------------------------------------------------------------------------------------------------\nmodule Arch\n#---------------------------------------------------------------------------------------------------\n# Arch::ARMv8\n#---------------------------------------------------------------------------------------------------\nmodule ARMv8\n# ARMv8 Table Descriptor.\nclass Stage1TableDescriptor < BitField\n    module NextLevelTableAddr\n        OFFSET = 16\n        NUMBITS = 32\n    end\n\n    module Type\n        OFFSET = 1\n        NUMBITS = 1\n\n        BLOCK = 0\n        TABLE = 1\n    end\n\n    module Valid\n        OFFSET = 0\n        NUMBITS = 1\n\n        FALSE = 0\n        TRUE = 1\n    end\n\n    attr_bitfield(:__next_level_table_addr, NextLevelTableAddr::OFFSET, NextLevelTableAddr::NUMBITS)\n    attr_bitfield(:type, Type::OFFSET, Type::NUMBITS)\n    attr_bitfield(:valid, Valid::OFFSET, Valid::NUMBITS)\n\n    def next_level_table_addr=(addr)\n        addr >>= Granule64KiB::SHIFT\n\n        self.__next_level_table_addr = addr\n    end\n\n    private :__next_level_table_addr=\nend\n\n# ARMv8 level 3 page descriptor.\nclass Stage1PageDescriptor < BitField\n    module UXN\n        OFFSET = 54\n        NUMBITS = 1\n\n        FALSE = 0\n        TRUE = 1\n    end\n\n    module PXN\n        OFFSET = 53\n        NUMBITS = 1\n\n        FALSE = 0\n        TRUE = 1\n    end\n\n    module OutputAddr\n        OFFSET = 16\n        NUMBITS = 32\n    end\n\n    module AF\n        OFFSET = 10\n        NUMBITS = 1\n\n        FALSE = 0\n        TRUE = 1\n    end\n\n    module SH\n        OFFSET = 8\n        NUMBITS = 2\n\n        INNER_SHAREABLE = 0b11\n    end\n\n    module AP\n        OFFSET = 6\n        NUMBITS = 2\n\n        RW_EL1 = 0b00\n        RO_EL1 = 0b10\n    end\n\n    module AttrIndx\n        OFFSET = 2\n        NUMBITS = 3\n    end\n\n    module Type\n        OFFSET = 1\n        NUMBITS = 1\n\n        RESERVED_INVALID = 0\n        PAGE = 1\n    end\n\n    module Valid\n        OFFSET = 0\n        NUMBITS = 1\n\n        FALSE = 0\n        TRUE = 1\n    end\n\n    attr_bitfield(:uxn, UXN::OFFSET, UXN::NUMBITS)\n    attr_bitfield(:pxn, PXN::OFFSET, PXN::NUMBITS)\n    attr_bitfield(:__output_addr, OutputAddr::OFFSET, OutputAddr::NUMBITS)\n    attr_bitfield(:af, AF::OFFSET, AF::NUMBITS)\n    attr_bitfield(:sh, SH::OFFSET, SH::NUMBITS)\n    attr_bitfield(:ap, AP::OFFSET, AP::NUMBITS)\n    attr_bitfield(:attr_indx, AttrIndx::OFFSET, AttrIndx::NUMBITS)\n    attr_bitfield(:type, Type::OFFSET, Type::NUMBITS)\n    attr_bitfield(:valid, Valid::OFFSET, Valid::NUMBITS)\n\n    def output_addr=(addr)\n        addr >>= Granule64KiB::SHIFT\n\n        self.__output_addr = addr\n    end\n\n    private :__output_addr=\nend\n\n# Translation table representing the structure defined in translation_table.rs.\nclass TranslationTable\n    module MAIR\n        NORMAL = 1\n    end\n\n    def initialize\n        do_sanity_checks\n\n        num_lvl2_tables = BSP.kernel_virt_addr_space_size >> Granule512MiB::SHIFT\n\n        @lvl3 = new_lvl3(num_lvl2_tables, BSP.phys_addr_of_kernel_tables)\n\n        @lvl2_phys_start_addr = @lvl3.phys_start_addr + @lvl3.size_in_byte\n        @lvl2 = new_lvl2(num_lvl2_tables, @lvl2_phys_start_addr)\n\n        populate_lvl2_entries\n    end\n\n    def map_at(virt_region, phys_region, attributes)\n        return if virt_region.empty?\n\n        raise if virt_region.size != phys_region.size\n        raise if phys_region.last > BSP.phys_addr_space_end_page\n\n        virt_region.zip(phys_region).each do |virt_page, phys_page|\n            desc = page_descriptor_from(virt_page)\n            set_lvl3_entry(desc, phys_page, attributes)\n        end\n    end\n\n    def to_binary\n        data = @lvl3.flatten.map(&:to_i) + @lvl2.map(&:to_i)\n        data.pack('Q<*') # \"Q\" == uint64_t, \"<\" == little endian\n    end\n\n    def phys_tables_base_addr_binary\n        [@lvl2_phys_start_addr].pack('Q<*') # \"Q\" == uint64_t, \"<\" == little endian\n    end\n\n    def phys_tables_base_addr\n        @lvl2_phys_start_addr\n    end\n\n    private\n\n    def do_sanity_checks\n        raise unless BSP.kernel_granule::SIZE == Granule64KiB::SIZE\n        raise unless (BSP.kernel_virt_addr_space_size % Granule512MiB::SIZE).zero?\n    end\n\n    def new_lvl3(num_lvl2_tables, start_addr)\n        CArray.new(start_addr, num_lvl2_tables) do\n            temp = CArray.new(start_addr, 8192) do\n                Stage1PageDescriptor.new\n            end\n            start_addr += temp.size_in_byte\n\n            temp\n        end\n    end\n\n    def new_lvl2(num_lvl2_tables, start_addr)\n        CArray.new(start_addr, num_lvl2_tables) do\n            Stage1TableDescriptor.new\n        end\n    end\n\n    def populate_lvl2_entries\n        @lvl2.each_with_index do |descriptor, i|\n            descriptor.next_level_table_addr = @lvl3[i].phys_start_addr\n            descriptor.type = Stage1TableDescriptor::Type::TABLE\n            descriptor.valid = Stage1TableDescriptor::Valid::TRUE\n        end\n    end\n\n    def lvl2_lvl3_index_from(addr)\n        addr -= BSP.kernel_virt_start_addr\n\n        lvl2_index = addr >> Granule512MiB::SHIFT\n        lvl3_index = (addr & Granule512MiB::MASK) >> Granule64KiB::SHIFT\n\n        raise unless lvl2_index < @lvl2.size\n\n        [lvl2_index, lvl3_index]\n    end\n\n    def page_descriptor_from(virt_addr)\n        lvl2_index, lvl3_index = lvl2_lvl3_index_from(virt_addr)\n\n        @lvl3[lvl2_index][lvl3_index]\n    end\n\n    # rubocop:disable Metrics/MethodLength\n    def set_attributes(desc, attributes)\n        case attributes.mem_attributes\n        when :CacheableDRAM\n            desc.sh = Stage1PageDescriptor::SH::INNER_SHAREABLE\n            desc.attr_indx = MAIR::NORMAL\n        else\n            raise 'Invalid input'\n        end\n\n        desc.ap = case attributes.acc_perms\n                  when :ReadOnly\n                      Stage1PageDescriptor::AP::RO_EL1\n                  when :ReadWrite\n                      Stage1PageDescriptor::AP::RW_EL1\n                  else\n                      raise 'Invalid input'\n\n                  end\n\n        desc.pxn = if attributes.execute_never\n                       Stage1PageDescriptor::PXN::TRUE\n                   else\n                       Stage1PageDescriptor::PXN::FALSE\n                   end\n\n        desc.uxn = Stage1PageDescriptor::UXN::TRUE\n    end\n    # rubocop:enable Metrics/MethodLength\n\n    def set_lvl3_entry(desc, output_addr, attributes)\n        desc.output_addr = output_addr\n        desc.af = Stage1PageDescriptor::AF::TRUE\n        desc.type = Stage1PageDescriptor::Type::PAGE\n        desc.valid = Stage1PageDescriptor::Valid::TRUE\n\n        set_attributes(desc, attributes)\n    end\nend\nend\nend\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/tools/translation_table_tool/bsp.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n# Raspberry Pi 3 + 4\nclass RaspberryPi\n    attr_reader :kernel_granule, :kernel_virt_addr_space_size, :kernel_virt_start_addr\n\n    MEMORY_SRC = File.read('kernel/src/bsp/raspberrypi/memory.rs').split(\"\\n\")\n\n    def initialize\n        @kernel_granule = Granule64KiB\n\n        @kernel_virt_addr_space_size = KERNEL_ELF.symbol_value('__kernel_virt_addr_space_size')\n        @kernel_virt_start_addr = KERNEL_ELF.symbol_value('__kernel_virt_start_addr')\n\n        @virt_addr_of_kernel_tables = KERNEL_ELF.symbol_value('KERNEL_TABLES')\n        @virt_addr_of_phys_kernel_tables_base_addr = KERNEL_ELF.symbol_value(\n            'PHYS_KERNEL_TABLES_BASE_ADDR'\n        )\n    end\n\n    def phys_addr_of_kernel_tables\n        KERNEL_ELF.virt_to_phys(@virt_addr_of_kernel_tables)\n    end\n\n    def kernel_tables_offset_in_file\n        KERNEL_ELF.virt_addr_to_file_offset(@virt_addr_of_kernel_tables)\n    end\n\n    def phys_kernel_tables_base_addr_offset_in_file\n        KERNEL_ELF.virt_addr_to_file_offset(@virt_addr_of_phys_kernel_tables_base_addr)\n    end\n\n    def phys_addr_space_end_page\n        x = MEMORY_SRC.grep(/pub const END/)\n        x = case BSP_TYPE\n            when :rpi3\n                x[0]\n            when :rpi4\n                x[1]\n            else\n                raise\n            end\n\n        x.scan(/\\d+/).join.to_i(16)\n    end\nend\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/tools/translation_table_tool/generic.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\nmodule Granule64KiB\n    SIZE = 64 * 1024\n    SHIFT = Math.log2(SIZE).to_i\nend\n\nmodule Granule512MiB\n    SIZE = 512 * 1024 * 1024\n    SHIFT = Math.log2(SIZE).to_i\n    MASK = SIZE - 1\nend\n\n# Monkey-patch Integer with some helper functions.\nclass Integer\n    def power_of_two?\n        self[0].zero?\n    end\n\n    def aligned?(alignment)\n        raise unless alignment.power_of_two?\n\n        (self & (alignment - 1)).zero?\n    end\n\n    def align_up(alignment)\n        raise unless alignment.power_of_two?\n\n        (self + alignment - 1) & ~(alignment - 1)\n    end\n\n    def to_hex_underscore(with_leading_zeros: false)\n        fmt = with_leading_zeros ? '%016x' : '%x'\n        value = format(fmt, self).to_s.reverse.scan(/.{4}|.+/).join('_').reverse\n\n        format('0x%s', value)\n    end\nend\n\n# An array where each value is the start address of a Page.\nclass MemoryRegion < Array\n    def initialize(start_addr, size, granule_size)\n        raise unless start_addr.aligned?(granule_size)\n        raise unless size.positive?\n        raise unless (size % granule_size).zero?\n\n        num_pages = size / granule_size\n        super(num_pages) do |i|\n            (i * granule_size) + start_addr\n        end\n    end\nend\n\n# Collection of memory attributes.\nclass AttributeFields\n    attr_reader :mem_attributes, :acc_perms, :execute_never\n\n    def initialize(mem_attributes, acc_perms, execute_never)\n        @mem_attributes = mem_attributes\n        @acc_perms = acc_perms\n        @execute_never = execute_never\n    end\n\n    def to_s\n        x = case @mem_attributes\n            when :CacheableDRAM\n                'C'\n            else\n                '?'\n            end\n\n        y = case @acc_perms\n            when :ReadWrite\n                'RW'\n            when :ReadOnly\n                'RO'\n            else\n                '??'\n            end\n\n        z = @execute_never ? 'XN' : 'X '\n\n        \"#{x} #{y} #{z}\"\n    end\nend\n\n# A container that describes a virt-to-phys region mapping.\nclass MappingDescriptor\n    @max_section_name_length = 'Sections'.length\n\n    class << self\n        attr_accessor :max_section_name_length\n\n        def update_max_section_name_length(length)\n            @max_section_name_length = [@max_section_name_length, length].max\n        end\n    end\n\n    attr_reader :name, :virt_region, :phys_region, :attributes\n\n    def initialize(name, virt_region, phys_region, attributes)\n        @name = name\n        @virt_region = virt_region\n        @phys_region = phys_region\n        @attributes = attributes\n    end\n\n    def to_s\n        name = @name.ljust(self.class.max_section_name_length)\n        virt_start = @virt_region.first.to_hex_underscore(with_leading_zeros: true)\n        phys_start = @phys_region.first.to_hex_underscore(with_leading_zeros: true)\n        size = ((@virt_region.size * 65_536) / 1024).to_s.rjust(3)\n\n        \"#{name} | #{virt_start} | #{phys_start} | #{size} KiB | #{@attributes}\"\n    end\n\n    def self.print_divider\n        print '             '\n        print '-' * max_section_name_length\n        puts '--------------------------------------------------------------------'\n    end\n\n    def self.print_header\n        print_divider\n        print '             '\n        print 'Sections'.center(max_section_name_length)\n        print '   '\n        print 'Virt Start Addr'.center(21)\n        print '   '\n        print 'Phys Start Addr'.center(21)\n        print '   '\n        print 'Size'.center(7)\n        print '   '\n        print 'Attr'.center(7)\n        puts\n        print_divider\n    end\nend\n\ndef kernel_map_binary\n    mapping_descriptors = KERNEL_ELF.generate_mapping_descriptors\n\n    # Generate_mapping_descriptors updates the header being printed with this call. So it must come\n    # afterwards.\n    MappingDescriptor.print_header\n\n    mapping_descriptors.each do |i|\n        print 'Generating'.rjust(12).green.bold\n        print ' '\n        puts i\n\n        TRANSLATION_TABLES.map_at(i.virt_region, i.phys_region, i.attributes)\n    end\n\n    MappingDescriptor.print_divider\nend\n\ndef kernel_patch_tables(kernel_elf_path)\n    print 'Patching'.rjust(12).green.bold\n    print ' Kernel table struct at ELF file offset '\n    puts BSP.kernel_tables_offset_in_file.to_hex_underscore\n\n    File.binwrite(kernel_elf_path, TRANSLATION_TABLES.to_binary, BSP.kernel_tables_offset_in_file)\nend\n\ndef kernel_patch_base_addr(kernel_elf_path)\n    print 'Patching'.rjust(12).green.bold\n    print ' Kernel tables physical base address start argument to value '\n    print TRANSLATION_TABLES.phys_tables_base_addr.to_hex_underscore\n    print ' at ELF file offset '\n    puts BSP.phys_kernel_tables_base_addr_offset_in_file.to_hex_underscore\n\n    File.binwrite(kernel_elf_path, TRANSLATION_TABLES.phys_tables_base_addr_binary,\n                  BSP.phys_kernel_tables_base_addr_offset_in_file)\nend\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/tools/translation_table_tool/kernel_elf.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n# KernelELF\nclass KernelELF\n    SECTION_FLAG_ALLOC = 2\n\n    def initialize(kernel_elf_path)\n        @elf = ELFTools::ELFFile.new(File.open(kernel_elf_path))\n        @symtab_section = @elf.section_by_name('.symtab')\n    end\n\n    def machine\n        @elf.machine.to_sym\n    end\n\n    def symbol_value(symbol_name)\n        @symtab_section.symbol_by_name(symbol_name).header.st_value\n    end\n\n    def segment_containing_virt_addr(virt_addr)\n        @elf.each_segments do |segment|\n            return segment if segment.vma_in?(virt_addr)\n        end\n    end\n\n    def virt_to_phys(virt_addr)\n        segment = segment_containing_virt_addr(virt_addr)\n        translation_offset = segment.header.p_vaddr - segment.header.p_paddr\n\n        virt_addr - translation_offset\n    end\n\n    def virt_addr_to_file_offset(virt_addr)\n        segment = segment_containing_virt_addr(virt_addr)\n        segment.vma_to_offset(virt_addr)\n    end\n\n    def sections_in_segment(segment)\n        head = segment.mem_head\n        tail = segment.mem_tail\n\n        sections = @elf.each_sections.select do |section|\n            file_offset = section.header.sh_addr\n            flags = section.header.sh_flags\n\n            file_offset >= head && file_offset < tail && (flags & SECTION_FLAG_ALLOC != 0)\n        end\n\n        sections.map(&:name).join(' ')\n    end\n\n    def select_load_segments\n        @elf.each_segments.select do |segment|\n            segment.instance_of?(ELFTools::Segments::LoadSegment)\n        end\n    end\n\n    def segment_get_acc_perms(segment)\n        if segment.readable? && segment.writable?\n            :ReadWrite\n        elsif segment.readable?\n            :ReadOnly\n        else\n            :Invalid\n        end\n    end\n\n    def update_max_section_name_length(descriptors)\n        MappingDescriptor.update_max_section_name_length(descriptors.map { |i| i.name.size }.max)\n    end\n\n    def generate_mapping_descriptors\n        descriptors = select_load_segments.map do |segment|\n            # Assume each segment is page aligned.\n            size = segment.mem_size.align_up(BSP.kernel_granule::SIZE)\n            virt_start_addr = segment.header.p_vaddr\n            phys_start_addr = segment.header.p_paddr\n            acc_perms = segment_get_acc_perms(segment)\n            execute_never = !segment.executable?\n            section_names = sections_in_segment(segment)\n\n            virt_region = MemoryRegion.new(virt_start_addr, size, BSP.kernel_granule::SIZE)\n            phys_region = MemoryRegion.new(phys_start_addr, size, BSP.kernel_granule::SIZE)\n            attributes = AttributeFields.new(:CacheableDRAM, acc_perms, execute_never)\n\n            MappingDescriptor.new(section_names, virt_region, phys_region, attributes)\n        end\n\n        update_max_section_name_length(descriptors)\n        descriptors\n    end\nend\n"
  },
  {
    "path": "16_virtual_mem_part4_higher_half_kernel/tools/translation_table_tool/main.rb",
    "content": "#!/usr/bin/env ruby\n# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'rubygems'\nrequire 'bundler/setup'\nrequire 'colorize'\nrequire 'elftools'\n\nrequire_relative 'generic'\nrequire_relative 'kernel_elf'\nrequire_relative 'bsp'\nrequire_relative 'arch'\n\nBSP_TYPE = ARGV[0].to_sym\nkernel_elf_path = ARGV[1]\n\nstart = Time.now\n\nKERNEL_ELF = KernelELF.new(kernel_elf_path)\n\nBSP = case BSP_TYPE\n      when :rpi3, :rpi4\n          RaspberryPi.new\n      else\n          raise\n      end\n\nTRANSLATION_TABLES = case KERNEL_ELF.machine\n                     when :AArch64\n                         Arch::ARMv8::TranslationTable.new\n                     else\n                         raise\n                     end\n\nkernel_map_binary\nkernel_patch_tables(kernel_elf_path)\nkernel_patch_base_addr(kernel_elf_path)\n\nelapsed = Time.now - start\n\nprint 'Finished'.rjust(12).green.bold\nputs \" in #{elapsed.round(2)}s\"\n"
  },
  {
    "path": "17_kernel_symbols/.cargo/config.toml",
    "content": "[target.'cfg(target_os = \"none\")']\nrunner = \"target/kernel_test_runner.sh\"\n"
  },
  {
    "path": "17_kernel_symbols/.vscode/settings.json",
    "content": "{\n    \"editor.formatOnSave\": true,\n    \"editor.rulers\": [100],\n    \"rust-analyzer.cargo.target\": \"aarch64-unknown-none-softfloat\",\n    \"rust-analyzer.cargo.features\": [\"bsp_rpi3\"],\n    \"rust-analyzer.checkOnSave.allTargets\": false,\n    \"rust-analyzer.checkOnSave.extraArgs\": [\"--lib\", \"--bins\"],\n    \"rust-analyzer.lens.debug\": false,\n    \"rust-analyzer.lens.run\": false\n}\n"
  },
  {
    "path": "17_kernel_symbols/Cargo.toml",
    "content": "[workspace]\n\nmembers = [\n        \"libraries/*\",\n        \"kernel\",\n        \"kernel_symbols\"\n]\n\n[profile.release]\nlto = true\n"
  },
  {
    "path": "17_kernel_symbols/Makefile",
    "content": "## SPDX-License-Identifier: MIT OR Apache-2.0\n##\n## Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\ninclude ../common/docker.mk\ninclude ../common/format.mk\ninclude ../common/operating_system.mk\n\n##--------------------------------------------------------------------------------------------------\n## Optional, user-provided configuration values\n##--------------------------------------------------------------------------------------------------\n\n# Default to the RPi3.\nBSP ?= rpi3\n\n# Default to a serial device name that is common in Linux.\nDEV_SERIAL ?= /dev/ttyUSB0\n\n# Optional integration test name.\nifdef TEST\n    TEST_ARG = --test $(TEST)\nelse\n    TEST_ARG = --test '*'\nendif\n\n\n\n##--------------------------------------------------------------------------------------------------\n## BSP-specific configuration values\n##--------------------------------------------------------------------------------------------------\nQEMU_MISSING_STRING = \"This board is not yet supported for QEMU.\"\n\nifeq ($(BSP),rpi3)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE = raspi3\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    QEMU_TEST_ARGS    = $(QEMU_RELEASE_ARGS) -semihosting\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    OPENOCD_ARG       = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi3.cfg\n    JTAG_BOOT_IMAGE   = ../X1_JTAG_boot/jtag_boot_rpi3.img\n    LD_SCRIPT_PATH    = $(shell pwd)/kernel/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a53\nelse ifeq ($(BSP),rpi4)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE =\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    QEMU_TEST_ARGS    = $(QEMU_RELEASE_ARGS) -semihosting\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    OPENOCD_ARG       = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi4.cfg\n    JTAG_BOOT_IMAGE   = ../X1_JTAG_boot/jtag_boot_rpi4.img\n    LD_SCRIPT_PATH    = $(shell pwd)/kernel/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a72\nendif\n\n# Export for build.rs.\nexport LD_SCRIPT_PATH\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets and Prerequisites\n##--------------------------------------------------------------------------------------------------\nKERNEL_MANIFEST      = kernel/Cargo.toml\nKERNEL_LINKER_SCRIPT = kernel.ld\nLAST_BUILD_CONFIG    = target/$(BSP).build_config\n\nKERNEL_ELF_RAW      = target/$(TARGET)/release/kernel\n# This parses cargo's dep-info file.\n# https://doc.rust-lang.org/cargo/guide/build-cache.html#dep-info-files\nKERNEL_ELF_RAW_DEPS = $(filter-out %: ,$(file < $(KERNEL_ELF_RAW).d)) $(KERNEL_MANIFEST) $(LAST_BUILD_CONFIG)\n\n##------------------------------------------------------------------------------\n## Translation tables\n##------------------------------------------------------------------------------\nTT_TOOL_PATH = tools/translation_table_tool\n\nKERNEL_ELF_TTABLES      = target/$(TARGET)/release/kernel+ttables\nKERNEL_ELF_TTABLES_DEPS = $(KERNEL_ELF_RAW) $(wildcard $(TT_TOOL_PATH)/*)\n\n##------------------------------------------------------------------------------\n## Kernel symbols\n##------------------------------------------------------------------------------\nexport KERNEL_SYMBOLS_TOOL_PATH = tools/kernel_symbols_tool\n\nKERNEL_ELF_TTABLES_SYMS = target/$(TARGET)/release/kernel+ttables+symbols\n\n# Unlike with KERNEL_ELF_RAW, we are not relying on dep-info here. One of the reasons being that the\n# name of the generated symbols file varies between runs, which can cause confusion.\nKERNEL_ELF_TTABLES_SYMS_DEPS = $(KERNEL_ELF_TTABLES) \\\n    $(wildcard kernel_symbols/*)                     \\\n    $(wildcard $(KERNEL_SYMBOLS_TOOL_PATH)/*)\n\nexport TARGET\nexport KERNEL_SYMBOLS_INPUT_ELF  = $(KERNEL_ELF_TTABLES)\nexport KERNEL_SYMBOLS_OUTPUT_ELF = $(KERNEL_ELF_TTABLES_SYMS)\n\nKERNEL_ELF = $(KERNEL_ELF_TTABLES_SYMS)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Command building blocks\n##--------------------------------------------------------------------------------------------------\nRUSTFLAGS = $(RUSTC_MISC_ARGS)                   \\\n    -C link-arg=--library-path=$(LD_SCRIPT_PATH) \\\n    -C link-arg=--script=$(KERNEL_LINKER_SCRIPT)\n\nRUSTFLAGS_PEDANTIC = $(RUSTFLAGS) \\\n    -D warnings                   \\\n    -D missing_docs\n\nFEATURES      = --features bsp_$(BSP)\nCOMPILER_ARGS = --target=$(TARGET) \\\n    $(FEATURES)                    \\\n    --release\n\nRUSTC_CMD   = cargo rustc $(COMPILER_ARGS) --manifest-path $(KERNEL_MANIFEST)\nDOC_CMD     = cargo doc $(COMPILER_ARGS)\nCLIPPY_CMD  = cargo clippy $(COMPILER_ARGS)\nTEST_CMD    = cargo test $(COMPILER_ARGS) --manifest-path $(KERNEL_MANIFEST)\nOBJCOPY_CMD = rust-objcopy \\\n    --strip-all            \\\n    -O binary\n\nEXEC_QEMU          = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE)\nEXEC_TT_TOOL       = ruby $(TT_TOOL_PATH)/main.rb\nEXEC_TEST_DISPATCH = ruby ../common/tests/dispatch.rb\nEXEC_MINIPUSH      = ruby ../common/serial/minipush.rb\n\n##------------------------------------------------------------------------------\n## Dockerization\n##------------------------------------------------------------------------------\nDOCKER_CMD            = docker run -t --rm -v $(shell pwd):/work/tutorial -w /work/tutorial\nDOCKER_CMD_INTERACT   = $(DOCKER_CMD) -i\nDOCKER_ARG_DIR_COMMON = -v $(shell pwd)/../common:/work/common\nDOCKER_ARG_DIR_JTAG   = -v $(shell pwd)/../X1_JTAG_boot:/work/X1_JTAG_boot\nDOCKER_ARG_DEV        = --privileged -v /dev:/dev\nDOCKER_ARG_NET        = --network host\n\n# DOCKER_IMAGE defined in include file (see top of this file).\nDOCKER_QEMU  = $(DOCKER_CMD_INTERACT) $(DOCKER_IMAGE)\nDOCKER_TOOLS = $(DOCKER_CMD) $(DOCKER_IMAGE)\nDOCKER_TEST  = $(DOCKER_CMD) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\nDOCKER_GDB   = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_NET) $(DOCKER_IMAGE)\n\n# Dockerize commands, which require USB device passthrough, only on Linux.\nifeq ($(shell uname -s),Linux)\n    DOCKER_CMD_DEV = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_DEV)\n\n    DOCKER_CHAINBOOT = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\n    DOCKER_JTAGBOOT  = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_ARG_DIR_JTAG) $(DOCKER_IMAGE)\n    DOCKER_OPENOCD   = $(DOCKER_CMD_DEV) $(DOCKER_ARG_NET) $(DOCKER_IMAGE)\nelse\n    DOCKER_OPENOCD   = echo \"Not yet supported on non-Linux systems.\"; \\#\nendif\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: all doc qemu chainboot clippy clean readelf objdump nm check\n\nall: $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Save the configuration as a file, so make understands if it changed.\n##------------------------------------------------------------------------------\n$(LAST_BUILD_CONFIG):\n\t@rm -f target/*.build_config\n\t@mkdir -p target\n\t@touch $(LAST_BUILD_CONFIG)\n\n##------------------------------------------------------------------------------\n## Compile the kernel ELF\n##------------------------------------------------------------------------------\n$(KERNEL_ELF_RAW): $(KERNEL_ELF_RAW_DEPS)\n\t$(call color_header, \"Compiling kernel ELF - $(BSP)\")\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(RUSTC_CMD)\n\n##------------------------------------------------------------------------------\n## Precompute the kernel translation tables and patch them into the kernel ELF\n##------------------------------------------------------------------------------\n$(KERNEL_ELF_TTABLES): $(KERNEL_ELF_TTABLES_DEPS)\n\t$(call color_header, \"Precomputing kernel translation tables and patching kernel ELF\")\n\t@cp $(KERNEL_ELF_RAW) $(KERNEL_ELF_TTABLES)\n\t@$(DOCKER_TOOLS) $(EXEC_TT_TOOL) $(BSP) $(KERNEL_ELF_TTABLES)\n\n##------------------------------------------------------------------------------\n## Generate kernel symbols and patch them into the kernel ELF\n##------------------------------------------------------------------------------\n$(KERNEL_ELF_TTABLES_SYMS): $(KERNEL_ELF_TTABLES_SYMS_DEPS)\n\t$(call color_header, \"Generating kernel symbols and patching kernel ELF\")\n\t@$(MAKE) --no-print-directory -f kernel_symbols.mk\n\n##------------------------------------------------------------------------------\n## Generate the stripped kernel binary\n##------------------------------------------------------------------------------\n$(KERNEL_BIN): $(KERNEL_ELF_TTABLES_SYMS)\n\t$(call color_header, \"Generating stripped binary\")\n\t@$(OBJCOPY_CMD) $(KERNEL_ELF_TTABLES_SYMS) $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Name\")\n\t@echo $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Size\")\n\t$(call disk_usage_KiB, $(KERNEL_BIN))\n\n##------------------------------------------------------------------------------\n## Generate the documentation\n##------------------------------------------------------------------------------\ndoc: clean\n\t$(call color_header, \"Generating docs\")\n\t@$(DOC_CMD) --document-private-items --open\n\n##------------------------------------------------------------------------------\n## Run the kernel in QEMU\n##------------------------------------------------------------------------------\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\nqemu:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\nqemu: $(KERNEL_BIN)\n\t$(call color_header, \"Launching QEMU\")\n\t@$(DOCKER_QEMU) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\nendif\n\n##------------------------------------------------------------------------------\n## Push the kernel to the real HW target\n##------------------------------------------------------------------------------\nchainboot: $(KERNEL_BIN)\n\t@$(DOCKER_CHAINBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run clippy\n##------------------------------------------------------------------------------\nclippy:\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(CLIPPY_CMD)\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(CLIPPY_CMD) --features test_build --tests \\\n                --manifest-path $(KERNEL_MANIFEST)\n\n##------------------------------------------------------------------------------\n## Clean\n##------------------------------------------------------------------------------\nclean:\n\trm -rf target $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run readelf\n##------------------------------------------------------------------------------\nreadelf: $(KERNEL_ELF)\n\t$(call color_header, \"Launching readelf\")\n\t@$(DOCKER_TOOLS) $(READELF_BINARY) --headers $(KERNEL_ELF)\n\n##------------------------------------------------------------------------------\n## Run objdump\n##------------------------------------------------------------------------------\nobjdump: $(KERNEL_ELF)\n\t$(call color_header, \"Launching objdump\")\n\t@$(DOCKER_TOOLS) $(OBJDUMP_BINARY) --disassemble --demangle \\\n                --section .text   \\\n                --section .rodata \\\n                $(KERNEL_ELF) | rustfilt\n\n##------------------------------------------------------------------------------\n## Run nm\n##------------------------------------------------------------------------------\nnm: $(KERNEL_ELF)\n\t$(call color_header, \"Launching nm\")\n\t@$(DOCKER_TOOLS) $(NM_BINARY) --demangle --print-size $(KERNEL_ELF) | sort | rustfilt\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Debugging targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: jtagboot openocd gdb gdb-opt0\n\n##------------------------------------------------------------------------------\n## Push the JTAG boot image to the real HW target\n##------------------------------------------------------------------------------\njtagboot:\n\t@$(DOCKER_JTAGBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(JTAG_BOOT_IMAGE)\n\n##------------------------------------------------------------------------------\n## Start OpenOCD session\n##------------------------------------------------------------------------------\nopenocd:\n\t$(call color_header, \"Launching OpenOCD\")\n\t@$(DOCKER_OPENOCD) openocd $(OPENOCD_ARG)\n\n##------------------------------------------------------------------------------\n## Start GDB session\n##------------------------------------------------------------------------------\ngdb: RUSTC_MISC_ARGS += -C debuginfo=2\ngdb-opt0: RUSTC_MISC_ARGS += -C debuginfo=2 -C opt-level=0\ngdb gdb-opt0: $(KERNEL_ELF)\n\t$(call color_header, \"Launching GDB\")\n\t@$(DOCKER_GDB) gdb-multiarch -q $(KERNEL_ELF)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Testing targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: test test_boot test_unit test_integration\n\ntest_unit test_integration: FEATURES += --features test_build\n\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\ntest_boot test_unit test_integration test:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\n##------------------------------------------------------------------------------\n## Run boot test\n##------------------------------------------------------------------------------\ntest_boot: $(KERNEL_BIN)\n\t$(call color_header, \"Boot test - $(BSP)\")\n\t@$(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Helpers for unit and integration test targets\n##------------------------------------------------------------------------------\ndefine KERNEL_TEST_RUNNER\n#!/usr/bin/env bash\n\n    # The cargo test runner seems to change into the crate under test's directory. Therefore, ensure\n    # this script executes from the root.\n    cd $(shell pwd)\n\n    TEST_ELF=$$(echo $$1 | sed -e 's/.*target/target/g')\n    TEST_ELF_SYMS=\"$${TEST_ELF}_syms\"\n    TEST_BINARY=$$(echo $$1.img | sed -e 's/.*target/target/g')\n\n    $(DOCKER_TOOLS) $(EXEC_TT_TOOL) $(BSP) $$TEST_ELF > /dev/null\n\n    # This overrides the two ENV variables. The other ENV variables that are required as input for\n    # the .mk file are set already because they are exported by this Makefile and this script is\n    # started by the same.\n    KERNEL_SYMBOLS_INPUT_ELF=$$TEST_ELF           \\\n        KERNEL_SYMBOLS_OUTPUT_ELF=$$TEST_ELF_SYMS \\\n        $(MAKE) --no-print-directory -f kernel_symbols.mk > /dev/null 2>&1\n\n    $(OBJCOPY_CMD) $$TEST_ELF_SYMS $$TEST_BINARY\n    $(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_TEST_ARGS) -kernel $$TEST_BINARY\nendef\n\nexport KERNEL_TEST_RUNNER\n\ndefine test_prepare\n    @mkdir -p target\n    @echo \"$$KERNEL_TEST_RUNNER\" > target/kernel_test_runner.sh\n    @chmod +x target/kernel_test_runner.sh\nendef\n\n##------------------------------------------------------------------------------\n## Run unit test(s)\n##------------------------------------------------------------------------------\ntest_unit:\n\t$(call color_header, \"Compiling unit test(s) - $(BSP)\")\n\t$(call test_prepare)\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(TEST_CMD) --lib\n\n##------------------------------------------------------------------------------\n## Run integration test(s)\n##------------------------------------------------------------------------------\ntest_integration:\n\t$(call color_header, \"Compiling integration test(s) - $(BSP)\")\n\t$(call test_prepare)\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(TEST_CMD) $(TEST_ARG)\n\ntest: test_boot test_unit test_integration\n\nendif\n"
  },
  {
    "path": "17_kernel_symbols/README.md",
    "content": "# Tutorial 17 - Kernel Symbols\n\n## tl;dr\n\n- To enrich and augment existing and future debugging code, we add support for `kernel symbol`\n  lookup.\n\n## Table of Contents\n\n- [Introduction](#introduction)\n- [Implementation](#implementation)\n  - [Linking Changes](#linking-changes)\n  - [Kernel Symbols Tool](#kernel-symbols-tool)\n  - [Lookup Code](#lookup-code)\n- [Test it](#test-it)\n- [Diff to previous](#diff-to-previous)\n\n## Introduction\n\nEver since the first tutorial, it was possible to execute the `make nm` target in order to view all\n`kernel symbols`. The kernel itself, however, does not have any means yet to correlate a virtual\naddress to a symbol during runtime. Gaining this capability would be useful for augmenting\ndebug-related prints. For example, when the kernel is handling an `exception`, it prints the content\nof the `exception link register`, which is the program address where the CPU was executing from when\nthe exception happened.\n\nUntil now, in order to understand to which function or code such an address belongs to, a manual\nlookup by the person debugging the issue was necessary. In this tutorial, we are adding a `data\nstructure` to the kernel which contains _all the symbol names and corresponding address ranges_.\nThis enables the kernel to print symbol names in existing and future debug-related code, which\nimproves triaging of issues by humans, because it does away with the manual lookup.\n\nThis tutorial is mostly is an enabler for the upcoming tutorial that will add [`backtracing`]\nsupport.\n\n[`backtracing`]: https://en.wikipedia.org/wiki/Stack_trace\n\n## Implementation\n\nFirst of all, a new support crate is added under `$ROOT/libraries/debug-symbol-types`. It contains\nthe definition for `struct Symbol`:\n\n```rust\n/// A symbol containing a size.\n#[repr(C)]\npub struct Symbol {\n    addr_range: Range<usize>,\n    name: &'static str,\n}\n```\n\nTo enable the kernel to lookup symbol names, we will add an `array` to the kernel binary that\ncontains all the kernel symbols. Because we can query the final symbol names and addresses only\n_after_ the kernel has been `linked`, the same approach as for the `translation tables` will be\nused: The symbols array will be patched into a `placeholder section` of the final kernel `ELF`.\n\n### Linking Changes\n\nIn the `kernel.ld` linker script, we define a new section named `kernel_symbols` and give it a size\nof `32 KiB`:\n\n```ld.s\n    .rodata         : ALIGN(8) { *(.rodata*) } :segment_code\n    .got            : ALIGN(8) { *(.got)     } :segment_code\n    .kernel_symbols : ALIGN(8) {\n        __kernel_symbols_start = .;\n        . += 32 * 1024;\n    } :segment_code\n```\n\nAlso, we are providing the start address of the section through the symbol `__kernel_symbols_start`,\nwhich will be used by our `Rust` code later on.\n\n### Kernel Symbols Tool\n\nUnder `$ROOT/tools/kernel_symbols_tool`, we are adding a helper tool that is able to dynamically\ngenerate an `array` of all the kernel symbols and patch it into the final kernel `ELF`. In our main\n`Makefile`, we are invoking the tool after the translation table generation. In the first step, the\ntool generates a temporary `Rust` file that instantiates the symbols array. Here is an example of\nhow this can look like:\n\n```console\n$ head ./target/aarch64-unknown-none-softfloat/release/kernel+ttables_symbols_demangled.rs\n```\n```rust\nuse debug_symbol_types::Symbol;\n\n# [no_mangle]\n# [link_section = \".rodata.symbol_desc\"]\nstatic KERNEL_SYMBOLS: [Symbol; 139] = [\n    Symbol::new(18446744072635809792, 124, \"_start\"),\n    Symbol::new(18446744072635809920, 8, \"BOOT_CORE_ID\"),\n    Symbol::new(18446744072635809928, 8, \"PHYS_KERNEL_TABLES_BASE_ADDR\"),\n    Symbol::new(18446744072635809936, 80, \"_start_rust\"),\n    Symbol::new(18446744072635813888, 84, \"__exception_restore_context\"),\n    // Many more\n```\n\nNext, the _helper crate_ `$ROOT/kernel_symbols` is compiled. This crate contains a single `main.rs`\nthat just includes the temporary symbols file shown above.\n\n```rust\n//! Generation of kernel symbols.\n\n#![no_std]\n#![no_main]\n\n#[cfg(feature = \"generated_symbols_available\")]\ninclude!(env!(\"KERNEL_SYMBOLS_DEMANGLED_RS\"));\n```\n\n`KERNEL_SYMBOLS_DEMANGLED_RS` is set by the corresponding `build.rs` file. The helper crate has its\nown `linker file`, which ensures that that just the array and the corresponding strings that it\nreferences are kept:\n\n```ld.s\nSECTIONS\n{\n    .rodata : {\n        ASSERT(. > 0xffffffff00000000, \"Expected higher half address\")\n\n        KEEP(*(.rodata.symbol_desc*))\n        . = ALIGN(8);\n        *(.rodata*)\n    }\n}\n```\n\nAfterwards, `objcopy` is used to strip the produced helper crate ELF. What remains is a small\n`binary blob` that just contains the symbols array and the `names` that are referenced. To ensure\nthat these references are valid kernel addresses (remember that those are defined as `name: &'static\nstr`, so basically a pointer to a kernel address), the sub-makefile compiling this helper crate\n(`$ROOT/kernel_symbols.mk`) did the following:\n\nIt used the `kernel_symbols_tool` to query the virtual address of the `kernel_symbols` **section**\n(of the final kernel ELF). This address was then supplied to the linker when the helper crate was\nlinked (emphasis on the `--section-start=.rodata=` part):\n\n```Makefile\nGET_SYMBOLS_SECTION_VIRT_ADDR = $(DOCKER_TOOLS) $(EXEC_SYMBOLS_TOOL) \\\n    --get_symbols_section_virt_addr $(KERNEL_SYMBOLS_OUTPUT_ELF)\n\nRUSTFLAGS = -C link-arg=--script=$(KERNEL_SYMBOLS_LINKER_SCRIPT) \\\n    -C link-arg=--section-start=.rodata=$$($(GET_SYMBOLS_SECTION_VIRT_ADDR))\n```\n\nThis might be a bit convoluted, but the main take away is: This ensures that the start address of\nthe `.rodata` section of the `kernel_symbols` helper crate is exactly the same address as the\n`placeholder section` of the final kernel ELF where the symbols `binary blob` will be patched into.\nThe latter is the last step done by the tool.\n\n### Lookup Code\n\nIn the kernel, we add the file `src/symbols.rs`. It makes the linker-provided symbol\n`__kernel_symbols_start` that we saw earlier accesible, and also defines `NUM_KERNEL_SYMBOLS`:\n\n```rust\n#[no_mangle]\nstatic NUM_KERNEL_SYMBOLS: u64 = 0;\n```\n\nWhen the `kernel_symbols_tool` patches the symbols blob into the kernel ELF, it also updates this\nvalue to reflect the number of symbols that are available. This is needed for the code that\ninternally crafts the slice of symbols that the kernel uses for lookup:\n\n```rust\nfn kernel_symbol_section_virt_start_addr() -> Address<Virtual> {\n    Address::new(unsafe { __kernel_symbols_start.get() as usize })\n}\n\nfn num_kernel_symbols() -> usize {\n    unsafe {\n        // Read volatile is needed here to prevent the compiler from optimizing NUM_KERNEL_SYMBOLS\n        // away.\n        core::ptr::read_volatile(&NUM_KERNEL_SYMBOLS as *const u64) as usize\n    }\n}\n\nfn kernel_symbols_slice() -> &'static [Symbol] {\n    let ptr = kernel_symbol_section_virt_start_addr().as_usize() as *const Symbol;\n\n    unsafe { slice::from_raw_parts(ptr, num_kernel_symbols()) }\n}\n```\n\nLookup is done by just iterating over the slice:\n\n```rust\n/// Retrieve the symbol corresponding to a virtual address, if any.\npub fn lookup_symbol(addr: Address<Virtual>) -> Option<&'static Symbol> {\n    kernel_symbols_slice()\n        .iter()\n        .find(|&i| i.contains(addr.as_usize()))\n}\n```\n\nAnd that's it for this tutorial. The upcoming tutorial on `backtracing` will put this code to more\nprominent use.\n\n## Test it\n\nFor now, symbol lookup can be observed in the integration test for synchronous exception handling.\nHere, the kernel now also prints the symbol name that corresponds to the value of `ELR_EL1`. In the\nfollowing case, this is `kernel_init()`, which is where the the exception is generated in the test:\n\n```console\n$ TEST=02_exception_sync_page_fault make test_integration\n[...]\n         -------------------------------------------------------------------\n         🦀 Testing synchronous exception handling by causing a page fault\n         -------------------------------------------------------------------\n\n         [    0.002640] Writing to bottom of address space to address 1 GiB...\n         [    0.004549] Kernel panic!\n\n         Panic location:\n               File 'kernel/src/_arch/aarch64/exception.rs', line 59, column 5\n\n         CPU Exception!\n\n         ESR_EL1: 0x96000004\n\n         ...\n\n         ELR_EL1: 0xffffffffc0001118\n               Symbol: kernel_init\n```\n\n## Diff to previous\n```diff\n\ndiff -uNr 16_virtual_mem_part4_higher_half_kernel/Cargo.toml 17_kernel_symbols/Cargo.toml\n--- 16_virtual_mem_part4_higher_half_kernel/Cargo.toml\n+++ 17_kernel_symbols/Cargo.toml\n@@ -2,7 +2,8 @@\n\n members = [\n         \"libraries/*\",\n-        \"kernel\"\n+        \"kernel\",\n+        \"kernel_symbols\"\n ]\n\n [profile.release]\n\ndiff -uNr 16_virtual_mem_part4_higher_half_kernel/kernel/Cargo.toml 17_kernel_symbols/kernel/Cargo.toml\n--- 16_virtual_mem_part4_higher_half_kernel/kernel/Cargo.toml\n+++ 17_kernel_symbols/kernel/Cargo.toml\n@@ -1,6 +1,6 @@\n [package]\n name = \"mingo\"\n-version = \"0.16.0\"\n+version = \"0.17.0\"\n authors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\n edition = \"2021\"\n\n@@ -16,6 +16,7 @@\n\n [dependencies]\n test-types = { path = \"../libraries/test-types\" }\n+debug-symbol-types = { path = \"../libraries/debug-symbol-types\" }\n\n # Optional dependencies\n tock-registers = { version = \"0.8.x\", default-features = false, features = [\"register_types\"], optional = true }\n\ndiff -uNr 16_virtual_mem_part4_higher_half_kernel/kernel/src/_arch/aarch64/exception.rs 17_kernel_symbols/kernel/src/_arch/aarch64/exception.rs\n--- 16_virtual_mem_part4_higher_half_kernel/kernel/src/_arch/aarch64/exception.rs\n+++ 17_kernel_symbols/kernel/src/_arch/aarch64/exception.rs\n@@ -11,7 +11,7 @@\n //!\n //! crate::exception::arch_exception\n\n-use crate::exception;\n+use crate::{exception, memory, symbols};\n use aarch64_cpu::{asm::barrier, registers::*};\n use core::{arch::global_asm, cell::UnsafeCell, fmt};\n use tock_registers::{\n@@ -260,6 +260,14 @@\n\n         writeln!(f, \"{}\", self.spsr_el1)?;\n         writeln!(f, \"ELR_EL1: {:#018x}\", self.elr_el1)?;\n+        writeln!(\n+            f,\n+            \"      Symbol: {}\",\n+            match symbols::lookup_symbol(memory::Address::new(self.elr_el1 as usize)) {\n+                Some(sym) => sym.name(),\n+                _ => \"Symbol not found\",\n+            }\n+        )?;\n         writeln!(f)?;\n         writeln!(f, \"General purpose register:\")?;\n\n\ndiff -uNr 16_virtual_mem_part4_higher_half_kernel/kernel/src/bsp/raspberrypi/kernel.ld 17_kernel_symbols/kernel/src/bsp/raspberrypi/kernel.ld\n--- 16_virtual_mem_part4_higher_half_kernel/kernel/src/bsp/raspberrypi/kernel.ld\n+++ 17_kernel_symbols/kernel/src/bsp/raspberrypi/kernel.ld\n@@ -56,7 +56,11 @@\n         *(.text*)                 /* Everything else */\n     } :segment_code\n\n-    .rodata : ALIGN(8) { *(.rodata*) } :segment_code\n+    .rodata         : ALIGN(8) { *(.rodata*) } :segment_code\n+    .kernel_symbols : ALIGN(8) {\n+        __kernel_symbols_start = .;\n+        . += 32 * 1024;\n+    } :segment_code\n\n     . = ALIGN(PAGE_SIZE);\n     __code_end_exclusive = .;\n\ndiff -uNr 16_virtual_mem_part4_higher_half_kernel/kernel/src/bsp/raspberrypi/memory.rs 17_kernel_symbols/kernel/src/bsp/raspberrypi/memory.rs\n--- 16_virtual_mem_part4_higher_half_kernel/kernel/src/bsp/raspberrypi/memory.rs\n+++ 17_kernel_symbols/kernel/src/bsp/raspberrypi/memory.rs\n@@ -20,6 +20,7 @@\n //! | .text                                 |\n //! | .rodata                               |\n //! | .got                                  |\n+//! | .kernel_symbols                       |\n //! |                                       |\n //! +---------------------------------------+\n //! |                                       | data_start == code_end_exclusive\n@@ -41,6 +42,7 @@\n //! | .text                                 |\n //! | .rodata                               |\n //! | .got                                  |\n+//! | .kernel_symbols                       |\n //! |                                       |\n //! +---------------------------------------+\n //! |                                       | data_start == code_end_exclusive\n\ndiff -uNr 16_virtual_mem_part4_higher_half_kernel/kernel/src/lib.rs 17_kernel_symbols/kernel/src/lib.rs\n--- 16_virtual_mem_part4_higher_half_kernel/kernel/src/lib.rs\n+++ 17_kernel_symbols/kernel/src/lib.rs\n@@ -142,6 +142,7 @@\n pub mod memory;\n pub mod print;\n pub mod state;\n+pub mod symbols;\n pub mod time;\n\n //--------------------------------------------------------------------------------------------------\n\ndiff -uNr 16_virtual_mem_part4_higher_half_kernel/kernel/src/symbols.rs 17_kernel_symbols/kernel/src/symbols.rs\n--- 16_virtual_mem_part4_higher_half_kernel/kernel/src/symbols.rs\n+++ 17_kernel_symbols/kernel/src/symbols.rs\n@@ -0,0 +1,88 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Debug symbol support.\n+\n+use crate::memory::{Address, Virtual};\n+use core::{cell::UnsafeCell, slice};\n+use debug_symbol_types::Symbol;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+// Symbol from the linker script.\n+extern \"Rust\" {\n+    static __kernel_symbols_start: UnsafeCell<()>;\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Global instances\n+//--------------------------------------------------------------------------------------------------\n+\n+/// This will be patched to the correct value by the \"kernel symbols tool\" after linking. This given\n+/// value here is just a (safe) dummy.\n+#[no_mangle]\n+static NUM_KERNEL_SYMBOLS: u64 = 0;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Code\n+//--------------------------------------------------------------------------------------------------\n+\n+fn kernel_symbol_section_virt_start_addr() -> Address<Virtual> {\n+    Address::new(unsafe { __kernel_symbols_start.get() as usize })\n+}\n+\n+fn num_kernel_symbols() -> usize {\n+    unsafe {\n+        // Read volatile is needed here to prevent the compiler from optimizing NUM_KERNEL_SYMBOLS\n+        // away.\n+        core::ptr::read_volatile(&NUM_KERNEL_SYMBOLS as *const u64) as usize\n+    }\n+}\n+\n+fn kernel_symbols_slice() -> &'static [Symbol] {\n+    let ptr = kernel_symbol_section_virt_start_addr().as_usize() as *const Symbol;\n+\n+    unsafe { slice::from_raw_parts(ptr, num_kernel_symbols()) }\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Retrieve the symbol corresponding to a virtual address, if any.\n+pub fn lookup_symbol(addr: Address<Virtual>) -> Option<&'static Symbol> {\n+    kernel_symbols_slice()\n+        .iter()\n+        .find(|&i| i.contains(addr.as_usize()))\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Testing\n+//--------------------------------------------------------------------------------------------------\n+\n+#[cfg(test)]\n+mod tests {\n+    use super::*;\n+    use test_macros::kernel_test;\n+\n+    /// Sanity of symbols module.\n+    #[kernel_test]\n+    fn symbols_sanity() {\n+        let first_sym = lookup_symbol(Address::new(\n+            crate::common::is_aligned as *const usize as usize,\n+        ))\n+        .unwrap()\n+        .name();\n+\n+        assert_eq!(first_sym, \"libkernel::common::is_aligned\");\n+\n+        let second_sym = lookup_symbol(Address::new(crate::version as *const usize as usize))\n+            .unwrap()\n+            .name();\n+\n+        assert_eq!(second_sym, \"libkernel::version\");\n+    }\n+}\n\ndiff -uNr 16_virtual_mem_part4_higher_half_kernel/kernel_symbols/build.rs 17_kernel_symbols/kernel_symbols/build.rs\n--- 16_virtual_mem_part4_higher_half_kernel/kernel_symbols/build.rs\n+++ 17_kernel_symbols/kernel_symbols/build.rs\n@@ -0,0 +1,14 @@\n+use std::{env, path::Path};\n+\n+fn main() {\n+    if let Ok(path) = env::var(\"KERNEL_SYMBOLS_DEMANGLED_RS\") {\n+        if Path::new(&path).exists() {\n+            println!(\"cargo:rustc-cfg=feature=\\\"generated_symbols_available\\\"\")\n+        }\n+    }\n+\n+    println!(\n+        \"cargo:rerun-if-changed={}\",\n+        Path::new(\"kernel_symbols.ld\").display()\n+    );\n+}\n\ndiff -uNr 16_virtual_mem_part4_higher_half_kernel/kernel_symbols/Cargo.toml 17_kernel_symbols/kernel_symbols/Cargo.toml\n--- 16_virtual_mem_part4_higher_half_kernel/kernel_symbols/Cargo.toml\n+++ 17_kernel_symbols/kernel_symbols/Cargo.toml\n@@ -0,0 +1,15 @@\n+[package]\n+name = \"kernel_symbols\"\n+version = \"0.1.0\"\n+edition = \"2021\"\n+\n+[features]\n+default = []\n+generated_symbols_available = []\n+\n+##--------------------------------------------------------------------------------------------------\n+## Dependencies\n+##--------------------------------------------------------------------------------------------------\n+\n+[dependencies]\n+debug-symbol-types = { path = \"../libraries/debug-symbol-types\" }\n\ndiff -uNr 16_virtual_mem_part4_higher_half_kernel/kernel_symbols/kernel_symbols.ld 17_kernel_symbols/kernel_symbols/kernel_symbols.ld\n--- 16_virtual_mem_part4_higher_half_kernel/kernel_symbols/kernel_symbols.ld\n+++ 17_kernel_symbols/kernel_symbols/kernel_symbols.ld\n@@ -0,0 +1,15 @@\n+/* SPDX-License-Identifier: MIT OR Apache-2.0\n+ *\n+ * Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n+ */\n+\n+SECTIONS\n+{\n+    .rodata : {\n+        ASSERT(. > 0xffffffff00000000, \"Expected higher half address\")\n+\n+        KEEP(*(.rodata.symbol_desc*))\n+        . = ALIGN(8);\n+        *(.rodata*)\n+    }\n+}\n\ndiff -uNr 16_virtual_mem_part4_higher_half_kernel/kernel_symbols/src/main.rs 17_kernel_symbols/kernel_symbols/src/main.rs\n--- 16_virtual_mem_part4_higher_half_kernel/kernel_symbols/src/main.rs\n+++ 17_kernel_symbols/kernel_symbols/src/main.rs\n@@ -0,0 +1,16 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Generation of kernel symbols.\n+\n+#![no_std]\n+#![no_main]\n+\n+#[cfg(feature = \"generated_symbols_available\")]\n+include!(env!(\"KERNEL_SYMBOLS_DEMANGLED_RS\"));\n+\n+#[panic_handler]\n+fn panic(_info: &core::panic::PanicInfo) -> ! {\n+    unimplemented!()\n+}\n\ndiff -uNr 16_virtual_mem_part4_higher_half_kernel/kernel_symbols.mk 17_kernel_symbols/kernel_symbols.mk\n--- 16_virtual_mem_part4_higher_half_kernel/kernel_symbols.mk\n+++ 17_kernel_symbols/kernel_symbols.mk\n@@ -0,0 +1,117 @@\n+## SPDX-License-Identifier: MIT OR Apache-2.0\n+##\n+## Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+include ../common/format.mk\n+include ../common/docker.mk\n+\n+##--------------------------------------------------------------------------------------------------\n+## Check for input variables that need be exported by the calling Makefile\n+##--------------------------------------------------------------------------------------------------\n+ifndef KERNEL_SYMBOLS_TOOL_PATH\n+$(error KERNEL_SYMBOLS_TOOL_PATH is not set)\n+endif\n+\n+ifndef TARGET\n+$(error TARGET is not set)\n+endif\n+\n+ifndef KERNEL_SYMBOLS_INPUT_ELF\n+$(error KERNEL_SYMBOLS_INPUT_ELF is not set)\n+endif\n+\n+ifndef KERNEL_SYMBOLS_OUTPUT_ELF\n+$(error KERNEL_SYMBOLS_OUTPUT_ELF is not set)\n+endif\n+\n+\n+\n+##--------------------------------------------------------------------------------------------------\n+## Targets and Prerequisites\n+##--------------------------------------------------------------------------------------------------\n+KERNEL_SYMBOLS_MANIFEST      = kernel_symbols/Cargo.toml\n+KERNEL_SYMBOLS_LINKER_SCRIPT = kernel_symbols/kernel_symbols.ld\n+\n+KERNEL_SYMBOLS_RS           = $(KERNEL_SYMBOLS_INPUT_ELF)_symbols.rs\n+KERNEL_SYMBOLS_DEMANGLED_RS = $(shell pwd)/$(KERNEL_SYMBOLS_INPUT_ELF)_symbols_demangled.rs\n+\n+KERNEL_SYMBOLS_ELF      = target/$(TARGET)/release/kernel_symbols\n+KERNEL_SYMBOLS_STRIPPED = target/$(TARGET)/release/kernel_symbols_stripped\n+\n+# Export for build.rs of kernel_symbols crate.\n+export KERNEL_SYMBOLS_DEMANGLED_RS\n+\n+\n+\n+##--------------------------------------------------------------------------------------------------\n+## Command building blocks\n+##--------------------------------------------------------------------------------------------------\n+GET_SYMBOLS_SECTION_VIRT_ADDR = $(DOCKER_TOOLS) $(EXEC_SYMBOLS_TOOL) \\\n+    --get_symbols_section_virt_addr $(KERNEL_SYMBOLS_OUTPUT_ELF)\n+\n+RUSTFLAGS = -C link-arg=--script=$(KERNEL_SYMBOLS_LINKER_SCRIPT) \\\n+    -C link-arg=--section-start=.rodata=$$($(GET_SYMBOLS_SECTION_VIRT_ADDR))\n+\n+RUSTFLAGS_PEDANTIC = $(RUSTFLAGS) \\\n+    -D warnings                   \\\n+    -D missing_docs\n+\n+COMPILER_ARGS = --target=$(TARGET) \\\n+    --release\n+\n+RUSTC_CMD   = cargo rustc $(COMPILER_ARGS) --manifest-path $(KERNEL_SYMBOLS_MANIFEST)\n+OBJCOPY_CMD = rust-objcopy \\\n+    --strip-all            \\\n+    -O binary\n+\n+EXEC_SYMBOLS_TOOL  = ruby $(KERNEL_SYMBOLS_TOOL_PATH)/main.rb\n+\n+##------------------------------------------------------------------------------\n+## Dockerization\n+##------------------------------------------------------------------------------\n+DOCKER_CMD = docker run -t --rm -v $(shell pwd):/work/tutorial -w /work/tutorial\n+\n+# DOCKER_IMAGE defined in include file (see top of this file).\n+DOCKER_TOOLS = $(DOCKER_CMD) $(DOCKER_IMAGE)\n+\n+\n+\n+##--------------------------------------------------------------------------------------------------\n+## Targets\n+##--------------------------------------------------------------------------------------------------\n+.PHONY: all symbols measure_time_start measure_time_finish\n+\n+all: measure_time_start symbols measure_time_finish\n+\n+symbols:\n+\t@cp $(KERNEL_SYMBOLS_INPUT_ELF) $(KERNEL_SYMBOLS_OUTPUT_ELF)\n+\n+\t@$(DOCKER_TOOLS) $(EXEC_SYMBOLS_TOOL) --gen_symbols $(KERNEL_SYMBOLS_OUTPUT_ELF) \\\n+                $(KERNEL_SYMBOLS_RS)\n+\n+\t$(call color_progress_prefix, \"Demangling\")\n+\t@echo Symbol names\n+\t@cat $(KERNEL_SYMBOLS_RS) | rustfilt > $(KERNEL_SYMBOLS_DEMANGLED_RS)\n+\n+\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(RUSTC_CMD)\n+\n+\t$(call color_progress_prefix, \"Stripping\")\n+\t@echo Symbols ELF file\n+\t@$(OBJCOPY_CMD) $(KERNEL_SYMBOLS_ELF) $(KERNEL_SYMBOLS_STRIPPED)\n+\n+\t@$(DOCKER_TOOLS) $(EXEC_SYMBOLS_TOOL) --patch_data $(KERNEL_SYMBOLS_OUTPUT_ELF) \\\n+                $(KERNEL_SYMBOLS_STRIPPED)\n+\n+# Note: The following is the only _trivial_ way I could think of that works out of the box on both\n+# Linux and macOS. Since macOS does not have the moduloN nanosecond format string option, the\n+# resolution is restricted to whole seconds.\n+measure_time_start:\n+\t@date +modulos > /tmp/kernel_symbols_start.date\n+\n+measure_time_finish:\n+\t@date +modulos > /tmp/kernel_symbols_end.date\n+\n+\t$(call color_progress_prefix, \"Finished\")\n+\t@echo \"in $$((`cat /tmp/kernel_symbols_end.date` - `cat /tmp/kernel_symbols_start.date`)).0s\"\n+\n+\t@rm /tmp/kernel_symbols_end.date /tmp/kernel_symbols_start.date\n\ndiff -uNr 16_virtual_mem_part4_higher_half_kernel/libraries/debug-symbol-types/Cargo.toml 17_kernel_symbols/libraries/debug-symbol-types/Cargo.toml\n--- 16_virtual_mem_part4_higher_half_kernel/libraries/debug-symbol-types/Cargo.toml\n+++ 17_kernel_symbols/libraries/debug-symbol-types/Cargo.toml\n@@ -0,0 +1,4 @@\n+[package]\n+name = \"debug-symbol-types\"\n+version = \"0.1.0\"\n+edition = \"2021\"\n\ndiff -uNr 16_virtual_mem_part4_higher_half_kernel/libraries/debug-symbol-types/src/lib.rs 17_kernel_symbols/libraries/debug-symbol-types/src/lib.rs\n--- 16_virtual_mem_part4_higher_half_kernel/libraries/debug-symbol-types/src/lib.rs\n+++ 17_kernel_symbols/libraries/debug-symbol-types/src/lib.rs\n@@ -0,0 +1,45 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Types for implementing debug symbol support.\n+\n+#![no_std]\n+\n+use core::ops::Range;\n+\n+/// A symbol containing a size.\n+#[repr(C)]\n+#[derive(Clone)]\n+pub struct Symbol {\n+    addr_range: Range<usize>,\n+    name: &'static str,\n+}\n+\n+impl Symbol {\n+    /// Create an instance.\n+    pub const fn new(start: usize, size: usize, name: &'static str) -> Symbol {\n+        Symbol {\n+            addr_range: Range {\n+                start,\n+                end: start + size,\n+            },\n+            name,\n+        }\n+    }\n+\n+    /// Returns true if addr is contained in the range.\n+    pub fn contains(&self, addr: usize) -> bool {\n+        self.addr_range.contains(&addr)\n+    }\n+\n+    /// Returns the symbol's name.\n+    pub fn name(&self) -> &'static str {\n+        self.name\n+    }\n+\n+    /// Returns the symbol's size.\n+    pub fn size(&self) -> usize {\n+        self.addr_range.end - self.addr_range.start\n+    }\n+}\n\ndiff -uNr 16_virtual_mem_part4_higher_half_kernel/Makefile 17_kernel_symbols/Makefile\n--- 16_virtual_mem_part4_higher_half_kernel/Makefile\n+++ 17_kernel_symbols/Makefile\n@@ -85,7 +85,24 @@\n KERNEL_ELF_TTABLES      = target/$(TARGET)/release/kernel+ttables\n KERNEL_ELF_TTABLES_DEPS = $(KERNEL_ELF_RAW) $(wildcard $(TT_TOOL_PATH)/*)\n\n-KERNEL_ELF = $(KERNEL_ELF_TTABLES)\n+##------------------------------------------------------------------------------\n+## Kernel symbols\n+##------------------------------------------------------------------------------\n+export KERNEL_SYMBOLS_TOOL_PATH = tools/kernel_symbols_tool\n+\n+KERNEL_ELF_TTABLES_SYMS = target/$(TARGET)/release/kernel+ttables+symbols\n+\n+# Unlike with KERNEL_ELF_RAW, we are not relying on dep-info here. One of the reasons being that the\n+# name of the generated symbols file varies between runs, which can cause confusion.\n+KERNEL_ELF_TTABLES_SYMS_DEPS = $(KERNEL_ELF_TTABLES) \\\n+    $(wildcard kernel_symbols/*)                     \\\n+    $(wildcard $(KERNEL_SYMBOLS_TOOL_PATH)/*)\n+\n+export TARGET\n+export KERNEL_SYMBOLS_INPUT_ELF  = $(KERNEL_ELF_TTABLES)\n+export KERNEL_SYMBOLS_OUTPUT_ELF = $(KERNEL_ELF_TTABLES_SYMS)\n+\n+KERNEL_ELF = $(KERNEL_ELF_TTABLES_SYMS)\n\n\n\n@@ -178,11 +195,18 @@\n \t@$(DOCKER_TOOLS) $(EXEC_TT_TOOL) $(BSP) $(KERNEL_ELF_TTABLES)\n\n ##------------------------------------------------------------------------------\n+## Generate kernel symbols and patch them into the kernel ELF\n+##------------------------------------------------------------------------------\n+$(KERNEL_ELF_TTABLES_SYMS): $(KERNEL_ELF_TTABLES_SYMS_DEPS)\n+\t$(call color_header, \"Generating kernel symbols and patching kernel ELF\")\n+\t@$(MAKE) --no-print-directory -f kernel_symbols.mk\n+\n+##------------------------------------------------------------------------------\n ## Generate the stripped kernel binary\n ##------------------------------------------------------------------------------\n-$(KERNEL_BIN): $(KERNEL_ELF_TTABLES)\n+$(KERNEL_BIN): $(KERNEL_ELF_TTABLES_SYMS)\n \t$(call color_header, \"Generating stripped binary\")\n-\t@$(OBJCOPY_CMD) $(KERNEL_ELF_TTABLES) $(KERNEL_BIN)\n+\t@$(OBJCOPY_CMD) $(KERNEL_ELF_TTABLES_SYMS) $(KERNEL_BIN)\n \t$(call color_progress_prefix, \"Name\")\n \t@echo $(KERNEL_BIN)\n \t$(call color_progress_prefix, \"Size\")\n@@ -191,7 +215,7 @@\n ##------------------------------------------------------------------------------\n ## Generate the documentation\n ##------------------------------------------------------------------------------\n-doc:\n+doc: clean\n \t$(call color_header, \"Generating docs\")\n \t@$(DOC_CMD) --document-private-items --open\n\n@@ -318,10 +342,19 @@\n     cd $(shell pwd)\n\n     TEST_ELF=$$(echo $$1 | sed -e 's/.*target/target/g')\n+    TEST_ELF_SYMS=\"$${TEST_ELF}_syms\"\n     TEST_BINARY=$$(echo $$1.img | sed -e 's/.*target/target/g')\n\n     $(DOCKER_TOOLS) $(EXEC_TT_TOOL) $(BSP) $$TEST_ELF > /dev/null\n-    $(OBJCOPY_CMD) $$TEST_ELF $$TEST_BINARY\n+\n+    # This overrides the two ENV variables. The other ENV variables that are required as input for\n+    # the .mk file are set already because they are exported by this Makefile and this script is\n+    # started by the same.\n+    KERNEL_SYMBOLS_INPUT_ELF=$$TEST_ELF           \\\n+        KERNEL_SYMBOLS_OUTPUT_ELF=$$TEST_ELF_SYMS \\\n+        $(MAKE) --no-print-directory -f kernel_symbols.mk > /dev/null 2>&1\n+\n+    $(OBJCOPY_CMD) $$TEST_ELF_SYMS $$TEST_BINARY\n     $(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_TEST_ARGS) -kernel $$TEST_BINARY\n endef\n\n\ndiff -uNr 16_virtual_mem_part4_higher_half_kernel/tools/kernel_symbols_tool/cmds.rb 17_kernel_symbols/tools/kernel_symbols_tool/cmds.rb\n--- 16_virtual_mem_part4_higher_half_kernel/tools/kernel_symbols_tool/cmds.rb\n+++ 17_kernel_symbols/tools/kernel_symbols_tool/cmds.rb\n@@ -0,0 +1,45 @@\n+# frozen_string_literal: true\n+\n+# SPDX-License-Identifier: MIT OR Apache-2.0\n+#\n+# Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+def generate_symbols(kernel_elf, output_file)\n+    File.open(output_file, 'w') do |file|\n+        header = <<~HEREDOC\n+            use debug_symbol_types::Symbol;\n+\n+            # [no_mangle]\n+            # [link_section = \".rodata.symbol_desc\"]\n+            static KERNEL_SYMBOLS: [Symbol; #{kernel_elf.num_symbols}] = [\n+        HEREDOC\n+\n+        file.write(header)\n+        kernel_elf.symbols.each do |sym|\n+            value = sym.header.st_value\n+            size = sym.header.st_size\n+            name = sym.name\n+\n+            file.write(\"    Symbol::new(#{value}, #{size}, \\\"#{name}\\\"),\\n\")\n+        end\n+        file.write(\"];\\n\")\n+    end\n+end\n+\n+def get_symbols_section_virt_addr(kernel_elf)\n+    kernel_elf.kernel_symbols_section_virt_addr\n+end\n+\n+def patch_symbol_data(kernel_elf, symbols_blob_path)\n+    symbols_blob = File.binread(symbols_blob_path)\n+\n+    raise if symbols_blob.size > kernel_elf.kernel_symbols_section_size\n+\n+    File.binwrite(kernel_elf.path, File.binread(symbols_blob_path),\n+                  kernel_elf.kernel_symbols_section_offset_in_file)\n+end\n+\n+def patch_num_symbols(kernel_elf)\n+    num_packed = [kernel_elf.num_symbols].pack('Q<*') # \"Q\" == uint64_t, \"<\" == little endian\n+    File.binwrite(kernel_elf.path, num_packed, kernel_elf.num_kernel_symbols_offset_in_file)\n+end\n\ndiff -uNr 16_virtual_mem_part4_higher_half_kernel/tools/kernel_symbols_tool/kernel_elf.rb 17_kernel_symbols/tools/kernel_symbols_tool/kernel_elf.rb\n--- 16_virtual_mem_part4_higher_half_kernel/tools/kernel_symbols_tool/kernel_elf.rb\n+++ 17_kernel_symbols/tools/kernel_symbols_tool/kernel_elf.rb\n@@ -0,0 +1,74 @@\n+# frozen_string_literal: true\n+\n+# SPDX-License-Identifier: MIT OR Apache-2.0\n+#\n+# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+# KernelELF\n+class KernelELF\n+    attr_reader :path\n+\n+    def initialize(kernel_elf_path, kernel_symbols_section, num_kernel_symbols)\n+        @elf = ELFTools::ELFFile.new(File.open(kernel_elf_path))\n+        @symtab_section = @elf.section_by_name('.symtab')\n+\n+        @path = kernel_elf_path\n+        fetch_values(kernel_symbols_section, num_kernel_symbols)\n+    end\n+\n+    private\n+\n+    def fetch_values(kernel_symbols_section, num_kernel_symbols)\n+        sym = @symtab_section.symbol_by_name(num_kernel_symbols)\n+        raise \"Symbol \\\"#{num_kernel_symbols}\\\" not found\" if sym.nil?\n+\n+        @num_kernel_symbols = sym\n+\n+        section = @elf.section_by_name(kernel_symbols_section)\n+        raise \"Section \\\"#{kernel_symbols_section}\\\" not found\" if section.nil?\n+\n+        @kernel_symbols_section = section\n+    end\n+\n+    def num_kernel_symbols_virt_addr\n+        @num_kernel_symbols.header.st_value\n+    end\n+\n+    def segment_containing_virt_addr(virt_addr)\n+        @elf.each_segments do |segment|\n+            return segment if segment.vma_in?(virt_addr)\n+        end\n+    end\n+\n+    def virt_addr_to_file_offset(virt_addr)\n+        segment = segment_containing_virt_addr(virt_addr)\n+        segment.vma_to_offset(virt_addr)\n+    end\n+\n+    public\n+\n+    def symbols\n+        non_zero_symbols = @symtab_section.symbols.reject { |sym| sym.header.st_size.zero? }\n+        non_zero_symbols.sort_by { |sym| sym.header.st_value }\n+    end\n+\n+    def num_symbols\n+        symbols.size\n+    end\n+\n+    def kernel_symbols_section_virt_addr\n+        @kernel_symbols_section.header.sh_addr.to_i\n+    end\n+\n+    def kernel_symbols_section_size\n+        @kernel_symbols_section.header.sh_size.to_i\n+    end\n+\n+    def kernel_symbols_section_offset_in_file\n+        virt_addr_to_file_offset(kernel_symbols_section_virt_addr)\n+    end\n+\n+    def num_kernel_symbols_offset_in_file\n+        virt_addr_to_file_offset(num_kernel_symbols_virt_addr)\n+    end\n+end\n\ndiff -uNr 16_virtual_mem_part4_higher_half_kernel/tools/kernel_symbols_tool/main.rb 17_kernel_symbols/tools/kernel_symbols_tool/main.rb\n--- 16_virtual_mem_part4_higher_half_kernel/tools/kernel_symbols_tool/main.rb\n+++ 17_kernel_symbols/tools/kernel_symbols_tool/main.rb\n@@ -0,0 +1,47 @@\n+#!/usr/bin/env ruby\n+# frozen_string_literal: true\n+\n+# SPDX-License-Identifier: MIT OR Apache-2.0\n+#\n+# Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+require 'rubygems'\n+require 'bundler/setup'\n+require 'colorize'\n+require 'elftools'\n+\n+require_relative 'kernel_elf'\n+require_relative 'cmds'\n+\n+KERNEL_SYMBOLS_SECTION = '.kernel_symbols'\n+NUM_KERNEL_SYMBOLS = 'NUM_KERNEL_SYMBOLS'\n+\n+cmd = ARGV[0]\n+\n+kernel_elf_path = ARGV[1]\n+kernel_elf = KernelELF.new(kernel_elf_path, KERNEL_SYMBOLS_SECTION, NUM_KERNEL_SYMBOLS)\n+\n+case cmd\n+when '--gen_symbols'\n+    output_file = ARGV[2]\n+\n+    print 'Generating'.rjust(12).green.bold\n+    puts ' Symbols source file'\n+\n+    generate_symbols(kernel_elf, output_file)\n+when '--get_symbols_section_virt_addr'\n+    addr = get_symbols_section_virt_addr(kernel_elf)\n+\n+    puts \"0x#{addr.to_s(16)}\"\n+when '--patch_data'\n+    symbols_blob_path = ARGV[2]\n+    num_symbols = kernel_elf.num_symbols\n+\n+    print 'Patching'.rjust(12).green.bold\n+    puts \" Symbols blob and number of symbols (#{num_symbols}) into ELF\"\n+\n+    patch_symbol_data(kernel_elf, symbols_blob_path)\n+    patch_num_symbols(kernel_elf)\n+else\n+    raise\n+end\n\n```\n"
  },
  {
    "path": "17_kernel_symbols/kernel/Cargo.toml",
    "content": "[package]\nname = \"mingo\"\nversion = \"0.17.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n\n[features]\ndefault = []\nbsp_rpi3 = [\"tock-registers\"]\nbsp_rpi4 = [\"tock-registers\"]\ntest_build = [\"qemu-exit\"]\n\n##--------------------------------------------------------------------------------------------------\n## Dependencies\n##--------------------------------------------------------------------------------------------------\n\n[dependencies]\ntest-types = { path = \"../libraries/test-types\" }\ndebug-symbol-types = { path = \"../libraries/debug-symbol-types\" }\n\n# Optional dependencies\ntock-registers = { version = \"0.8.x\", default-features = false, features = [\"register_types\"], optional = true }\nqemu-exit = { version = \"3.x.x\", optional = true }\n\n# Platform specific dependencies\n[target.'cfg(target_arch = \"aarch64\")'.dependencies]\naarch64-cpu = { version = \"9.x.x\" }\n\n##--------------------------------------------------------------------------------------------------\n## Testing\n##--------------------------------------------------------------------------------------------------\n\n[dev-dependencies]\ntest-macros = { path = \"../libraries/test-macros\" }\n\n# Unit tests are done in the library part of the kernel.\n[lib]\nname = \"libkernel\"\ntest = true\n\n# Disable unit tests for the kernel binary.\n[[bin]]\nname = \"kernel\"\npath = \"src/main.rs\"\ntest = false\n\n# List of tests without harness.\n[[test]]\nname = \"00_console_sanity\"\nharness = false\n\n[[test]]\nname = \"02_exception_sync_page_fault\"\nharness = false\n\n[[test]]\nname = \"03_exception_restore_sanity\"\nharness = false\n"
  },
  {
    "path": "17_kernel_symbols/kernel/build.rs",
    "content": "use std::{env, fs, process};\n\nfn main() {\n    let ld_script_path = match env::var(\"LD_SCRIPT_PATH\") {\n        Ok(var) => var,\n        _ => process::exit(0),\n    };\n\n    let files = fs::read_dir(ld_script_path).unwrap();\n    files\n        .filter_map(Result::ok)\n        .filter(|d| {\n            if let Some(e) = d.path().extension() {\n                e == \"ld\"\n            } else {\n                false\n            }\n        })\n        .for_each(|f| println!(\"cargo:rerun-if-changed={}\", f.path().display()));\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/_arch/aarch64/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural boot code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::boot::arch_boot\n\nuse crate::{memory, memory::Address};\nuse aarch64_cpu::{asm, registers::*};\nuse core::arch::global_asm;\nuse tock_registers::interfaces::Writeable;\n\n// Assembly counterpart to this file.\nglobal_asm!(\n    include_str!(\"boot.s\"),\n    CONST_CURRENTEL_EL2 = const 0x8,\n    CONST_CORE_ID_MASK = const 0b11\n);\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Prepares the transition from EL2 to EL1.\n///\n/// # Safety\n///\n/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.\n/// - The HW state of EL1 must be prepared in a sound way.\n#[inline(always)]\nunsafe fn prepare_el2_to_el1_transition(\n    virt_boot_core_stack_end_exclusive_addr: u64,\n    virt_kernel_init_addr: u64,\n) {\n    // Enable timer counter registers for EL1.\n    CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET);\n\n    // No offset for reading the counters.\n    CNTVOFF_EL2.set(0);\n\n    // Set EL1 execution state to AArch64.\n    HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64);\n\n    // Set up a simulated exception return.\n    //\n    // First, fake a saved program status where all interrupts were masked and SP_EL1 was used as a\n    // stack pointer.\n    SPSR_EL2.write(\n        SPSR_EL2::D::Masked\n            + SPSR_EL2::A::Masked\n            + SPSR_EL2::I::Masked\n            + SPSR_EL2::F::Masked\n            + SPSR_EL2::M::EL1h,\n    );\n\n    // Second, let the link register point to kernel_init().\n    ELR_EL2.set(virt_kernel_init_addr);\n\n    // Set up SP_EL1 (stack pointer), which will be used by EL1 once we \"return\" to it. Since there\n    // are no plans to ever return to EL2, just re-use the same stack.\n    SP_EL1.set(virt_boot_core_stack_end_exclusive_addr);\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The Rust entry of the `kernel` binary.\n///\n/// The function is called from the assembly `_start` function.\n///\n/// # Safety\n///\n/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`.\n#[no_mangle]\npub unsafe extern \"C\" fn _start_rust(\n    phys_kernel_tables_base_addr: u64,\n    virt_boot_core_stack_end_exclusive_addr: u64,\n    virt_kernel_init_addr: u64,\n) -> ! {\n    prepare_el2_to_el1_transition(\n        virt_boot_core_stack_end_exclusive_addr,\n        virt_kernel_init_addr,\n    );\n\n    // Turn on the MMU for EL1.\n    let addr = Address::new(phys_kernel_tables_base_addr as usize);\n    memory::mmu::enable_mmu_and_caching(addr).unwrap();\n\n    // Use `eret` to \"return\" to EL1. Since virtual memory will already be enabled, this results in\n    // execution of kernel_init() in EL1 from its _virtual address_.\n    asm::eret()\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/_arch/aarch64/cpu/boot.s",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2022 Andre Richter <andre.o.richter@gmail.com>\n\n//--------------------------------------------------------------------------------------------------\n// Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Load the address of a symbol into a register, PC-relative.\n//\n// The symbol must lie within +/- 4 GiB of the Program Counter.\n//\n// # Resources\n//\n// - https://sourceware.org/binutils/docs-2.36/as/AArch64_002dRelocations.html\n.macro ADR_REL register, symbol\n\tadrp\t\\register, \\symbol\n\tadd\t\\register, \\register, #:lo12:\\symbol\n.endm\n\n// Load the address of a symbol into a register, absolute.\n//\n// # Resources\n//\n// - https://sourceware.org/binutils/docs-2.36/as/AArch64_002dRelocations.html\n.macro ADR_ABS register, symbol\n\tmovz\t\\register, #:abs_g3:\\symbol\n\tmovk\t\\register, #:abs_g2_nc:\\symbol\n\tmovk\t\\register, #:abs_g1_nc:\\symbol\n\tmovk\t\\register, #:abs_g0_nc:\\symbol\n.endm\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n.section .text._start\n\n//------------------------------------------------------------------------------\n// fn _start()\n//------------------------------------------------------------------------------\n_start:\n\t// Only proceed if the core executes in EL2. Park it otherwise.\n\tmrs\tx0, CurrentEL\n\tcmp\tx0, {CONST_CURRENTEL_EL2}\n\tb.ne\t.L_parking_loop\n\n\t// Only proceed on the boot core. Park it otherwise.\n\tmrs\tx1, MPIDR_EL1\n\tand\tx1, x1, {CONST_CORE_ID_MASK}\n\tldr\tx2, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs\n\tcmp\tx1, x2\n\tb.ne\t.L_parking_loop\n\n\t// If execution reaches here, it is the boot core.\n\n\t// Initialize DRAM.\n\tADR_REL\tx0, __bss_start\n\tADR_REL x1, __bss_end_exclusive\n\n.L_bss_init_loop:\n\tcmp\tx0, x1\n\tb.eq\t.L_prepare_rust\n\tstp\txzr, xzr, [x0], #16\n\tb\t.L_bss_init_loop\n\n\t// Prepare the jump to Rust code.\n.L_prepare_rust:\n\t// Load the base address of the kernel's translation tables.\n\tldr\tx0, PHYS_KERNEL_TABLES_BASE_ADDR // provided by bsp/__board_name__/memory/mmu.rs\n\n\t// Load the _absolute_ addresses of the following symbols. Since the kernel is linked at\n\t// the top of the 64 bit address space, these are effectively virtual addresses.\n\tADR_ABS\tx1, __boot_core_stack_end_exclusive\n\tADR_ABS\tx2, kernel_init\n\n\t// Load the PC-relative address of the stack and set the stack pointer.\n\t//\n\t// Since _start() is the first function that runs after the firmware has loaded the kernel\n\t// into memory, retrieving this symbol PC-relative returns the \"physical\" address.\n\t//\n\t// Setting the stack pointer to this value ensures that anything that still runs in EL2,\n\t// until the kernel returns to EL1 with the MMU enabled, works as well. After the return to\n\t// EL1, the virtual address of the stack retrieved above will be used.\n\tADR_REL\tx3, __boot_core_stack_end_exclusive\n\tmov\tsp, x3\n\n\t// Read the CPU's timer counter frequency and store it in ARCH_TIMER_COUNTER_FREQUENCY.\n\t// Abort if the frequency read back as 0.\n\tADR_REL\tx4, ARCH_TIMER_COUNTER_FREQUENCY // provided by aarch64/time.rs\n\tmrs\tx5, CNTFRQ_EL0\n\tcmp\tx5, xzr\n\tb.eq\t.L_parking_loop\n\tstr\tw5, [x4]\n\n\t// Jump to Rust code. x0, x1 and x2 hold the function arguments provided to _start_rust().\n\tb\t_start_rust\n\n\t// Infinitely wait for events (aka \"park the core\").\n.L_parking_loop:\n\twfe\n\tb\t.L_parking_loop\n\n.size\t_start, . - _start\n.type\t_start, function\n.global\t_start\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/_arch/aarch64/cpu/smp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural symmetric multiprocessing.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::smp::arch_smp\n\nuse aarch64_cpu::registers::*;\nuse tock_registers::interfaces::Readable;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return the executing core's id.\n#[inline(always)]\npub fn core_id<T>() -> T\nwhere\n    T: From<u8>,\n{\n    const CORE_MASK: u64 = 0b11;\n\n    T::from((MPIDR_EL1.get() & CORE_MASK) as u8)\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/_arch/aarch64/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural processor code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::arch_cpu\n\nuse aarch64_cpu::asm;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\npub use asm::nop;\n\n/// Pause execution on the core.\n#[inline(always)]\npub fn wait_forever() -> ! {\n    loop {\n        asm::wfe()\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n#[cfg(feature = \"test_build\")]\nuse qemu_exit::QEMUExit;\n\n#[cfg(feature = \"test_build\")]\nconst QEMU_EXIT_HANDLE: qemu_exit::AArch64 = qemu_exit::AArch64::new();\n\n/// Make the host QEMU binary execute `exit(1)`.\n#[cfg(feature = \"test_build\")]\npub fn qemu_exit_failure() -> ! {\n    QEMU_EXIT_HANDLE.exit_failure()\n}\n\n/// Make the host QEMU binary execute `exit(0)`.\n#[cfg(feature = \"test_build\")]\npub fn qemu_exit_success() -> ! {\n    QEMU_EXIT_HANDLE.exit_success()\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/_arch/aarch64/exception/asynchronous.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural asynchronous exception handling.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::exception::asynchronous::arch_asynchronous\n\nuse aarch64_cpu::registers::*;\nuse core::arch::asm;\nuse tock_registers::interfaces::{Readable, Writeable};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nmod daif_bits {\n    pub const IRQ: u8 = 0b0010;\n}\n\ntrait DaifField {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register>;\n}\n\nstruct Debug;\nstruct SError;\nstruct IRQ;\nstruct FIQ;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl DaifField for Debug {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::D\n    }\n}\n\nimpl DaifField for SError {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::A\n    }\n}\n\nimpl DaifField for IRQ {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::I\n    }\n}\n\nimpl DaifField for FIQ {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::F\n    }\n}\n\nfn is_masked<T>() -> bool\nwhere\n    T: DaifField,\n{\n    DAIF.is_set(T::daif_field())\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Returns whether IRQs are masked on the executing core.\npub fn is_local_irq_masked() -> bool {\n    !is_masked::<IRQ>()\n}\n\n/// Unmask IRQs on the executing core.\n///\n/// It is not needed to place an explicit instruction synchronization barrier after the `msr`.\n/// Quoting the Architecture Reference Manual for ARMv8-A, section C5.1.3:\n///\n/// \"Writes to PSTATE.{PAN, D, A, I, F} occur in program order without the need for additional\n/// synchronization.\"\n#[inline(always)]\npub fn local_irq_unmask() {\n    unsafe {\n        asm!(\n            \"msr DAIFClr, {arg}\",\n            arg = const daif_bits::IRQ,\n            options(nomem, nostack, preserves_flags)\n        );\n    }\n}\n\n/// Mask IRQs on the executing core.\n#[inline(always)]\npub fn local_irq_mask() {\n    unsafe {\n        asm!(\n            \"msr DAIFSet, {arg}\",\n            arg = const daif_bits::IRQ,\n            options(nomem, nostack, preserves_flags)\n        );\n    }\n}\n\n/// Mask IRQs on the executing core and return the previously saved interrupt mask bits (DAIF).\n#[inline(always)]\npub fn local_irq_mask_save() -> u64 {\n    let saved = DAIF.get();\n    local_irq_mask();\n\n    saved\n}\n\n/// Restore the interrupt mask bits (DAIF) using the callee's argument.\n///\n/// # Invariant\n///\n/// - No sanity checks on the input.\n#[inline(always)]\npub fn local_irq_restore(saved: u64) {\n    DAIF.set(saved);\n}\n\n/// Print the AArch64 exceptions status.\n#[rustfmt::skip]\npub fn print_state() {\n    use crate::info;\n\n    let to_mask_str = |x| -> _ {\n        if x { \"Masked\" } else { \"Unmasked\" }\n    };\n\n    info!(\"      Debug:  {}\", to_mask_str(is_masked::<Debug>()));\n    info!(\"      SError: {}\", to_mask_str(is_masked::<SError>()));\n    info!(\"      IRQ:    {}\", to_mask_str(is_masked::<IRQ>()));\n    info!(\"      FIQ:    {}\", to_mask_str(is_masked::<FIQ>()));\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/_arch/aarch64/exception.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural synchronous and asynchronous exception handling.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::exception::arch_exception\n\nuse crate::{exception, memory, symbols};\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::{arch::global_asm, cell::UnsafeCell, fmt};\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    registers::InMemoryRegister,\n};\n\n// Assembly counterpart to this file.\nglobal_asm!(include_str!(\"exception.s\"));\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Wrapper structs for memory copies of registers.\n#[repr(transparent)]\nstruct SpsrEL1(InMemoryRegister<u64, SPSR_EL1::Register>);\nstruct EsrEL1(InMemoryRegister<u64, ESR_EL1::Register>);\n\n/// The exception context as it is stored on the stack on exception entry.\n#[repr(C)]\nstruct ExceptionContext {\n    /// General Purpose Registers.\n    gpr: [u64; 30],\n\n    /// The link register, aka x30.\n    lr: u64,\n\n    /// Exception link register. The program counter at the time the exception happened.\n    elr_el1: u64,\n\n    /// Saved program status.\n    spsr_el1: SpsrEL1,\n\n    /// Exception syndrome register.\n    esr_el1: EsrEL1,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Prints verbose information about the exception and then panics.\nfn default_exception_handler(exc: &ExceptionContext) {\n    panic!(\n        \"CPU Exception!\\n\\n\\\n        {}\",\n        exc\n    );\n}\n\n//------------------------------------------------------------------------------\n// Current, EL0\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn current_el0_synchronous(_e: &mut ExceptionContext) {\n    panic!(\"Should not be here. Use of SP_EL0 in EL1 is not supported.\")\n}\n\n#[no_mangle]\nextern \"C\" fn current_el0_irq(_e: &mut ExceptionContext) {\n    panic!(\"Should not be here. Use of SP_EL0 in EL1 is not supported.\")\n}\n\n#[no_mangle]\nextern \"C\" fn current_el0_serror(_e: &mut ExceptionContext) {\n    panic!(\"Should not be here. Use of SP_EL0 in EL1 is not supported.\")\n}\n\n//------------------------------------------------------------------------------\n// Current, ELx\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn current_elx_synchronous(e: &mut ExceptionContext) {\n    #[cfg(feature = \"test_build\")]\n    {\n        const TEST_SVC_ID: u64 = 0x1337;\n\n        if let Some(ESR_EL1::EC::Value::SVC64) = e.esr_el1.exception_class() {\n            if e.esr_el1.iss() == TEST_SVC_ID {\n                return;\n            }\n        }\n    }\n\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn current_elx_irq(_e: &mut ExceptionContext) {\n    let token = unsafe { &exception::asynchronous::IRQContext::new() };\n    exception::asynchronous::irq_manager().handle_pending_irqs(token);\n}\n\n#[no_mangle]\nextern \"C\" fn current_elx_serror(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n//------------------------------------------------------------------------------\n// Lower, AArch64\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn lower_aarch64_synchronous(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch64_irq(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch64_serror(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n//------------------------------------------------------------------------------\n// Lower, AArch32\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn lower_aarch32_synchronous(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch32_irq(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch32_serror(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n//------------------------------------------------------------------------------\n// Misc\n//------------------------------------------------------------------------------\n\n/// Human readable SPSR_EL1.\n#[rustfmt::skip]\nimpl fmt::Display for SpsrEL1 {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        // Raw value.\n        writeln!(f, \"SPSR_EL1: {:#010x}\", self.0.get())?;\n\n        let to_flag_str = |x| -> _ {\n            if x { \"Set\" } else { \"Not set\" }\n         };\n\n        writeln!(f, \"      Flags:\")?;\n        writeln!(f, \"            Negative (N): {}\", to_flag_str(self.0.is_set(SPSR_EL1::N)))?;\n        writeln!(f, \"            Zero     (Z): {}\", to_flag_str(self.0.is_set(SPSR_EL1::Z)))?;\n        writeln!(f, \"            Carry    (C): {}\", to_flag_str(self.0.is_set(SPSR_EL1::C)))?;\n        writeln!(f, \"            Overflow (V): {}\", to_flag_str(self.0.is_set(SPSR_EL1::V)))?;\n\n        let to_mask_str = |x| -> _ {\n            if x { \"Masked\" } else { \"Unmasked\" }\n        };\n\n        writeln!(f, \"      Exception handling state:\")?;\n        writeln!(f, \"            Debug  (D): {}\", to_mask_str(self.0.is_set(SPSR_EL1::D)))?;\n        writeln!(f, \"            SError (A): {}\", to_mask_str(self.0.is_set(SPSR_EL1::A)))?;\n        writeln!(f, \"            IRQ    (I): {}\", to_mask_str(self.0.is_set(SPSR_EL1::I)))?;\n        writeln!(f, \"            FIQ    (F): {}\", to_mask_str(self.0.is_set(SPSR_EL1::F)))?;\n\n        write!(f, \"      Illegal Execution State (IL): {}\",\n            to_flag_str(self.0.is_set(SPSR_EL1::IL))\n        )\n    }\n}\n\nimpl EsrEL1 {\n    #[inline(always)]\n    fn exception_class(&self) -> Option<ESR_EL1::EC::Value> {\n        self.0.read_as_enum(ESR_EL1::EC)\n    }\n\n    #[cfg(feature = \"test_build\")]\n    #[inline(always)]\n    fn iss(&self) -> u64 {\n        self.0.read(ESR_EL1::ISS)\n    }\n}\n\n/// Human readable ESR_EL1.\n#[rustfmt::skip]\nimpl fmt::Display for EsrEL1 {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        // Raw print of whole register.\n        writeln!(f, \"ESR_EL1: {:#010x}\", self.0.get())?;\n\n        // Raw print of exception class.\n        write!(f, \"      Exception Class         (EC) : {:#x}\", self.0.read(ESR_EL1::EC))?;\n\n        // Exception class.\n        let ec_translation = match self.exception_class() {\n            Some(ESR_EL1::EC::Value::DataAbortCurrentEL) => \"Data Abort, current EL\",\n            _ => \"N/A\",\n        };\n        writeln!(f, \" - {}\", ec_translation)?;\n\n        // Raw print of instruction specific syndrome.\n        write!(f, \"      Instr Specific Syndrome (ISS): {:#x}\", self.0.read(ESR_EL1::ISS))\n    }\n}\n\nimpl ExceptionContext {\n    #[inline(always)]\n    fn exception_class(&self) -> Option<ESR_EL1::EC::Value> {\n        self.esr_el1.exception_class()\n    }\n\n    #[inline(always)]\n    fn fault_address_valid(&self) -> bool {\n        use ESR_EL1::EC::Value::*;\n\n        match self.exception_class() {\n            None => false,\n            Some(ec) => matches!(\n                ec,\n                InstrAbortLowerEL\n                    | InstrAbortCurrentEL\n                    | PCAlignmentFault\n                    | DataAbortLowerEL\n                    | DataAbortCurrentEL\n                    | WatchpointLowerEL\n                    | WatchpointCurrentEL\n            ),\n        }\n    }\n}\n\n/// Human readable print of the exception context.\nimpl fmt::Display for ExceptionContext {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        writeln!(f, \"{}\", self.esr_el1)?;\n\n        if self.fault_address_valid() {\n            writeln!(f, \"FAR_EL1: {:#018x}\", FAR_EL1.get() as usize)?;\n        }\n\n        writeln!(f, \"{}\", self.spsr_el1)?;\n        writeln!(f, \"ELR_EL1: {:#018x}\", self.elr_el1)?;\n        writeln!(\n            f,\n            \"      Symbol: {}\",\n            match symbols::lookup_symbol(memory::Address::new(self.elr_el1 as usize)) {\n                Some(sym) => sym.name(),\n                _ => \"Symbol not found\",\n            }\n        )?;\n        writeln!(f)?;\n        writeln!(f, \"General purpose register:\")?;\n\n        #[rustfmt::skip]\n        let alternating = |x| -> _ {\n            if x % 2 == 0 { \"   \" } else { \"\\n\" }\n        };\n\n        // Print two registers per line.\n        for (i, reg) in self.gpr.iter().enumerate() {\n            write!(f, \"      x{: <2}: {: >#018x}{}\", i, reg, alternating(i))?;\n        }\n        write!(f, \"      lr : {:#018x}\", self.lr)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse crate::exception::PrivilegeLevel;\n\n/// The processing element's current privilege level.\npub fn current_privilege_level() -> (PrivilegeLevel, &'static str) {\n    let el = CurrentEL.read_as_enum(CurrentEL::EL);\n    match el {\n        Some(CurrentEL::EL::Value::EL2) => (PrivilegeLevel::Hypervisor, \"EL2\"),\n        Some(CurrentEL::EL::Value::EL1) => (PrivilegeLevel::Kernel, \"EL1\"),\n        Some(CurrentEL::EL::Value::EL0) => (PrivilegeLevel::User, \"EL0\"),\n        _ => (PrivilegeLevel::Unknown, \"Unknown\"),\n    }\n}\n\n/// Init exception handling by setting the exception vector base address register.\n///\n/// # Safety\n///\n/// - Changes the HW state of the executing core.\n/// - The vector table and the symbol `__exception_vector_table_start` from the linker script must\n///   adhere to the alignment and size constraints demanded by the ARMv8-A Architecture Reference\n///   Manual.\npub unsafe fn handling_init() {\n    // Provided by exception.S.\n    extern \"Rust\" {\n        static __exception_vector_start: UnsafeCell<()>;\n    }\n\n    VBAR_EL1.set(__exception_vector_start.get() as u64);\n\n    // Force VBAR update to complete before next instruction.\n    barrier::isb(barrier::SY);\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/_arch/aarch64/exception.s",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2022 Andre Richter <andre.o.richter@gmail.com>\n\n//--------------------------------------------------------------------------------------------------\n// Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Call the function provided by parameter `\\handler` after saving the exception context. Provide\n/// the context as the first parameter to '\\handler'.\n.macro CALL_WITH_CONTEXT handler\n__vector_\\handler:\n\t// Make room on the stack for the exception context.\n\tsub\tsp,  sp,  #16 * 17\n\n\t// Store all general purpose registers on the stack.\n\tstp\tx0,  x1,  [sp, #16 * 0]\n\tstp\tx2,  x3,  [sp, #16 * 1]\n\tstp\tx4,  x5,  [sp, #16 * 2]\n\tstp\tx6,  x7,  [sp, #16 * 3]\n\tstp\tx8,  x9,  [sp, #16 * 4]\n\tstp\tx10, x11, [sp, #16 * 5]\n\tstp\tx12, x13, [sp, #16 * 6]\n\tstp\tx14, x15, [sp, #16 * 7]\n\tstp\tx16, x17, [sp, #16 * 8]\n\tstp\tx18, x19, [sp, #16 * 9]\n\tstp\tx20, x21, [sp, #16 * 10]\n\tstp\tx22, x23, [sp, #16 * 11]\n\tstp\tx24, x25, [sp, #16 * 12]\n\tstp\tx26, x27, [sp, #16 * 13]\n\tstp\tx28, x29, [sp, #16 * 14]\n\n\t// Add the exception link register (ELR_EL1), saved program status (SPSR_EL1) and exception\n\t// syndrome register (ESR_EL1).\n\tmrs\tx1,  ELR_EL1\n\tmrs\tx2,  SPSR_EL1\n\tmrs\tx3,  ESR_EL1\n\n\tstp\tlr,  x1,  [sp, #16 * 15]\n\tstp\tx2,  x3,  [sp, #16 * 16]\n\n\t// x0 is the first argument for the function called through `\\handler`.\n\tmov\tx0,  sp\n\n\t// Call `\\handler`.\n\tbl\t\\handler\n\n\t// After returning from exception handling code, replay the saved context and return via\n\t// `eret`.\n\tb\t__exception_restore_context\n\n.size\t__vector_\\handler, . - __vector_\\handler\n.type\t__vector_\\handler, function\n.endm\n\n.macro FIQ_SUSPEND\n1:\twfe\n\tb\t1b\n.endm\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n.section .text\n\n//------------------------------------------------------------------------------\n// The exception vector table.\n//------------------------------------------------------------------------------\n\n// Align by 2^11 bytes, as demanded by ARMv8-A. Same as ALIGN(2048) in an ld script.\n.align 11\n\n// Export a symbol for the Rust code to use.\n__exception_vector_start:\n\n// Current exception level with SP_EL0.\n//\n// .org sets the offset relative to section start.\n//\n// # Safety\n//\n// - It must be ensured that `CALL_WITH_CONTEXT` <= 0x80 bytes.\n.org 0x000\n\tCALL_WITH_CONTEXT current_el0_synchronous\n.org 0x080\n\tCALL_WITH_CONTEXT current_el0_irq\n.org 0x100\n\tFIQ_SUSPEND\n.org 0x180\n\tCALL_WITH_CONTEXT current_el0_serror\n\n// Current exception level with SP_ELx, x > 0.\n.org 0x200\n\tCALL_WITH_CONTEXT current_elx_synchronous\n.org 0x280\n\tCALL_WITH_CONTEXT current_elx_irq\n.org 0x300\n\tFIQ_SUSPEND\n.org 0x380\n\tCALL_WITH_CONTEXT current_elx_serror\n\n// Lower exception level, AArch64\n.org 0x400\n\tCALL_WITH_CONTEXT lower_aarch64_synchronous\n.org 0x480\n\tCALL_WITH_CONTEXT lower_aarch64_irq\n.org 0x500\n\tFIQ_SUSPEND\n.org 0x580\n\tCALL_WITH_CONTEXT lower_aarch64_serror\n\n// Lower exception level, AArch32\n.org 0x600\n\tCALL_WITH_CONTEXT lower_aarch32_synchronous\n.org 0x680\n\tCALL_WITH_CONTEXT lower_aarch32_irq\n.org 0x700\n\tFIQ_SUSPEND\n.org 0x780\n\tCALL_WITH_CONTEXT lower_aarch32_serror\n.org 0x800\n\n//------------------------------------------------------------------------------\n// fn __exception_restore_context()\n//------------------------------------------------------------------------------\n__exception_restore_context:\n\tldr\tw19,      [sp, #16 * 16]\n\tldp\tlr,  x20, [sp, #16 * 15]\n\n\tmsr\tSPSR_EL1, x19\n\tmsr\tELR_EL1,  x20\n\n\tldp\tx0,  x1,  [sp, #16 * 0]\n\tldp\tx2,  x3,  [sp, #16 * 1]\n\tldp\tx4,  x5,  [sp, #16 * 2]\n\tldp\tx6,  x7,  [sp, #16 * 3]\n\tldp\tx8,  x9,  [sp, #16 * 4]\n\tldp\tx10, x11, [sp, #16 * 5]\n\tldp\tx12, x13, [sp, #16 * 6]\n\tldp\tx14, x15, [sp, #16 * 7]\n\tldp\tx16, x17, [sp, #16 * 8]\n\tldp\tx18, x19, [sp, #16 * 9]\n\tldp\tx20, x21, [sp, #16 * 10]\n\tldp\tx22, x23, [sp, #16 * 11]\n\tldp\tx24, x25, [sp, #16 * 12]\n\tldp\tx26, x27, [sp, #16 * 13]\n\tldp\tx28, x29, [sp, #16 * 14]\n\n\tadd\tsp,  sp,  #16 * 17\n\n\teret\n\n.size\t__exception_restore_context, . - __exception_restore_context\n.type\t__exception_restore_context, function\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/_arch/aarch64/memory/mmu/translation_table.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural translation table.\n//!\n//! Only 64 KiB granule is supported.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::memory::mmu::translation_table::arch_translation_table\n\nuse crate::{\n    bsp,\n    memory::{\n        self,\n        mmu::{\n            arch_mmu::{Granule512MiB, Granule64KiB},\n            AccessPermissions, AttributeFields, MemAttributes, MemoryRegion, PageAddress,\n        },\n        Address, Physical, Virtual,\n    },\n};\nuse core::convert;\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields,\n    registers::InMemoryRegister,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// A table descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-15.\nregister_bitfields! {u64,\n    STAGE1_TABLE_DESCRIPTOR [\n        /// Physical address of the next descriptor.\n        NEXT_LEVEL_TABLE_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16]\n\n        TYPE  OFFSET(1) NUMBITS(1) [\n            Block = 0,\n            Table = 1\n        ],\n\n        VALID OFFSET(0) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ]\n    ]\n}\n\n// A level 3 page descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-17.\nregister_bitfields! {u64,\n    STAGE1_PAGE_DESCRIPTOR [\n        /// Unprivileged execute-never.\n        UXN      OFFSET(54) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ],\n\n        /// Privileged execute-never.\n        PXN      OFFSET(53) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ],\n\n        /// Physical address of the next table descriptor (lvl2) or the page descriptor (lvl3).\n        OUTPUT_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16]\n\n        /// Access flag.\n        AF       OFFSET(10) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ],\n\n        /// Shareability field.\n        SH       OFFSET(8) NUMBITS(2) [\n            OuterShareable = 0b10,\n            InnerShareable = 0b11\n        ],\n\n        /// Access Permissions.\n        AP       OFFSET(6) NUMBITS(2) [\n            RW_EL1 = 0b00,\n            RW_EL1_EL0 = 0b01,\n            RO_EL1 = 0b10,\n            RO_EL1_EL0 = 0b11\n        ],\n\n        /// Memory attributes index into the MAIR_EL1 register.\n        AttrIndx OFFSET(2) NUMBITS(3) [],\n\n        TYPE     OFFSET(1) NUMBITS(1) [\n            Reserved_Invalid = 0,\n            Page = 1\n        ],\n\n        VALID    OFFSET(0) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ]\n    ]\n}\n\n/// A table descriptor for 64 KiB aperture.\n///\n/// The output points to the next table.\n#[derive(Copy, Clone)]\n#[repr(C)]\nstruct TableDescriptor {\n    value: u64,\n}\n\n/// A page descriptor with 64 KiB aperture.\n///\n/// The output points to physical memory.\n#[derive(Copy, Clone)]\n#[repr(C)]\nstruct PageDescriptor {\n    value: u64,\n}\n\ntrait StartAddr {\n    fn virt_start_addr(&self) -> Address<Virtual>;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB\n/// aligned, so the lvl3 is put first.\n#[repr(C)]\n#[repr(align(65536))]\npub struct FixedSizeTranslationTable<const NUM_TABLES: usize, const START_FROM_TOP: bool> {\n    /// Page descriptors, covering 64 KiB windows per entry.\n    lvl3: [[PageDescriptor; 8192]; NUM_TABLES],\n\n    /// Table descriptors, covering 512 MiB windows.\n    lvl2: [TableDescriptor; NUM_TABLES],\n\n    /// Have the tables been initialized?\n    initialized: bool,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T, const N: usize> StartAddr for [T; N] {\n    fn virt_start_addr(&self) -> Address<Virtual> {\n        Address::new(self as *const _ as usize)\n    }\n}\n\nimpl TableDescriptor {\n    /// Create an instance.\n    ///\n    /// Descriptor is invalid by default.\n    pub const fn new_zeroed() -> Self {\n        Self { value: 0 }\n    }\n\n    /// Create an instance pointing to the supplied address.\n    pub fn from_next_lvl_table_addr(phys_next_lvl_table_addr: Address<Physical>) -> Self {\n        let val = InMemoryRegister::<u64, STAGE1_TABLE_DESCRIPTOR::Register>::new(0);\n\n        let shifted = phys_next_lvl_table_addr.as_usize() >> Granule64KiB::SHIFT;\n        val.write(\n            STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64)\n                + STAGE1_TABLE_DESCRIPTOR::TYPE::Table\n                + STAGE1_TABLE_DESCRIPTOR::VALID::True,\n        );\n\n        TableDescriptor { value: val.get() }\n    }\n}\n\n/// Convert the kernel's generic memory attributes to HW-specific attributes of the MMU.\nimpl convert::From<AttributeFields>\n    for tock_registers::fields::FieldValue<u64, STAGE1_PAGE_DESCRIPTOR::Register>\n{\n    fn from(attribute_fields: AttributeFields) -> Self {\n        // Memory attributes.\n        let mut desc = match attribute_fields.mem_attributes {\n            MemAttributes::CacheableDRAM => {\n                STAGE1_PAGE_DESCRIPTOR::SH::InnerShareable\n                    + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::NORMAL)\n            }\n            MemAttributes::Device => {\n                STAGE1_PAGE_DESCRIPTOR::SH::OuterShareable\n                    + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::DEVICE)\n            }\n        };\n\n        // Access Permissions.\n        desc += match attribute_fields.acc_perms {\n            AccessPermissions::ReadOnly => STAGE1_PAGE_DESCRIPTOR::AP::RO_EL1,\n            AccessPermissions::ReadWrite => STAGE1_PAGE_DESCRIPTOR::AP::RW_EL1,\n        };\n\n        // The execute-never attribute is mapped to PXN in AArch64.\n        desc += if attribute_fields.execute_never {\n            STAGE1_PAGE_DESCRIPTOR::PXN::True\n        } else {\n            STAGE1_PAGE_DESCRIPTOR::PXN::False\n        };\n\n        // Always set unprivileged exectue-never as long as userspace is not implemented yet.\n        desc += STAGE1_PAGE_DESCRIPTOR::UXN::True;\n\n        desc\n    }\n}\n\n/// Convert the HW-specific attributes of the MMU to kernel's generic memory attributes.\nimpl convert::TryFrom<InMemoryRegister<u64, STAGE1_PAGE_DESCRIPTOR::Register>> for AttributeFields {\n    type Error = &'static str;\n\n    fn try_from(\n        desc: InMemoryRegister<u64, STAGE1_PAGE_DESCRIPTOR::Register>,\n    ) -> Result<AttributeFields, Self::Error> {\n        let mem_attributes = match desc.read(STAGE1_PAGE_DESCRIPTOR::AttrIndx) {\n            memory::mmu::arch_mmu::mair::NORMAL => MemAttributes::CacheableDRAM,\n            memory::mmu::arch_mmu::mair::DEVICE => MemAttributes::Device,\n            _ => return Err(\"Unexpected memory attribute\"),\n        };\n\n        let acc_perms = match desc.read_as_enum(STAGE1_PAGE_DESCRIPTOR::AP) {\n            Some(STAGE1_PAGE_DESCRIPTOR::AP::Value::RO_EL1) => AccessPermissions::ReadOnly,\n            Some(STAGE1_PAGE_DESCRIPTOR::AP::Value::RW_EL1) => AccessPermissions::ReadWrite,\n            _ => return Err(\"Unexpected access permission\"),\n        };\n\n        let execute_never = desc.read(STAGE1_PAGE_DESCRIPTOR::PXN) > 0;\n\n        Ok(AttributeFields {\n            mem_attributes,\n            acc_perms,\n            execute_never,\n        })\n    }\n}\n\nimpl PageDescriptor {\n    /// Create an instance.\n    ///\n    /// Descriptor is invalid by default.\n    pub const fn new_zeroed() -> Self {\n        Self { value: 0 }\n    }\n\n    /// Create an instance.\n    pub fn from_output_page_addr(\n        phys_output_page_addr: PageAddress<Physical>,\n        attribute_fields: &AttributeFields,\n    ) -> Self {\n        let val = InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(0);\n\n        let shifted = phys_output_page_addr.into_inner().as_usize() >> Granule64KiB::SHIFT;\n        val.write(\n            STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted as u64)\n                + STAGE1_PAGE_DESCRIPTOR::AF::True\n                + STAGE1_PAGE_DESCRIPTOR::TYPE::Page\n                + STAGE1_PAGE_DESCRIPTOR::VALID::True\n                + (*attribute_fields).into(),\n        );\n\n        Self { value: val.get() }\n    }\n\n    /// Returns the valid bit.\n    fn is_valid(&self) -> bool {\n        InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(self.value)\n            .is_set(STAGE1_PAGE_DESCRIPTOR::VALID)\n    }\n\n    /// Returns the output page.\n    fn output_page_addr(&self) -> PageAddress<Physical> {\n        let shifted = InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(self.value)\n            .read(STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB) as usize;\n\n        PageAddress::from(shifted << Granule64KiB::SHIFT)\n    }\n\n    /// Returns the attributes.\n    fn try_attributes(&self) -> Result<AttributeFields, &'static str> {\n        InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(self.value).try_into()\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<const AS_SIZE: usize> memory::mmu::AssociatedTranslationTable\n    for memory::mmu::AddressSpace<AS_SIZE>\nwhere\n    [u8; Self::SIZE >> Granule512MiB::SHIFT]: Sized,\n{\n    type TableStartFromTop =\n        FixedSizeTranslationTable<{ Self::SIZE >> Granule512MiB::SHIFT }, true>;\n\n    type TableStartFromBottom =\n        FixedSizeTranslationTable<{ Self::SIZE >> Granule512MiB::SHIFT }, false>;\n}\n\nimpl<const NUM_TABLES: usize, const START_FROM_TOP: bool>\n    FixedSizeTranslationTable<NUM_TABLES, START_FROM_TOP>\n{\n    const START_FROM_TOP_OFFSET: Address<Virtual> =\n        Address::new((usize::MAX - (Granule512MiB::SIZE * NUM_TABLES)) + 1);\n\n    /// Create an instance.\n    #[allow(clippy::assertions_on_constants)]\n    const fn _new(for_precompute: bool) -> Self {\n        assert!(bsp::memory::mmu::KernelGranule::SIZE == Granule64KiB::SIZE);\n\n        // Can't have a zero-sized address space.\n        assert!(NUM_TABLES > 0);\n\n        Self {\n            lvl3: [[PageDescriptor::new_zeroed(); 8192]; NUM_TABLES],\n            lvl2: [TableDescriptor::new_zeroed(); NUM_TABLES],\n            initialized: for_precompute,\n        }\n    }\n\n    pub const fn new_for_precompute() -> Self {\n        Self::_new(true)\n    }\n\n    #[cfg(test)]\n    pub fn new_for_runtime() -> Self {\n        Self::_new(false)\n    }\n\n    /// Helper to calculate the lvl2 and lvl3 indices from an address.\n    #[inline(always)]\n    fn lvl2_lvl3_index_from_page_addr(\n        &self,\n        virt_page_addr: PageAddress<Virtual>,\n    ) -> Result<(usize, usize), &'static str> {\n        let mut addr = virt_page_addr.into_inner();\n\n        if START_FROM_TOP {\n            addr = addr - Self::START_FROM_TOP_OFFSET;\n        }\n\n        let lvl2_index = addr.as_usize() >> Granule512MiB::SHIFT;\n        let lvl3_index = (addr.as_usize() & Granule512MiB::MASK) >> Granule64KiB::SHIFT;\n\n        if lvl2_index > (NUM_TABLES - 1) {\n            return Err(\"Virtual page is out of bounds of translation table\");\n        }\n\n        Ok((lvl2_index, lvl3_index))\n    }\n\n    /// Returns the PageDescriptor corresponding to the supplied page address.\n    #[inline(always)]\n    fn page_descriptor_from_page_addr(\n        &self,\n        virt_page_addr: PageAddress<Virtual>,\n    ) -> Result<&PageDescriptor, &'static str> {\n        let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from_page_addr(virt_page_addr)?;\n        let desc = &self.lvl3[lvl2_index][lvl3_index];\n\n        Ok(desc)\n    }\n\n    /// Sets the PageDescriptor corresponding to the supplied page address.\n    ///\n    /// Doesn't allow overriding an already valid page.\n    #[inline(always)]\n    fn set_page_descriptor_from_page_addr(\n        &mut self,\n        virt_page_addr: PageAddress<Virtual>,\n        new_desc: &PageDescriptor,\n    ) -> Result<(), &'static str> {\n        let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from_page_addr(virt_page_addr)?;\n        let desc = &mut self.lvl3[lvl2_index][lvl3_index];\n\n        if desc.is_valid() {\n            return Err(\"Virtual page is already mapped\");\n        }\n\n        *desc = *new_desc;\n        Ok(())\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\n\nimpl<const NUM_TABLES: usize, const START_FROM_TOP: bool>\n    memory::mmu::translation_table::interface::TranslationTable\n    for FixedSizeTranslationTable<NUM_TABLES, START_FROM_TOP>\n{\n    fn init(&mut self) -> Result<(), &'static str> {\n        if self.initialized {\n            return Ok(());\n        }\n\n        // Populate the l2 entries.\n        for (lvl2_nr, lvl2_entry) in self.lvl2.iter_mut().enumerate() {\n            let virt_table_addr = self.lvl3[lvl2_nr].virt_start_addr();\n            let phys_table_addr = memory::mmu::try_kernel_virt_addr_to_phys_addr(virt_table_addr)?;\n\n            let new_desc = TableDescriptor::from_next_lvl_table_addr(phys_table_addr);\n            *lvl2_entry = new_desc;\n        }\n\n        self.initialized = true;\n\n        Ok(())\n    }\n\n    unsafe fn map_at(\n        &mut self,\n        virt_region: &MemoryRegion<Virtual>,\n        phys_region: &MemoryRegion<Physical>,\n        attr: &AttributeFields,\n    ) -> Result<(), &'static str> {\n        assert!(self.initialized, \"Translation tables not initialized\");\n\n        if virt_region.size() != phys_region.size() {\n            return Err(\"Tried to map memory regions with unequal sizes\");\n        }\n\n        if phys_region.end_exclusive_page_addr() > bsp::memory::phys_addr_space_end_exclusive_addr()\n        {\n            return Err(\"Tried to map outside of physical address space\");\n        }\n\n        let iter = phys_region.into_iter().zip(virt_region.into_iter());\n        for (phys_page_addr, virt_page_addr) in iter {\n            let new_desc = PageDescriptor::from_output_page_addr(phys_page_addr, attr);\n            let virt_page = virt_page_addr;\n\n            self.set_page_descriptor_from_page_addr(virt_page, &new_desc)?;\n        }\n\n        Ok(())\n    }\n\n    fn try_virt_page_addr_to_phys_page_addr(\n        &self,\n        virt_page_addr: PageAddress<Virtual>,\n    ) -> Result<PageAddress<Physical>, &'static str> {\n        let page_desc = self.page_descriptor_from_page_addr(virt_page_addr)?;\n\n        if !page_desc.is_valid() {\n            return Err(\"Page marked invalid\");\n        }\n\n        Ok(page_desc.output_page_addr())\n    }\n\n    fn try_page_attributes(\n        &self,\n        virt_page_addr: PageAddress<Virtual>,\n    ) -> Result<AttributeFields, &'static str> {\n        let page_desc = self.page_descriptor_from_page_addr(virt_page_addr)?;\n\n        if !page_desc.is_valid() {\n            return Err(\"Page marked invalid\");\n        }\n\n        page_desc.try_attributes()\n    }\n\n    /// Try to translate a virtual address to a physical address.\n    ///\n    /// Will only succeed if there exists a valid mapping for the input address.\n    fn try_virt_addr_to_phys_addr(\n        &self,\n        virt_addr: Address<Virtual>,\n    ) -> Result<Address<Physical>, &'static str> {\n        let virt_page = PageAddress::from(virt_addr.align_down_page());\n        let phys_page = self.try_virt_page_addr_to_phys_page_addr(virt_page)?;\n\n        Ok(phys_page.into_inner() + virt_addr.offset_into_page())\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\npub type MinSizeTranslationTable = FixedSizeTranslationTable<1, true>;\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// Check if the size of `struct TableDescriptor` is as expected.\n    #[kernel_test]\n    fn size_of_tabledescriptor_equals_64_bit() {\n        assert_eq!(\n            core::mem::size_of::<TableDescriptor>(),\n            core::mem::size_of::<u64>()\n        );\n    }\n\n    /// Check if the size of `struct PageDescriptor` is as expected.\n    #[kernel_test]\n    fn size_of_pagedescriptor_equals_64_bit() {\n        assert_eq!(\n            core::mem::size_of::<PageDescriptor>(),\n            core::mem::size_of::<u64>()\n        );\n    }\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/_arch/aarch64/memory/mmu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management Unit Driver.\n//!\n//! Only 64 KiB granule is supported.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::memory::mmu::arch_mmu\n\nuse crate::{\n    bsp, memory,\n    memory::{mmu::TranslationGranule, Address, Physical},\n};\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::intrinsics::unlikely;\nuse tock_registers::interfaces::{ReadWriteable, Readable, Writeable};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Memory Management Unit type.\nstruct MemoryManagementUnit;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub type Granule512MiB = TranslationGranule<{ 512 * 1024 * 1024 }>;\npub type Granule64KiB = TranslationGranule<{ 64 * 1024 }>;\n\n/// Constants for indexing the MAIR_EL1.\n#[allow(dead_code)]\npub mod mair {\n    pub const DEVICE: u64 = 0;\n    pub const NORMAL: u64 = 1;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic MMU: MemoryManagementUnit = MemoryManagementUnit;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<const AS_SIZE: usize> memory::mmu::AddressSpace<AS_SIZE> {\n    /// Checks for architectural restrictions.\n    pub const fn arch_address_space_size_sanity_checks() {\n        // Size must be at least one full 512 MiB table.\n        assert!((AS_SIZE % Granule512MiB::SIZE) == 0);\n\n        // Check for 48 bit virtual address size as maximum, which is supported by any ARMv8\n        // version.\n        assert!(AS_SIZE <= (1 << 48));\n    }\n}\n\nimpl MemoryManagementUnit {\n    /// Setup function for the MAIR_EL1 register.\n    #[inline(always)]\n    fn set_up_mair(&self) {\n        // Define the memory types being mapped.\n        MAIR_EL1.write(\n            // Attribute 1 - Cacheable normal DRAM.\n            MAIR_EL1::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc +\n        MAIR_EL1::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc +\n\n        // Attribute 0 - Device.\n        MAIR_EL1::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck,\n        );\n    }\n\n    /// Configure various settings of stage 1 of the EL1 translation regime.\n    #[inline(always)]\n    fn configure_translation_control(&self) {\n        let t1sz = (64 - bsp::memory::mmu::KernelVirtAddrSpace::SIZE_SHIFT) as u64;\n\n        TCR_EL1.write(\n            TCR_EL1::TBI1::Used\n                + TCR_EL1::IPS::Bits_40\n                + TCR_EL1::TG1::KiB_64\n                + TCR_EL1::SH1::Inner\n                + TCR_EL1::ORGN1::WriteBack_ReadAlloc_WriteAlloc_Cacheable\n                + TCR_EL1::IRGN1::WriteBack_ReadAlloc_WriteAlloc_Cacheable\n                + TCR_EL1::EPD1::EnableTTBR1Walks\n                + TCR_EL1::A1::TTBR1\n                + TCR_EL1::T1SZ.val(t1sz)\n                + TCR_EL1::EPD0::DisableTTBR0Walks,\n        );\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the MMU instance.\npub fn mmu() -> &'static impl memory::mmu::interface::MMU {\n    &MMU\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse memory::mmu::MMUEnableError;\n\nimpl memory::mmu::interface::MMU for MemoryManagementUnit {\n    unsafe fn enable_mmu_and_caching(\n        &self,\n        phys_tables_base_addr: Address<Physical>,\n    ) -> Result<(), MMUEnableError> {\n        if unlikely(self.is_enabled()) {\n            return Err(MMUEnableError::AlreadyEnabled);\n        }\n\n        // Fail early if translation granule is not supported.\n        if unlikely(!ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran64::Supported)) {\n            return Err(MMUEnableError::Other(\n                \"Translation granule not supported in HW\",\n            ));\n        }\n\n        // Prepare the memory attribute indirection register.\n        self.set_up_mair();\n\n        // Set the \"Translation Table Base Register\".\n        TTBR1_EL1.set_baddr(phys_tables_base_addr.as_usize() as u64);\n\n        self.configure_translation_control();\n\n        // Switch the MMU on.\n        //\n        // First, force all previous changes to be seen before the MMU is enabled.\n        barrier::isb(barrier::SY);\n\n        // Enable the MMU and turn on data and instruction caching.\n        SCTLR_EL1.modify(SCTLR_EL1::M::Enable + SCTLR_EL1::C::Cacheable + SCTLR_EL1::I::Cacheable);\n\n        // Force MMU init to complete before next instruction.\n        barrier::isb(barrier::SY);\n\n        Ok(())\n    }\n\n    #[inline(always)]\n    fn is_enabled(&self) -> bool {\n        SCTLR_EL1.matches_all(SCTLR_EL1::M::Enable)\n    }\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/_arch/aarch64/time.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural timer primitives.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::time::arch_time\n\nuse crate::warn;\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::{\n    num::{NonZeroU128, NonZeroU32, NonZeroU64},\n    ops::{Add, Div},\n    time::Duration,\n};\nuse tock_registers::interfaces::Readable;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nconst NANOSEC_PER_SEC: NonZeroU64 = NonZeroU64::new(1_000_000_000).unwrap();\n\n#[derive(Copy, Clone, PartialOrd, PartialEq)]\nstruct GenericTimerCounterValue(u64);\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n/// Boot assembly code overwrites this value with the value of CNTFRQ_EL0 before any Rust code is\n/// executed. This given value here is just a (safe) dummy.\n#[no_mangle]\nstatic ARCH_TIMER_COUNTER_FREQUENCY: NonZeroU32 = NonZeroU32::MIN;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nfn arch_timer_counter_frequency() -> NonZeroU32 {\n    // Read volatile is needed here to prevent the compiler from optimizing\n    // ARCH_TIMER_COUNTER_FREQUENCY away.\n    //\n    // This is safe, because all the safety requirements as stated in read_volatile()'s\n    // documentation are fulfilled.\n    unsafe { core::ptr::read_volatile(&ARCH_TIMER_COUNTER_FREQUENCY) }\n}\n\nimpl GenericTimerCounterValue {\n    pub const MAX: Self = GenericTimerCounterValue(u64::MAX);\n}\n\nimpl Add for GenericTimerCounterValue {\n    type Output = Self;\n\n    fn add(self, other: Self) -> Self {\n        GenericTimerCounterValue(self.0.wrapping_add(other.0))\n    }\n}\n\nimpl From<GenericTimerCounterValue> for Duration {\n    fn from(counter_value: GenericTimerCounterValue) -> Self {\n        if counter_value.0 == 0 {\n            return Duration::ZERO;\n        }\n\n        let frequency: NonZeroU64 = arch_timer_counter_frequency().into();\n\n        // Div<NonZeroU64> implementation for u64 cannot panic.\n        let secs = counter_value.0.div(frequency);\n\n        // This is safe, because frequency can never be greater than u32::MAX, which means the\n        // largest theoretical value for sub_second_counter_value is (u32::MAX - 1). Therefore,\n        // (sub_second_counter_value * NANOSEC_PER_SEC) cannot overflow an u64.\n        //\n        // The subsequent division ensures the result fits into u32, since the max result is smaller\n        // than NANOSEC_PER_SEC. Therefore, just cast it to u32 using `as`.\n        let sub_second_counter_value = counter_value.0 % frequency;\n        let nanos = unsafe { sub_second_counter_value.unchecked_mul(u64::from(NANOSEC_PER_SEC)) }\n            .div(frequency) as u32;\n\n        Duration::new(secs, nanos)\n    }\n}\n\nfn max_duration() -> Duration {\n    Duration::from(GenericTimerCounterValue::MAX)\n}\n\nimpl TryFrom<Duration> for GenericTimerCounterValue {\n    type Error = &'static str;\n\n    fn try_from(duration: Duration) -> Result<Self, Self::Error> {\n        if duration < resolution() {\n            return Ok(GenericTimerCounterValue(0));\n        }\n\n        if duration > max_duration() {\n            return Err(\"Conversion error. Duration too big\");\n        }\n\n        let frequency: u128 = u32::from(arch_timer_counter_frequency()) as u128;\n        let duration: u128 = duration.as_nanos();\n\n        // This is safe, because frequency can never be greater than u32::MAX, and\n        // (Duration::MAX.as_nanos() * u32::MAX) < u128::MAX.\n        let counter_value =\n            unsafe { duration.unchecked_mul(frequency) }.div(NonZeroU128::from(NANOSEC_PER_SEC));\n\n        // Since we checked above that we are <= max_duration(), just cast to u64.\n        Ok(GenericTimerCounterValue(counter_value as u64))\n    }\n}\n\n#[inline(always)]\nfn read_cntpct() -> GenericTimerCounterValue {\n    // Prevent that the counter is read ahead of time due to out-of-order execution.\n    barrier::isb(barrier::SY);\n    let cnt = CNTPCT_EL0.get();\n\n    GenericTimerCounterValue(cnt)\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The timer's resolution.\npub fn resolution() -> Duration {\n    Duration::from(GenericTimerCounterValue(1))\n}\n\n/// The uptime since power-on of the device.\n///\n/// This includes time consumed by firmware and bootloaders.\npub fn uptime() -> Duration {\n    read_cntpct().into()\n}\n\n/// Spin for a given duration.\npub fn spin_for(duration: Duration) {\n    let curr_counter_value = read_cntpct();\n\n    let counter_value_delta: GenericTimerCounterValue = match duration.try_into() {\n        Err(msg) => {\n            warn!(\"spin_for: {}. Skipping\", msg);\n            return;\n        }\n        Ok(val) => val,\n    };\n    let counter_value_target = curr_counter_value + counter_value_delta;\n\n    // Busy wait.\n    //\n    // Read CNTPCT_EL0 directly to avoid the ISB that is part of [`read_cntpct`].\n    while GenericTimerCounterValue(CNTPCT_EL0.get()) < counter_value_target {}\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/bsp/device_driver/arm/gicv2/gicc.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GICC Driver - GIC CPU interface.\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    exception,\n    memory::{Address, Virtual},\n};\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields, register_structs,\n    registers::ReadWrite,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nregister_bitfields! {\n    u32,\n\n    /// CPU Interface Control Register\n    CTLR [\n        Enable OFFSET(0) NUMBITS(1) []\n    ],\n\n    /// Interrupt Priority Mask Register\n    PMR [\n        Priority OFFSET(0) NUMBITS(8) []\n    ],\n\n    /// Interrupt Acknowledge Register\n    IAR [\n        InterruptID OFFSET(0) NUMBITS(10) []\n    ],\n\n    /// End of Interrupt Register\n    EOIR [\n        EOIINTID OFFSET(0) NUMBITS(10) []\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    pub RegisterBlock {\n        (0x000 => CTLR: ReadWrite<u32, CTLR::Register>),\n        (0x004 => PMR: ReadWrite<u32, PMR::Register>),\n        (0x008 => _reserved1),\n        (0x00C => IAR: ReadWrite<u32, IAR::Register>),\n        (0x010 => EOIR: ReadWrite<u32, EOIR::Register>),\n        (0x014  => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the GIC CPU interface.\npub struct GICC {\n    registers: Registers,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GICC {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n        }\n    }\n\n    /// Accept interrupts of any priority.\n    ///\n    /// Quoting the GICv2 Architecture Specification:\n    ///\n    ///   \"Writing 255 to the GICC_PMR always sets it to the largest supported priority field\n    ///    value.\"\n    ///\n    /// # Safety\n    ///\n    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n    ///   of `&mut self`.\n    pub fn priority_accept_all(&self) {\n        self.registers.PMR.write(PMR::Priority.val(255)); // Comment in arch spec.\n    }\n\n    /// Enable the interface - start accepting IRQs.\n    ///\n    /// # Safety\n    ///\n    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n    ///   of `&mut self`.\n    pub fn enable(&self) {\n        self.registers.CTLR.write(CTLR::Enable::SET);\n    }\n\n    /// Extract the number of the highest-priority pending IRQ.\n    ///\n    /// Can only be called from IRQ context, which is ensured by taking an `IRQContext` token.\n    ///\n    /// # Safety\n    ///\n    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n    ///   of `&mut self`.\n    #[allow(clippy::trivially_copy_pass_by_ref)]\n    pub fn pending_irq_number<'irq_context>(\n        &self,\n        _ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) -> usize {\n        self.registers.IAR.read(IAR::InterruptID) as usize\n    }\n\n    /// Complete handling of the currently active IRQ.\n    ///\n    /// Can only be called from IRQ context, which is ensured by taking an `IRQContext` token.\n    ///\n    /// To be called after `pending_irq_number()`.\n    ///\n    /// # Safety\n    ///\n    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n    ///   of `&mut self`.\n    #[allow(clippy::trivially_copy_pass_by_ref)]\n    pub fn mark_comleted<'irq_context>(\n        &self,\n        irq_number: u32,\n        _ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) {\n        self.registers.EOIR.write(EOIR::EOIINTID.val(irq_number));\n    }\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/bsp/device_driver/arm/gicv2/gicd.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GICD Driver - GIC Distributor.\n//!\n//! # Glossary\n//!   - SPI - Shared Peripheral Interrupt.\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    memory::{Address, Virtual},\n    state, synchronization,\n    synchronization::IRQSafeNullLock,\n};\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields, register_structs,\n    registers::{ReadOnly, ReadWrite},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nregister_bitfields! {\n    u32,\n\n    /// Distributor Control Register\n    CTLR [\n        Enable OFFSET(0) NUMBITS(1) []\n    ],\n\n    /// Interrupt Controller Type Register\n    TYPER [\n        ITLinesNumber OFFSET(0)  NUMBITS(5) []\n    ],\n\n    /// Interrupt Processor Targets Registers\n    ITARGETSR [\n        Offset3 OFFSET(24) NUMBITS(8) [],\n        Offset2 OFFSET(16) NUMBITS(8) [],\n        Offset1 OFFSET(8)  NUMBITS(8) [],\n        Offset0 OFFSET(0)  NUMBITS(8) []\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    SharedRegisterBlock {\n        (0x000 => CTLR: ReadWrite<u32, CTLR::Register>),\n        (0x004 => TYPER: ReadOnly<u32, TYPER::Register>),\n        (0x008 => _reserved1),\n        (0x104 => ISENABLER: [ReadWrite<u32>; 31]),\n        (0x180 => _reserved2),\n        (0x820 => ITARGETSR: [ReadWrite<u32, ITARGETSR::Register>; 248]),\n        (0xC00 => @END),\n    }\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    BankedRegisterBlock {\n        (0x000 => _reserved1),\n        (0x100 => ISENABLER: ReadWrite<u32>),\n        (0x104 => _reserved2),\n        (0x800 => ITARGETSR: [ReadOnly<u32, ITARGETSR::Register>; 8]),\n        (0x820 => @END),\n    }\n}\n\n/// Abstraction for the non-banked parts of the associated MMIO registers.\ntype SharedRegisters = MMIODerefWrapper<SharedRegisterBlock>;\n\n/// Abstraction for the banked parts of the associated MMIO registers.\ntype BankedRegisters = MMIODerefWrapper<BankedRegisterBlock>;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the GIC Distributor.\npub struct GICD {\n    /// Access to shared registers is guarded with a lock.\n    shared_registers: IRQSafeNullLock<SharedRegisters>,\n\n    /// Access to banked registers is unguarded.\n    banked_registers: BankedRegisters,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl SharedRegisters {\n    /// Return the number of IRQs that this HW implements.\n    #[inline(always)]\n    fn num_irqs(&mut self) -> usize {\n        // Query number of implemented IRQs.\n        //\n        // Refer to GICv2 Architecture Specification, Section 4.3.2.\n        ((self.TYPER.read(TYPER::ITLinesNumber) as usize) + 1) * 32\n    }\n\n    /// Return a slice of the implemented ITARGETSR.\n    #[inline(always)]\n    fn implemented_itargets_slice(&mut self) -> &[ReadWrite<u32, ITARGETSR::Register>] {\n        assert!(self.num_irqs() >= 36);\n\n        // Calculate the max index of the shared ITARGETSR array.\n        //\n        // The first 32 IRQs are private, so not included in `shared_registers`. Each ITARGETS\n        // register has four entries, so shift right by two. Subtract one because we start\n        // counting at zero.\n        let spi_itargetsr_max_index = ((self.num_irqs() - 32) >> 2) - 1;\n\n        // Rust automatically inserts slice range sanity check, i.e. max >= min.\n        &self.ITARGETSR[0..spi_itargetsr_max_index]\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl GICD {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            shared_registers: IRQSafeNullLock::new(SharedRegisters::new(mmio_start_addr)),\n            banked_registers: BankedRegisters::new(mmio_start_addr),\n        }\n    }\n\n    /// Use a banked ITARGETSR to retrieve the executing core's GIC target mask.\n    ///\n    /// Quoting the GICv2 Architecture Specification:\n    ///\n    ///   \"GICD_ITARGETSR0 to GICD_ITARGETSR7 are read-only, and each field returns a value that\n    ///    corresponds only to the processor reading the register.\"\n    fn local_gic_target_mask(&self) -> u32 {\n        self.banked_registers.ITARGETSR[0].read(ITARGETSR::Offset0)\n    }\n\n    /// Route all SPIs to the boot core and enable the distributor.\n    pub fn boot_core_init(&self) {\n        assert!(\n            state::state_manager().is_init(),\n            \"Only allowed during kernel init phase\"\n        );\n\n        // Target all SPIs to the boot core only.\n        let mask = self.local_gic_target_mask();\n\n        self.shared_registers.lock(|regs| {\n            for i in regs.implemented_itargets_slice().iter() {\n                i.write(\n                    ITARGETSR::Offset3.val(mask)\n                        + ITARGETSR::Offset2.val(mask)\n                        + ITARGETSR::Offset1.val(mask)\n                        + ITARGETSR::Offset0.val(mask),\n                );\n            }\n\n            regs.CTLR.write(CTLR::Enable::SET);\n        });\n    }\n\n    /// Enable an interrupt.\n    pub fn enable(&self, irq_num: &super::IRQNumber) {\n        let irq_num = irq_num.get();\n\n        // Each bit in the u32 enable register corresponds to one IRQ number. Shift right by 5\n        // (division by 32) and arrive at the index for the respective ISENABLER[i].\n        let enable_reg_index = irq_num >> 5;\n        let enable_bit: u32 = 1u32 << (irq_num % 32);\n\n        // Check if we are handling a private or shared IRQ.\n        match irq_num {\n            // Private.\n            0..=31 => {\n                let enable_reg = &self.banked_registers.ISENABLER;\n                enable_reg.set(enable_reg.get() | enable_bit);\n            }\n            // Shared.\n            _ => {\n                let enable_reg_index_shared = enable_reg_index - 1;\n\n                self.shared_registers.lock(|regs| {\n                    let enable_reg = &regs.ISENABLER[enable_reg_index_shared];\n                    enable_reg.set(enable_reg.get() | enable_bit);\n                });\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/bsp/device_driver/arm/gicv2.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GICv2 Driver - ARM Generic Interrupt Controller v2.\n//!\n//! The following is a collection of excerpts with useful information from\n//!   - `Programmer's Guide for ARMv8-A`\n//!   - `ARM Generic Interrupt Controller Architecture Specification`\n//!\n//! # Programmer's Guide - 10.6.1 Configuration\n//!\n//! The GIC is accessed as a memory-mapped peripheral.\n//!\n//! All cores can access the common Distributor, but the CPU interface is banked, that is, each core\n//! uses the same address to access its own private CPU interface.\n//!\n//! It is not possible for a core to access the CPU interface of another core.\n//!\n//! # Architecture Specification - 10.6.2 Initialization\n//!\n//! Both the Distributor and the CPU interfaces are disabled at reset. The GIC must be initialized\n//! after reset before it can deliver interrupts to the core.\n//!\n//! In the Distributor, software must configure the priority, target, security and enable individual\n//! interrupts. The Distributor must subsequently be enabled through its control register\n//! (GICD_CTLR). For each CPU interface, software must program the priority mask and preemption\n//! settings.\n//!\n//! Each CPU interface block itself must be enabled through its control register (GICD_CTLR). This\n//! prepares the GIC to deliver interrupts to the core.\n//!\n//! Before interrupts are expected in the core, software prepares the core to take interrupts by\n//! setting a valid interrupt vector in the vector table, and clearing interrupt mask bits in\n//! PSTATE, and setting the routing controls.\n//!\n//! The entire interrupt mechanism in the system can be disabled by disabling the Distributor.\n//! Interrupt delivery to an individual core can be disabled by disabling its CPU interface.\n//! Individual interrupts can also be disabled (or enabled) in the distributor.\n//!\n//! For an interrupt to reach the core, the individual interrupt, Distributor and CPU interface must\n//! all be enabled. The interrupt also needs to be of sufficient priority, that is, higher than the\n//! core's priority mask.\n//!\n//! # Architecture Specification - 1.4.2 Interrupt types\n//!\n//! - Peripheral interrupt\n//!     - Private Peripheral Interrupt (PPI)\n//!         - This is a peripheral interrupt that is specific to a single processor.\n//!     - Shared Peripheral Interrupt (SPI)\n//!         - This is a peripheral interrupt that the Distributor can route to any of a specified\n//!           combination of processors.\n//!\n//! - Software-generated interrupt (SGI)\n//!     - This is an interrupt generated by software writing to a GICD_SGIR register in the GIC. The\n//!       system uses SGIs for interprocessor communication.\n//!     - An SGI has edge-triggered properties. The software triggering of the interrupt is\n//!       equivalent to the edge transition of the interrupt request signal.\n//!     - When an SGI occurs in a multiprocessor implementation, the CPUID field in the Interrupt\n//!       Acknowledge Register, GICC_IAR, or the Aliased Interrupt Acknowledge Register, GICC_AIAR,\n//!       identifies the processor that requested the interrupt.\n//!\n//! # Architecture Specification - 2.2.1 Interrupt IDs\n//!\n//! Interrupts from sources are identified using ID numbers. Each CPU interface can see up to 1020\n//! interrupts. The banking of SPIs and PPIs increases the total number of interrupts supported by\n//! the Distributor.\n//!\n//! The GIC assigns interrupt ID numbers ID0-ID1019 as follows:\n//!   - Interrupt numbers 32..1019 are used for SPIs.\n//!   - Interrupt numbers 0..31 are used for interrupts that are private to a CPU interface. These\n//!     interrupts are banked in the Distributor.\n//!       - A banked interrupt is one where the Distributor can have multiple interrupts with the\n//!         same ID. A banked interrupt is identified uniquely by its ID number and its associated\n//!         CPU interface number. Of the banked interrupt IDs:\n//!           - 00..15 SGIs\n//!           - 16..31 PPIs\n\nmod gicc;\nmod gicd;\n\nuse crate::{\n    bsp::{self, device_driver::common::BoundedUsize},\n    cpu, driver, exception,\n    memory::{Address, Virtual},\n    synchronization,\n    synchronization::InitStateLock,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\ntype HandlerTable = [Option<exception::asynchronous::IRQHandlerDescriptor<IRQNumber>>;\n    IRQNumber::MAX_INCLUSIVE + 1];\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Used for the associated type of trait [`exception::asynchronous::interface::IRQManager`].\npub type IRQNumber = BoundedUsize<{ GICv2::MAX_IRQ_NUMBER }>;\n\n/// Representation of the GIC.\npub struct GICv2 {\n    /// The Distributor.\n    gicd: gicd::GICD,\n\n    /// The CPU Interface.\n    gicc: gicc::GICC,\n\n    /// Stores registered IRQ handlers. Writable only during kernel init. RO afterwards.\n    handler_table: InitStateLock<HandlerTable>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GICv2 {\n    const MAX_IRQ_NUMBER: usize = 300; // Normally 1019, but keep it lower to save some space.\n\n    pub const COMPATIBLE: &'static str = \"GICv2 (ARM Generic Interrupt Controller v2)\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(\n        gicd_mmio_start_addr: Address<Virtual>,\n        gicc_mmio_start_addr: Address<Virtual>,\n    ) -> Self {\n        Self {\n            gicd: gicd::GICD::new(gicd_mmio_start_addr),\n            gicc: gicc::GICC::new(gicc_mmio_start_addr),\n            handler_table: InitStateLock::new([None; IRQNumber::MAX_INCLUSIVE + 1]),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::ReadWriteEx;\n\nimpl driver::interface::DeviceDriver for GICv2 {\n    type IRQNumberType = IRQNumber;\n\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n\n    unsafe fn init(&self) -> Result<(), &'static str> {\n        if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() {\n            self.gicd.boot_core_init();\n        }\n\n        self.gicc.priority_accept_all();\n        self.gicc.enable();\n\n        Ok(())\n    }\n}\n\nimpl exception::asynchronous::interface::IRQManager for GICv2 {\n    type IRQNumberType = IRQNumber;\n\n    fn register_handler(\n        &self,\n        irq_handler_descriptor: exception::asynchronous::IRQHandlerDescriptor<Self::IRQNumberType>,\n    ) -> Result<(), &'static str> {\n        self.handler_table.write(|table| {\n            let irq_number = irq_handler_descriptor.number().get();\n\n            if table[irq_number].is_some() {\n                return Err(\"IRQ handler already registered\");\n            }\n\n            table[irq_number] = Some(irq_handler_descriptor);\n\n            Ok(())\n        })\n    }\n\n    fn enable(&self, irq_number: &Self::IRQNumberType) {\n        self.gicd.enable(irq_number);\n    }\n\n    fn handle_pending_irqs<'irq_context>(\n        &'irq_context self,\n        ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) {\n        // Extract the highest priority pending IRQ number from the Interrupt Acknowledge Register\n        // (IAR).\n        let irq_number = self.gicc.pending_irq_number(ic);\n\n        // Guard against spurious interrupts.\n        if irq_number > GICv2::MAX_IRQ_NUMBER {\n            return;\n        }\n\n        // Call the IRQ handler. Panic if there is none.\n        self.handler_table.read(|table| {\n            match table[irq_number] {\n                None => panic!(\"No handler registered for IRQ {}\", irq_number),\n                Some(descriptor) => {\n                    // Call the IRQ handler. Panics on failure.\n                    descriptor.handler().handle().expect(\"Error handling IRQ\");\n                }\n            }\n        });\n\n        // Signal completion of handling.\n        self.gicc.mark_comleted(irq_number as u32, ic);\n    }\n\n    fn print_handler(&self) {\n        use crate::info;\n\n        info!(\"      Peripheral handler:\");\n\n        self.handler_table.read(|table| {\n            for (i, opt) in table.iter().skip(32).enumerate() {\n                if let Some(handler) = opt {\n                    info!(\"            {: >3}. {}\", i + 32, handler.name());\n                }\n            }\n        });\n    }\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/bsp/device_driver/arm.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! ARM driver top level.\n\npub mod gicv2;\n\npub use gicv2::*;\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GPIO Driver.\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    driver,\n    exception::asynchronous::IRQNumber,\n    memory::{Address, Virtual},\n    synchronization,\n    synchronization::IRQSafeNullLock,\n};\nuse tock_registers::{\n    interfaces::{ReadWriteable, Writeable},\n    register_bitfields, register_structs,\n    registers::ReadWrite,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// GPIO registers.\n//\n// Descriptions taken from\n// - https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf\n// - https://datasheets.raspberrypi.org/bcm2711/bcm2711-peripherals.pdf\nregister_bitfields! {\n    u32,\n\n    /// GPIO Function Select 1\n    GPFSEL1 [\n        /// Pin 15\n        FSEL15 OFFSET(15) NUMBITS(3) [\n            Input = 0b000,\n            Output = 0b001,\n            AltFunc0 = 0b100  // PL011 UART RX\n\n        ],\n\n        /// Pin 14\n        FSEL14 OFFSET(12) NUMBITS(3) [\n            Input = 0b000,\n            Output = 0b001,\n            AltFunc0 = 0b100  // PL011 UART TX\n        ]\n    ],\n\n    /// GPIO Pull-up/down Register\n    ///\n    /// BCM2837 only.\n    GPPUD [\n        /// Controls the actuation of the internal pull-up/down control line to ALL the GPIO pins.\n        PUD OFFSET(0) NUMBITS(2) [\n            Off = 0b00,\n            PullDown = 0b01,\n            PullUp = 0b10\n        ]\n    ],\n\n    /// GPIO Pull-up/down Clock Register 0\n    ///\n    /// BCM2837 only.\n    GPPUDCLK0 [\n        /// Pin 15\n        PUDCLK15 OFFSET(15) NUMBITS(1) [\n            NoEffect = 0,\n            AssertClock = 1\n        ],\n\n        /// Pin 14\n        PUDCLK14 OFFSET(14) NUMBITS(1) [\n            NoEffect = 0,\n            AssertClock = 1\n        ]\n    ],\n\n    /// GPIO Pull-up / Pull-down Register 0\n    ///\n    /// BCM2711 only.\n    GPIO_PUP_PDN_CNTRL_REG0 [\n        /// Pin 15\n        GPIO_PUP_PDN_CNTRL15 OFFSET(30) NUMBITS(2) [\n            NoResistor = 0b00,\n            PullUp = 0b01\n        ],\n\n        /// Pin 14\n        GPIO_PUP_PDN_CNTRL14 OFFSET(28) NUMBITS(2) [\n            NoResistor = 0b00,\n            PullUp = 0b01\n        ]\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    RegisterBlock {\n        (0x00 => _reserved1),\n        (0x04 => GPFSEL1: ReadWrite<u32, GPFSEL1::Register>),\n        (0x08 => _reserved2),\n        (0x94 => GPPUD: ReadWrite<u32, GPPUD::Register>),\n        (0x98 => GPPUDCLK0: ReadWrite<u32, GPPUDCLK0::Register>),\n        (0x9C => _reserved3),\n        (0xE4 => GPIO_PUP_PDN_CNTRL_REG0: ReadWrite<u32, GPIO_PUP_PDN_CNTRL_REG0::Register>),\n        (0xE8 => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\nstruct GPIOInner {\n    registers: Registers,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the GPIO HW.\npub struct GPIO {\n    inner: IRQSafeNullLock<GPIOInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GPIOInner {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n        }\n    }\n\n    /// Disable pull-up/down on pins 14 and 15.\n    #[cfg(feature = \"bsp_rpi3\")]\n    fn disable_pud_14_15_bcm2837(&mut self) {\n        use crate::time;\n        use core::time::Duration;\n\n        // The Linux 2837 GPIO driver waits 1 µs between the steps.\n        const DELAY: Duration = Duration::from_micros(1);\n\n        self.registers.GPPUD.write(GPPUD::PUD::Off);\n        time::time_manager().spin_for(DELAY);\n\n        self.registers\n            .GPPUDCLK0\n            .write(GPPUDCLK0::PUDCLK15::AssertClock + GPPUDCLK0::PUDCLK14::AssertClock);\n        time::time_manager().spin_for(DELAY);\n\n        self.registers.GPPUD.write(GPPUD::PUD::Off);\n        self.registers.GPPUDCLK0.set(0);\n    }\n\n    /// Disable pull-up/down on pins 14 and 15.\n    #[cfg(feature = \"bsp_rpi4\")]\n    fn disable_pud_14_15_bcm2711(&mut self) {\n        self.registers.GPIO_PUP_PDN_CNTRL_REG0.write(\n            GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL15::PullUp\n                + GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL14::PullUp,\n        );\n    }\n\n    /// Map PL011 UART as standard output.\n    ///\n    /// TX to pin 14\n    /// RX to pin 15\n    pub fn map_pl011_uart(&mut self) {\n        // Select the UART on pins 14 and 15.\n        self.registers\n            .GPFSEL1\n            .modify(GPFSEL1::FSEL15::AltFunc0 + GPFSEL1::FSEL14::AltFunc0);\n\n        // Disable pull-up/down on pins 14 and 15.\n        #[cfg(feature = \"bsp_rpi3\")]\n        self.disable_pud_14_15_bcm2837();\n\n        #[cfg(feature = \"bsp_rpi4\")]\n        self.disable_pud_14_15_bcm2711();\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GPIO {\n    pub const COMPATIBLE: &'static str = \"BCM GPIO\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            inner: IRQSafeNullLock::new(GPIOInner::new(mmio_start_addr)),\n        }\n    }\n\n    /// Concurrency safe version of `GPIOInner.map_pl011_uart()`\n    pub fn map_pl011_uart(&self) {\n        self.inner.lock(|inner| inner.map_pl011_uart())\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl driver::interface::DeviceDriver for GPIO {\n    type IRQNumberType = IRQNumber;\n\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Peripheral Interrupt Controller Driver.\n//!\n//! # Resources\n//!\n//! - <https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf>\n\nuse super::{PendingIRQs, PeripheralIRQ};\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    exception,\n    memory::{Address, Virtual},\n    synchronization,\n    synchronization::{IRQSafeNullLock, InitStateLock},\n};\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_structs,\n    registers::{ReadOnly, WriteOnly},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    WORegisterBlock {\n        (0x00 => _reserved1),\n        (0x10 => ENABLE_1: WriteOnly<u32>),\n        (0x14 => ENABLE_2: WriteOnly<u32>),\n        (0x18 => @END),\n    }\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    RORegisterBlock {\n        (0x00 => _reserved1),\n        (0x04 => PENDING_1: ReadOnly<u32>),\n        (0x08 => PENDING_2: ReadOnly<u32>),\n        (0x0c => @END),\n    }\n}\n\n/// Abstraction for the WriteOnly parts of the associated MMIO registers.\ntype WriteOnlyRegisters = MMIODerefWrapper<WORegisterBlock>;\n\n/// Abstraction for the ReadOnly parts of the associated MMIO registers.\ntype ReadOnlyRegisters = MMIODerefWrapper<RORegisterBlock>;\n\ntype HandlerTable = [Option<exception::asynchronous::IRQHandlerDescriptor<PeripheralIRQ>>;\n    PeripheralIRQ::MAX_INCLUSIVE + 1];\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the peripheral interrupt controller.\npub struct PeripheralIC {\n    /// Access to write registers is guarded with a lock.\n    wo_registers: IRQSafeNullLock<WriteOnlyRegisters>,\n\n    /// Register read access is unguarded.\n    ro_registers: ReadOnlyRegisters,\n\n    /// Stores registered IRQ handlers. Writable only during kernel init. RO afterwards.\n    handler_table: InitStateLock<HandlerTable>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PeripheralIC {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            wo_registers: IRQSafeNullLock::new(WriteOnlyRegisters::new(mmio_start_addr)),\n            ro_registers: ReadOnlyRegisters::new(mmio_start_addr),\n            handler_table: InitStateLock::new([None; PeripheralIRQ::MAX_INCLUSIVE + 1]),\n        }\n    }\n\n    /// Query the list of pending IRQs.\n    fn pending_irqs(&self) -> PendingIRQs {\n        let pending_mask: u64 = (u64::from(self.ro_registers.PENDING_2.get()) << 32)\n            | u64::from(self.ro_registers.PENDING_1.get());\n\n        PendingIRQs::new(pending_mask)\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::{Mutex, ReadWriteEx};\n\nimpl exception::asynchronous::interface::IRQManager for PeripheralIC {\n    type IRQNumberType = PeripheralIRQ;\n\n    fn register_handler(\n        &self,\n        irq_handler_descriptor: exception::asynchronous::IRQHandlerDescriptor<Self::IRQNumberType>,\n    ) -> Result<(), &'static str> {\n        self.handler_table.write(|table| {\n            let irq_number = irq_handler_descriptor.number().get();\n\n            if table[irq_number].is_some() {\n                return Err(\"IRQ handler already registered\");\n            }\n\n            table[irq_number] = Some(irq_handler_descriptor);\n\n            Ok(())\n        })\n    }\n\n    fn enable(&self, irq: &Self::IRQNumberType) {\n        self.wo_registers.lock(|regs| {\n            let enable_reg = if irq.get() <= 31 {\n                &regs.ENABLE_1\n            } else {\n                &regs.ENABLE_2\n            };\n\n            let enable_bit: u32 = 1 << (irq.get() % 32);\n\n            // Writing a 1 to a bit will set the corresponding IRQ enable bit. All other IRQ enable\n            // bits are unaffected. So we don't need read and OR'ing here.\n            enable_reg.set(enable_bit);\n        });\n    }\n\n    fn handle_pending_irqs<'irq_context>(\n        &'irq_context self,\n        _ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) {\n        self.handler_table.read(|table| {\n            for irq_number in self.pending_irqs() {\n                match table[irq_number] {\n                    None => panic!(\"No handler registered for IRQ {}\", irq_number),\n                    Some(descriptor) => {\n                        // Call the IRQ handler. Panics on failure.\n                        descriptor.handler().handle().expect(\"Error handling IRQ\");\n                    }\n                }\n            }\n        })\n    }\n\n    fn print_handler(&self) {\n        use crate::info;\n\n        info!(\"      Peripheral handler:\");\n\n        self.handler_table.read(|table| {\n            for (i, opt) in table.iter().enumerate() {\n                if let Some(handler) = opt {\n                    info!(\"            {: >3}. {}\", i, handler.name());\n                }\n            }\n        });\n    }\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Interrupt Controller Driver.\n\nmod peripheral_ic;\n\nuse crate::{\n    bsp::device_driver::common::BoundedUsize,\n    driver,\n    exception::{self, asynchronous::IRQHandlerDescriptor},\n    memory::{Address, Virtual},\n};\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Wrapper struct for a bitmask indicating pending IRQ numbers.\nstruct PendingIRQs {\n    bitmask: u64,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub type LocalIRQ = BoundedUsize<{ InterruptController::MAX_LOCAL_IRQ_NUMBER }>;\npub type PeripheralIRQ = BoundedUsize<{ InterruptController::MAX_PERIPHERAL_IRQ_NUMBER }>;\n\n/// Used for the associated type of trait [`exception::asynchronous::interface::IRQManager`].\n#[derive(Copy, Clone)]\n#[allow(missing_docs)]\npub enum IRQNumber {\n    Local(LocalIRQ),\n    Peripheral(PeripheralIRQ),\n}\n\n/// Representation of the Interrupt Controller.\npub struct InterruptController {\n    periph: peripheral_ic::PeripheralIC,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PendingIRQs {\n    pub fn new(bitmask: u64) -> Self {\n        Self { bitmask }\n    }\n}\n\nimpl Iterator for PendingIRQs {\n    type Item = usize;\n\n    fn next(&mut self) -> Option<Self::Item> {\n        if self.bitmask == 0 {\n            return None;\n        }\n\n        let next = self.bitmask.trailing_zeros() as usize;\n        self.bitmask &= self.bitmask.wrapping_sub(1);\n        Some(next)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl fmt::Display for IRQNumber {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        match self {\n            Self::Local(number) => write!(f, \"Local({})\", number),\n            Self::Peripheral(number) => write!(f, \"Peripheral({})\", number),\n        }\n    }\n}\n\nimpl InterruptController {\n    // Restrict to 3 for now. This makes future code for local_ic.rs more straight forward.\n    const MAX_LOCAL_IRQ_NUMBER: usize = 3;\n    const MAX_PERIPHERAL_IRQ_NUMBER: usize = 63;\n\n    pub const COMPATIBLE: &'static str = \"BCM Interrupt Controller\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(periph_mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            periph: peripheral_ic::PeripheralIC::new(periph_mmio_start_addr),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\n\nimpl driver::interface::DeviceDriver for InterruptController {\n    type IRQNumberType = IRQNumber;\n\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n}\n\nimpl exception::asynchronous::interface::IRQManager for InterruptController {\n    type IRQNumberType = IRQNumber;\n\n    fn register_handler(\n        &self,\n        irq_handler_descriptor: exception::asynchronous::IRQHandlerDescriptor<Self::IRQNumberType>,\n    ) -> Result<(), &'static str> {\n        match irq_handler_descriptor.number() {\n            IRQNumber::Local(_) => unimplemented!(\"Local IRQ controller not implemented.\"),\n            IRQNumber::Peripheral(pirq) => {\n                let periph_descriptor = IRQHandlerDescriptor::new(\n                    pirq,\n                    irq_handler_descriptor.name(),\n                    irq_handler_descriptor.handler(),\n                );\n\n                self.periph.register_handler(periph_descriptor)\n            }\n        }\n    }\n\n    fn enable(&self, irq: &Self::IRQNumberType) {\n        match irq {\n            IRQNumber::Local(_) => unimplemented!(\"Local IRQ controller not implemented.\"),\n            IRQNumber::Peripheral(pirq) => self.periph.enable(pirq),\n        }\n    }\n\n    fn handle_pending_irqs<'irq_context>(\n        &'irq_context self,\n        ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) {\n        // It can only be a peripheral IRQ pending because enable() does not support local IRQs yet.\n        self.periph.handle_pending_irqs(ic)\n    }\n\n    fn print_handler(&self) {\n        self.periph.print_handler();\n    }\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! PL011 UART driver.\n//!\n//! # Resources\n//!\n//! - <https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf>\n//! - <https://developer.arm.com/documentation/ddi0183/latest>\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    console, cpu, driver,\n    exception::{self, asynchronous::IRQNumber},\n    memory::{Address, Virtual},\n    synchronization,\n    synchronization::IRQSafeNullLock,\n};\nuse core::fmt;\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields, register_structs,\n    registers::{ReadOnly, ReadWrite, WriteOnly},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// PL011 UART registers.\n//\n// Descriptions taken from \"PrimeCell UART (PL011) Technical Reference Manual\" r1p5.\nregister_bitfields! {\n    u32,\n\n    /// Flag Register.\n    FR [\n        /// Transmit FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n        /// Line Control Register, LCR_H.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the transmit holding register is empty.\n        /// - If the FIFO is enabled, the TXFE bit is set when the transmit FIFO is empty.\n        /// - This bit does not indicate if there is data in the transmit shift register.\n        TXFE OFFSET(7) NUMBITS(1) [],\n\n        /// Transmit FIFO full. The meaning of this bit depends on the state of the FEN bit in the\n        /// LCR_H Register.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the transmit holding register is full.\n        /// - If the FIFO is enabled, the TXFF bit is set when the transmit FIFO is full.\n        TXFF OFFSET(5) NUMBITS(1) [],\n\n        /// Receive FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n        /// LCR_H Register.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the receive holding register is empty.\n        /// - If the FIFO is enabled, the RXFE bit is set when the receive FIFO is empty.\n        RXFE OFFSET(4) NUMBITS(1) [],\n\n        /// UART busy. If this bit is set to 1, the UART is busy transmitting data. This bit remains\n        /// set until the complete byte, including all the stop bits, has been sent from the shift\n        /// register.\n        ///\n        /// This bit is set as soon as the transmit FIFO becomes non-empty, regardless of whether\n        /// the UART is enabled or not.\n        BUSY OFFSET(3) NUMBITS(1) []\n    ],\n\n    /// Integer Baud Rate Divisor.\n    IBRD [\n        /// The integer baud rate divisor.\n        BAUD_DIVINT OFFSET(0) NUMBITS(16) []\n    ],\n\n    /// Fractional Baud Rate Divisor.\n    FBRD [\n        ///  The fractional baud rate divisor.\n        BAUD_DIVFRAC OFFSET(0) NUMBITS(6) []\n    ],\n\n    /// Line Control Register.\n    LCR_H [\n        /// Word length. These bits indicate the number of data bits transmitted or received in a\n        /// frame.\n        #[allow(clippy::enum_variant_names)]\n        WLEN OFFSET(5) NUMBITS(2) [\n            FiveBit = 0b00,\n            SixBit = 0b01,\n            SevenBit = 0b10,\n            EightBit = 0b11\n        ],\n\n        /// Enable FIFOs:\n        ///\n        /// 0 = FIFOs are disabled (character mode) that is, the FIFOs become 1-byte-deep holding\n        /// registers.\n        ///\n        /// 1 = Transmit and receive FIFO buffers are enabled (FIFO mode).\n        FEN  OFFSET(4) NUMBITS(1) [\n            FifosDisabled = 0,\n            FifosEnabled = 1\n        ]\n    ],\n\n    /// Control Register.\n    CR [\n        /// Receive enable. If this bit is set to 1, the receive section of the UART is enabled.\n        /// Data reception occurs for either UART signals or SIR signals depending on the setting of\n        /// the SIREN bit. When the UART is disabled in the middle of reception, it completes the\n        /// current character before stopping.\n        RXE OFFSET(9) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// Transmit enable. If this bit is set to 1, the transmit section of the UART is enabled.\n        /// Data transmission occurs for either UART signals, or SIR signals depending on the\n        /// setting of the SIREN bit. When the UART is disabled in the middle of transmission, it\n        /// completes the current character before stopping.\n        TXE OFFSET(8) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// UART enable:\n        ///\n        /// 0 = UART is disabled. If the UART is disabled in the middle of transmission or\n        /// reception, it completes the current character before stopping.\n        ///\n        /// 1 = The UART is enabled. Data transmission and reception occurs for either UART signals\n        /// or SIR signals depending on the setting of the SIREN bit\n        UARTEN OFFSET(0) NUMBITS(1) [\n            /// If the UART is disabled in the middle of transmission or reception, it completes the\n            /// current character before stopping.\n            Disabled = 0,\n            Enabled = 1\n        ]\n    ],\n\n    /// Interrupt FIFO Level Select Register.\n    IFLS [\n        /// Receive interrupt FIFO level select. The trigger points for the receive interrupt are as\n        /// follows.\n        RXIFLSEL OFFSET(3) NUMBITS(5) [\n            OneEigth = 0b000,\n            OneQuarter = 0b001,\n            OneHalf = 0b010,\n            ThreeQuarters = 0b011,\n            SevenEights = 0b100\n        ]\n    ],\n\n    /// Interrupt Mask Set/Clear Register.\n    IMSC [\n        /// Receive timeout interrupt mask. A read returns the current mask for the UARTRTINTR\n        /// interrupt.\n        ///\n        /// - On a write of 1, the mask of the UARTRTINTR interrupt is set.\n        /// - A write of 0 clears the mask.\n        RTIM OFFSET(6) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// Receive interrupt mask. A read returns the current mask for the UARTRXINTR interrupt.\n        ///\n        /// - On a write of 1, the mask of the UARTRXINTR interrupt is set.\n        /// - A write of 0 clears the mask.\n        RXIM OFFSET(4) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ]\n    ],\n\n    /// Masked Interrupt Status Register.\n    MIS [\n        /// Receive timeout masked interrupt status. Returns the masked interrupt state of the\n        /// UARTRTINTR interrupt.\n        RTMIS OFFSET(6) NUMBITS(1) [],\n\n        /// Receive masked interrupt status. Returns the masked interrupt state of the UARTRXINTR\n        /// interrupt.\n        RXMIS OFFSET(4) NUMBITS(1) []\n    ],\n\n    /// Interrupt Clear Register.\n    ICR [\n        /// Meta field for all pending interrupts.\n        ALL OFFSET(0) NUMBITS(11) []\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    pub RegisterBlock {\n        (0x00 => DR: ReadWrite<u32>),\n        (0x04 => _reserved1),\n        (0x18 => FR: ReadOnly<u32, FR::Register>),\n        (0x1c => _reserved2),\n        (0x24 => IBRD: WriteOnly<u32, IBRD::Register>),\n        (0x28 => FBRD: WriteOnly<u32, FBRD::Register>),\n        (0x2c => LCR_H: WriteOnly<u32, LCR_H::Register>),\n        (0x30 => CR: WriteOnly<u32, CR::Register>),\n        (0x34 => IFLS: ReadWrite<u32, IFLS::Register>),\n        (0x38 => IMSC: ReadWrite<u32, IMSC::Register>),\n        (0x3C => _reserved3),\n        (0x40 => MIS: ReadOnly<u32, MIS::Register>),\n        (0x44 => ICR: WriteOnly<u32, ICR::Register>),\n        (0x48 => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\n#[derive(PartialEq)]\nenum BlockingMode {\n    Blocking,\n    NonBlocking,\n}\n\nstruct PL011UartInner {\n    registers: Registers,\n    chars_written: usize,\n    chars_read: usize,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the UART.\npub struct PL011Uart {\n    inner: IRQSafeNullLock<PL011UartInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PL011UartInner {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n            chars_written: 0,\n            chars_read: 0,\n        }\n    }\n\n    /// Set up baud rate and characteristics.\n    ///\n    /// This results in 8N1 and 921_600 baud.\n    ///\n    /// The calculation for the BRD is (we set the clock to 48 MHz in config.txt):\n    /// `(48_000_000 / 16) / 921_600 = 3.2552083`.\n    ///\n    /// This means the integer part is `3` and goes into the `IBRD`.\n    /// The fractional part is `0.2552083`.\n    ///\n    /// `FBRD` calculation according to the PL011 Technical Reference Manual:\n    /// `INTEGER((0.2552083 * 64) + 0.5) = 16`.\n    ///\n    /// Therefore, the generated baud rate divider is: `3 + 16/64 = 3.25`. Which results in a\n    /// genrated baud rate of `48_000_000 / (16 * 3.25) = 923_077`.\n    ///\n    /// Error = `((923_077 - 921_600) / 921_600) * 100 = 0.16%`.\n    pub fn init(&mut self) {\n        // Execution can arrive here while there are still characters queued in the TX FIFO and\n        // actively being sent out by the UART hardware. If the UART is turned off in this case,\n        // those queued characters would be lost.\n        //\n        // For example, this can happen during runtime on a call to panic!(), because panic!()\n        // initializes its own UART instance and calls init().\n        //\n        // Hence, flush first to ensure all pending characters are transmitted.\n        self.flush();\n\n        // Turn the UART off temporarily.\n        self.registers.CR.set(0);\n\n        // Clear all pending interrupts.\n        self.registers.ICR.write(ICR::ALL::CLEAR);\n\n        // From the PL011 Technical Reference Manual:\n        //\n        // The LCR_H, IBRD, and FBRD registers form the single 30-bit wide LCR Register that is\n        // updated on a single write strobe generated by a LCR_H write. So, to internally update the\n        // contents of IBRD or FBRD, a LCR_H write must always be performed at the end.\n        //\n        // Set the baud rate, 8N1 and FIFO enabled.\n        self.registers.IBRD.write(IBRD::BAUD_DIVINT.val(3));\n        self.registers.FBRD.write(FBRD::BAUD_DIVFRAC.val(16));\n        self.registers\n            .LCR_H\n            .write(LCR_H::WLEN::EightBit + LCR_H::FEN::FifosEnabled);\n\n        // Set RX FIFO fill level at 1/8.\n        self.registers.IFLS.write(IFLS::RXIFLSEL::OneEigth);\n\n        // Enable RX IRQ + RX timeout IRQ.\n        self.registers\n            .IMSC\n            .write(IMSC::RXIM::Enabled + IMSC::RTIM::Enabled);\n\n        // Turn the UART on.\n        self.registers\n            .CR\n            .write(CR::UARTEN::Enabled + CR::TXE::Enabled + CR::RXE::Enabled);\n    }\n\n    /// Send a character.\n    fn write_char(&mut self, c: char) {\n        // Spin while TX FIFO full is set, waiting for an empty slot.\n        while self.registers.FR.matches_all(FR::TXFF::SET) {\n            cpu::nop();\n        }\n\n        // Write the character to the buffer.\n        self.registers.DR.set(c as u32);\n\n        self.chars_written += 1;\n    }\n\n    /// Block execution until the last buffered character has been physically put on the TX wire.\n    fn flush(&self) {\n        // Spin until the busy bit is cleared.\n        while self.registers.FR.matches_all(FR::BUSY::SET) {\n            cpu::nop();\n        }\n    }\n\n    /// Retrieve a character.\n    fn read_char_converting(&mut self, blocking_mode: BlockingMode) -> Option<char> {\n        // If RX FIFO is empty,\n        if self.registers.FR.matches_all(FR::RXFE::SET) {\n            // immediately return in non-blocking mode.\n            if blocking_mode == BlockingMode::NonBlocking {\n                return None;\n            }\n\n            // Otherwise, wait until a char was received.\n            while self.registers.FR.matches_all(FR::RXFE::SET) {\n                cpu::nop();\n            }\n        }\n\n        // Read one character.\n        let mut ret = self.registers.DR.get() as u8 as char;\n\n        // Convert carrige return to newline.\n        if ret == '\\r' {\n            ret = '\\n'\n        }\n\n        // Update statistics.\n        self.chars_read += 1;\n\n        Some(ret)\n    }\n}\n\n/// Implementing `core::fmt::Write` enables usage of the `format_args!` macros, which in turn are\n/// used to implement the `kernel`'s `print!` and `println!` macros. By implementing `write_str()`,\n/// we get `write_fmt()` automatically.\n///\n/// The function takes an `&mut self`, so it must be implemented for the inner struct.\n///\n/// See [`src/print.rs`].\n///\n/// [`src/print.rs`]: ../../print/index.html\nimpl fmt::Write for PL011UartInner {\n    fn write_str(&mut self, s: &str) -> fmt::Result {\n        for c in s.chars() {\n            self.write_char(c);\n        }\n\n        Ok(())\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PL011Uart {\n    pub const COMPATIBLE: &'static str = \"BCM PL011 UART\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            inner: IRQSafeNullLock::new(PL011UartInner::new(mmio_start_addr)),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl driver::interface::DeviceDriver for PL011Uart {\n    type IRQNumberType = IRQNumber;\n\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n\n    unsafe fn init(&self) -> Result<(), &'static str> {\n        self.inner.lock(|inner| inner.init());\n\n        Ok(())\n    }\n\n    fn register_and_enable_irq_handler(\n        &'static self,\n        irq_number: &Self::IRQNumberType,\n    ) -> Result<(), &'static str> {\n        use exception::asynchronous::{irq_manager, IRQHandlerDescriptor};\n\n        let descriptor = IRQHandlerDescriptor::new(*irq_number, Self::COMPATIBLE, self);\n\n        irq_manager().register_handler(descriptor)?;\n        irq_manager().enable(irq_number);\n\n        Ok(())\n    }\n}\n\nimpl console::interface::Write for PL011Uart {\n    /// Passthrough of `args` to the `core::fmt::Write` implementation, but guarded by a Mutex to\n    /// serialize access.\n    fn write_char(&self, c: char) {\n        self.inner.lock(|inner| inner.write_char(c));\n    }\n\n    fn write_fmt(&self, args: core::fmt::Arguments) -> fmt::Result {\n        // Fully qualified syntax for the call to `core::fmt::Write::write_fmt()` to increase\n        // readability.\n        self.inner.lock(|inner| fmt::Write::write_fmt(inner, args))\n    }\n\n    fn flush(&self) {\n        // Spin until TX FIFO empty is set.\n        self.inner.lock(|inner| inner.flush());\n    }\n}\n\nimpl console::interface::Read for PL011Uart {\n    fn read_char(&self) -> char {\n        self.inner\n            .lock(|inner| inner.read_char_converting(BlockingMode::Blocking).unwrap())\n    }\n\n    fn clear_rx(&self) {\n        // Read from the RX FIFO until it is indicating empty.\n        while self\n            .inner\n            .lock(|inner| inner.read_char_converting(BlockingMode::NonBlocking))\n            .is_some()\n        {}\n    }\n}\n\nimpl console::interface::Statistics for PL011Uart {\n    fn chars_written(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_written)\n    }\n\n    fn chars_read(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_read)\n    }\n}\n\nimpl console::interface::All for PL011Uart {}\n\nimpl exception::asynchronous::interface::IRQHandler for PL011Uart {\n    fn handle(&self) -> Result<(), &'static str> {\n        self.inner.lock(|inner| {\n            let pending = inner.registers.MIS.extract();\n\n            // Clear all pending IRQs.\n            inner.registers.ICR.write(ICR::ALL::CLEAR);\n\n            // Check for any kind of RX interrupt.\n            if pending.matches_any(MIS::RXMIS::SET + MIS::RTMIS::SET) {\n                // Echo any received characters.\n                while let Some(c) = inner.read_char_converting(BlockingMode::NonBlocking) {\n                    inner.write_char(c)\n                }\n            }\n        });\n\n        Ok(())\n    }\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/bsp/device_driver/bcm.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BCM driver top level.\n\nmod bcm2xxx_gpio;\n#[cfg(feature = \"bsp_rpi3\")]\nmod bcm2xxx_interrupt_controller;\nmod bcm2xxx_pl011_uart;\n\npub use bcm2xxx_gpio::*;\n#[cfg(feature = \"bsp_rpi3\")]\npub use bcm2xxx_interrupt_controller::*;\npub use bcm2xxx_pl011_uart::*;\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/bsp/device_driver/common.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Common device driver code.\n\nuse crate::memory::{Address, Virtual};\nuse core::{fmt, marker::PhantomData, ops};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct MMIODerefWrapper<T> {\n    start_addr: Address<Virtual>,\n    phantom: PhantomData<fn() -> T>,\n}\n\n/// A wrapper type for usize with integrated range bound check.\n#[derive(Copy, Clone)]\npub struct BoundedUsize<const MAX_INCLUSIVE: usize>(usize);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T> MMIODerefWrapper<T> {\n    /// Create an instance.\n    pub const unsafe fn new(start_addr: Address<Virtual>) -> Self {\n        Self {\n            start_addr,\n            phantom: PhantomData,\n        }\n    }\n}\n\nimpl<T> ops::Deref for MMIODerefWrapper<T> {\n    type Target = T;\n\n    fn deref(&self) -> &Self::Target {\n        unsafe { &*(self.start_addr.as_usize() as *const _) }\n    }\n}\n\nimpl<const MAX_INCLUSIVE: usize> BoundedUsize<{ MAX_INCLUSIVE }> {\n    pub const MAX_INCLUSIVE: usize = MAX_INCLUSIVE;\n\n    /// Creates a new instance if number <= MAX_INCLUSIVE.\n    pub const fn new(number: usize) -> Self {\n        assert!(number <= MAX_INCLUSIVE);\n\n        Self(number)\n    }\n\n    /// Return the wrapped number.\n    pub const fn get(self) -> usize {\n        self.0\n    }\n}\n\nimpl<const MAX_INCLUSIVE: usize> fmt::Display for BoundedUsize<{ MAX_INCLUSIVE }> {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        write!(f, \"{}\", self.0)\n    }\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/bsp/device_driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Device driver.\n\n#[cfg(feature = \"bsp_rpi4\")]\nmod arm;\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod bcm;\nmod common;\n\n#[cfg(feature = \"bsp_rpi4\")]\npub use arm::*;\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use bcm::*;\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/bsp/raspberrypi/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Processor code.\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Used by `arch` code to find the early boot core.\n#[no_mangle]\n#[link_section = \".text._start_arguments\"]\npub static BOOT_CORE_ID: u64 = 0;\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/bsp/raspberrypi/driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP driver support.\n\nuse super::{exception, memory::map::mmio};\nuse crate::{\n    bsp::device_driver,\n    console, driver as generic_driver,\n    exception::{self as generic_exception},\n    memory,\n    memory::mmu::MMIODescriptor,\n};\nuse core::{\n    mem::MaybeUninit,\n    sync::atomic::{AtomicBool, Ordering},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic mut PL011_UART: MaybeUninit<device_driver::PL011Uart> = MaybeUninit::uninit();\nstatic mut GPIO: MaybeUninit<device_driver::GPIO> = MaybeUninit::uninit();\n\n#[cfg(feature = \"bsp_rpi3\")]\nstatic mut INTERRUPT_CONTROLLER: MaybeUninit<device_driver::InterruptController> =\n    MaybeUninit::uninit();\n\n#[cfg(feature = \"bsp_rpi4\")]\nstatic mut INTERRUPT_CONTROLLER: MaybeUninit<device_driver::GICv2> = MaybeUninit::uninit();\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// This must be called only after successful init of the memory subsystem.\nunsafe fn instantiate_uart() -> Result<(), &'static str> {\n    let mmio_descriptor = MMIODescriptor::new(mmio::PL011_UART_START, mmio::PL011_UART_SIZE);\n    let virt_addr =\n        memory::mmu::kernel_map_mmio(device_driver::PL011Uart::COMPATIBLE, &mmio_descriptor)?;\n\n    PL011_UART.write(device_driver::PL011Uart::new(virt_addr));\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the UART driver.\nunsafe fn post_init_uart() -> Result<(), &'static str> {\n    console::register_console(PL011_UART.assume_init_ref());\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the memory subsystem.\nunsafe fn instantiate_gpio() -> Result<(), &'static str> {\n    let mmio_descriptor = MMIODescriptor::new(mmio::GPIO_START, mmio::GPIO_SIZE);\n    let virt_addr =\n        memory::mmu::kernel_map_mmio(device_driver::GPIO::COMPATIBLE, &mmio_descriptor)?;\n\n    GPIO.write(device_driver::GPIO::new(virt_addr));\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the GPIO driver.\nunsafe fn post_init_gpio() -> Result<(), &'static str> {\n    GPIO.assume_init_ref().map_pl011_uart();\n    Ok(())\n}\n\n/// This must be called only after successful init of the memory subsystem.\n#[cfg(feature = \"bsp_rpi3\")]\nunsafe fn instantiate_interrupt_controller() -> Result<(), &'static str> {\n    let periph_mmio_descriptor =\n        MMIODescriptor::new(mmio::PERIPHERAL_IC_START, mmio::PERIPHERAL_IC_SIZE);\n    let periph_virt_addr = memory::mmu::kernel_map_mmio(\n        device_driver::InterruptController::COMPATIBLE,\n        &periph_mmio_descriptor,\n    )?;\n\n    INTERRUPT_CONTROLLER.write(device_driver::InterruptController::new(periph_virt_addr));\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the memory subsystem.\n#[cfg(feature = \"bsp_rpi4\")]\nunsafe fn instantiate_interrupt_controller() -> Result<(), &'static str> {\n    let gicd_mmio_descriptor = MMIODescriptor::new(mmio::GICD_START, mmio::GICD_SIZE);\n    let gicd_virt_addr = memory::mmu::kernel_map_mmio(\"GICv2 GICD\", &gicd_mmio_descriptor)?;\n\n    let gicc_mmio_descriptor = MMIODescriptor::new(mmio::GICC_START, mmio::GICC_SIZE);\n    let gicc_virt_addr = memory::mmu::kernel_map_mmio(\"GICV2 GICC\", &gicc_mmio_descriptor)?;\n\n    INTERRUPT_CONTROLLER.write(device_driver::GICv2::new(gicd_virt_addr, gicc_virt_addr));\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the interrupt controller driver.\nunsafe fn post_init_interrupt_controller() -> Result<(), &'static str> {\n    generic_exception::asynchronous::register_irq_manager(INTERRUPT_CONTROLLER.assume_init_ref());\n\n    Ok(())\n}\n\n/// Function needs to ensure that driver registration happens only after correct instantiation.\nunsafe fn driver_uart() -> Result<(), &'static str> {\n    instantiate_uart()?;\n\n    let uart_descriptor = generic_driver::DeviceDriverDescriptor::new(\n        PL011_UART.assume_init_ref(),\n        Some(post_init_uart),\n        Some(exception::asynchronous::irq_map::PL011_UART),\n    );\n    generic_driver::driver_manager().register_driver(uart_descriptor);\n\n    Ok(())\n}\n\n/// Function needs to ensure that driver registration happens only after correct instantiation.\nunsafe fn driver_gpio() -> Result<(), &'static str> {\n    instantiate_gpio()?;\n\n    let gpio_descriptor = generic_driver::DeviceDriverDescriptor::new(\n        GPIO.assume_init_ref(),\n        Some(post_init_gpio),\n        None,\n    );\n    generic_driver::driver_manager().register_driver(gpio_descriptor);\n\n    Ok(())\n}\n\n/// Function needs to ensure that driver registration happens only after correct instantiation.\nunsafe fn driver_interrupt_controller() -> Result<(), &'static str> {\n    instantiate_interrupt_controller()?;\n\n    let interrupt_controller_descriptor = generic_driver::DeviceDriverDescriptor::new(\n        INTERRUPT_CONTROLLER.assume_init_ref(),\n        Some(post_init_interrupt_controller),\n        None,\n    );\n    generic_driver::driver_manager().register_driver(interrupt_controller_descriptor);\n\n    Ok(())\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Initialize the driver subsystem.\n///\n/// # Safety\n///\n/// See child function calls.\npub unsafe fn init() -> Result<(), &'static str> {\n    static INIT_DONE: AtomicBool = AtomicBool::new(false);\n    if INIT_DONE.load(Ordering::Relaxed) {\n        return Err(\"Init already done\");\n    }\n\n    driver_uart()?;\n    driver_gpio()?;\n    driver_interrupt_controller()?;\n\n    INIT_DONE.store(true, Ordering::Relaxed);\n    Ok(())\n}\n\n/// Minimal code needed to bring up the console in QEMU (for testing only). This is often less steps\n/// than on real hardware due to QEMU's abstractions.\n#[cfg(feature = \"test_build\")]\npub fn qemu_bring_up_console() {\n    use crate::cpu;\n\n    unsafe {\n        instantiate_uart().unwrap_or_else(|_| cpu::qemu_exit_failure());\n        console::register_console(PL011_UART.assume_init_ref());\n    };\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/bsp/raspberrypi/exception/asynchronous.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP asynchronous exception handling.\n\nuse crate::bsp;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Export for reuse in generic asynchronous.rs.\npub use bsp::device_driver::IRQNumber;\n\n#[cfg(feature = \"bsp_rpi3\")]\npub(in crate::bsp) mod irq_map {\n    use super::bsp::device_driver::{IRQNumber, PeripheralIRQ};\n\n    pub const PL011_UART: IRQNumber = IRQNumber::Peripheral(PeripheralIRQ::new(57));\n}\n\n#[cfg(feature = \"bsp_rpi4\")]\npub(in crate::bsp) mod irq_map {\n    use super::bsp::device_driver::IRQNumber;\n\n    pub const PL011_UART: IRQNumber = IRQNumber::new(153);\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/bsp/raspberrypi/exception.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP synchronous and asynchronous exception handling.\n\npub mod asynchronous;\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/bsp/raspberrypi/kernel.ld",
    "content": "/* SPDX-License-Identifier: MIT OR Apache-2.0\n *\n * Copyright (c) 2018-2022 Andre Richter <andre.o.richter@gmail.com>\n */\n\nINCLUDE kernel_virt_addr_space_size.ld;\n\nPAGE_SIZE = 64K;\nPAGE_MASK = PAGE_SIZE - 1;\n\n/* The kernel's virtual address range will be:\n *\n * [END_ADDRESS_INCLUSIVE, START_ADDRESS]\n * [u64::MAX             , (u64::MAX - __kernel_virt_addr_space_size) + 1]\n */\n__kernel_virt_start_addr = ((0xffffffffffffffff - __kernel_virt_addr_space_size) + 1);\n\n__rpi_phys_dram_start_addr = 0;\n\n/* The physical address at which the the kernel binary will be loaded by the Raspberry's firmware */\n__rpi_phys_binary_load_addr = 0x80000;\n\n\nENTRY(__rpi_phys_binary_load_addr)\n\n/* Flags:\n *     4 == R\n *     5 == RX\n *     6 == RW\n *\n * Segments are marked PT_LOAD below so that the ELF file provides virtual and physical addresses.\n * It doesn't mean all of them need actually be loaded.\n */\nPHDRS\n{\n    segment_code            PT_LOAD FLAGS(5);\n    segment_data            PT_LOAD FLAGS(6);\n    segment_boot_core_stack PT_LOAD FLAGS(6);\n}\n\nSECTIONS\n{\n    . =  __kernel_virt_start_addr;\n\n    ASSERT((. & PAGE_MASK) == 0, \"Start of address space is not page aligned\")\n\n    /***********************************************************************************************\n    * Code + RO Data + Global Offset Table\n    ***********************************************************************************************/\n    __code_start = .;\n    .text : AT(__rpi_phys_binary_load_addr)\n    {\n        KEEP(*(.text._start))\n        *(.text._start_arguments) /* Constants (or statics in Rust speak) read by _start(). */\n        *(.text._start_rust)      /* The Rust entry point */\n        *(.text*)                 /* Everything else */\n    } :segment_code\n\n    .rodata         : ALIGN(8) { *(.rodata*) } :segment_code\n    .kernel_symbols : ALIGN(8) {\n        __kernel_symbols_start = .;\n        . += 32 * 1024;\n    } :segment_code\n\n    . = ALIGN(PAGE_SIZE);\n    __code_end_exclusive = .;\n\n    /***********************************************************************************************\n    * Data + BSS\n    ***********************************************************************************************/\n    __data_start = .;\n    .data : { *(.data*) } :segment_data\n\n    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */\n    .bss (NOLOAD) : ALIGN(16)\n    {\n        __bss_start = .;\n        *(.bss*);\n        . = ALIGN(16);\n        __bss_end_exclusive = .;\n    } :segment_data\n\n    . = ALIGN(PAGE_SIZE);\n    __data_end_exclusive = .;\n\n    /***********************************************************************************************\n    * MMIO Remap Reserved\n    ***********************************************************************************************/\n    __mmio_remap_start = .;\n    . += 8 * 1024 * 1024;\n    __mmio_remap_end_exclusive = .;\n\n    ASSERT((. & PAGE_MASK) == 0, \"MMIO remap reservation is not page aligned\")\n\n    /***********************************************************************************************\n    * Guard Page\n    ***********************************************************************************************/\n    . += PAGE_SIZE;\n\n    /***********************************************************************************************\n    * Boot Core Stack\n    ***********************************************************************************************/\n    .boot_core_stack (NOLOAD) : AT(__rpi_phys_dram_start_addr)\n    {\n        __boot_core_stack_start = .;         /*   ^             */\n                                             /*   | stack       */\n        . += __rpi_phys_binary_load_addr;    /*   | growth      */\n                                             /*   | direction   */\n        __boot_core_stack_end_exclusive = .; /*   |             */\n    } :segment_boot_core_stack\n\n    ASSERT((. & PAGE_MASK) == 0, \"End of boot core stack is not page aligned\")\n\n    /***********************************************************************************************\n    * Misc\n    ***********************************************************************************************/\n    .got : { *(.got*) }\n    ASSERT(SIZEOF(.got) == 0, \"Relocation support not expected\")\n\n    /DISCARD/ : { *(.comment*) }\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/bsp/raspberrypi/kernel_virt_addr_space_size.ld",
    "content": "__kernel_virt_addr_space_size = 1024 * 1024 * 1024\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/bsp/raspberrypi/memory/mmu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Memory Management Unit.\n\nuse crate::{\n    memory::{\n        mmu::{\n            self as generic_mmu, AddressSpace, AssociatedTranslationTable, AttributeFields,\n            MemoryRegion, PageAddress, TranslationGranule,\n        },\n        Physical, Virtual,\n    },\n    synchronization::InitStateLock,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\ntype KernelTranslationTable =\n    <KernelVirtAddrSpace as AssociatedTranslationTable>::TableStartFromTop;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// The translation granule chosen by this BSP. This will be used everywhere else in the kernel to\n/// derive respective data structures and their sizes. For example, the `crate::memory::mmu::Page`.\npub type KernelGranule = TranslationGranule<{ 64 * 1024 }>;\n\n/// The kernel's virtual address space defined by this BSP.\npub type KernelVirtAddrSpace = AddressSpace<{ kernel_virt_addr_space_size() }>;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n/// The kernel translation tables.\n///\n/// It is mandatory that InitStateLock is transparent.\n///\n/// That is, `size_of(InitStateLock<KernelTranslationTable>) == size_of(KernelTranslationTable)`.\n/// There is a unit tests that checks this porperty.\n#[link_section = \".data\"]\n#[no_mangle]\nstatic KERNEL_TABLES: InitStateLock<KernelTranslationTable> =\n    InitStateLock::new(KernelTranslationTable::new_for_precompute());\n\n/// This value is needed during early boot for MMU setup.\n///\n/// This will be patched to the correct value by the \"translation table tool\" after linking. This\n/// given value here is just a dummy.\n#[link_section = \".text._start_arguments\"]\n#[no_mangle]\nstatic PHYS_KERNEL_TABLES_BASE_ADDR: u64 = 0xCCCCAAAAFFFFEEEE;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// This is a hack for retrieving the value for the kernel's virtual address space size as a\n/// constant from a common place, since it is needed as a compile-time/link-time constant in both,\n/// the linker script and the Rust sources.\n#[allow(clippy::needless_late_init)]\nconst fn kernel_virt_addr_space_size() -> usize {\n    let __kernel_virt_addr_space_size;\n\n    include!(\"../kernel_virt_addr_space_size.ld\");\n\n    __kernel_virt_addr_space_size\n}\n\n/// Helper function for calculating the number of pages the given parameter spans.\nconst fn size_to_num_pages(size: usize) -> usize {\n    assert!(size > 0);\n    assert!(size % KernelGranule::SIZE == 0);\n\n    size >> KernelGranule::SHIFT\n}\n\n/// The code pages of the kernel binary.\nfn virt_code_region() -> MemoryRegion<Virtual> {\n    let num_pages = size_to_num_pages(super::code_size());\n\n    let start_page_addr = super::virt_code_start();\n    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n\n    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n}\n\n/// The data pages of the kernel binary.\nfn virt_data_region() -> MemoryRegion<Virtual> {\n    let num_pages = size_to_num_pages(super::data_size());\n\n    let start_page_addr = super::virt_data_start();\n    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n\n    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n}\n\n/// The boot core stack pages.\nfn virt_boot_core_stack_region() -> MemoryRegion<Virtual> {\n    let num_pages = size_to_num_pages(super::boot_core_stack_size());\n\n    let start_page_addr = super::virt_boot_core_stack_start();\n    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n\n    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n}\n\n// There is no reason to expect the following conversions to fail, since they were generated offline\n// by the `translation table tool`. If it doesn't work, a panic due to the unwraps is justified.\nfn kernel_virt_to_phys_region(virt_region: MemoryRegion<Virtual>) -> MemoryRegion<Physical> {\n    let phys_start_page_addr =\n        generic_mmu::try_kernel_virt_page_addr_to_phys_page_addr(virt_region.start_page_addr())\n            .unwrap();\n\n    let phys_end_exclusive_page_addr = phys_start_page_addr\n        .checked_offset(virt_region.num_pages() as isize)\n        .unwrap();\n\n    MemoryRegion::new(phys_start_page_addr, phys_end_exclusive_page_addr)\n}\n\nfn kernel_page_attributes(virt_page_addr: PageAddress<Virtual>) -> AttributeFields {\n    generic_mmu::try_kernel_page_attributes(virt_page_addr).unwrap()\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the kernel's translation tables.\npub fn kernel_translation_tables() -> &'static InitStateLock<KernelTranslationTable> {\n    &KERNEL_TABLES\n}\n\n/// The MMIO remap pages.\npub fn virt_mmio_remap_region() -> MemoryRegion<Virtual> {\n    let num_pages = size_to_num_pages(super::mmio_remap_size());\n\n    let start_page_addr = super::virt_mmio_remap_start();\n    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n\n    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n}\n\n/// Add mapping records for the kernel binary.\n///\n/// The actual translation table entries for the kernel binary are generated using the offline\n/// `translation table tool` and patched into the kernel binary. This function just adds the mapping\n/// record entries.\npub fn kernel_add_mapping_records_for_precomputed() {\n    let virt_code_region = virt_code_region();\n    generic_mmu::kernel_add_mapping_record(\n        \"Kernel code and RO data\",\n        &virt_code_region,\n        &kernel_virt_to_phys_region(virt_code_region),\n        &kernel_page_attributes(virt_code_region.start_page_addr()),\n    );\n\n    let virt_data_region = virt_data_region();\n    generic_mmu::kernel_add_mapping_record(\n        \"Kernel data and bss\",\n        &virt_data_region,\n        &kernel_virt_to_phys_region(virt_data_region),\n        &kernel_page_attributes(virt_data_region.start_page_addr()),\n    );\n\n    let virt_boot_core_stack_region = virt_boot_core_stack_region();\n    generic_mmu::kernel_add_mapping_record(\n        \"Kernel boot-core stack\",\n        &virt_boot_core_stack_region,\n        &kernel_virt_to_phys_region(virt_boot_core_stack_region),\n        &kernel_page_attributes(virt_boot_core_stack_region.start_page_addr()),\n    );\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/bsp/raspberrypi/memory.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Memory Management.\n//!\n//! The physical memory layout.\n//!\n//! The Raspberry's firmware copies the kernel binary to 0x8_0000. The preceding region will be used\n//! as the boot core's stack.\n//!\n//! +---------------------------------------+\n//! |                                       | boot_core_stack_start @ 0x0\n//! |                                       |                                ^\n//! | Boot-core Stack                       |                                | stack\n//! |                                       |                                | growth\n//! |                                       |                                | direction\n//! +---------------------------------------+\n//! |                                       | code_start @ 0x8_0000 == boot_core_stack_end_exclusive\n//! | .text                                 |\n//! | .rodata                               |\n//! | .got                                  |\n//! | .kernel_symbols                       |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       | data_start == code_end_exclusive\n//! | .data                                 |\n//! | .bss                                  |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       | data_end_exclusive\n//! |                                       |\n//!\n//!\n//!\n//!\n//!\n//! The virtual memory layout is as follows:\n//!\n//! +---------------------------------------+\n//! |                                       | code_start @ __kernel_virt_start_addr\n//! | .text                                 |\n//! | .rodata                               |\n//! | .got                                  |\n//! | .kernel_symbols                       |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       | data_start == code_end_exclusive\n//! | .data                                 |\n//! | .bss                                  |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       |  mmio_remap_start == data_end_exclusive\n//! | VA region for MMIO remapping          |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       |  mmio_remap_end_exclusive\n//! | Unmapped guard page                   |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       | boot_core_stack_start\n//! |                                       |                                ^\n//! | Boot-core Stack                       |                                | stack\n//! |                                       |                                | growth\n//! |                                       |                                | direction\n//! +---------------------------------------+\n//! |                                       | boot_core_stack_end_exclusive\n//! |                                       |\npub mod mmu;\n\nuse crate::memory::{mmu::PageAddress, Address, Physical, Virtual};\nuse core::cell::UnsafeCell;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Symbols from the linker script.\nextern \"Rust\" {\n    static __code_start: UnsafeCell<()>;\n    static __code_end_exclusive: UnsafeCell<()>;\n\n    static __data_start: UnsafeCell<()>;\n    static __data_end_exclusive: UnsafeCell<()>;\n\n    static __mmio_remap_start: UnsafeCell<()>;\n    static __mmio_remap_end_exclusive: UnsafeCell<()>;\n\n    static __boot_core_stack_start: UnsafeCell<()>;\n    static __boot_core_stack_end_exclusive: UnsafeCell<()>;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// The board's physical memory map.\n#[rustfmt::skip]\npub(super) mod map {\n    use super::*;\n\n    /// Physical devices.\n    #[cfg(feature = \"bsp_rpi3\")]\n    pub mod mmio {\n        use super::*;\n\n        pub const PERIPHERAL_IC_START: Address<Physical> = Address::new(0x3F00_B200);\n        pub const PERIPHERAL_IC_SIZE:  usize             =              0x24;\n\n        pub const GPIO_START:          Address<Physical> = Address::new(0x3F20_0000);\n        pub const GPIO_SIZE:           usize             =              0xA0;\n\n        pub const PL011_UART_START:    Address<Physical> = Address::new(0x3F20_1000);\n        pub const PL011_UART_SIZE:     usize             =              0x48;\n\n        pub const END:                 Address<Physical> = Address::new(0x4001_0000);\n    }\n\n    /// Physical devices.\n    #[cfg(feature = \"bsp_rpi4\")]\n    pub mod mmio {\n        use super::*;\n\n        pub const GPIO_START:       Address<Physical> = Address::new(0xFE20_0000);\n        pub const GPIO_SIZE:        usize             =              0xA0;\n\n        pub const PL011_UART_START: Address<Physical> = Address::new(0xFE20_1000);\n        pub const PL011_UART_SIZE:  usize             =              0x48;\n\n        pub const GICD_START:       Address<Physical> = Address::new(0xFF84_1000);\n        pub const GICD_SIZE:        usize             =              0x824;\n\n        pub const GICC_START:       Address<Physical> = Address::new(0xFF84_2000);\n        pub const GICC_SIZE:        usize             =              0x14;\n\n        pub const END:              Address<Physical> = Address::new(0xFF85_0000);\n    }\n\n    pub const END: Address<Physical> = mmio::END;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Start page address of the code segment.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn virt_code_start() -> PageAddress<Virtual> {\n    PageAddress::from(unsafe { __code_start.get() as usize })\n}\n\n/// Size of the code segment.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn code_size() -> usize {\n    unsafe { (__code_end_exclusive.get() as usize) - (__code_start.get() as usize) }\n}\n\n/// Start page address of the data segment.\n#[inline(always)]\nfn virt_data_start() -> PageAddress<Virtual> {\n    PageAddress::from(unsafe { __data_start.get() as usize })\n}\n\n/// Size of the data segment.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn data_size() -> usize {\n    unsafe { (__data_end_exclusive.get() as usize) - (__data_start.get() as usize) }\n}\n\n/// Start page address of the MMIO remap reservation.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn virt_mmio_remap_start() -> PageAddress<Virtual> {\n    PageAddress::from(unsafe { __mmio_remap_start.get() as usize })\n}\n\n/// Size of the MMIO remap reservation.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn mmio_remap_size() -> usize {\n    unsafe { (__mmio_remap_end_exclusive.get() as usize) - (__mmio_remap_start.get() as usize) }\n}\n\n/// Start page address of the boot core's stack.\n#[inline(always)]\nfn virt_boot_core_stack_start() -> PageAddress<Virtual> {\n    PageAddress::from(unsafe { __boot_core_stack_start.get() as usize })\n}\n\n/// Size of the boot core's stack.\n#[inline(always)]\nfn boot_core_stack_size() -> usize {\n    unsafe {\n        (__boot_core_stack_end_exclusive.get() as usize) - (__boot_core_stack_start.get() as usize)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Exclusive end address of the physical address space.\n#[inline(always)]\npub fn phys_addr_space_end_exclusive_addr() -> PageAddress<Physical> {\n    PageAddress::from(map::END)\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/bsp/raspberrypi.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Top-level BSP file for the Raspberry Pi 3 and 4.\n\npub mod cpu;\npub mod driver;\npub mod exception;\npub mod memory;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Board identification.\npub fn board_name() -> &'static str {\n    #[cfg(feature = \"bsp_rpi3\")]\n    {\n        \"Raspberry Pi 3\"\n    }\n\n    #[cfg(feature = \"bsp_rpi4\")]\n    {\n        \"Raspberry Pi 4\"\n    }\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/bsp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Conditional reexporting of Board Support Packages.\n\nmod device_driver;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod raspberrypi;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use raspberrypi::*;\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/common.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! General purpose code.\n\n/// Check if a value is aligned to a given size.\n#[inline(always)]\npub const fn is_aligned(value: usize, alignment: usize) -> bool {\n    assert!(alignment.is_power_of_two());\n\n    (value & (alignment - 1)) == 0\n}\n\n/// Align down.\n#[inline(always)]\npub const fn align_down(value: usize, alignment: usize) -> usize {\n    assert!(alignment.is_power_of_two());\n\n    value & !(alignment - 1)\n}\n\n/// Align up.\n#[inline(always)]\npub const fn align_up(value: usize, alignment: usize) -> usize {\n    assert!(alignment.is_power_of_two());\n\n    (value + alignment - 1) & !(alignment - 1)\n}\n\n/// Convert a size into human readable format.\npub const fn size_human_readable_ceil(size: usize) -> (usize, &'static str) {\n    const KIB: usize = 1024;\n    const MIB: usize = 1024 * 1024;\n    const GIB: usize = 1024 * 1024 * 1024;\n\n    if (size / GIB) > 0 {\n        (size.div_ceil(GIB), \"GiB\")\n    } else if (size / MIB) > 0 {\n        (size.div_ceil(MIB), \"MiB\")\n    } else if (size / KIB) > 0 {\n        (size.div_ceil(KIB), \"KiB\")\n    } else {\n        (size, \"Byte\")\n    }\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/console/null_console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Null console.\n\nuse super::interface;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct NullConsole;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\npub static NULL_CONSOLE: NullConsole = NullConsole {};\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl interface::Write for NullConsole {\n    fn write_char(&self, _c: char) {}\n\n    fn write_fmt(&self, _args: fmt::Arguments) -> fmt::Result {\n        fmt::Result::Ok(())\n    }\n\n    fn flush(&self) {}\n}\n\nimpl interface::Read for NullConsole {\n    fn clear_rx(&self) {}\n}\n\nimpl interface::Statistics for NullConsole {}\nimpl interface::All for NullConsole {}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! System console.\n\nmod null_console;\n\nuse crate::synchronization;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Console interfaces.\npub mod interface {\n    use core::fmt;\n\n    /// Console write functions.\n    pub trait Write {\n        /// Write a single character.\n        fn write_char(&self, c: char);\n\n        /// Write a Rust format string.\n        fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result;\n\n        /// Block until the last buffered character has been physically put on the TX wire.\n        fn flush(&self);\n    }\n\n    /// Console read functions.\n    pub trait Read {\n        /// Read a single character.\n        fn read_char(&self) -> char {\n            ' '\n        }\n\n        /// Clear RX buffers, if any.\n        fn clear_rx(&self);\n    }\n\n    /// Console statistics.\n    pub trait Statistics {\n        /// Return the number of characters written.\n        fn chars_written(&self) -> usize {\n            0\n        }\n\n        /// Return the number of characters read.\n        fn chars_read(&self) -> usize {\n            0\n        }\n    }\n\n    /// Trait alias for a full-fledged console.\n    pub trait All: Write + Read + Statistics {}\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic CUR_CONSOLE: InitStateLock<&'static (dyn interface::All + Sync)> =\n    InitStateLock::new(&null_console::NULL_CONSOLE);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::{interface::ReadWriteEx, InitStateLock};\n\n/// Register a new console.\npub fn register_console(new_console: &'static (dyn interface::All + Sync)) {\n    CUR_CONSOLE.write(|con| *con = new_console);\n}\n\n/// Return a reference to the currently registered console.\n///\n/// This is the global console used by all printing macros.\npub fn console() -> &'static dyn interface::All {\n    CUR_CONSOLE.read(|con| *con)\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Boot code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/cpu/boot.rs\"]\nmod arch_boot;\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/cpu/smp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Symmetric multiprocessing.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/cpu/smp.rs\"]\nmod arch_smp;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_smp::core_id;\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Processor code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/cpu.rs\"]\nmod arch_cpu;\n\nmod boot;\n\npub mod smp;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_cpu::{nop, wait_forever};\n\n#[cfg(feature = \"test_build\")]\npub use arch_cpu::{qemu_exit_failure, qemu_exit_success};\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Driver support.\n\nuse crate::{\n    exception, info,\n    synchronization::{interface::ReadWriteEx, InitStateLock},\n};\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nconst NUM_DRIVERS: usize = 5;\n\nstruct DriverManagerInner<T>\nwhere\n    T: 'static,\n{\n    next_index: usize,\n    descriptors: [Option<DeviceDriverDescriptor<T>>; NUM_DRIVERS],\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Driver interfaces.\npub mod interface {\n    /// Device Driver functions.\n    pub trait DeviceDriver {\n        /// Different interrupt controllers might use different types for IRQ number.\n        type IRQNumberType: super::fmt::Display;\n\n        /// Return a compatibility string for identifying the driver.\n        fn compatible(&self) -> &'static str;\n\n        /// Called by the kernel to bring up the device.\n        ///\n        /// # Safety\n        ///\n        /// - During init, drivers might do stuff with system-wide impact.\n        unsafe fn init(&self) -> Result<(), &'static str> {\n            Ok(())\n        }\n\n        /// Called by the kernel to register and enable the device's IRQ handler.\n        ///\n        /// Rust's type system will prevent a call to this function unless the calling instance\n        /// itself has static lifetime.\n        fn register_and_enable_irq_handler(\n            &'static self,\n            irq_number: &Self::IRQNumberType,\n        ) -> Result<(), &'static str> {\n            panic!(\n                \"Attempt to enable IRQ {} for device {}, but driver does not support this\",\n                irq_number,\n                self.compatible()\n            )\n        }\n    }\n}\n\n/// Tpye to be used as an optional callback after a driver's init() has run.\npub type DeviceDriverPostInitCallback = unsafe fn() -> Result<(), &'static str>;\n\n/// A descriptor for device drivers.\n#[derive(Copy, Clone)]\npub struct DeviceDriverDescriptor<T>\nwhere\n    T: 'static,\n{\n    device_driver: &'static (dyn interface::DeviceDriver<IRQNumberType = T> + Sync),\n    post_init_callback: Option<DeviceDriverPostInitCallback>,\n    irq_number: Option<T>,\n}\n\n/// Provides device driver management functions.\npub struct DriverManager<T>\nwhere\n    T: 'static,\n{\n    inner: InitStateLock<DriverManagerInner<T>>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic DRIVER_MANAGER: DriverManager<exception::asynchronous::IRQNumber> = DriverManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T> DriverManagerInner<T>\nwhere\n    T: 'static + Copy,\n{\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            next_index: 0,\n            descriptors: [None; NUM_DRIVERS],\n        }\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T> DeviceDriverDescriptor<T> {\n    /// Create an instance.\n    pub fn new(\n        device_driver: &'static (dyn interface::DeviceDriver<IRQNumberType = T> + Sync),\n        post_init_callback: Option<DeviceDriverPostInitCallback>,\n        irq_number: Option<T>,\n    ) -> Self {\n        Self {\n            device_driver,\n            post_init_callback,\n            irq_number,\n        }\n    }\n}\n\n/// Return a reference to the global DriverManager.\npub fn driver_manager() -> &'static DriverManager<exception::asynchronous::IRQNumber> {\n    &DRIVER_MANAGER\n}\n\nimpl<T> DriverManager<T>\nwhere\n    T: fmt::Display + Copy,\n{\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            inner: InitStateLock::new(DriverManagerInner::new()),\n        }\n    }\n\n    /// Register a device driver with the kernel.\n    pub fn register_driver(&self, descriptor: DeviceDriverDescriptor<T>) {\n        self.inner.write(|inner| {\n            inner.descriptors[inner.next_index] = Some(descriptor);\n            inner.next_index += 1;\n        })\n    }\n\n    /// Helper for iterating over registered drivers.\n    fn for_each_descriptor<'a>(&'a self, f: impl FnMut(&'a DeviceDriverDescriptor<T>)) {\n        self.inner.read(|inner| {\n            inner\n                .descriptors\n                .iter()\n                .filter_map(|x| x.as_ref())\n                .for_each(f)\n        })\n    }\n\n    /// Fully initialize all drivers and their interrupts handlers.\n    ///\n    /// # Safety\n    ///\n    /// - During init, drivers might do stuff with system-wide impact.\n    pub unsafe fn init_drivers_and_irqs(&self) {\n        self.for_each_descriptor(|descriptor| {\n            // 1. Initialize driver.\n            if let Err(x) = descriptor.device_driver.init() {\n                panic!(\n                    \"Error initializing driver: {}: {}\",\n                    descriptor.device_driver.compatible(),\n                    x\n                );\n            }\n\n            // 2. Call corresponding post init callback.\n            if let Some(callback) = &descriptor.post_init_callback {\n                if let Err(x) = callback() {\n                    panic!(\n                        \"Error during driver post-init callback: {}: {}\",\n                        descriptor.device_driver.compatible(),\n                        x\n                    );\n                }\n            }\n        });\n\n        // 3. After all post-init callbacks were done, the interrupt controller should be\n        //    registered and functional. So let drivers register with it now.\n        self.for_each_descriptor(|descriptor| {\n            if let Some(irq_number) = &descriptor.irq_number {\n                if let Err(x) = descriptor\n                    .device_driver\n                    .register_and_enable_irq_handler(irq_number)\n                {\n                    panic!(\n                        \"Error during driver interrupt handler registration: {}: {}\",\n                        descriptor.device_driver.compatible(),\n                        x\n                    );\n                }\n            }\n        });\n    }\n\n    /// Enumerate all registered device drivers.\n    pub fn enumerate(&self) {\n        let mut i: usize = 1;\n        self.for_each_descriptor(|descriptor| {\n            info!(\"      {}. {}\", i, descriptor.device_driver.compatible());\n\n            i += 1;\n        });\n    }\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/exception/asynchronous/null_irq_manager.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Null IRQ Manager.\n\nuse super::{interface, IRQContext, IRQHandlerDescriptor};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct NullIRQManager;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\npub static NULL_IRQ_MANAGER: NullIRQManager = NullIRQManager {};\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl interface::IRQManager for NullIRQManager {\n    type IRQNumberType = super::IRQNumber;\n\n    fn register_handler(\n        &self,\n        _descriptor: IRQHandlerDescriptor<Self::IRQNumberType>,\n    ) -> Result<(), &'static str> {\n        panic!(\"No IRQ Manager registered yet\");\n    }\n\n    fn enable(&self, _irq_number: &Self::IRQNumberType) {\n        panic!(\"No IRQ Manager registered yet\");\n    }\n\n    fn handle_pending_irqs<'irq_context>(&'irq_context self, _ic: &IRQContext<'irq_context>) {\n        panic!(\"No IRQ Manager registered yet\");\n    }\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/exception/asynchronous.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Asynchronous exception handling.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/exception/asynchronous.rs\"]\nmod arch_asynchronous;\nmod null_irq_manager;\n\nuse crate::{bsp, synchronization};\nuse core::marker::PhantomData;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_asynchronous::{\n    is_local_irq_masked, local_irq_mask, local_irq_mask_save, local_irq_restore, local_irq_unmask,\n    print_state,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Interrupt number as defined by the BSP.\npub type IRQNumber = bsp::exception::asynchronous::IRQNumber;\n\n/// Interrupt descriptor.\n#[derive(Copy, Clone)]\npub struct IRQHandlerDescriptor<T>\nwhere\n    T: Copy,\n{\n    /// The IRQ number.\n    number: T,\n\n    /// Descriptive name.\n    name: &'static str,\n\n    /// Reference to handler trait object.\n    handler: &'static (dyn interface::IRQHandler + Sync),\n}\n\n/// IRQContext token.\n///\n/// An instance of this type indicates that the local core is currently executing in IRQ\n/// context, aka executing an interrupt vector or subcalls of it.\n///\n/// Concept and implementation derived from the `CriticalSection` introduced in\n/// <https://github.com/rust-embedded/bare-metal>\n#[derive(Clone, Copy)]\npub struct IRQContext<'irq_context> {\n    _0: PhantomData<&'irq_context ()>,\n}\n\n/// Asynchronous exception handling interfaces.\npub mod interface {\n\n    /// Implemented by types that handle IRQs.\n    pub trait IRQHandler {\n        /// Called when the corresponding interrupt is asserted.\n        fn handle(&self) -> Result<(), &'static str>;\n    }\n\n    /// IRQ management functions.\n    ///\n    /// The `BSP` is supposed to supply one global instance. Typically implemented by the\n    /// platform's interrupt controller.\n    pub trait IRQManager {\n        /// The IRQ number type depends on the implementation.\n        type IRQNumberType: Copy;\n\n        /// Register a handler.\n        fn register_handler(\n            &self,\n            irq_handler_descriptor: super::IRQHandlerDescriptor<Self::IRQNumberType>,\n        ) -> Result<(), &'static str>;\n\n        /// Enable an interrupt in the controller.\n        fn enable(&self, irq_number: &Self::IRQNumberType);\n\n        /// Handle pending interrupts.\n        ///\n        /// This function is called directly from the CPU's IRQ exception vector. On AArch64,\n        /// this means that the respective CPU core has disabled exception handling.\n        /// This function can therefore not be preempted and runs start to finish.\n        ///\n        /// Takes an IRQContext token to ensure it can only be called from IRQ context.\n        #[allow(clippy::trivially_copy_pass_by_ref)]\n        fn handle_pending_irqs<'irq_context>(\n            &'irq_context self,\n            ic: &super::IRQContext<'irq_context>,\n        );\n\n        /// Print list of registered handlers.\n        fn print_handler(&self) {}\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic CUR_IRQ_MANAGER: InitStateLock<\n    &'static (dyn interface::IRQManager<IRQNumberType = IRQNumber> + Sync),\n> = InitStateLock::new(&null_irq_manager::NULL_IRQ_MANAGER);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::{interface::ReadWriteEx, InitStateLock};\n\nimpl<T> IRQHandlerDescriptor<T>\nwhere\n    T: Copy,\n{\n    /// Create an instance.\n    pub const fn new(\n        number: T,\n        name: &'static str,\n        handler: &'static (dyn interface::IRQHandler + Sync),\n    ) -> Self {\n        Self {\n            number,\n            name,\n            handler,\n        }\n    }\n\n    /// Return the number.\n    pub const fn number(&self) -> T {\n        self.number\n    }\n\n    /// Return the name.\n    pub const fn name(&self) -> &'static str {\n        self.name\n    }\n\n    /// Return the handler.\n    pub const fn handler(&self) -> &'static (dyn interface::IRQHandler + Sync) {\n        self.handler\n    }\n}\n\nimpl<'irq_context> IRQContext<'irq_context> {\n    /// Creates an IRQContext token.\n    ///\n    /// # Safety\n    ///\n    /// - This must only be called when the current core is in an interrupt context and will not\n    ///   live beyond the end of it. That is, creation is allowed in interrupt vector functions. For\n    ///   example, in the ARMv8-A case, in `extern \"C\" fn current_elx_irq()`.\n    /// - Note that the lifetime `'irq_context` of the returned instance is unconstrained. User code\n    ///   must not be able to influence the lifetime picked for this type, since that might cause it\n    ///   to be inferred to `'static`.\n    #[inline(always)]\n    pub unsafe fn new() -> Self {\n        IRQContext { _0: PhantomData }\n    }\n}\n\n/// Executes the provided closure while IRQs are masked on the executing core.\n///\n/// While the function temporarily changes the HW state of the executing core, it restores it to the\n/// previous state before returning, so this is deemed safe.\n#[inline(always)]\npub fn exec_with_irq_masked<T>(f: impl FnOnce() -> T) -> T {\n    let saved = local_irq_mask_save();\n    let ret = f();\n    local_irq_restore(saved);\n\n    ret\n}\n\n/// Register a new IRQ manager.\npub fn register_irq_manager(\n    new_manager: &'static (dyn interface::IRQManager<IRQNumberType = IRQNumber> + Sync),\n) {\n    CUR_IRQ_MANAGER.write(|manager| *manager = new_manager);\n}\n\n/// Return a reference to the currently registered IRQ manager.\n///\n/// This is the IRQ manager used by the architectural interrupt handling code.\npub fn irq_manager() -> &'static dyn interface::IRQManager<IRQNumberType = IRQNumber> {\n    CUR_IRQ_MANAGER.read(|manager| *manager)\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/exception.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Synchronous and asynchronous exception handling.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/exception.rs\"]\nmod arch_exception;\n\npub mod asynchronous;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_exception::{current_privilege_level, handling_init};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Kernel privilege levels.\n#[allow(missing_docs)]\n#[derive(Eq, PartialEq)]\npub enum PrivilegeLevel {\n    User,\n    Kernel,\n    Hypervisor,\n    Unknown,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// Libkernel unit tests must execute in kernel mode.\n    #[kernel_test]\n    fn test_runner_executes_in_kernel_mode() {\n        let (level, _) = current_privilege_level();\n\n        assert!(level == PrivilegeLevel::Kernel)\n    }\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/lib.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n// Rust embedded logo for `make doc`.\n#![doc(\n    html_logo_url = \"https://raw.githubusercontent.com/rust-embedded/wg/master/assets/logo/ewg-logo-blue-white-on-transparent.png\"\n)]\n\n//! The `kernel` library.\n//!\n//! Used to compose the final kernel binary.\n//!\n//! # Code organization and architecture\n//!\n//! The code is divided into different *modules*, each representing a typical **subsystem** of the\n//! `kernel`. Top-level module files of subsystems reside directly in the `src` folder. For example,\n//! `src/memory.rs` contains code that is concerned with all things memory management.\n//!\n//! ## Visibility of processor architecture code\n//!\n//! Some of the `kernel`'s subsystems depend on low-level code that is specific to the target\n//! processor architecture. For each supported processor architecture, there exists a subfolder in\n//! `src/_arch`, for example, `src/_arch/aarch64`.\n//!\n//! The architecture folders mirror the subsystem modules laid out in `src`. For example,\n//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go\n//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in\n//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic\n//! module's name prefixed with `arch_`.\n//!\n//! For example, this is the top of `src/memory/mmu.rs`:\n//!\n//! ```\n//! #[cfg(target_arch = \"aarch64\")]\n//! #[path = \"../_arch/aarch64/memory/mmu.rs\"]\n//! mod arch_mmu;\n//! ```\n//!\n//! Often times, items from the `arch_ module` will be publicly reexported by the parent module.\n//! This way, each architecture specific module can provide its implementation of an item, while the\n//! caller must not be concerned which architecture has been conditionally compiled.\n//!\n//! ## BSP code\n//!\n//! `BSP` stands for Board Support Package. `BSP` code is organized under `src/bsp.rs` and contains\n//! target board specific definitions and functions. These are things such as the board's memory map\n//! or instances of drivers for devices that are featured on the respective board.\n//!\n//! Just like processor architecture code, the `BSP` code's module structure tries to mirror the\n//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is\n//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`.\n//!\n//! ## Kernel interfaces\n//!\n//! Both `arch` and `bsp` contain code that is conditionally compiled depending on the actual target\n//! and board for which the kernel is compiled. For example, the `interrupt controller` hardware of\n//! the `Raspberry Pi 3` and the `Raspberry Pi 4` is different, but we want the rest of the `kernel`\n//! code to play nicely with any of the two without much hassle.\n//!\n//! In order to provide a clean abstraction between `arch`, `bsp` and `generic kernel code`,\n//! `interface` traits are provided *whenever possible* and *where it makes sense*. They are defined\n//! in the respective subsystem module and help to enforce the idiom of *program to an interface,\n//! not an implementation*. For example, there will be a common IRQ handling interface which the two\n//! different interrupt controller `drivers` of both Raspberrys will implement, and only export the\n//! interface to the rest of the `kernel`.\n//!\n//! ```\n//!         +-------------------+\n//!         | Interface (Trait) |\n//!         |                   |\n//!         +--+-------------+--+\n//!            ^             ^\n//!            |             |\n//!            |             |\n//! +----------+--+       +--+----------+\n//! | kernel code |       |  bsp code   |\n//! |             |       |  arch code  |\n//! +-------------+       +-------------+\n//! ```\n//!\n//! # Summary\n//!\n//! For a logical `kernel` subsystem, corresponding code can be distributed over several physical\n//! locations. Here is an example for the **memory** subsystem:\n//!\n//! - `src/memory.rs` and `src/memory/**/*`\n//!   - Common code that is agnostic of target processor architecture and `BSP` characteristics.\n//!     - Example: A function to zero a chunk of memory.\n//!   - Interfaces for the memory subsystem that are implemented by `arch` or `BSP` code.\n//!     - Example: An `MMU` interface that defines `MMU` function prototypes.\n//! - `src/bsp/__board_name__/memory.rs` and `src/bsp/__board_name__/memory/**/*`\n//!   - `BSP` specific code.\n//!   - Example: The board's memory map (physical addresses of DRAM and MMIO devices).\n//! - `src/_arch/__arch_name__/memory.rs` and `src/_arch/__arch_name__/memory/**/*`\n//!   - Processor architecture specific code.\n//!   - Example: Implementation of the `MMU` interface for the `__arch_name__` processor\n//!     architecture.\n//!\n//! From a namespace perspective, **memory** subsystem code lives in:\n//!\n//! - `crate::memory::*`\n//! - `crate::bsp::memory::*`\n//!\n//! # Boot flow\n//!\n//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.\n//!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.\n//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.\n\n#![allow(clippy::upper_case_acronyms)]\n#![allow(incomplete_features)]\n#![feature(asm_const)]\n#![feature(const_option)]\n#![feature(core_intrinsics)]\n#![feature(format_args_nl)]\n#![feature(generic_const_exprs)]\n#![feature(int_roundings)]\n#![feature(is_sorted)]\n#![feature(linkage)]\n#![feature(nonzero_min_max)]\n#![feature(panic_info_message)]\n#![feature(step_trait)]\n#![feature(trait_alias)]\n#![feature(unchecked_math)]\n#![no_std]\n// Testing\n#![cfg_attr(test, no_main)]\n#![feature(custom_test_frameworks)]\n#![reexport_test_harness_main = \"test_main\"]\n#![test_runner(crate::test_runner)]\n\nmod panic_wait;\nmod synchronization;\n\npub mod bsp;\npub mod common;\npub mod console;\npub mod cpu;\npub mod driver;\npub mod exception;\npub mod memory;\npub mod print;\npub mod state;\npub mod symbols;\npub mod time;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Version string.\npub fn version() -> &'static str {\n    concat!(\n        env!(\"CARGO_PKG_NAME\"),\n        \" version \",\n        env!(\"CARGO_PKG_VERSION\")\n    )\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n/// The default runner for unit tests.\npub fn test_runner(tests: &[&test_types::UnitTest]) {\n    // This line will be printed as the test header.\n    println!(\"Running {} tests\", tests.len());\n\n    for (i, test) in tests.iter().enumerate() {\n        print!(\"{:>3}. {:.<58}\", i + 1, test.name);\n\n        // Run the actual test.\n        (test.test_func)();\n\n        // Failed tests call panic!(). Execution reaches here only if the test has passed.\n        println!(\"[ok]\")\n    }\n}\n\n/// The `kernel_init()` for unit tests.\n#[cfg(test)]\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    test_main();\n\n    cpu::qemu_exit_success()\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/main.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n// Rust embedded logo for `make doc`.\n#![doc(\n    html_logo_url = \"https://raw.githubusercontent.com/rust-embedded/wg/master/assets/logo/ewg-logo-blue-white-on-transparent.png\"\n)]\n\n//! The `kernel` binary.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\nuse libkernel::{bsp, cpu, driver, exception, info, memory, state, time};\n\n/// Early init code.\n///\n/// When this code runs, virtual memory is already enabled.\n///\n/// # Safety\n///\n/// - Only a single core must be active and running this function.\n/// - Printing will not work until the respective driver's MMIO is remapped.\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n\n    // Initialize the BSP driver subsystem.\n    if let Err(x) = bsp::driver::init() {\n        panic!(\"Error initializing BSP driver subsystem: {}\", x);\n    }\n\n    // Initialize all device drivers.\n    driver::driver_manager().init_drivers_and_irqs();\n\n    bsp::memory::mmu::kernel_add_mapping_records_for_precomputed();\n\n    // Unmask interrupts on the boot CPU core.\n    exception::asynchronous::local_irq_unmask();\n\n    // Announce conclusion of the kernel_init() phase.\n    state::state_manager().transition_to_single_core_main();\n\n    // Transition from unsafe to safe.\n    kernel_main()\n}\n\n/// The main function running after the early init.\nfn kernel_main() -> ! {\n    info!(\"{}\", libkernel::version());\n    info!(\"Booting on: {}\", bsp::board_name());\n\n    info!(\"MMU online:\");\n    memory::mmu::kernel_print_mappings();\n\n    let (_, privilege_level) = exception::current_privilege_level();\n    info!(\"Current privilege level: {}\", privilege_level);\n\n    info!(\"Exception handling state:\");\n    exception::asynchronous::print_state();\n\n    info!(\n        \"Architectural timer resolution: {} ns\",\n        time::time_manager().resolution().as_nanos()\n    );\n\n    info!(\"Drivers loaded:\");\n    driver::driver_manager().enumerate();\n\n    info!(\"Registered IRQ handlers:\");\n    exception::asynchronous::irq_manager().print_handler();\n\n    info!(\"Echoing input now\");\n    cpu::wait_forever();\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/memory/mmu/mapping_record.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A record of mapped pages.\n\nuse super::{\n    AccessPermissions, Address, AttributeFields, MMIODescriptor, MemAttributes, MemoryRegion,\n    Physical, Virtual,\n};\nuse crate::{bsp, common, info, synchronization, synchronization::InitStateLock, warn};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Type describing a virtual memory mapping.\n#[allow(missing_docs)]\n#[derive(Copy, Clone)]\nstruct MappingRecordEntry {\n    pub users: [Option<&'static str>; 5],\n    pub phys_start_addr: Address<Physical>,\n    pub virt_start_addr: Address<Virtual>,\n    pub num_pages: usize,\n    pub attribute_fields: AttributeFields,\n}\n\nstruct MappingRecord {\n    inner: [Option<MappingRecordEntry>; 12],\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic KERNEL_MAPPING_RECORD: InitStateLock<MappingRecord> =\n    InitStateLock::new(MappingRecord::new());\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl MappingRecordEntry {\n    pub fn new(\n        name: &'static str,\n        virt_region: &MemoryRegion<Virtual>,\n        phys_region: &MemoryRegion<Physical>,\n        attr: &AttributeFields,\n    ) -> Self {\n        Self {\n            users: [Some(name), None, None, None, None],\n            phys_start_addr: phys_region.start_addr(),\n            virt_start_addr: virt_region.start_addr(),\n            num_pages: phys_region.num_pages(),\n            attribute_fields: *attr,\n        }\n    }\n\n    fn find_next_free_user(&mut self) -> Result<&mut Option<&'static str>, &'static str> {\n        if let Some(x) = self.users.iter_mut().find(|x| x.is_none()) {\n            return Ok(x);\n        };\n\n        Err(\"Storage for user info exhausted\")\n    }\n\n    pub fn add_user(&mut self, user: &'static str) -> Result<(), &'static str> {\n        let x = self.find_next_free_user()?;\n        *x = Some(user);\n        Ok(())\n    }\n}\n\nimpl MappingRecord {\n    pub const fn new() -> Self {\n        Self { inner: [None; 12] }\n    }\n\n    fn size(&self) -> usize {\n        self.inner.iter().filter(|x| x.is_some()).count()\n    }\n\n    fn sort(&mut self) {\n        let upper_bound_exclusive = self.size();\n        let entries = &mut self.inner[0..upper_bound_exclusive];\n\n        if !entries.is_sorted_by_key(|item| item.unwrap().virt_start_addr) {\n            entries.sort_unstable_by_key(|item| item.unwrap().virt_start_addr)\n        }\n    }\n\n    fn find_next_free(&mut self) -> Result<&mut Option<MappingRecordEntry>, &'static str> {\n        if let Some(x) = self.inner.iter_mut().find(|x| x.is_none()) {\n            return Ok(x);\n        }\n\n        Err(\"Storage for mapping info exhausted\")\n    }\n\n    fn find_duplicate(\n        &mut self,\n        phys_region: &MemoryRegion<Physical>,\n    ) -> Option<&mut MappingRecordEntry> {\n        self.inner\n            .iter_mut()\n            .filter_map(|x| x.as_mut())\n            .filter(|x| x.attribute_fields.mem_attributes == MemAttributes::Device)\n            .find(|x| {\n                if x.phys_start_addr != phys_region.start_addr() {\n                    return false;\n                }\n\n                if x.num_pages != phys_region.num_pages() {\n                    return false;\n                }\n\n                true\n            })\n    }\n\n    pub fn add(\n        &mut self,\n        name: &'static str,\n        virt_region: &MemoryRegion<Virtual>,\n        phys_region: &MemoryRegion<Physical>,\n        attr: &AttributeFields,\n    ) -> Result<(), &'static str> {\n        let x = self.find_next_free()?;\n\n        *x = Some(MappingRecordEntry::new(\n            name,\n            virt_region,\n            phys_region,\n            attr,\n        ));\n\n        self.sort();\n\n        Ok(())\n    }\n\n    pub fn print(&self) {\n        info!(\"      -------------------------------------------------------------------------------------------------------------------------------------------\");\n        info!(\n            \"      {:^44}     {:^30}   {:^7}   {:^9}   {:^35}\",\n            \"Virtual\", \"Physical\", \"Size\", \"Attr\", \"Entity\"\n        );\n        info!(\"      -------------------------------------------------------------------------------------------------------------------------------------------\");\n\n        for i in self.inner.iter().flatten() {\n            let size = i.num_pages * bsp::memory::mmu::KernelGranule::SIZE;\n            let virt_start = i.virt_start_addr;\n            let virt_end_inclusive = virt_start + (size - 1);\n            let phys_start = i.phys_start_addr;\n            let phys_end_inclusive = phys_start + (size - 1);\n\n            let (size, unit) = common::size_human_readable_ceil(size);\n\n            let attr = match i.attribute_fields.mem_attributes {\n                MemAttributes::CacheableDRAM => \"C\",\n                MemAttributes::Device => \"Dev\",\n            };\n\n            let acc_p = match i.attribute_fields.acc_perms {\n                AccessPermissions::ReadOnly => \"RO\",\n                AccessPermissions::ReadWrite => \"RW\",\n            };\n\n            let xn = if i.attribute_fields.execute_never {\n                \"XN\"\n            } else {\n                \"X\"\n            };\n\n            info!(\n                \"      {}..{} --> {}..{} | {:>3} {} | {:<3} {} {:<2} | {}\",\n                virt_start,\n                virt_end_inclusive,\n                phys_start,\n                phys_end_inclusive,\n                size,\n                unit,\n                attr,\n                acc_p,\n                xn,\n                i.users[0].unwrap()\n            );\n\n            for k in i.users[1..].iter() {\n                if let Some(additional_user) = *k {\n                    info!(\n                        \"                                                                                                            | {}\",\n                        additional_user\n                    );\n                }\n            }\n        }\n\n        info!(\"      -------------------------------------------------------------------------------------------------------------------------------------------\");\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::interface::ReadWriteEx;\n\n/// Add an entry to the mapping info record.\npub fn kernel_add(\n    name: &'static str,\n    virt_region: &MemoryRegion<Virtual>,\n    phys_region: &MemoryRegion<Physical>,\n    attr: &AttributeFields,\n) -> Result<(), &'static str> {\n    KERNEL_MAPPING_RECORD.write(|mr| mr.add(name, virt_region, phys_region, attr))\n}\n\npub fn kernel_find_and_insert_mmio_duplicate(\n    mmio_descriptor: &MMIODescriptor,\n    new_user: &'static str,\n) -> Option<Address<Virtual>> {\n    let phys_region: MemoryRegion<Physical> = (*mmio_descriptor).into();\n\n    KERNEL_MAPPING_RECORD.write(|mr| {\n        let dup = mr.find_duplicate(&phys_region)?;\n\n        if let Err(x) = dup.add_user(new_user) {\n            warn!(\"{}\", x);\n        }\n\n        Some(dup.virt_start_addr)\n    })\n}\n\n/// Human-readable print of all recorded kernel mappings.\npub fn kernel_print() {\n    KERNEL_MAPPING_RECORD.read(|mr| mr.print());\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/memory/mmu/page_alloc.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Page allocation.\n\nuse super::MemoryRegion;\nuse crate::{\n    memory::{AddressType, Virtual},\n    synchronization::IRQSafeNullLock,\n    warn,\n};\nuse core::num::NonZeroUsize;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// A page allocator that can be lazyily initialized.\npub struct PageAllocator<ATYPE: AddressType> {\n    pool: Option<MemoryRegion<ATYPE>>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic KERNEL_MMIO_VA_ALLOCATOR: IRQSafeNullLock<PageAllocator<Virtual>> =\n    IRQSafeNullLock::new(PageAllocator::new());\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the kernel's MMIO virtual address allocator.\npub fn kernel_mmio_va_allocator() -> &'static IRQSafeNullLock<PageAllocator<Virtual>> {\n    &KERNEL_MMIO_VA_ALLOCATOR\n}\n\nimpl<ATYPE: AddressType> PageAllocator<ATYPE> {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self { pool: None }\n    }\n\n    /// Initialize the allocator.\n    pub fn init(&mut self, pool: MemoryRegion<ATYPE>) {\n        if self.pool.is_some() {\n            warn!(\"Already initialized\");\n            return;\n        }\n\n        self.pool = Some(pool);\n    }\n\n    /// Allocate a number of pages.\n    pub fn alloc(\n        &mut self,\n        num_requested_pages: NonZeroUsize,\n    ) -> Result<MemoryRegion<ATYPE>, &'static str> {\n        if self.pool.is_none() {\n            return Err(\"Allocator not initialized\");\n        }\n\n        self.pool\n            .as_mut()\n            .unwrap()\n            .take_first_n_pages(num_requested_pages)\n    }\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/memory/mmu/translation_table.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Translation table.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../../_arch/aarch64/memory/mmu/translation_table.rs\"]\nmod arch_translation_table;\n\nuse super::{AttributeFields, MemoryRegion};\nuse crate::memory::{Address, Physical, Virtual};\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\n#[cfg(target_arch = \"aarch64\")]\npub use arch_translation_table::FixedSizeTranslationTable;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Translation table interfaces.\npub mod interface {\n    use crate::memory::mmu::PageAddress;\n\n    use super::*;\n\n    /// Translation table operations.\n    pub trait TranslationTable {\n        /// Anything that needs to run before any of the other provided functions can be used.\n        ///\n        /// # Safety\n        ///\n        /// - Implementor must ensure that this function can run only once or is harmless if invoked\n        ///   multiple times.\n        fn init(&mut self) -> Result<(), &'static str>;\n\n        /// Map the given virtual memory region to the given physical memory region.\n        ///\n        /// # Safety\n        ///\n        /// - Using wrong attributes can cause multiple issues of different nature in the system.\n        /// - It is not required that the architectural implementation prevents aliasing. That is,\n        ///   mapping to the same physical memory using multiple virtual addresses, which would\n        ///   break Rust's ownership assumptions. This should be protected against in the kernel's\n        ///   generic MMU code.\n        unsafe fn map_at(\n            &mut self,\n            virt_region: &MemoryRegion<Virtual>,\n            phys_region: &MemoryRegion<Physical>,\n            attr: &AttributeFields,\n        ) -> Result<(), &'static str>;\n\n        /// Try to translate a virtual page address to a physical page address.\n        ///\n        /// Will only succeed if there exists a valid mapping for the input page.\n        fn try_virt_page_addr_to_phys_page_addr(\n            &self,\n            virt_page_addr: PageAddress<Virtual>,\n        ) -> Result<PageAddress<Physical>, &'static str>;\n\n        /// Try to get the attributes of a page.\n        ///\n        /// Will only succeed if there exists a valid mapping for the input page.\n        fn try_page_attributes(\n            &self,\n            virt_page_addr: PageAddress<Virtual>,\n        ) -> Result<AttributeFields, &'static str>;\n\n        /// Try to translate a virtual address to a physical address.\n        ///\n        /// Will only succeed if there exists a valid mapping for the input address.\n        fn try_virt_addr_to_phys_addr(\n            &self,\n            virt_addr: Address<Virtual>,\n        ) -> Result<Address<Physical>, &'static str>;\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use crate::memory::mmu::{AccessPermissions, MemAttributes, PageAddress};\n    use arch_translation_table::MinSizeTranslationTable;\n    use interface::TranslationTable;\n    use test_macros::kernel_test;\n\n    /// Sanity checks for the TranslationTable implementation.\n    #[kernel_test]\n    fn translationtable_implementation_sanity() {\n        // This will occupy a lot of space on the stack.\n        let mut tables = MinSizeTranslationTable::new_for_runtime();\n\n        assert_eq!(tables.init(), Ok(()));\n\n        let virt_end_exclusive_page_addr: PageAddress<Virtual> = PageAddress::MAX;\n        let virt_start_page_addr: PageAddress<Virtual> =\n            virt_end_exclusive_page_addr.checked_offset(-5).unwrap();\n\n        let phys_start_page_addr: PageAddress<Physical> = PageAddress::from(0);\n        let phys_end_exclusive_page_addr: PageAddress<Physical> =\n            phys_start_page_addr.checked_offset(5).unwrap();\n\n        let virt_region = MemoryRegion::new(virt_start_page_addr, virt_end_exclusive_page_addr);\n        let phys_region = MemoryRegion::new(phys_start_page_addr, phys_end_exclusive_page_addr);\n\n        let attr = AttributeFields {\n            mem_attributes: MemAttributes::CacheableDRAM,\n            acc_perms: AccessPermissions::ReadWrite,\n            execute_never: true,\n        };\n\n        unsafe { assert_eq!(tables.map_at(&virt_region, &phys_region, &attr), Ok(())) };\n\n        assert_eq!(\n            tables.try_virt_page_addr_to_phys_page_addr(virt_start_page_addr),\n            Ok(phys_start_page_addr)\n        );\n\n        assert_eq!(\n            tables.try_page_attributes(virt_start_page_addr.checked_offset(-1).unwrap()),\n            Err(\"Page marked invalid\")\n        );\n\n        assert_eq!(tables.try_page_attributes(virt_start_page_addr), Ok(attr));\n\n        let virt_addr = virt_start_page_addr.into_inner() + 0x100;\n        let phys_addr = phys_start_page_addr.into_inner() + 0x100;\n        assert_eq!(tables.try_virt_addr_to_phys_addr(virt_addr), Ok(phys_addr));\n    }\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/memory/mmu/types.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management Unit types.\n\nuse crate::{\n    bsp, common,\n    memory::{Address, AddressType, Physical},\n};\nuse core::{convert::From, iter::Step, num::NonZeroUsize, ops::Range};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// A wrapper type around [Address] that ensures page alignment.\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub struct PageAddress<ATYPE: AddressType> {\n    inner: Address<ATYPE>,\n}\n\n/// A type that describes a region of memory in quantities of pages.\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub struct MemoryRegion<ATYPE: AddressType> {\n    start: PageAddress<ATYPE>,\n    end_exclusive: PageAddress<ATYPE>,\n}\n\n/// Architecture agnostic memory attributes.\n#[allow(missing_docs)]\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub enum MemAttributes {\n    CacheableDRAM,\n    Device,\n}\n\n/// Architecture agnostic access permissions.\n#[allow(missing_docs)]\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub enum AccessPermissions {\n    ReadOnly,\n    ReadWrite,\n}\n\n/// Collection of memory attributes.\n#[allow(missing_docs)]\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub struct AttributeFields {\n    pub mem_attributes: MemAttributes,\n    pub acc_perms: AccessPermissions,\n    pub execute_never: bool,\n}\n\n/// An MMIO descriptor for use in device drivers.\n#[derive(Copy, Clone)]\npub struct MMIODescriptor {\n    start_addr: Address<Physical>,\n    end_addr_exclusive: Address<Physical>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n//------------------------------------------------------------------------------\n// PageAddress\n//------------------------------------------------------------------------------\nimpl<ATYPE: AddressType> PageAddress<ATYPE> {\n    /// The largest value that can be represented by this type.\n    pub const MAX: Self = PageAddress {\n        inner: Address::new(usize::MAX).align_down_page(),\n    };\n\n    /// Unwraps the value.\n    pub fn into_inner(self) -> Address<ATYPE> {\n        self.inner\n    }\n\n    /// Calculates the offset from the page address.\n    ///\n    /// `count` is in units of [PageAddress]. For example, a count of 2 means `result = self + 2 *\n    /// page_size`.\n    pub fn checked_offset(self, count: isize) -> Option<Self> {\n        if count == 0 {\n            return Some(self);\n        }\n\n        let delta = count\n            .unsigned_abs()\n            .checked_mul(bsp::memory::mmu::KernelGranule::SIZE)?;\n        let result = if count.is_positive() {\n            self.inner.as_usize().checked_add(delta)?\n        } else {\n            self.inner.as_usize().checked_sub(delta)?\n        };\n\n        Some(Self {\n            inner: Address::new(result),\n        })\n    }\n}\n\nimpl<ATYPE: AddressType> From<usize> for PageAddress<ATYPE> {\n    fn from(addr: usize) -> Self {\n        assert!(\n            common::is_aligned(addr, bsp::memory::mmu::KernelGranule::SIZE),\n            \"Input usize not page aligned\"\n        );\n\n        Self {\n            inner: Address::new(addr),\n        }\n    }\n}\n\nimpl<ATYPE: AddressType> From<Address<ATYPE>> for PageAddress<ATYPE> {\n    fn from(addr: Address<ATYPE>) -> Self {\n        assert!(addr.is_page_aligned(), \"Input Address not page aligned\");\n\n        Self { inner: addr }\n    }\n}\n\nimpl<ATYPE: AddressType> Step for PageAddress<ATYPE> {\n    fn steps_between(start: &Self, end: &Self) -> Option<usize> {\n        if start > end {\n            return None;\n        }\n\n        // Since start <= end, do unchecked arithmetic.\n        Some(\n            (end.inner.as_usize() - start.inner.as_usize())\n                >> bsp::memory::mmu::KernelGranule::SHIFT,\n        )\n    }\n\n    fn forward_checked(start: Self, count: usize) -> Option<Self> {\n        start.checked_offset(count as isize)\n    }\n\n    fn backward_checked(start: Self, count: usize) -> Option<Self> {\n        start.checked_offset(-(count as isize))\n    }\n}\n\n//------------------------------------------------------------------------------\n// MemoryRegion\n//------------------------------------------------------------------------------\nimpl<ATYPE: AddressType> MemoryRegion<ATYPE> {\n    /// Create an instance.\n    pub fn new(start: PageAddress<ATYPE>, end_exclusive: PageAddress<ATYPE>) -> Self {\n        assert!(start <= end_exclusive);\n\n        Self {\n            start,\n            end_exclusive,\n        }\n    }\n\n    fn as_range(&self) -> Range<PageAddress<ATYPE>> {\n        self.into_iter()\n    }\n\n    /// Returns the start page address.\n    pub fn start_page_addr(&self) -> PageAddress<ATYPE> {\n        self.start\n    }\n\n    /// Returns the start address.\n    pub fn start_addr(&self) -> Address<ATYPE> {\n        self.start.into_inner()\n    }\n\n    /// Returns the exclusive end page address.\n    pub fn end_exclusive_page_addr(&self) -> PageAddress<ATYPE> {\n        self.end_exclusive\n    }\n\n    /// Returns the exclusive end page address.\n    pub fn end_inclusive_page_addr(&self) -> PageAddress<ATYPE> {\n        self.end_exclusive.checked_offset(-1).unwrap()\n    }\n\n    /// Checks if self contains an address.\n    pub fn contains(&self, addr: Address<ATYPE>) -> bool {\n        let page_addr = PageAddress::from(addr.align_down_page());\n        self.as_range().contains(&page_addr)\n    }\n\n    /// Checks if there is an overlap with another memory region.\n    pub fn overlaps(&self, other_region: &Self) -> bool {\n        let self_range = self.as_range();\n\n        self_range.contains(&other_region.start_page_addr())\n            || self_range.contains(&other_region.end_inclusive_page_addr())\n    }\n\n    /// Returns the number of pages contained in this region.\n    pub fn num_pages(&self) -> usize {\n        PageAddress::steps_between(&self.start, &self.end_exclusive).unwrap()\n    }\n\n    /// Returns the size in bytes of this region.\n    pub fn size(&self) -> usize {\n        // Invariant: start <= end_exclusive, so do unchecked arithmetic.\n        let end_exclusive = self.end_exclusive.into_inner().as_usize();\n        let start = self.start.into_inner().as_usize();\n\n        end_exclusive - start\n    }\n\n    /// Splits the MemoryRegion like:\n    ///\n    /// --------------------------------------------------------------------------------\n    /// |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |\n    /// --------------------------------------------------------------------------------\n    ///   ^                               ^                                       ^\n    ///   |                               |                                       |\n    ///   left_start     left_end_exclusive                                       |\n    ///                                                                           |\n    ///                                   ^                                       |\n    ///                                   |                                       |\n    ///                                   right_start           right_end_exclusive\n    ///\n    /// Left region is returned to the caller. Right region is the new region for this struct.\n    pub fn take_first_n_pages(&mut self, num_pages: NonZeroUsize) -> Result<Self, &'static str> {\n        let count: usize = num_pages.into();\n\n        let left_end_exclusive = self.start.checked_offset(count as isize);\n        let left_end_exclusive = match left_end_exclusive {\n            None => return Err(\"Overflow while calculating left_end_exclusive\"),\n            Some(x) => x,\n        };\n\n        if left_end_exclusive > self.end_exclusive {\n            return Err(\"Not enough free pages\");\n        }\n\n        let allocation = Self {\n            start: self.start,\n            end_exclusive: left_end_exclusive,\n        };\n        self.start = left_end_exclusive;\n\n        Ok(allocation)\n    }\n}\n\nimpl<ATYPE: AddressType> IntoIterator for MemoryRegion<ATYPE> {\n    type Item = PageAddress<ATYPE>;\n    type IntoIter = Range<Self::Item>;\n\n    fn into_iter(self) -> Self::IntoIter {\n        Range {\n            start: self.start,\n            end: self.end_exclusive,\n        }\n    }\n}\n\nimpl From<MMIODescriptor> for MemoryRegion<Physical> {\n    fn from(desc: MMIODescriptor) -> Self {\n        let start = PageAddress::from(desc.start_addr.align_down_page());\n        let end_exclusive = PageAddress::from(desc.end_addr_exclusive().align_up_page());\n\n        Self {\n            start,\n            end_exclusive,\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// MMIODescriptor\n//------------------------------------------------------------------------------\n\nimpl MMIODescriptor {\n    /// Create an instance.\n    pub const fn new(start_addr: Address<Physical>, size: usize) -> Self {\n        assert!(size > 0);\n        let end_addr_exclusive = Address::new(start_addr.as_usize() + size);\n\n        Self {\n            start_addr,\n            end_addr_exclusive,\n        }\n    }\n\n    /// Return the start address.\n    pub const fn start_addr(&self) -> Address<Physical> {\n        self.start_addr\n    }\n\n    /// Return the exclusive end address.\n    pub fn end_addr_exclusive(&self) -> Address<Physical> {\n        self.end_addr_exclusive\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use crate::memory::Virtual;\n    use test_macros::kernel_test;\n\n    /// Sanity of [PageAddress] methods.\n    #[kernel_test]\n    fn pageaddress_type_method_sanity() {\n        let page_addr: PageAddress<Virtual> =\n            PageAddress::from(bsp::memory::mmu::KernelGranule::SIZE * 2);\n\n        assert_eq!(\n            page_addr.checked_offset(-2),\n            Some(PageAddress::<Virtual>::from(0))\n        );\n\n        assert_eq!(\n            page_addr.checked_offset(2),\n            Some(PageAddress::<Virtual>::from(\n                bsp::memory::mmu::KernelGranule::SIZE * 4\n            ))\n        );\n\n        assert_eq!(\n            PageAddress::<Virtual>::from(0).checked_offset(0),\n            Some(PageAddress::<Virtual>::from(0))\n        );\n        assert_eq!(PageAddress::<Virtual>::from(0).checked_offset(-1), None);\n\n        let max_page_addr = Address::<Virtual>::new(usize::MAX).align_down_page();\n        assert_eq!(\n            PageAddress::<Virtual>::from(max_page_addr).checked_offset(1),\n            None\n        );\n\n        let zero = PageAddress::<Virtual>::from(0);\n        let three = PageAddress::<Virtual>::from(bsp::memory::mmu::KernelGranule::SIZE * 3);\n        assert_eq!(PageAddress::steps_between(&zero, &three), Some(3));\n    }\n\n    /// Sanity of [MemoryRegion] methods.\n    #[kernel_test]\n    fn memoryregion_type_method_sanity() {\n        let zero = PageAddress::<Virtual>::from(0);\n        let zero_region = MemoryRegion::new(zero, zero);\n        assert_eq!(zero_region.num_pages(), 0);\n        assert_eq!(zero_region.size(), 0);\n\n        let one = PageAddress::<Virtual>::from(bsp::memory::mmu::KernelGranule::SIZE);\n        let one_region = MemoryRegion::new(zero, one);\n        assert_eq!(one_region.num_pages(), 1);\n        assert_eq!(one_region.size(), bsp::memory::mmu::KernelGranule::SIZE);\n\n        let three = PageAddress::<Virtual>::from(bsp::memory::mmu::KernelGranule::SIZE * 3);\n        let mut three_region = MemoryRegion::new(zero, three);\n        assert!(three_region.contains(zero.into_inner()));\n        assert!(!three_region.contains(three.into_inner()));\n        assert!(three_region.overlaps(&one_region));\n\n        let allocation = three_region\n            .take_first_n_pages(NonZeroUsize::new(2).unwrap())\n            .unwrap();\n        assert_eq!(allocation.num_pages(), 2);\n        assert_eq!(three_region.num_pages(), 1);\n\n        for (i, alloc) in allocation.into_iter().enumerate() {\n            assert_eq!(\n                alloc.into_inner().as_usize(),\n                i * bsp::memory::mmu::KernelGranule::SIZE\n            );\n        }\n    }\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/memory/mmu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management Unit.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/memory/mmu.rs\"]\nmod arch_mmu;\n\nmod mapping_record;\nmod page_alloc;\nmod translation_table;\nmod types;\n\nuse crate::{\n    bsp,\n    memory::{Address, Physical, Virtual},\n    synchronization::{self, interface::Mutex},\n    warn,\n};\nuse core::{fmt, num::NonZeroUsize};\n\npub use types::*;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// MMU enable errors variants.\n#[allow(missing_docs)]\n#[derive(Debug)]\npub enum MMUEnableError {\n    AlreadyEnabled,\n    Other(&'static str),\n}\n\n/// Memory Management interfaces.\npub mod interface {\n    use super::*;\n\n    /// MMU functions.\n    pub trait MMU {\n        /// Turns on the MMU for the first time and enables data and instruction caching.\n        ///\n        /// # Safety\n        ///\n        /// - Changes the HW's global state.\n        unsafe fn enable_mmu_and_caching(\n            &self,\n            phys_tables_base_addr: Address<Physical>,\n        ) -> Result<(), MMUEnableError>;\n\n        /// Returns true if the MMU is enabled, false otherwise.\n        fn is_enabled(&self) -> bool;\n    }\n}\n\n/// Describes the characteristics of a translation granule.\npub struct TranslationGranule<const GRANULE_SIZE: usize>;\n\n/// Describes properties of an address space.\npub struct AddressSpace<const AS_SIZE: usize>;\n\n/// Intended to be implemented for [`AddressSpace`].\npub trait AssociatedTranslationTable {\n    /// A translation table whose address range is:\n    ///\n    /// [u64::MAX, (u64::MAX - AS_SIZE) + 1]\n    type TableStartFromTop;\n\n    /// A translation table whose address range is:\n    ///\n    /// [AS_SIZE - 1, 0]\n    type TableStartFromBottom;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\nuse interface::MMU;\nuse synchronization::interface::ReadWriteEx;\nuse translation_table::interface::TranslationTable;\n\n/// Map a region in the kernel's translation tables.\n///\n/// No input checks done, input is passed through to the architectural implementation.\n///\n/// # Safety\n///\n/// - See `map_at()`.\n/// - Does not prevent aliasing.\nunsafe fn kernel_map_at_unchecked(\n    name: &'static str,\n    virt_region: &MemoryRegion<Virtual>,\n    phys_region: &MemoryRegion<Physical>,\n    attr: &AttributeFields,\n) -> Result<(), &'static str> {\n    bsp::memory::mmu::kernel_translation_tables()\n        .write(|tables| tables.map_at(virt_region, phys_region, attr))?;\n\n    kernel_add_mapping_record(name, virt_region, phys_region, attr);\n\n    Ok(())\n}\n\n/// Try to translate a kernel virtual address to a physical address.\n///\n/// Will only succeed if there exists a valid mapping for the input address.\nfn try_kernel_virt_addr_to_phys_addr(\n    virt_addr: Address<Virtual>,\n) -> Result<Address<Physical>, &'static str> {\n    bsp::memory::mmu::kernel_translation_tables()\n        .read(|tables| tables.try_virt_addr_to_phys_addr(virt_addr))\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl fmt::Display for MMUEnableError {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        match self {\n            MMUEnableError::AlreadyEnabled => write!(f, \"MMU is already enabled\"),\n            MMUEnableError::Other(x) => write!(f, \"{}\", x),\n        }\n    }\n}\n\nimpl<const GRANULE_SIZE: usize> TranslationGranule<GRANULE_SIZE> {\n    /// The granule's size.\n    pub const SIZE: usize = Self::size_checked();\n\n    /// The granule's mask.\n    pub const MASK: usize = Self::SIZE - 1;\n\n    /// The granule's shift, aka log2(size).\n    pub const SHIFT: usize = Self::SIZE.trailing_zeros() as usize;\n\n    const fn size_checked() -> usize {\n        assert!(GRANULE_SIZE.is_power_of_two());\n\n        GRANULE_SIZE\n    }\n}\n\nimpl<const AS_SIZE: usize> AddressSpace<AS_SIZE> {\n    /// The address space size.\n    pub const SIZE: usize = Self::size_checked();\n\n    /// The address space shift, aka log2(size).\n    pub const SIZE_SHIFT: usize = Self::SIZE.trailing_zeros() as usize;\n\n    const fn size_checked() -> usize {\n        assert!(AS_SIZE.is_power_of_two());\n\n        // Check for architectural restrictions as well.\n        Self::arch_address_space_size_sanity_checks();\n\n        AS_SIZE\n    }\n}\n\n/// Query the BSP for the reserved virtual addresses for MMIO remapping and initialize the kernel's\n/// MMIO VA allocator with it.\npub fn kernel_init_mmio_va_allocator() {\n    let region = bsp::memory::mmu::virt_mmio_remap_region();\n\n    page_alloc::kernel_mmio_va_allocator().lock(|allocator| allocator.init(region));\n}\n\n/// Add an entry to the mapping info record.\npub fn kernel_add_mapping_record(\n    name: &'static str,\n    virt_region: &MemoryRegion<Virtual>,\n    phys_region: &MemoryRegion<Physical>,\n    attr: &AttributeFields,\n) {\n    if let Err(x) = mapping_record::kernel_add(name, virt_region, phys_region, attr) {\n        warn!(\"{}\", x);\n    }\n}\n\n/// MMIO remapping in the kernel translation tables.\n///\n/// Typically used by device drivers.\n///\n/// # Safety\n///\n/// - Same as `kernel_map_at_unchecked()`, minus the aliasing part.\npub unsafe fn kernel_map_mmio(\n    name: &'static str,\n    mmio_descriptor: &MMIODescriptor,\n) -> Result<Address<Virtual>, &'static str> {\n    let phys_region = MemoryRegion::from(*mmio_descriptor);\n    let offset_into_start_page = mmio_descriptor.start_addr().offset_into_page();\n\n    // Check if an identical region has been mapped for another driver. If so, reuse it.\n    let virt_addr = if let Some(addr) =\n        mapping_record::kernel_find_and_insert_mmio_duplicate(mmio_descriptor, name)\n    {\n        addr\n    // Otherwise, allocate a new region and map it.\n    } else {\n        let num_pages = match NonZeroUsize::new(phys_region.num_pages()) {\n            None => return Err(\"Requested 0 pages\"),\n            Some(x) => x,\n        };\n\n        let virt_region =\n            page_alloc::kernel_mmio_va_allocator().lock(|allocator| allocator.alloc(num_pages))?;\n\n        kernel_map_at_unchecked(\n            name,\n            &virt_region,\n            &phys_region,\n            &AttributeFields {\n                mem_attributes: MemAttributes::Device,\n                acc_perms: AccessPermissions::ReadWrite,\n                execute_never: true,\n            },\n        )?;\n\n        virt_region.start_addr()\n    };\n\n    Ok(virt_addr + offset_into_start_page)\n}\n\n/// Try to translate a kernel virtual page address to a physical page address.\n///\n/// Will only succeed if there exists a valid mapping for the input page.\npub fn try_kernel_virt_page_addr_to_phys_page_addr(\n    virt_page_addr: PageAddress<Virtual>,\n) -> Result<PageAddress<Physical>, &'static str> {\n    bsp::memory::mmu::kernel_translation_tables()\n        .read(|tables| tables.try_virt_page_addr_to_phys_page_addr(virt_page_addr))\n}\n\n/// Try to get the attributes of a kernel page.\n///\n/// Will only succeed if there exists a valid mapping for the input page.\npub fn try_kernel_page_attributes(\n    virt_page_addr: PageAddress<Virtual>,\n) -> Result<AttributeFields, &'static str> {\n    bsp::memory::mmu::kernel_translation_tables()\n        .read(|tables| tables.try_page_attributes(virt_page_addr))\n}\n\n/// Human-readable print of all recorded kernel mappings.\npub fn kernel_print_mappings() {\n    mapping_record::kernel_print()\n}\n\n/// Enable the MMU and data + instruction caching.\n///\n/// # Safety\n///\n/// - Crucial function during kernel init. Changes the the complete memory view of the processor.\n#[inline(always)]\npub unsafe fn enable_mmu_and_caching(\n    phys_tables_base_addr: Address<Physical>,\n) -> Result<(), MMUEnableError> {\n    arch_mmu::mmu().enable_mmu_and_caching(phys_tables_base_addr)\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/memory.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management.\n\npub mod mmu;\n\nuse crate::{bsp, common};\nuse core::{\n    fmt,\n    marker::PhantomData,\n    ops::{Add, Sub},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Metadata trait for marking the type of an address.\npub trait AddressType: Copy + Clone + PartialOrd + PartialEq + Ord + Eq {}\n\n/// Zero-sized type to mark a physical address.\n#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq)]\npub enum Physical {}\n\n/// Zero-sized type to mark a virtual address.\n#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq)]\npub enum Virtual {}\n\n/// Generic address type.\n#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq)]\npub struct Address<ATYPE: AddressType> {\n    value: usize,\n    _address_type: PhantomData<fn() -> ATYPE>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl AddressType for Physical {}\nimpl AddressType for Virtual {}\n\nimpl<ATYPE: AddressType> Address<ATYPE> {\n    /// Create an instance.\n    pub const fn new(value: usize) -> Self {\n        Self {\n            value,\n            _address_type: PhantomData,\n        }\n    }\n\n    /// Convert to usize.\n    pub const fn as_usize(self) -> usize {\n        self.value\n    }\n\n    /// Align down to page size.\n    #[must_use]\n    pub const fn align_down_page(self) -> Self {\n        let aligned = common::align_down(self.value, bsp::memory::mmu::KernelGranule::SIZE);\n\n        Self::new(aligned)\n    }\n\n    /// Align up to page size.\n    #[must_use]\n    pub const fn align_up_page(self) -> Self {\n        let aligned = common::align_up(self.value, bsp::memory::mmu::KernelGranule::SIZE);\n\n        Self::new(aligned)\n    }\n\n    /// Checks if the address is page aligned.\n    pub const fn is_page_aligned(&self) -> bool {\n        common::is_aligned(self.value, bsp::memory::mmu::KernelGranule::SIZE)\n    }\n\n    /// Return the address' offset into the corresponding page.\n    pub const fn offset_into_page(&self) -> usize {\n        self.value & bsp::memory::mmu::KernelGranule::MASK\n    }\n}\n\nimpl<ATYPE: AddressType> Add<usize> for Address<ATYPE> {\n    type Output = Self;\n\n    #[inline(always)]\n    fn add(self, rhs: usize) -> Self::Output {\n        match self.value.checked_add(rhs) {\n            None => panic!(\"Overflow on Address::add\"),\n            Some(x) => Self::new(x),\n        }\n    }\n}\n\nimpl<ATYPE: AddressType> Sub<Address<ATYPE>> for Address<ATYPE> {\n    type Output = Self;\n\n    #[inline(always)]\n    fn sub(self, rhs: Address<ATYPE>) -> Self::Output {\n        match self.value.checked_sub(rhs.value) {\n            None => panic!(\"Overflow on Address::sub\"),\n            Some(x) => Self::new(x),\n        }\n    }\n}\n\nimpl fmt::Display for Address<Physical> {\n    // Don't expect to see physical addresses greater than 40 bit.\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        let q3: u8 = ((self.value >> 32) & 0xff) as u8;\n        let q2: u16 = ((self.value >> 16) & 0xffff) as u16;\n        let q1: u16 = (self.value & 0xffff) as u16;\n\n        write!(f, \"0x\")?;\n        write!(f, \"{:02x}_\", q3)?;\n        write!(f, \"{:04x}_\", q2)?;\n        write!(f, \"{:04x}\", q1)\n    }\n}\n\nimpl fmt::Display for Address<Virtual> {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        let q4: u16 = ((self.value >> 48) & 0xffff) as u16;\n        let q3: u16 = ((self.value >> 32) & 0xffff) as u16;\n        let q2: u16 = ((self.value >> 16) & 0xffff) as u16;\n        let q1: u16 = (self.value & 0xffff) as u16;\n\n        write!(f, \"0x\")?;\n        write!(f, \"{:04x}_\", q4)?;\n        write!(f, \"{:04x}_\", q3)?;\n        write!(f, \"{:04x}_\", q2)?;\n        write!(f, \"{:04x}\", q1)\n    }\n}\n\n/// Initialize the memory subsystem.\npub fn init() {\n    mmu::kernel_init_mmio_va_allocator();\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// Sanity of [Address] methods.\n    #[kernel_test]\n    fn address_type_method_sanity() {\n        let addr = Address::<Virtual>::new(bsp::memory::mmu::KernelGranule::SIZE + 100);\n\n        assert_eq!(\n            addr.align_down_page().as_usize(),\n            bsp::memory::mmu::KernelGranule::SIZE\n        );\n\n        assert_eq!(\n            addr.align_up_page().as_usize(),\n            bsp::memory::mmu::KernelGranule::SIZE * 2\n        );\n\n        assert!(!addr.is_page_aligned());\n\n        assert_eq!(addr.offset_into_page(), 100);\n    }\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/panic_wait.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A panic handler that infinitely waits.\n\nuse crate::{cpu, exception, println};\nuse core::panic::PanicInfo;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// The point of exit for `libkernel`.\n///\n/// It is linked weakly, so that the integration tests can overload its standard behavior.\n#[linkage = \"weak\"]\n#[no_mangle]\nfn _panic_exit() -> ! {\n    #[cfg(not(feature = \"test_build\"))]\n    {\n        cpu::wait_forever()\n    }\n\n    #[cfg(feature = \"test_build\")]\n    {\n        cpu::qemu_exit_failure()\n    }\n}\n\n/// Stop immediately if called a second time.\n///\n/// # Note\n///\n/// Using atomics here relieves us from needing to use `unsafe` for the static variable.\n///\n/// On `AArch64`, which is the only implemented architecture at the time of writing this,\n/// [`AtomicBool::load`] and [`AtomicBool::store`] are lowered to ordinary load and store\n/// instructions. They are therefore safe to use even with MMU + caching deactivated.\n///\n/// [`AtomicBool::load`]: core::sync::atomic::AtomicBool::load\n/// [`AtomicBool::store`]: core::sync::atomic::AtomicBool::store\nfn panic_prevent_reenter() {\n    use core::sync::atomic::{AtomicBool, Ordering};\n\n    #[cfg(not(target_arch = \"aarch64\"))]\n    compile_error!(\"Add the target_arch to above's check if the following code is safe to use\");\n\n    static PANIC_IN_PROGRESS: AtomicBool = AtomicBool::new(false);\n\n    if !PANIC_IN_PROGRESS.load(Ordering::Relaxed) {\n        PANIC_IN_PROGRESS.store(true, Ordering::Relaxed);\n\n        return;\n    }\n\n    _panic_exit()\n}\n\n#[panic_handler]\nfn panic(info: &PanicInfo) -> ! {\n    exception::asynchronous::local_irq_mask();\n\n    // Protect against panic infinite loops if any of the following code panics itself.\n    panic_prevent_reenter();\n\n    let timestamp = crate::time::time_manager().uptime();\n    let (location, line, column) = match info.location() {\n        Some(loc) => (loc.file(), loc.line(), loc.column()),\n        _ => (\"???\", 0, 0),\n    };\n\n    println!(\n        \"[  {:>3}.{:06}] Kernel panic!\\n\\n\\\n        Panic location:\\n      File '{}', line {}, column {}\\n\\n\\\n        {}\",\n        timestamp.as_secs(),\n        timestamp.subsec_micros(),\n        location,\n        line,\n        column,\n        info.message().unwrap_or(&format_args!(\"\")),\n    );\n\n    _panic_exit()\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/print.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Printing.\n\nuse crate::console;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n#[doc(hidden)]\npub fn _print(args: fmt::Arguments) {\n    console::console().write_fmt(args).unwrap();\n}\n\n/// Prints without a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! print {\n    ($($arg:tt)*) => ($crate::print::_print(format_args!($($arg)*)));\n}\n\n/// Prints with a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! println {\n    () => ($crate::print!(\"\\n\"));\n    ($($arg:tt)*) => ({\n        $crate::print::_print(format_args_nl!($($arg)*));\n    })\n}\n\n/// Prints an info, with a newline.\n#[macro_export]\nmacro_rules! info {\n    ($string:expr) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[  {:>3}.{:06}] \", $string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n        ));\n    });\n    ($format_string:expr, $($arg:tt)*) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[  {:>3}.{:06}] \", $format_string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n            $($arg)*\n        ));\n    })\n}\n\n/// Prints a warning, with a newline.\n#[macro_export]\nmacro_rules! warn {\n    ($string:expr) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[W {:>3}.{:06}] \", $string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n        ));\n    });\n    ($format_string:expr, $($arg:tt)*) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[W {:>3}.{:06}] \", $format_string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n            $($arg)*\n        ));\n    })\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/state.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! State information about the kernel itself.\n\nuse core::sync::atomic::{AtomicU8, Ordering};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Different stages in the kernel execution.\n#[derive(Copy, Clone, Eq, PartialEq)]\nenum State {\n    /// The kernel starts booting in this state.\n    Init,\n\n    /// The kernel transitions to this state when jumping to `kernel_main()` (at the end of\n    /// `kernel_init()`, after all init calls are done).\n    SingleCoreMain,\n\n    /// The kernel transitions to this state when it boots the secondary cores, aka switches\n    /// exectution mode to symmetric multiprocessing (SMP).\n    MultiCoreMain,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Maintains the kernel state and state transitions.\npub struct StateManager(AtomicU8);\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic STATE_MANAGER: StateManager = StateManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the global StateManager.\npub fn state_manager() -> &'static StateManager {\n    &STATE_MANAGER\n}\n\nimpl StateManager {\n    const INIT: u8 = 0;\n    const SINGLE_CORE_MAIN: u8 = 1;\n    const MULTI_CORE_MAIN: u8 = 2;\n\n    /// Create a new instance.\n    pub const fn new() -> Self {\n        Self(AtomicU8::new(Self::INIT))\n    }\n\n    /// Return the current state.\n    fn state(&self) -> State {\n        let state = self.0.load(Ordering::Acquire);\n\n        match state {\n            Self::INIT => State::Init,\n            Self::SINGLE_CORE_MAIN => State::SingleCoreMain,\n            Self::MULTI_CORE_MAIN => State::MultiCoreMain,\n            _ => panic!(\"Invalid KERNEL_STATE\"),\n        }\n    }\n\n    /// Return if the kernel is init state.\n    pub fn is_init(&self) -> bool {\n        self.state() == State::Init\n    }\n\n    /// Transition from Init to SingleCoreMain.\n    pub fn transition_to_single_core_main(&self) {\n        if self\n            .0\n            .compare_exchange(\n                Self::INIT,\n                Self::SINGLE_CORE_MAIN,\n                Ordering::Acquire,\n                Ordering::Relaxed,\n            )\n            .is_err()\n        {\n            panic!(\"transition_to_single_core_main() called while state != Init\");\n        }\n    }\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/symbols.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Debug symbol support.\n\nuse crate::memory::{Address, Virtual};\nuse core::{cell::UnsafeCell, slice};\nuse debug_symbol_types::Symbol;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Symbol from the linker script.\nextern \"Rust\" {\n    static __kernel_symbols_start: UnsafeCell<()>;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n/// This will be patched to the correct value by the \"kernel symbols tool\" after linking. This given\n/// value here is just a (safe) dummy.\n#[no_mangle]\nstatic NUM_KERNEL_SYMBOLS: u64 = 0;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nfn kernel_symbol_section_virt_start_addr() -> Address<Virtual> {\n    Address::new(unsafe { __kernel_symbols_start.get() as usize })\n}\n\nfn num_kernel_symbols() -> usize {\n    unsafe {\n        // Read volatile is needed here to prevent the compiler from optimizing NUM_KERNEL_SYMBOLS\n        // away.\n        core::ptr::read_volatile(&NUM_KERNEL_SYMBOLS as *const u64) as usize\n    }\n}\n\nfn kernel_symbols_slice() -> &'static [Symbol] {\n    let ptr = kernel_symbol_section_virt_start_addr().as_usize() as *const Symbol;\n\n    unsafe { slice::from_raw_parts(ptr, num_kernel_symbols()) }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Retrieve the symbol corresponding to a virtual address, if any.\npub fn lookup_symbol(addr: Address<Virtual>) -> Option<&'static Symbol> {\n    kernel_symbols_slice()\n        .iter()\n        .find(|&i| i.contains(addr.as_usize()))\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// Sanity of symbols module.\n    #[kernel_test]\n    fn symbols_sanity() {\n        let first_sym = lookup_symbol(Address::new(\n            crate::common::is_aligned as *const usize as usize,\n        ))\n        .unwrap()\n        .name();\n\n        assert_eq!(first_sym, \"libkernel::common::is_aligned\");\n\n        let second_sym = lookup_symbol(Address::new(crate::version as *const usize as usize))\n            .unwrap()\n            .name();\n\n        assert_eq!(second_sym, \"libkernel::version\");\n    }\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/synchronization.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Synchronization primitives.\n//!\n//! # Resources\n//!\n//!   - <https://doc.rust-lang.org/book/ch16-04-extensible-concurrency-sync-and-send.html>\n//!   - <https://stackoverflow.com/questions/59428096/understanding-the-send-trait>\n//!   - <https://doc.rust-lang.org/std/cell/index.html>\n\nuse core::cell::UnsafeCell;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Synchronization interfaces.\npub mod interface {\n\n    /// Any object implementing this trait guarantees exclusive access to the data wrapped within\n    /// the Mutex for the duration of the provided closure.\n    pub trait Mutex {\n        /// The type of the data that is wrapped by this mutex.\n        type Data;\n\n        /// Locks the mutex and grants the closure temporary mutable access to the wrapped data.\n        fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R;\n    }\n\n    /// A reader-writer exclusion type.\n    ///\n    /// The implementing object allows either a number of readers or at most one writer at any point\n    /// in time.\n    pub trait ReadWriteEx {\n        /// The type of encapsulated data.\n        type Data;\n\n        /// Grants temporary mutable access to the encapsulated data.\n        fn write<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R;\n\n        /// Grants temporary immutable access to the encapsulated data.\n        fn read<'a, R>(&'a self, f: impl FnOnce(&'a Self::Data) -> R) -> R;\n    }\n}\n\n/// A pseudo-lock for teaching purposes.\n///\n/// In contrast to a real Mutex implementation, does not protect against concurrent access from\n/// other cores to the contained data. This part is preserved for later lessons.\n///\n/// The lock will only be used as long as it is safe to do so, i.e. as long as the kernel is\n/// executing on a single core.\npub struct IRQSafeNullLock<T>\nwhere\n    T: ?Sized,\n{\n    data: UnsafeCell<T>,\n}\n\n/// A pseudo-lock that is RW during the single-core kernel init phase and RO afterwards.\n///\n/// Intended to encapsulate data that is populated during kernel init when no concurrency exists.\npub struct InitStateLock<T>\nwhere\n    T: ?Sized,\n{\n    data: UnsafeCell<T>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nunsafe impl<T> Send for IRQSafeNullLock<T> where T: ?Sized + Send {}\nunsafe impl<T> Sync for IRQSafeNullLock<T> where T: ?Sized + Send {}\n\nimpl<T> IRQSafeNullLock<T> {\n    /// Create an instance.\n    pub const fn new(data: T) -> Self {\n        Self {\n            data: UnsafeCell::new(data),\n        }\n    }\n}\n\nunsafe impl<T> Send for InitStateLock<T> where T: ?Sized + Send {}\nunsafe impl<T> Sync for InitStateLock<T> where T: ?Sized + Send {}\n\nimpl<T> InitStateLock<T> {\n    /// Create an instance.\n    pub const fn new(data: T) -> Self {\n        Self {\n            data: UnsafeCell::new(data),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse crate::{exception, state};\n\nimpl<T> interface::Mutex for IRQSafeNullLock<T> {\n    type Data = T;\n\n    fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R {\n        // In a real lock, there would be code encapsulating this line that ensures that this\n        // mutable reference will ever only be given out once at a time.\n        let data = unsafe { &mut *self.data.get() };\n\n        // Execute the closure while IRQs are masked.\n        exception::asynchronous::exec_with_irq_masked(|| f(data))\n    }\n}\n\nimpl<T> interface::ReadWriteEx for InitStateLock<T> {\n    type Data = T;\n\n    fn write<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R {\n        assert!(\n            state::state_manager().is_init(),\n            \"InitStateLock::write called after kernel init phase\"\n        );\n        assert!(\n            !exception::asynchronous::is_local_irq_masked(),\n            \"InitStateLock::write called with IRQs unmasked\"\n        );\n\n        let data = unsafe { &mut *self.data.get() };\n\n        f(data)\n    }\n\n    fn read<'a, R>(&'a self, f: impl FnOnce(&'a Self::Data) -> R) -> R {\n        let data = unsafe { &*self.data.get() };\n\n        f(data)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// InitStateLock must be transparent.\n    #[kernel_test]\n    fn init_state_lock_is_transparent() {\n        use core::mem::size_of;\n\n        assert_eq!(size_of::<InitStateLock<u64>>(), size_of::<u64>());\n    }\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/src/time.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Timer primitives.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/time.rs\"]\nmod arch_time;\n\nuse core::time::Duration;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Provides time management functions.\npub struct TimeManager;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic TIME_MANAGER: TimeManager = TimeManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the global TimeManager.\npub fn time_manager() -> &'static TimeManager {\n    &TIME_MANAGER\n}\n\nimpl TimeManager {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self\n    }\n\n    /// The timer's resolution.\n    pub fn resolution(&self) -> Duration {\n        arch_time::resolution()\n    }\n\n    /// The uptime since power-on of the device.\n    ///\n    /// This includes time consumed by firmware and bootloaders.\n    pub fn uptime(&self) -> Duration {\n        arch_time::uptime()\n    }\n\n    /// Spin for a given duration.\n    pub fn spin_for(&self, duration: Duration) {\n        arch_time::spin_for(duration)\n    }\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/tests/00_console_sanity.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'console_io_test'\n\n# Verify sending and receiving works as expected.\nclass TxRxHandshakeTest < SubtestBase\n    def name\n        'Transmit and Receive handshake'\n    end\n\n    def run(qemu_out, qemu_in)\n        qemu_in.write_nonblock('ABC')\n        expect_or_raise(qemu_out, 'OK1234')\n    end\nend\n\n# Check for correct TX statistics implementation. Depends on test 1 being run first.\nclass TxStatisticsTest < SubtestBase\n    def name\n        'Transmit statistics'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, '6')\n    end\nend\n\n# Check for correct RX statistics implementation. Depends on test 1 being run first.\nclass RxStatisticsTest < SubtestBase\n    def name\n        'Receive statistics'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, '3')\n    end\nend\n\n## -------------------------------------------------------------------------------------------------\n## Test registration\n## -------------------------------------------------------------------------------------------------\ndef subtest_collection\n    [TxRxHandshakeTest.new, TxStatisticsTest.new, RxStatisticsTest.new]\nend\n"
  },
  {
    "path": "17_kernel_symbols/kernel/tests/00_console_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Console sanity tests - RX, TX and statistics.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Console tests should time out on the I/O harness in case of panic.\nmod panic_wait_forever;\n\nuse libkernel::{bsp, console, cpu, exception, memory, print};\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    use console::console;\n\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    // Handshake\n    assert_eq!(console().read_char(), 'A');\n    assert_eq!(console().read_char(), 'B');\n    assert_eq!(console().read_char(), 'C');\n    print!(\"OK1234\");\n\n    // 6\n    print!(\"{}\", console().chars_written());\n\n    // 3\n    print!(\"{}\", console().chars_read());\n\n    // The QEMU process running this test will be closed by the I/O test harness.\n    cpu::wait_forever();\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/tests/01_timer_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Timer sanity tests.\n\n#![feature(custom_test_frameworks)]\n#![no_main]\n#![no_std]\n#![reexport_test_harness_main = \"test_main\"]\n#![test_runner(libkernel::test_runner)]\n\nuse core::time::Duration;\nuse libkernel::{bsp, cpu, exception, memory, time};\nuse test_macros::kernel_test;\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    // Depending on CPU arch, some timer bring-up code could go here. Not needed for the RPi.\n\n    test_main();\n\n    cpu::qemu_exit_success()\n}\n\n/// Simple check that the timer is running.\n#[kernel_test]\nfn timer_is_counting() {\n    assert!(time::time_manager().uptime().as_nanos() > 0)\n}\n\n/// Timer resolution must be sufficient.\n#[kernel_test]\nfn timer_resolution_is_sufficient() {\n    assert!(time::time_manager().resolution().as_nanos() > 0);\n    assert!(time::time_manager().resolution().as_nanos() < 100)\n}\n\n/// Sanity check spin_for() implementation.\n#[kernel_test]\nfn spin_accuracy_check_1_second() {\n    let t1 = time::time_manager().uptime();\n    time::time_manager().spin_for(Duration::from_secs(1));\n    let t2 = time::time_manager().uptime();\n\n    assert_eq!((t2 - t1).as_secs(), 1)\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/tests/02_exception_sync_page_fault.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Page faults must result in synchronous exceptions.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Overwrites libkernel's `panic_wait::_panic_exit()` so that it returns a \"success\" code.\n///\n/// In this test, reaching the panic is a success, because it is called from the synchronous\n/// exception handler, which is what this test wants to achieve.\n///\n/// It also means that this integration test can not use any other code that calls panic!() directly\n/// or indirectly.\nmod panic_exit_success;\n\nuse libkernel::{bsp, cpu, exception, info, memory, println};\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    // This line will be printed as the test header.\n    println!(\"Testing synchronous exception handling by causing a page fault\");\n\n    info!(\"Writing to bottom of address space to address 1 GiB...\");\n    let big_addr: u64 = 1024 * 1024 * 1024;\n    core::ptr::read_volatile(big_addr as *mut u64);\n\n    // If execution reaches here, the memory access above did not cause a page fault exception.\n    cpu::qemu_exit_failure()\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/tests/03_exception_restore_sanity.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'console_io_test'\n\n# Verify that exception restore works.\nclass ExceptionRestoreTest < SubtestBase\n    def name\n        'Exception restore'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, 'Back from system call!')\n    end\nend\n\n## -------------------------------------------------------------------------------------------------\n## Test registration\n## -------------------------------------------------------------------------------------------------\ndef subtest_collection\n    [ExceptionRestoreTest.new]\nend\n"
  },
  {
    "path": "17_kernel_symbols/kernel/tests/03_exception_restore_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A simple sanity test to see if exception restore code works.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Console tests should time out on the I/O harness in case of panic.\nmod panic_wait_forever;\n\nuse core::arch::asm;\nuse libkernel::{bsp, cpu, exception, info, memory, println};\n\n#[inline(never)]\nfn nested_system_call() {\n    #[cfg(target_arch = \"aarch64\")]\n    unsafe {\n        asm!(\"svc #0x1337\", options(nomem, nostack, preserves_flags));\n    }\n\n    #[cfg(not(target_arch = \"aarch64\"))]\n    {\n        info!(\"Not supported yet\");\n        cpu::wait_forever();\n    }\n}\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    // This line will be printed as the test header.\n    println!(\"Testing exception restore\");\n\n    info!(\"Making a dummy system call\");\n\n    // Calling this inside a function indirectly tests if the link register is restored properly.\n    nested_system_call();\n\n    info!(\"Back from system call!\");\n\n    // The QEMU process running this test will be closed by the I/O test harness.\n    cpu::wait_forever();\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/tests/04_exception_irq_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! IRQ handling sanity tests.\n\n#![feature(custom_test_frameworks)]\n#![no_main]\n#![no_std]\n#![reexport_test_harness_main = \"test_main\"]\n#![test_runner(libkernel::test_runner)]\n\nuse libkernel::{bsp, cpu, exception, memory};\nuse test_macros::kernel_test;\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    exception::handling_init();\n    exception::asynchronous::local_irq_unmask();\n\n    test_main();\n\n    cpu::qemu_exit_success()\n}\n\n/// Check that IRQ masking works.\n#[kernel_test]\nfn local_irq_mask_works() {\n    // Precondition: IRQs are unmasked.\n    assert!(exception::asynchronous::is_local_irq_masked());\n\n    exception::asynchronous::local_irq_mask();\n    assert!(!exception::asynchronous::is_local_irq_masked());\n\n    // Restore earlier state.\n    exception::asynchronous::local_irq_unmask();\n}\n\n/// Check that IRQ unmasking works.\n#[kernel_test]\nfn local_irq_unmask_works() {\n    // Precondition: IRQs are masked.\n    exception::asynchronous::local_irq_mask();\n    assert!(!exception::asynchronous::is_local_irq_masked());\n\n    exception::asynchronous::local_irq_unmask();\n    assert!(exception::asynchronous::is_local_irq_masked());\n}\n\n/// Check that IRQ mask save is saving \"something\".\n#[kernel_test]\nfn local_irq_mask_save_works() {\n    // Precondition: IRQs are unmasked.\n    assert!(exception::asynchronous::is_local_irq_masked());\n\n    let first = exception::asynchronous::local_irq_mask_save();\n    assert!(!exception::asynchronous::is_local_irq_masked());\n\n    let second = exception::asynchronous::local_irq_mask_save();\n    assert_ne!(first, second);\n\n    exception::asynchronous::local_irq_restore(first);\n    assert!(exception::asynchronous::is_local_irq_masked());\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/tests/boot_test_string.rb",
    "content": "# frozen_string_literal: true\n\nEXPECTED_PRINT = 'Echoing input now'\n"
  },
  {
    "path": "17_kernel_symbols/kernel/tests/panic_exit_success/mod.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n/// Overwrites libkernel's `panic_wait::_panic_exit()` with the QEMU-exit version.\n#[no_mangle]\nfn _panic_exit() -> ! {\n    libkernel::cpu::qemu_exit_success()\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel/tests/panic_wait_forever/mod.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n/// Overwrites libkernel's `panic_wait::_panic_exit()` with wait_forever.\n#[no_mangle]\nfn _panic_exit() -> ! {\n    libkernel::cpu::wait_forever()\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel_symbols/Cargo.toml",
    "content": "[package]\nname = \"kernel_symbols\"\nversion = \"0.1.0\"\nedition = \"2021\"\n\n[features]\ndefault = []\ngenerated_symbols_available = []\n\n##--------------------------------------------------------------------------------------------------\n## Dependencies\n##--------------------------------------------------------------------------------------------------\n\n[dependencies]\ndebug-symbol-types = { path = \"../libraries/debug-symbol-types\" }\n"
  },
  {
    "path": "17_kernel_symbols/kernel_symbols/build.rs",
    "content": "use std::{env, path::Path};\n\nfn main() {\n    if let Ok(path) = env::var(\"KERNEL_SYMBOLS_DEMANGLED_RS\") {\n        if Path::new(&path).exists() {\n            println!(\"cargo:rustc-cfg=feature=\\\"generated_symbols_available\\\"\")\n        }\n    }\n\n    println!(\n        \"cargo:rerun-if-changed={}\",\n        Path::new(\"kernel_symbols.ld\").display()\n    );\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel_symbols/kernel_symbols.ld",
    "content": "/* SPDX-License-Identifier: MIT OR Apache-2.0\n *\n * Copyright (c) 2022 Andre Richter <andre.o.richter@gmail.com>\n */\n\nSECTIONS\n{\n    .rodata : {\n        ASSERT(. > 0xffffffff00000000, \"Expected higher half address\")\n\n        KEEP(*(.rodata.symbol_desc*))\n        . = ALIGN(8);\n        *(.rodata*)\n    }\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel_symbols/src/main.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Generation of kernel symbols.\n\n#![no_std]\n#![no_main]\n\n#[cfg(feature = \"generated_symbols_available\")]\ninclude!(env!(\"KERNEL_SYMBOLS_DEMANGLED_RS\"));\n\n#[panic_handler]\nfn panic(_info: &core::panic::PanicInfo) -> ! {\n    unimplemented!()\n}\n"
  },
  {
    "path": "17_kernel_symbols/kernel_symbols.mk",
    "content": "## SPDX-License-Identifier: MIT OR Apache-2.0\n##\n## Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\ninclude ../common/format.mk\ninclude ../common/docker.mk\n\n##--------------------------------------------------------------------------------------------------\n## Check for input variables that need be exported by the calling Makefile\n##--------------------------------------------------------------------------------------------------\nifndef KERNEL_SYMBOLS_TOOL_PATH\n$(error KERNEL_SYMBOLS_TOOL_PATH is not set)\nendif\n\nifndef TARGET\n$(error TARGET is not set)\nendif\n\nifndef KERNEL_SYMBOLS_INPUT_ELF\n$(error KERNEL_SYMBOLS_INPUT_ELF is not set)\nendif\n\nifndef KERNEL_SYMBOLS_OUTPUT_ELF\n$(error KERNEL_SYMBOLS_OUTPUT_ELF is not set)\nendif\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets and Prerequisites\n##--------------------------------------------------------------------------------------------------\nKERNEL_SYMBOLS_MANIFEST      = kernel_symbols/Cargo.toml\nKERNEL_SYMBOLS_LINKER_SCRIPT = kernel_symbols/kernel_symbols.ld\n\nKERNEL_SYMBOLS_RS           = $(KERNEL_SYMBOLS_INPUT_ELF)_symbols.rs\nKERNEL_SYMBOLS_DEMANGLED_RS = $(shell pwd)/$(KERNEL_SYMBOLS_INPUT_ELF)_symbols_demangled.rs\n\nKERNEL_SYMBOLS_ELF      = target/$(TARGET)/release/kernel_symbols\nKERNEL_SYMBOLS_STRIPPED = target/$(TARGET)/release/kernel_symbols_stripped\n\n# Export for build.rs of kernel_symbols crate.\nexport KERNEL_SYMBOLS_DEMANGLED_RS\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Command building blocks\n##--------------------------------------------------------------------------------------------------\nGET_SYMBOLS_SECTION_VIRT_ADDR = $(DOCKER_TOOLS) $(EXEC_SYMBOLS_TOOL) \\\n    --get_symbols_section_virt_addr $(KERNEL_SYMBOLS_OUTPUT_ELF)\n\nRUSTFLAGS = -C link-arg=--script=$(KERNEL_SYMBOLS_LINKER_SCRIPT) \\\n    -C link-arg=--section-start=.rodata=$$($(GET_SYMBOLS_SECTION_VIRT_ADDR))\n\nRUSTFLAGS_PEDANTIC = $(RUSTFLAGS) \\\n    -D warnings                   \\\n    -D missing_docs\n\nCOMPILER_ARGS = --target=$(TARGET) \\\n    --release\n\nRUSTC_CMD   = cargo rustc $(COMPILER_ARGS) --manifest-path $(KERNEL_SYMBOLS_MANIFEST)\nOBJCOPY_CMD = rust-objcopy \\\n    --strip-all            \\\n    -O binary\n\nEXEC_SYMBOLS_TOOL  = ruby $(KERNEL_SYMBOLS_TOOL_PATH)/main.rb\n\n##------------------------------------------------------------------------------\n## Dockerization\n##------------------------------------------------------------------------------\nDOCKER_CMD = docker run -t --rm -v $(shell pwd):/work/tutorial -w /work/tutorial\n\n# DOCKER_IMAGE defined in include file (see top of this file).\nDOCKER_TOOLS = $(DOCKER_CMD) $(DOCKER_IMAGE)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: all symbols measure_time_start measure_time_finish\n\nall: measure_time_start symbols measure_time_finish\n\nsymbols:\n\t@cp $(KERNEL_SYMBOLS_INPUT_ELF) $(KERNEL_SYMBOLS_OUTPUT_ELF)\n\n\t@$(DOCKER_TOOLS) $(EXEC_SYMBOLS_TOOL) --gen_symbols $(KERNEL_SYMBOLS_OUTPUT_ELF) \\\n                $(KERNEL_SYMBOLS_RS)\n\n\t$(call color_progress_prefix, \"Demangling\")\n\t@echo Symbol names\n\t@cat $(KERNEL_SYMBOLS_RS) | rustfilt > $(KERNEL_SYMBOLS_DEMANGLED_RS)\n\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(RUSTC_CMD)\n\n\t$(call color_progress_prefix, \"Stripping\")\n\t@echo Symbols ELF file\n\t@$(OBJCOPY_CMD) $(KERNEL_SYMBOLS_ELF) $(KERNEL_SYMBOLS_STRIPPED)\n\n\t@$(DOCKER_TOOLS) $(EXEC_SYMBOLS_TOOL) --patch_data $(KERNEL_SYMBOLS_OUTPUT_ELF) \\\n                $(KERNEL_SYMBOLS_STRIPPED)\n\n# Note: The following is the only _trivial_ way I could think of that works out of the box on both\n# Linux and macOS. Since macOS does not have the %N nanosecond format string option, the\n# resolution is restricted to whole seconds.\nmeasure_time_start:\n\t@date +%s > /tmp/kernel_symbols_start.date\n\nmeasure_time_finish:\n\t@date +%s > /tmp/kernel_symbols_end.date\n\n\t$(call color_progress_prefix, \"Finished\")\n\t@echo \"in $$((`cat /tmp/kernel_symbols_end.date` - `cat /tmp/kernel_symbols_start.date`)).0s\"\n\n\t@rm /tmp/kernel_symbols_end.date /tmp/kernel_symbols_start.date\n"
  },
  {
    "path": "17_kernel_symbols/libraries/debug-symbol-types/Cargo.toml",
    "content": "[package]\nname = \"debug-symbol-types\"\nversion = \"0.1.0\"\nedition = \"2021\"\n"
  },
  {
    "path": "17_kernel_symbols/libraries/debug-symbol-types/src/lib.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Types for implementing debug symbol support.\n\n#![no_std]\n\nuse core::ops::Range;\n\n/// A symbol containing a size.\n#[repr(C)]\n#[derive(Clone)]\npub struct Symbol {\n    addr_range: Range<usize>,\n    name: &'static str,\n}\n\nimpl Symbol {\n    /// Create an instance.\n    pub const fn new(start: usize, size: usize, name: &'static str) -> Symbol {\n        Symbol {\n            addr_range: Range {\n                start,\n                end: start + size,\n            },\n            name,\n        }\n    }\n\n    /// Returns true if addr is contained in the range.\n    pub fn contains(&self, addr: usize) -> bool {\n        self.addr_range.contains(&addr)\n    }\n\n    /// Returns the symbol's name.\n    pub fn name(&self) -> &'static str {\n        self.name\n    }\n\n    /// Returns the symbol's size.\n    pub fn size(&self) -> usize {\n        self.addr_range.end - self.addr_range.start\n    }\n}\n"
  },
  {
    "path": "17_kernel_symbols/libraries/test-macros/Cargo.toml",
    "content": "[package]\nname = \"test-macros\"\nversion = \"0.1.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n\n[lib]\nproc-macro = true\n\n[dependencies]\nproc-macro2 = \"1.x\"\nquote = \"1.x\"\nsyn = { version = \"1.x\", features = [\"full\"] }\ntest-types = { path = \"../test-types\" }\n"
  },
  {
    "path": "17_kernel_symbols/libraries/test-macros/src/lib.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\nuse proc_macro::TokenStream;\nuse proc_macro2::Span;\nuse quote::quote;\nuse syn::{parse_macro_input, Ident, ItemFn};\n\n#[proc_macro_attribute]\npub fn kernel_test(_attr: TokenStream, input: TokenStream) -> TokenStream {\n    let f = parse_macro_input!(input as ItemFn);\n\n    let test_name = &format!(\"{}\", f.sig.ident);\n    let test_ident = Ident::new(\n        &format!(\"{}_TEST_CONTAINER\", f.sig.ident.to_string().to_uppercase()),\n        Span::call_site(),\n    );\n    let test_code_block = f.block;\n\n    quote!(\n        #[test_case]\n        const #test_ident: test_types::UnitTest = test_types::UnitTest {\n            name: #test_name,\n            test_func: || #test_code_block,\n        };\n    )\n    .into()\n}\n"
  },
  {
    "path": "17_kernel_symbols/libraries/test-types/Cargo.toml",
    "content": "[package]\nname = \"test-types\"\nversion = \"0.1.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n"
  },
  {
    "path": "17_kernel_symbols/libraries/test-types/src/lib.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Types for the `custom_test_frameworks` implementation.\n\n#![no_std]\n\n/// Unit test container.\npub struct UnitTest {\n    /// Name of the test.\n    pub name: &'static str,\n\n    /// Function pointer to the test.\n    pub test_func: fn(),\n}\n"
  },
  {
    "path": "17_kernel_symbols/tools/kernel_symbols_tool/cmds.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\ndef generate_symbols(kernel_elf, output_file)\n    File.open(output_file, 'w') do |file|\n        header = <<~HEREDOC\n            use debug_symbol_types::Symbol;\n\n            # [no_mangle]\n            # [link_section = \".rodata.symbol_desc\"]\n            static KERNEL_SYMBOLS: [Symbol; #{kernel_elf.num_symbols}] = [\n        HEREDOC\n\n        file.write(header)\n        kernel_elf.symbols.each do |sym|\n            value = sym.header.st_value\n            size = sym.header.st_size\n            name = sym.name\n\n            file.write(\"    Symbol::new(#{value}, #{size}, \\\"#{name}\\\"),\\n\")\n        end\n        file.write(\"];\\n\")\n    end\nend\n\ndef get_symbols_section_virt_addr(kernel_elf)\n    kernel_elf.kernel_symbols_section_virt_addr\nend\n\ndef patch_symbol_data(kernel_elf, symbols_blob_path)\n    symbols_blob = File.binread(symbols_blob_path)\n\n    raise if symbols_blob.size > kernel_elf.kernel_symbols_section_size\n\n    File.binwrite(kernel_elf.path, File.binread(symbols_blob_path),\n                  kernel_elf.kernel_symbols_section_offset_in_file)\nend\n\ndef patch_num_symbols(kernel_elf)\n    num_packed = [kernel_elf.num_symbols].pack('Q<*') # \"Q\" == uint64_t, \"<\" == little endian\n    File.binwrite(kernel_elf.path, num_packed, kernel_elf.num_kernel_symbols_offset_in_file)\nend\n"
  },
  {
    "path": "17_kernel_symbols/tools/kernel_symbols_tool/kernel_elf.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n# KernelELF\nclass KernelELF\n    attr_reader :path\n\n    def initialize(kernel_elf_path, kernel_symbols_section, num_kernel_symbols)\n        @elf = ELFTools::ELFFile.new(File.open(kernel_elf_path))\n        @symtab_section = @elf.section_by_name('.symtab')\n\n        @path = kernel_elf_path\n        fetch_values(kernel_symbols_section, num_kernel_symbols)\n    end\n\n    private\n\n    def fetch_values(kernel_symbols_section, num_kernel_symbols)\n        sym = @symtab_section.symbol_by_name(num_kernel_symbols)\n        raise \"Symbol \\\"#{num_kernel_symbols}\\\" not found\" if sym.nil?\n\n        @num_kernel_symbols = sym\n\n        section = @elf.section_by_name(kernel_symbols_section)\n        raise \"Section \\\"#{kernel_symbols_section}\\\" not found\" if section.nil?\n\n        @kernel_symbols_section = section\n    end\n\n    def num_kernel_symbols_virt_addr\n        @num_kernel_symbols.header.st_value\n    end\n\n    def segment_containing_virt_addr(virt_addr)\n        @elf.each_segments do |segment|\n            return segment if segment.vma_in?(virt_addr)\n        end\n    end\n\n    def virt_addr_to_file_offset(virt_addr)\n        segment = segment_containing_virt_addr(virt_addr)\n        segment.vma_to_offset(virt_addr)\n    end\n\n    public\n\n    def symbols\n        non_zero_symbols = @symtab_section.symbols.reject { |sym| sym.header.st_size.zero? }\n        non_zero_symbols.sort_by { |sym| sym.header.st_value }\n    end\n\n    def num_symbols\n        symbols.size\n    end\n\n    def kernel_symbols_section_virt_addr\n        @kernel_symbols_section.header.sh_addr.to_i\n    end\n\n    def kernel_symbols_section_size\n        @kernel_symbols_section.header.sh_size.to_i\n    end\n\n    def kernel_symbols_section_offset_in_file\n        virt_addr_to_file_offset(kernel_symbols_section_virt_addr)\n    end\n\n    def num_kernel_symbols_offset_in_file\n        virt_addr_to_file_offset(num_kernel_symbols_virt_addr)\n    end\nend\n"
  },
  {
    "path": "17_kernel_symbols/tools/kernel_symbols_tool/main.rb",
    "content": "#!/usr/bin/env ruby\n# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'rubygems'\nrequire 'bundler/setup'\nrequire 'colorize'\nrequire 'elftools'\n\nrequire_relative 'kernel_elf'\nrequire_relative 'cmds'\n\nKERNEL_SYMBOLS_SECTION = '.kernel_symbols'\nNUM_KERNEL_SYMBOLS = 'NUM_KERNEL_SYMBOLS'\n\ncmd = ARGV[0]\n\nkernel_elf_path = ARGV[1]\nkernel_elf = KernelELF.new(kernel_elf_path, KERNEL_SYMBOLS_SECTION, NUM_KERNEL_SYMBOLS)\n\ncase cmd\nwhen '--gen_symbols'\n    output_file = ARGV[2]\n\n    print 'Generating'.rjust(12).green.bold\n    puts ' Symbols source file'\n\n    generate_symbols(kernel_elf, output_file)\nwhen '--get_symbols_section_virt_addr'\n    addr = get_symbols_section_virt_addr(kernel_elf)\n\n    puts \"0x#{addr.to_s(16)}\"\nwhen '--patch_data'\n    symbols_blob_path = ARGV[2]\n    num_symbols = kernel_elf.num_symbols\n\n    print 'Patching'.rjust(12).green.bold\n    puts \" Symbols blob and number of symbols (#{num_symbols}) into ELF\"\n\n    patch_symbol_data(kernel_elf, symbols_blob_path)\n    patch_num_symbols(kernel_elf)\nelse\n    raise\nend\n"
  },
  {
    "path": "17_kernel_symbols/tools/translation_table_tool/arch.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n# Bitfield manipulation.\nclass BitField\n    def initialize\n        @value = 0\n    end\n\n    def self.attr_bitfield(name, offset, num_bits)\n        define_method(\"#{name}=\") do |bits|\n            mask = (2**num_bits) - 1\n\n            raise \"Input out of range: #{name} = 0x#{bits.to_s(16)}\" if (bits & ~mask).positive?\n\n            # Clear bitfield\n            @value &= ~(mask << offset)\n\n            # Set it\n            @value |= (bits << offset)\n        end\n    end\n\n    def to_i\n        @value\n    end\n\n    def size_in_byte\n        8\n    end\nend\n\n# An array class that knows its memory location.\nclass CArray < Array\n    attr_reader :phys_start_addr\n\n    def initialize(phys_start_addr, size, &block)\n        @phys_start_addr = phys_start_addr\n\n        super(size, &block)\n    end\n\n    def size_in_byte\n        inject(0) { |sum, n| sum + n.size_in_byte }\n    end\nend\n\n#---------------------------------------------------------------------------------------------------\n# Arch::\n#---------------------------------------------------------------------------------------------------\nmodule Arch\n#---------------------------------------------------------------------------------------------------\n# Arch::ARMv8\n#---------------------------------------------------------------------------------------------------\nmodule ARMv8\n# ARMv8 Table Descriptor.\nclass Stage1TableDescriptor < BitField\n    module NextLevelTableAddr\n        OFFSET = 16\n        NUMBITS = 32\n    end\n\n    module Type\n        OFFSET = 1\n        NUMBITS = 1\n\n        BLOCK = 0\n        TABLE = 1\n    end\n\n    module Valid\n        OFFSET = 0\n        NUMBITS = 1\n\n        FALSE = 0\n        TRUE = 1\n    end\n\n    attr_bitfield(:__next_level_table_addr, NextLevelTableAddr::OFFSET, NextLevelTableAddr::NUMBITS)\n    attr_bitfield(:type, Type::OFFSET, Type::NUMBITS)\n    attr_bitfield(:valid, Valid::OFFSET, Valid::NUMBITS)\n\n    def next_level_table_addr=(addr)\n        addr >>= Granule64KiB::SHIFT\n\n        self.__next_level_table_addr = addr\n    end\n\n    private :__next_level_table_addr=\nend\n\n# ARMv8 level 3 page descriptor.\nclass Stage1PageDescriptor < BitField\n    module UXN\n        OFFSET = 54\n        NUMBITS = 1\n\n        FALSE = 0\n        TRUE = 1\n    end\n\n    module PXN\n        OFFSET = 53\n        NUMBITS = 1\n\n        FALSE = 0\n        TRUE = 1\n    end\n\n    module OutputAddr\n        OFFSET = 16\n        NUMBITS = 32\n    end\n\n    module AF\n        OFFSET = 10\n        NUMBITS = 1\n\n        FALSE = 0\n        TRUE = 1\n    end\n\n    module SH\n        OFFSET = 8\n        NUMBITS = 2\n\n        INNER_SHAREABLE = 0b11\n    end\n\n    module AP\n        OFFSET = 6\n        NUMBITS = 2\n\n        RW_EL1 = 0b00\n        RO_EL1 = 0b10\n    end\n\n    module AttrIndx\n        OFFSET = 2\n        NUMBITS = 3\n    end\n\n    module Type\n        OFFSET = 1\n        NUMBITS = 1\n\n        RESERVED_INVALID = 0\n        PAGE = 1\n    end\n\n    module Valid\n        OFFSET = 0\n        NUMBITS = 1\n\n        FALSE = 0\n        TRUE = 1\n    end\n\n    attr_bitfield(:uxn, UXN::OFFSET, UXN::NUMBITS)\n    attr_bitfield(:pxn, PXN::OFFSET, PXN::NUMBITS)\n    attr_bitfield(:__output_addr, OutputAddr::OFFSET, OutputAddr::NUMBITS)\n    attr_bitfield(:af, AF::OFFSET, AF::NUMBITS)\n    attr_bitfield(:sh, SH::OFFSET, SH::NUMBITS)\n    attr_bitfield(:ap, AP::OFFSET, AP::NUMBITS)\n    attr_bitfield(:attr_indx, AttrIndx::OFFSET, AttrIndx::NUMBITS)\n    attr_bitfield(:type, Type::OFFSET, Type::NUMBITS)\n    attr_bitfield(:valid, Valid::OFFSET, Valid::NUMBITS)\n\n    def output_addr=(addr)\n        addr >>= Granule64KiB::SHIFT\n\n        self.__output_addr = addr\n    end\n\n    private :__output_addr=\nend\n\n# Translation table representing the structure defined in translation_table.rs.\nclass TranslationTable\n    module MAIR\n        NORMAL = 1\n    end\n\n    def initialize\n        do_sanity_checks\n\n        num_lvl2_tables = BSP.kernel_virt_addr_space_size >> Granule512MiB::SHIFT\n\n        @lvl3 = new_lvl3(num_lvl2_tables, BSP.phys_addr_of_kernel_tables)\n\n        @lvl2_phys_start_addr = @lvl3.phys_start_addr + @lvl3.size_in_byte\n        @lvl2 = new_lvl2(num_lvl2_tables, @lvl2_phys_start_addr)\n\n        populate_lvl2_entries\n    end\n\n    def map_at(virt_region, phys_region, attributes)\n        return if virt_region.empty?\n\n        raise if virt_region.size != phys_region.size\n        raise if phys_region.last > BSP.phys_addr_space_end_page\n\n        virt_region.zip(phys_region).each do |virt_page, phys_page|\n            desc = page_descriptor_from(virt_page)\n            set_lvl3_entry(desc, phys_page, attributes)\n        end\n    end\n\n    def to_binary\n        data = @lvl3.flatten.map(&:to_i) + @lvl2.map(&:to_i)\n        data.pack('Q<*') # \"Q\" == uint64_t, \"<\" == little endian\n    end\n\n    def phys_tables_base_addr_binary\n        [@lvl2_phys_start_addr].pack('Q<*') # \"Q\" == uint64_t, \"<\" == little endian\n    end\n\n    def phys_tables_base_addr\n        @lvl2_phys_start_addr\n    end\n\n    private\n\n    def do_sanity_checks\n        raise unless BSP.kernel_granule::SIZE == Granule64KiB::SIZE\n        raise unless (BSP.kernel_virt_addr_space_size % Granule512MiB::SIZE).zero?\n    end\n\n    def new_lvl3(num_lvl2_tables, start_addr)\n        CArray.new(start_addr, num_lvl2_tables) do\n            temp = CArray.new(start_addr, 8192) do\n                Stage1PageDescriptor.new\n            end\n            start_addr += temp.size_in_byte\n\n            temp\n        end\n    end\n\n    def new_lvl2(num_lvl2_tables, start_addr)\n        CArray.new(start_addr, num_lvl2_tables) do\n            Stage1TableDescriptor.new\n        end\n    end\n\n    def populate_lvl2_entries\n        @lvl2.each_with_index do |descriptor, i|\n            descriptor.next_level_table_addr = @lvl3[i].phys_start_addr\n            descriptor.type = Stage1TableDescriptor::Type::TABLE\n            descriptor.valid = Stage1TableDescriptor::Valid::TRUE\n        end\n    end\n\n    def lvl2_lvl3_index_from(addr)\n        addr -= BSP.kernel_virt_start_addr\n\n        lvl2_index = addr >> Granule512MiB::SHIFT\n        lvl3_index = (addr & Granule512MiB::MASK) >> Granule64KiB::SHIFT\n\n        raise unless lvl2_index < @lvl2.size\n\n        [lvl2_index, lvl3_index]\n    end\n\n    def page_descriptor_from(virt_addr)\n        lvl2_index, lvl3_index = lvl2_lvl3_index_from(virt_addr)\n\n        @lvl3[lvl2_index][lvl3_index]\n    end\n\n    # rubocop:disable Metrics/MethodLength\n    def set_attributes(desc, attributes)\n        case attributes.mem_attributes\n        when :CacheableDRAM\n            desc.sh = Stage1PageDescriptor::SH::INNER_SHAREABLE\n            desc.attr_indx = MAIR::NORMAL\n        else\n            raise 'Invalid input'\n        end\n\n        desc.ap = case attributes.acc_perms\n                  when :ReadOnly\n                      Stage1PageDescriptor::AP::RO_EL1\n                  when :ReadWrite\n                      Stage1PageDescriptor::AP::RW_EL1\n                  else\n                      raise 'Invalid input'\n\n                  end\n\n        desc.pxn = if attributes.execute_never\n                       Stage1PageDescriptor::PXN::TRUE\n                   else\n                       Stage1PageDescriptor::PXN::FALSE\n                   end\n\n        desc.uxn = Stage1PageDescriptor::UXN::TRUE\n    end\n    # rubocop:enable Metrics/MethodLength\n\n    def set_lvl3_entry(desc, output_addr, attributes)\n        desc.output_addr = output_addr\n        desc.af = Stage1PageDescriptor::AF::TRUE\n        desc.type = Stage1PageDescriptor::Type::PAGE\n        desc.valid = Stage1PageDescriptor::Valid::TRUE\n\n        set_attributes(desc, attributes)\n    end\nend\nend\nend\n"
  },
  {
    "path": "17_kernel_symbols/tools/translation_table_tool/bsp.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n# Raspberry Pi 3 + 4\nclass RaspberryPi\n    attr_reader :kernel_granule, :kernel_virt_addr_space_size, :kernel_virt_start_addr\n\n    MEMORY_SRC = File.read('kernel/src/bsp/raspberrypi/memory.rs').split(\"\\n\")\n\n    def initialize\n        @kernel_granule = Granule64KiB\n\n        @kernel_virt_addr_space_size = KERNEL_ELF.symbol_value('__kernel_virt_addr_space_size')\n        @kernel_virt_start_addr = KERNEL_ELF.symbol_value('__kernel_virt_start_addr')\n\n        @virt_addr_of_kernel_tables = KERNEL_ELF.symbol_value('KERNEL_TABLES')\n        @virt_addr_of_phys_kernel_tables_base_addr = KERNEL_ELF.symbol_value(\n            'PHYS_KERNEL_TABLES_BASE_ADDR'\n        )\n    end\n\n    def phys_addr_of_kernel_tables\n        KERNEL_ELF.virt_to_phys(@virt_addr_of_kernel_tables)\n    end\n\n    def kernel_tables_offset_in_file\n        KERNEL_ELF.virt_addr_to_file_offset(@virt_addr_of_kernel_tables)\n    end\n\n    def phys_kernel_tables_base_addr_offset_in_file\n        KERNEL_ELF.virt_addr_to_file_offset(@virt_addr_of_phys_kernel_tables_base_addr)\n    end\n\n    def phys_addr_space_end_page\n        x = MEMORY_SRC.grep(/pub const END/)\n        x = case BSP_TYPE\n            when :rpi3\n                x[0]\n            when :rpi4\n                x[1]\n            else\n                raise\n            end\n\n        x.scan(/\\d+/).join.to_i(16)\n    end\nend\n"
  },
  {
    "path": "17_kernel_symbols/tools/translation_table_tool/generic.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\nmodule Granule64KiB\n    SIZE = 64 * 1024\n    SHIFT = Math.log2(SIZE).to_i\nend\n\nmodule Granule512MiB\n    SIZE = 512 * 1024 * 1024\n    SHIFT = Math.log2(SIZE).to_i\n    MASK = SIZE - 1\nend\n\n# Monkey-patch Integer with some helper functions.\nclass Integer\n    def power_of_two?\n        self[0].zero?\n    end\n\n    def aligned?(alignment)\n        raise unless alignment.power_of_two?\n\n        (self & (alignment - 1)).zero?\n    end\n\n    def align_up(alignment)\n        raise unless alignment.power_of_two?\n\n        (self + alignment - 1) & ~(alignment - 1)\n    end\n\n    def to_hex_underscore(with_leading_zeros: false)\n        fmt = with_leading_zeros ? '%016x' : '%x'\n        value = format(fmt, self).to_s.reverse.scan(/.{4}|.+/).join('_').reverse\n\n        format('0x%s', value)\n    end\nend\n\n# An array where each value is the start address of a Page.\nclass MemoryRegion < Array\n    def initialize(start_addr, size, granule_size)\n        raise unless start_addr.aligned?(granule_size)\n        raise unless size.positive?\n        raise unless (size % granule_size).zero?\n\n        num_pages = size / granule_size\n        super(num_pages) do |i|\n            (i * granule_size) + start_addr\n        end\n    end\nend\n\n# Collection of memory attributes.\nclass AttributeFields\n    attr_reader :mem_attributes, :acc_perms, :execute_never\n\n    def initialize(mem_attributes, acc_perms, execute_never)\n        @mem_attributes = mem_attributes\n        @acc_perms = acc_perms\n        @execute_never = execute_never\n    end\n\n    def to_s\n        x = case @mem_attributes\n            when :CacheableDRAM\n                'C'\n            else\n                '?'\n            end\n\n        y = case @acc_perms\n            when :ReadWrite\n                'RW'\n            when :ReadOnly\n                'RO'\n            else\n                '??'\n            end\n\n        z = @execute_never ? 'XN' : 'X '\n\n        \"#{x} #{y} #{z}\"\n    end\nend\n\n# A container that describes a virt-to-phys region mapping.\nclass MappingDescriptor\n    @max_section_name_length = 'Sections'.length\n\n    class << self\n        attr_accessor :max_section_name_length\n\n        def update_max_section_name_length(length)\n            @max_section_name_length = [@max_section_name_length, length].max\n        end\n    end\n\n    attr_reader :name, :virt_region, :phys_region, :attributes\n\n    def initialize(name, virt_region, phys_region, attributes)\n        @name = name\n        @virt_region = virt_region\n        @phys_region = phys_region\n        @attributes = attributes\n    end\n\n    def to_s\n        name = @name.ljust(self.class.max_section_name_length)\n        virt_start = @virt_region.first.to_hex_underscore(with_leading_zeros: true)\n        phys_start = @phys_region.first.to_hex_underscore(with_leading_zeros: true)\n        size = ((@virt_region.size * 65_536) / 1024).to_s.rjust(3)\n\n        \"#{name} | #{virt_start} | #{phys_start} | #{size} KiB | #{@attributes}\"\n    end\n\n    def self.print_divider\n        print '             '\n        print '-' * max_section_name_length\n        puts '--------------------------------------------------------------------'\n    end\n\n    def self.print_header\n        print_divider\n        print '             '\n        print 'Sections'.center(max_section_name_length)\n        print '   '\n        print 'Virt Start Addr'.center(21)\n        print '   '\n        print 'Phys Start Addr'.center(21)\n        print '   '\n        print 'Size'.center(7)\n        print '   '\n        print 'Attr'.center(7)\n        puts\n        print_divider\n    end\nend\n\ndef kernel_map_binary\n    mapping_descriptors = KERNEL_ELF.generate_mapping_descriptors\n\n    # Generate_mapping_descriptors updates the header being printed with this call. So it must come\n    # afterwards.\n    MappingDescriptor.print_header\n\n    mapping_descriptors.each do |i|\n        print 'Generating'.rjust(12).green.bold\n        print ' '\n        puts i\n\n        TRANSLATION_TABLES.map_at(i.virt_region, i.phys_region, i.attributes)\n    end\n\n    MappingDescriptor.print_divider\nend\n\ndef kernel_patch_tables(kernel_elf_path)\n    print 'Patching'.rjust(12).green.bold\n    print ' Kernel table struct at ELF file offset '\n    puts BSP.kernel_tables_offset_in_file.to_hex_underscore\n\n    File.binwrite(kernel_elf_path, TRANSLATION_TABLES.to_binary, BSP.kernel_tables_offset_in_file)\nend\n\ndef kernel_patch_base_addr(kernel_elf_path)\n    print 'Patching'.rjust(12).green.bold\n    print ' Kernel tables physical base address start argument to value '\n    print TRANSLATION_TABLES.phys_tables_base_addr.to_hex_underscore\n    print ' at ELF file offset '\n    puts BSP.phys_kernel_tables_base_addr_offset_in_file.to_hex_underscore\n\n    File.binwrite(kernel_elf_path, TRANSLATION_TABLES.phys_tables_base_addr_binary,\n                  BSP.phys_kernel_tables_base_addr_offset_in_file)\nend\n"
  },
  {
    "path": "17_kernel_symbols/tools/translation_table_tool/kernel_elf.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n# KernelELF\nclass KernelELF\n    SECTION_FLAG_ALLOC = 2\n\n    def initialize(kernel_elf_path)\n        @elf = ELFTools::ELFFile.new(File.open(kernel_elf_path))\n        @symtab_section = @elf.section_by_name('.symtab')\n    end\n\n    def machine\n        @elf.machine.to_sym\n    end\n\n    def symbol_value(symbol_name)\n        @symtab_section.symbol_by_name(symbol_name).header.st_value\n    end\n\n    def segment_containing_virt_addr(virt_addr)\n        @elf.each_segments do |segment|\n            return segment if segment.vma_in?(virt_addr)\n        end\n    end\n\n    def virt_to_phys(virt_addr)\n        segment = segment_containing_virt_addr(virt_addr)\n        translation_offset = segment.header.p_vaddr - segment.header.p_paddr\n\n        virt_addr - translation_offset\n    end\n\n    def virt_addr_to_file_offset(virt_addr)\n        segment = segment_containing_virt_addr(virt_addr)\n        segment.vma_to_offset(virt_addr)\n    end\n\n    def sections_in_segment(segment)\n        head = segment.mem_head\n        tail = segment.mem_tail\n\n        sections = @elf.each_sections.select do |section|\n            file_offset = section.header.sh_addr\n            flags = section.header.sh_flags\n\n            file_offset >= head && file_offset < tail && (flags & SECTION_FLAG_ALLOC != 0)\n        end\n\n        sections.map(&:name).join(' ')\n    end\n\n    def select_load_segments\n        @elf.each_segments.select do |segment|\n            segment.instance_of?(ELFTools::Segments::LoadSegment)\n        end\n    end\n\n    def segment_get_acc_perms(segment)\n        if segment.readable? && segment.writable?\n            :ReadWrite\n        elsif segment.readable?\n            :ReadOnly\n        else\n            :Invalid\n        end\n    end\n\n    def update_max_section_name_length(descriptors)\n        MappingDescriptor.update_max_section_name_length(descriptors.map { |i| i.name.size }.max)\n    end\n\n    def generate_mapping_descriptors\n        descriptors = select_load_segments.map do |segment|\n            # Assume each segment is page aligned.\n            size = segment.mem_size.align_up(BSP.kernel_granule::SIZE)\n            virt_start_addr = segment.header.p_vaddr\n            phys_start_addr = segment.header.p_paddr\n            acc_perms = segment_get_acc_perms(segment)\n            execute_never = !segment.executable?\n            section_names = sections_in_segment(segment)\n\n            virt_region = MemoryRegion.new(virt_start_addr, size, BSP.kernel_granule::SIZE)\n            phys_region = MemoryRegion.new(phys_start_addr, size, BSP.kernel_granule::SIZE)\n            attributes = AttributeFields.new(:CacheableDRAM, acc_perms, execute_never)\n\n            MappingDescriptor.new(section_names, virt_region, phys_region, attributes)\n        end\n\n        update_max_section_name_length(descriptors)\n        descriptors\n    end\nend\n"
  },
  {
    "path": "17_kernel_symbols/tools/translation_table_tool/main.rb",
    "content": "#!/usr/bin/env ruby\n# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'rubygems'\nrequire 'bundler/setup'\nrequire 'colorize'\nrequire 'elftools'\n\nrequire_relative 'generic'\nrequire_relative 'kernel_elf'\nrequire_relative 'bsp'\nrequire_relative 'arch'\n\nBSP_TYPE = ARGV[0].to_sym\nkernel_elf_path = ARGV[1]\n\nstart = Time.now\n\nKERNEL_ELF = KernelELF.new(kernel_elf_path)\n\nBSP = case BSP_TYPE\n      when :rpi3, :rpi4\n          RaspberryPi.new\n      else\n          raise\n      end\n\nTRANSLATION_TABLES = case KERNEL_ELF.machine\n                     when :AArch64\n                         Arch::ARMv8::TranslationTable.new\n                     else\n                         raise\n                     end\n\nkernel_map_binary\nkernel_patch_tables(kernel_elf_path)\nkernel_patch_base_addr(kernel_elf_path)\n\nelapsed = Time.now - start\n\nprint 'Finished'.rjust(12).green.bold\nputs \" in #{elapsed.round(2)}s\"\n"
  },
  {
    "path": "18_backtrace/.cargo/config.toml",
    "content": "[target.'cfg(target_os = \"none\")']\nrunner = \"target/kernel_test_runner.sh\"\n"
  },
  {
    "path": "18_backtrace/.vscode/settings.json",
    "content": "{\n    \"editor.formatOnSave\": true,\n    \"editor.rulers\": [100],\n    \"rust-analyzer.cargo.target\": \"aarch64-unknown-none-softfloat\",\n    \"rust-analyzer.cargo.features\": [\"bsp_rpi3\"],\n    \"rust-analyzer.checkOnSave.allTargets\": false,\n    \"rust-analyzer.checkOnSave.extraArgs\": [\"--lib\", \"--bins\"],\n    \"rust-analyzer.lens.debug\": false,\n    \"rust-analyzer.lens.run\": false\n}\n"
  },
  {
    "path": "18_backtrace/Cargo.toml",
    "content": "[workspace]\n\nmembers = [\n        \"libraries/*\",\n        \"kernel\",\n        \"kernel_symbols\"\n]\n\n[profile.release]\nlto = true\ndebug = true\n"
  },
  {
    "path": "18_backtrace/Makefile",
    "content": "## SPDX-License-Identifier: MIT OR Apache-2.0\n##\n## Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\ninclude ../common/docker.mk\ninclude ../common/format.mk\ninclude ../common/operating_system.mk\n\n##--------------------------------------------------------------------------------------------------\n## Optional, user-provided configuration values\n##--------------------------------------------------------------------------------------------------\n\n# Default to the RPi3.\nBSP ?= rpi3\n\n# Default to a serial device name that is common in Linux.\nDEV_SERIAL ?= /dev/ttyUSB0\n\n# Optional integration test name.\nifdef TEST\n    TEST_ARG = --test $(TEST)\nelse\n    TEST_ARG = --test '*'\nendif\n\n\n\n##--------------------------------------------------------------------------------------------------\n## BSP-specific configuration values\n##--------------------------------------------------------------------------------------------------\nQEMU_MISSING_STRING = \"This board is not yet supported for QEMU.\"\n\nifeq ($(BSP),rpi3)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE = raspi3\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    QEMU_TEST_ARGS    = $(QEMU_RELEASE_ARGS) -semihosting\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    OPENOCD_ARG       = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi3.cfg\n    JTAG_BOOT_IMAGE   = ../X1_JTAG_boot/jtag_boot_rpi3.img\n    LD_SCRIPT_PATH    = $(shell pwd)/kernel/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a53 -C force-frame-pointers\nelse ifeq ($(BSP),rpi4)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE =\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    QEMU_TEST_ARGS    = $(QEMU_RELEASE_ARGS) -semihosting\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    OPENOCD_ARG       = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi4.cfg\n    JTAG_BOOT_IMAGE   = ../X1_JTAG_boot/jtag_boot_rpi4.img\n    LD_SCRIPT_PATH    = $(shell pwd)/kernel/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a72 -C force-frame-pointers\nendif\n\n# Export for build.rs.\nexport LD_SCRIPT_PATH\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets and Prerequisites\n##--------------------------------------------------------------------------------------------------\nKERNEL_MANIFEST      = kernel/Cargo.toml\nKERNEL_LINKER_SCRIPT = kernel.ld\nLAST_BUILD_CONFIG    = target/$(BSP).build_config\n\nKERNEL_ELF_RAW      = target/$(TARGET)/release/kernel\n# This parses cargo's dep-info file.\n# https://doc.rust-lang.org/cargo/guide/build-cache.html#dep-info-files\nKERNEL_ELF_RAW_DEPS = $(filter-out %: ,$(file < $(KERNEL_ELF_RAW).d)) $(KERNEL_MANIFEST) $(LAST_BUILD_CONFIG)\n\n##------------------------------------------------------------------------------\n## Translation tables\n##------------------------------------------------------------------------------\nTT_TOOL_PATH = tools/translation_table_tool\n\nKERNEL_ELF_TTABLES      = target/$(TARGET)/release/kernel+ttables\nKERNEL_ELF_TTABLES_DEPS = $(KERNEL_ELF_RAW) $(wildcard $(TT_TOOL_PATH)/*)\n\n##------------------------------------------------------------------------------\n## Kernel symbols\n##------------------------------------------------------------------------------\nexport KERNEL_SYMBOLS_TOOL_PATH = tools/kernel_symbols_tool\n\nKERNEL_ELF_TTABLES_SYMS = target/$(TARGET)/release/kernel+ttables+symbols\n\n# Unlike with KERNEL_ELF_RAW, we are not relying on dep-info here. One of the reasons being that the\n# name of the generated symbols file varies between runs, which can cause confusion.\nKERNEL_ELF_TTABLES_SYMS_DEPS = $(KERNEL_ELF_TTABLES) \\\n    $(wildcard kernel_symbols/*)                     \\\n    $(wildcard $(KERNEL_SYMBOLS_TOOL_PATH)/*)\n\nexport TARGET\nexport KERNEL_SYMBOLS_INPUT_ELF  = $(KERNEL_ELF_TTABLES)\nexport KERNEL_SYMBOLS_OUTPUT_ELF = $(KERNEL_ELF_TTABLES_SYMS)\n\nKERNEL_ELF = $(KERNEL_ELF_TTABLES_SYMS)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Command building blocks\n##--------------------------------------------------------------------------------------------------\nRUSTFLAGS = $(RUSTC_MISC_ARGS)                   \\\n    -C link-arg=--library-path=$(LD_SCRIPT_PATH) \\\n    -C link-arg=--script=$(KERNEL_LINKER_SCRIPT)\n\nRUSTFLAGS_PEDANTIC = $(RUSTFLAGS) \\\n    -D warnings                   \\\n    -D missing_docs\n\nFEATURES      = --features bsp_$(BSP)\nCOMPILER_ARGS = --target=$(TARGET) \\\n    $(FEATURES)                    \\\n    --release\n\n# build-std can be skipped for helper commands that do not rely on correct stack frames and other\n# custom compiler options. This results in a huge speedup.\nRUSTC_CMD   = cargo rustc $(COMPILER_ARGS) -Z build-std=core --manifest-path $(KERNEL_MANIFEST)\nDOC_CMD     = cargo doc $(COMPILER_ARGS)\nCLIPPY_CMD  = cargo clippy $(COMPILER_ARGS)\nTEST_CMD    = cargo test $(COMPILER_ARGS) -Z build-std=core --manifest-path $(KERNEL_MANIFEST)\nOBJCOPY_CMD = rust-objcopy \\\n    --strip-all            \\\n    -O binary\n\nEXEC_QEMU          = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE)\nEXEC_TT_TOOL       = ruby $(TT_TOOL_PATH)/main.rb\nEXEC_TEST_DISPATCH = ruby ../common/tests/dispatch.rb\nEXEC_MINIPUSH      = ruby ../common/serial/minipush.rb\n\n##------------------------------------------------------------------------------\n## Dockerization\n##------------------------------------------------------------------------------\nDOCKER_CMD            = docker run -t --rm -v $(shell pwd):/work/tutorial -w /work/tutorial\nDOCKER_CMD_INTERACT   = $(DOCKER_CMD) -i\nDOCKER_ARG_DIR_COMMON = -v $(shell pwd)/../common:/work/common\nDOCKER_ARG_DIR_JTAG   = -v $(shell pwd)/../X1_JTAG_boot:/work/X1_JTAG_boot\nDOCKER_ARG_DEV        = --privileged -v /dev:/dev\nDOCKER_ARG_NET        = --network host\n\n# DOCKER_IMAGE defined in include file (see top of this file).\nDOCKER_QEMU  = $(DOCKER_CMD_INTERACT) $(DOCKER_IMAGE)\nDOCKER_TOOLS = $(DOCKER_CMD) $(DOCKER_IMAGE)\nDOCKER_TEST  = $(DOCKER_CMD) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\nDOCKER_GDB   = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_NET) $(DOCKER_IMAGE)\n\n# Dockerize commands, which require USB device passthrough, only on Linux.\nifeq ($(shell uname -s),Linux)\n    DOCKER_CMD_DEV = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_DEV)\n\n    DOCKER_CHAINBOOT = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\n    DOCKER_JTAGBOOT  = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_ARG_DIR_JTAG) $(DOCKER_IMAGE)\n    DOCKER_OPENOCD   = $(DOCKER_CMD_DEV) $(DOCKER_ARG_NET) $(DOCKER_IMAGE)\nelse\n    DOCKER_OPENOCD   = echo \"Not yet supported on non-Linux systems.\"; \\#\nendif\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: all doc qemu chainboot clippy clean readelf objdump nm check\n\nall: $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Save the configuration as a file, so make understands if it changed.\n##------------------------------------------------------------------------------\n$(LAST_BUILD_CONFIG):\n\t@rm -f target/*.build_config\n\t@mkdir -p target\n\t@touch $(LAST_BUILD_CONFIG)\n\n##------------------------------------------------------------------------------\n## Compile the kernel ELF\n##------------------------------------------------------------------------------\n$(KERNEL_ELF_RAW): $(KERNEL_ELF_RAW_DEPS)\n\t$(call color_header, \"Compiling kernel ELF - $(BSP)\")\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(RUSTC_CMD)\n\n##------------------------------------------------------------------------------\n## Precompute the kernel translation tables and patch them into the kernel ELF\n##------------------------------------------------------------------------------\n$(KERNEL_ELF_TTABLES): $(KERNEL_ELF_TTABLES_DEPS)\n\t$(call color_header, \"Precomputing kernel translation tables and patching kernel ELF\")\n\t@cp $(KERNEL_ELF_RAW) $(KERNEL_ELF_TTABLES)\n\t@$(DOCKER_TOOLS) $(EXEC_TT_TOOL) $(BSP) $(KERNEL_ELF_TTABLES)\n\n##------------------------------------------------------------------------------\n## Generate kernel symbols and patch them into the kernel ELF\n##------------------------------------------------------------------------------\n$(KERNEL_ELF_TTABLES_SYMS): $(KERNEL_ELF_TTABLES_SYMS_DEPS)\n\t$(call color_header, \"Generating kernel symbols and patching kernel ELF\")\n\t@$(MAKE) --no-print-directory -f kernel_symbols.mk\n\n##------------------------------------------------------------------------------\n## Generate the stripped kernel binary\n##------------------------------------------------------------------------------\n$(KERNEL_BIN): $(KERNEL_ELF_TTABLES_SYMS)\n\t$(call color_header, \"Generating stripped binary\")\n\t@$(OBJCOPY_CMD) $(KERNEL_ELF_TTABLES_SYMS) $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Name\")\n\t@echo $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Size\")\n\t$(call disk_usage_KiB, $(KERNEL_BIN))\n\n##------------------------------------------------------------------------------\n## Generate the documentation\n##------------------------------------------------------------------------------\ndoc: clean\n\t$(call color_header, \"Generating docs\")\n\t@$(DOC_CMD) --document-private-items --open\n\n##------------------------------------------------------------------------------\n## Run the kernel in QEMU\n##------------------------------------------------------------------------------\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\nqemu:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\nqemu: $(KERNEL_BIN)\n\t$(call color_header, \"Launching QEMU\")\n\t@$(DOCKER_QEMU) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\nendif\n\n##------------------------------------------------------------------------------\n## Push the kernel to the real HW target\n##------------------------------------------------------------------------------\nchainboot: $(KERNEL_BIN)\n\t@$(DOCKER_CHAINBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run clippy\n##------------------------------------------------------------------------------\nclippy:\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(CLIPPY_CMD)\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(CLIPPY_CMD) --features test_build --tests \\\n                --manifest-path $(KERNEL_MANIFEST)\n\n##------------------------------------------------------------------------------\n## Clean\n##------------------------------------------------------------------------------\nclean:\n\trm -rf target $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run readelf\n##------------------------------------------------------------------------------\nreadelf: $(KERNEL_ELF)\n\t$(call color_header, \"Launching readelf\")\n\t@$(DOCKER_TOOLS) $(READELF_BINARY) --headers $(KERNEL_ELF)\n\n##------------------------------------------------------------------------------\n## Run objdump\n##------------------------------------------------------------------------------\nobjdump: $(KERNEL_ELF)\n\t$(call color_header, \"Launching objdump\")\n\t@$(DOCKER_TOOLS) $(OBJDUMP_BINARY) --disassemble --demangle \\\n                --section .text   \\\n                --section .rodata \\\n                $(KERNEL_ELF) | rustfilt\n\n##------------------------------------------------------------------------------\n## Run nm\n##------------------------------------------------------------------------------\nnm: $(KERNEL_ELF)\n\t$(call color_header, \"Launching nm\")\n\t@$(DOCKER_TOOLS) $(NM_BINARY) --demangle --print-size $(KERNEL_ELF) | sort | rustfilt\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Debugging targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: jtagboot openocd gdb gdb-opt0\n\n##------------------------------------------------------------------------------\n## Push the JTAG boot image to the real HW target\n##------------------------------------------------------------------------------\njtagboot:\n\t@$(DOCKER_JTAGBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(JTAG_BOOT_IMAGE)\n\n##------------------------------------------------------------------------------\n## Start OpenOCD session\n##------------------------------------------------------------------------------\nopenocd:\n\t$(call color_header, \"Launching OpenOCD\")\n\t@$(DOCKER_OPENOCD) openocd $(OPENOCD_ARG)\n\n##------------------------------------------------------------------------------\n## Start GDB session\n##------------------------------------------------------------------------------\ngdb-opt0: RUSTC_MISC_ARGS += -C opt-level=0\ngdb gdb-opt0: $(KERNEL_ELF)\n\t$(call color_header, \"Launching GDB\")\n\t@$(DOCKER_GDB) gdb-multiarch -q $(KERNEL_ELF)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Testing targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: test test_boot test_unit test_integration\n\ntest_unit test_integration: FEATURES += --features test_build\n\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\ntest_boot test_unit test_integration test:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\n##------------------------------------------------------------------------------\n## Run boot test\n##------------------------------------------------------------------------------\ntest_boot: $(KERNEL_BIN)\n\t$(call color_header, \"Boot test - $(BSP)\")\n\t@$(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Helpers for unit and integration test targets\n##------------------------------------------------------------------------------\ndefine KERNEL_TEST_RUNNER\n#!/usr/bin/env bash\n\n    # The cargo test runner seems to change into the crate under test's directory. Therefore, ensure\n    # this script executes from the root.\n    cd $(shell pwd)\n\n    TEST_ELF=$$(echo $$1 | sed -e 's/.*target/target/g')\n    TEST_ELF_SYMS=\"$${TEST_ELF}_syms\"\n    TEST_BINARY=$$(echo $$1.img | sed -e 's/.*target/target/g')\n\n    $(DOCKER_TOOLS) $(EXEC_TT_TOOL) $(BSP) $$TEST_ELF > /dev/null\n\n    # This overrides the two ENV variables. The other ENV variables that are required as input for\n    # the .mk file are set already because they are exported by this Makefile and this script is\n    # started by the same.\n    KERNEL_SYMBOLS_INPUT_ELF=$$TEST_ELF           \\\n        KERNEL_SYMBOLS_OUTPUT_ELF=$$TEST_ELF_SYMS \\\n        $(MAKE) --no-print-directory -f kernel_symbols.mk > /dev/null 2>&1\n\n    $(OBJCOPY_CMD) $$TEST_ELF_SYMS $$TEST_BINARY\n    $(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_TEST_ARGS) -kernel $$TEST_BINARY\nendef\n\nexport KERNEL_TEST_RUNNER\n\ndefine test_prepare\n    @mkdir -p target\n    @echo \"$$KERNEL_TEST_RUNNER\" > target/kernel_test_runner.sh\n    @chmod +x target/kernel_test_runner.sh\nendef\n\n##------------------------------------------------------------------------------\n## Run unit test(s)\n##------------------------------------------------------------------------------\ntest_unit:\n\t$(call color_header, \"Compiling unit test(s) - $(BSP)\")\n\t$(call test_prepare)\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(TEST_CMD) --lib\n\n##------------------------------------------------------------------------------\n## Run integration test(s)\n##------------------------------------------------------------------------------\ntest_integration:\n\t$(call color_header, \"Compiling integration test(s) - $(BSP)\")\n\t$(call test_prepare)\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(TEST_CMD) $(TEST_ARG)\n\ntest: test_boot test_unit test_integration\n\nendif\n"
  },
  {
    "path": "18_backtrace/README.md",
    "content": "# Tutorial 18 - Backtracing\n\n## tl;dr\n\n- Support for [`backtracing`] is implemented into the kernel.\n\n```console\n[    0.002782] Writing to bottom of address space to address 1 GiB...\n[    0.004623] Kernel panic!\n\nPanic location:\n      File 'kernel/src/_arch/aarch64/exception.rs', line 59, column 5\n\n[...]\n\nBacktrace:\n      ----------------------------------------------------------------------------------------------\n          Address            Function containing address\n      ----------------------------------------------------------------------------------------------\n       1. ffffffffc0005560 | libkernel::panic_wait::_panic_print\n       2. ffffffffc00054a0 | rust_begin_unwind\n       3. ffffffffc0002950 | core::panicking::panic_fmt\n       4. ffffffffc0004898 | current_elx_synchronous\n       5. ffffffffc0000a74 | __vector_current_elx_synchronous\n       6. ffffffffc000111c | kernel_init\n      ----------------------------------------------------------------------------------------------\n```\n\n[`backtracing`]: https://en.wikipedia.org/wiki/Stack_trace\n\n## Table of Contents\n\n- [Introduction](#introduction)\n- [Implementation](#implementation)\n  - [Chasing Frames](#chasing-frames)\n  - [Compiler Changes](#compiler-changes)\n  - [Supporting Changes](#supporting-changes)\n- [Test it](#test-it)\n- [Diff to previous](#diff-to-previous)\n\n## Introduction\n\nSince the kernel gained support for looking up `symbol names` in the previous tutorial, it is now\npossible to implement support for printing meaningful backtraces (also called `stack traces`). The\nprimary use-case will be printing backtraces during a `panic`, which will ease debugging. This is a\ngood time to add this feature, since some of the upcoming tutorials will cover complex topics and\ncode changes, so that this will come in handy during development.\n\n## Implementation\n\nSince backtracing is a scheme that is usually defined in the [`calling-convention`], and therefore\ntightly coupled to the `processor architecture `, the heart of the backtracing code will live in the\n`_arch` folder. What can be shared between different architectures is the formatting and printing\npart. Hence, the code will be organized as follows:\n\n[`calling-convention`]: https://en.wikipedia.org/wiki/Calling_convention\n\n- `src/backtrace.rs` makes a generic definition of a `BacktraceItem`. It also provides code that\n  uses an `Iterator<Item = BacktraceItem>` to format and print the backtrace.\n- `src/__arch_name__/backtrace.rs` contains the code that generates the actual iterator.\n\nHere is the definition of `BacktraceItem`:\n\n```rust\npub enum BacktraceItem {\n    InvalidFramePointer(Address<Virtual>),\n    InvalidLink(Address<Virtual>),\n    Link(Address<Virtual>),\n}\n```\n\nIn summary, it has two error cases and one valid case. This will become clearer in a minute when we\nlook at what a `stack frame` and a `frame pointer` is.\n\n### Chasing Frames\n\nFor `AArch64`, we need to consult the [Procedure Call Standard for the Arm® 64-bit Architecture]\n(`AAPCS64`). It has the following to say:\n\n> Conforming code shall construct a *linked list* of stack-frames. Each frame shall link to the\n> frame of its caller by means of a frame record of two 64-bit values on the stack (independent of\n> the data model). The frame record for the innermost frame (belonging to the most recent routine\n> invocation) shall be pointed to by the frame pointer register (FP). The lowest addressed\n> double-word shall point to the previous frame record and the highest addressed double-word shall\n> contain the value passed in LR on entry to the current function [...]. The location of the frame\n> record within a stack frame is not specified.\n\n[Procedure Call Standard for the Arm® 64-bit Architecture]: https://github.com/ARM-software/abi-aa/blob/main/aapcs64/aapcs64.rst\n\nThe nature of the `linked list` becomes a bit clearer when we look into the corresponding section in\nthe [ARM Cortex-A Series Programmer’s Guide for ARMv8-A] as well. Here are text and picture\nsnippets:\n\n> An AAPC64 stack frame shown in Figure 9-2. The frame pointer (X29) should point to the previous\n> frame pointer saved on stack, with the saved LR (X30) stored after it. The final frame pointer in\n> the chain should be set to 0. The Stack Pointer must always be aligned on a 16 byte boundary.\n\n[ARM Cortex-A Series Programmer’s Guide for ARMv8-A]: https://developer.arm.com/documentation/den0024/latest/\n\n<p align=\"center\">\n    <img src=\"../doc/18_stack_frames.png\" height=\"500\" align=\"center\">\n</p>\n\nHence, we can define the following struct in `src/__arch_name__/backtrace.rs` for the stack frame\nrecord:\n\n```rust\n#[repr(C)]\nstruct StackFrameRecord<'a> {\n    previous_record: Option<&'a StackFrameRecord<'a>>,\n    link: Address<Virtual>,\n}\n```\n\nThe interesting part is the `previous_record` member. We learned from the two documents which we\ninspected above that the lowest addressed double-word is either:\n\n- Zero.\n- Or pointing to the previous stack frame record.\n\nThanks to Rust's null pointer optimization [[1]][[2]], this allows us to conveniently type this as\nan `Option<&StackFrameRecord>`. So whenever we inspect `previous_record` and observe it to be\n`None`, we know that we've reached the end of the backtrace.\n\n[1]: https://doc.rust-lang.org/std/option/#representation\n[2]: https://stackoverflow.com/a/46557737\n\nThe start of the backtrace is trivially accessed through `x29` (aka the `Frame Pointer Register`).\nThis is used to generate a `StackFrameIterator`:\n\n```rust\nstruct StackFrameRecordIterator<'a> {\n    cur: &'a StackFrameRecord<'a>,\n}\n\n/// [...]\n\nfn stack_frame_record_iterator<'a>() -> Option<StackFrameRecordIterator<'a>> {\n    let fp = Address::<Virtual>::new(FP.get() as usize);\n    if !fp.is_valid_stack_addr() {\n        return None;\n    }\n\n    Some(StackFrameRecordIterator {\n        cur: unsafe { &*(fp.as_usize() as *const _) },\n    })\n}\n```\n\nAlthough it should be guaranteed by the compiler (and any hand-written assembly) that `x29` points\nto a valid stack address, it makes sense to double-check this before generating a reference. There\nis always a chance that corruption happens. The implementation of the iterator itself does this\nsanity check as well whenever the iterator is advanced. Additionally, it is also checked whether the\n`link` address points to a valid `code` section in the kernel before the address is passed on to\nthe caller of the iterator:\n\n```rust\nimpl<'a> Iterator for StackFrameRecordIterator<'a> {\n    type Item = BacktraceItem;\n\n    fn next(&mut self) -> Option<Self::Item> {\n        static ABORT_FRAME: StackFrameRecord = StackFrameRecord {\n            previous_record: None,\n            link: Address::new(0),\n        };\n\n        // If previous is None, this is the root frame, so iteration will stop here.\n        let previous = self.cur.previous_record?;\n\n        // Need to abort if the pointer to the previous frame record is invalid.\n        let prev_addr = Address::<Virtual>::new(previous as *const _ as usize);\n        if !prev_addr.is_valid_stack_addr() {\n            // This allows to return the error and then stop on the next iteration.\n            self.cur = &ABORT_FRAME;\n            return Some(BacktraceItem::InvalidFramePointer(prev_addr));\n        }\n\n        let ret = if !self.cur.link.is_valid_code_addr() {\n            Some(BacktraceItem::InvalidLink(self.cur.link))\n        } else {\n            // The link points to the instruction to be executed _after_ returning from a branch.\n            // However, we want to show the instruction that caused the branch, so subtract by one\n            // instruction.\n            //\n            // This might be called from panic!, so it must not panic itself on the subtraction.\n            let link = if self.cur.link >= Address::new(4) {\n                self.cur.link - 4\n            } else {\n                self.cur.link\n            };\n\n            Some(BacktraceItem::Link(link))\n        };\n\n        // Advance the iterator.\n        self.cur = previous;\n\n        ret\n    }\n}\n```\n\nThis already was the gist of the architectural part of the implementation! In the generic part,\nwhere the backtrace is printed, the address returned in `BacktraceItem::Link` is additionally used\nto look up the corresponding `symbol`, so that this is conveniently printed together:\n\n```rust\nmatch backtrace_res {\n\n    // omitted\n\n    BacktraceItem::Link(addr) => {\n        fmt_res = writeln!(\n            f,\n            \"      {:>2}. {:016x} | {:<50}\",\n            i + 1,\n            addr.as_usize(),\n            match symbols::lookup_symbol(addr) {\n                Some(sym) => sym.name(),\n                _ => \"Symbol not found\",\n            }\n        )\n    }\n};\n```\n\nFinally, we add printing of a backtrace to `panic!`:\n\n```rust\nprintln!(\n    \"[  {:>3}.{:06}] Kernel panic!\\n\\n\\\n    Panic location:\\n      File '{}', line {}, column {}\\n\\n\\\n    {}\\n\\n\\\n    {}\",\n    timestamp.as_secs(),\n    timestamp.subsec_micros(),\n    location,\n    line,\n    column,\n    info.message().unwrap_or(&format_args!(\"\")),\n    backtrace::Backtrace\n);\n```\n\n### Compiler Changes\n\nBy default, the `aarch64-unknown-none*` targets *do not* guarantee that a stack frame record is\ngenerated on each function call. Without, the backtracing code will not work. Fortunately,\ngeneration can be forced by modifying the `rustc codegen options`. We add the following to the\n`Makefile`:\n\n```makefile\nifeq ($(BSP),rpi3)\n\n    # omitted\n\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a53 -C force-frame-pointers\n```\n\nBut there is more! Until now, when we compiled the kernel, cargo was using a **precompiled** version\nof the `Rust core library` that comes with rustup whenever a target is added. This is usually very\nbeneficial in terms of speeding up compilation. Unfortunately, the precompiled version was not\ncompiled with `-C force-frame-pointers` either. This can be solved using cargo's [`build-std`\nfeature]. We set it in the Makefile so that cargo also compiles the core library using our compiler\nsettings, which means we get the frame records thanks to `-C force-frame-pointers` for any core\nlibrary functions as well.\n\n```Makefile\n# build-std can be skipped for helper commands that do not rely on correct stack frames and other\n# custom compiler options. This results in a huge speedup.\nRUSTC_CMD   = cargo rustc $(COMPILER_ARGS) -Z build-std=core --manifest-path $(KERNEL_MANIFEST)\nDOC_CMD     = cargo doc $(COMPILER_ARGS)\nCLIPPY_CMD  = cargo clippy $(COMPILER_ARGS)\nTEST_CMD    = cargo test $(COMPILER_ARGS) -Z build-std=core --manifest-path $(KERNEL_MANIFEST)\n```\n\n[`build-std` feature]: https://doc.rust-lang.org/cargo/reference/unstable.html#build-std\n\n### Supporting Changes\n\nThere's a couple of changes not covered in this tutorial text, but the reader should ideally skim\nthrough them:\n\n- [`src/_arch/aarch64/exception.s`](`kernel/src/_arch/aarch64/exception.s`) adds some tricky code to\n  generate a stack frame record on exception entry. The file includes elaborate comments that can be\n  inspected.\n- [`src/_arch/aarch64/cpu/boot.rs`](`kernel/src/_arch/aarch64/cpu/boot.rs`) adds some code that\n  ensures that `kernel_init()` becomes the root of the backtrace (meaning its is ensured that\n  `previous_frame` will be zero for `kernel_init()`'s frame record).\n- In `$ROOT/Cargo.toml`, `debug = true` has been set, which ensures that the kernel ELF includes the\n  maximum amount of debug information. Please note that this does *not* change anything for the\n  kernel at runtime. However, it will allow to dig even deeper on an address that has been reported\n  by a kernel backtrace. For example, using the `addr2line` tool. The following two snippets show\n  what `addr2line` reports when the debug flag is not or is set, respectively.\n\n```console\n$ # debug = false\n$ addr2line -p -f -s -i -e target/aarch64-unknown-none-softfloat/release/kernel+ttables+symbols 0xffffffffc0001da8 | rustfilt\nkernel::kernel_main at kernel.c562062a-cgu.1:?\n```\n\n```console\n$ # debug = true\n$ addr2line -p -f -s -i -e target/aarch64-unknown-none-softfloat/release/kernel+ttables+symbols 0xffffffffc0001da8 | rustfilt\nlibkernel::memory::mmu::mapping_record::MappingRecord::print at mapping_record.rs:136\n (inlined by) libkernel::memory::mmu::mapping_record::kernel_print::{{closure}} at mapping_record.rs:232\n (inlined by) <libkernel::synchronization::InitStateLock<T> as libkernel::synchronization::interface::ReadWriteEx>::read at synchronization.rs:139\n (inlined by) libkernel::memory::mmu::mapping_record::kernel_print at mapping_record.rs:232\n (inlined by) libkernel::memory::mmu::kernel_print_mappings at mmu.rs:269\n (inlined by) kernel::kernel_main at main.rs:84\n```\n\n## Test it\n\nThree tests were added that check the sanity of the backtracing code. Also, any previous tests that\nprint a `panic` will now also include a backtrace. For example, `02_exception_sync_page_fault.rs`:\n\n```console\n$ TEST=02_exception_sync_page_fault make test_integration\n[...]\n         -------------------------------------------------------------------\n         🦀 Testing synchronous exception handling by causing a page fault\n         -------------------------------------------------------------------\n\n         [    0.002782] Writing to bottom of address space to address 1 GiB...\n         [    0.004623] Kernel panic!\n\n         Panic location:\n               File 'kernel/src/_arch/aarch64/exception.rs', line 59, column 5\n\n         CPU Exception!\n\n         ESR_EL1: 0x96000004\n               Exception Class         (EC) : 0x25 - Data Abort, current EL\n               Instr Specific Syndrome (ISS): 0x4\n         FAR_EL1: 0x0000000040000000\n\n         [...]\n\n         Backtrace:\n               ----------------------------------------------------------------------------------------------\n                   Address            Function containing address\n               ----------------------------------------------------------------------------------------------\n                1. ffffffffc0005560 | libkernel::panic_wait::_panic_print\n                2. ffffffffc00054a0 | rust_begin_unwind\n                3. ffffffffc0002950 | core::panicking::panic_fmt\n                4. ffffffffc0004898 | current_elx_synchronous\n                5. ffffffffc0000a74 | __vector_current_elx_synchronous\n                6. ffffffffc000111c | kernel_init\n               ----------------------------------------------------------------------------------------------\n\n         -------------------------------------------------------------------\n         ✅ Success: 02_exception_sync_page_fault.rs\n         -------------------------------------------------------------------\n```\n\n## Diff to previous\n```diff\n\ndiff -uNr 17_kernel_symbols/Cargo.toml 18_backtrace/Cargo.toml\n--- 17_kernel_symbols/Cargo.toml\n+++ 18_backtrace/Cargo.toml\n@@ -8,3 +8,4 @@\n\n [profile.release]\n lto = true\n+debug = true\n\ndiff -uNr 17_kernel_symbols/kernel/Cargo.toml 18_backtrace/kernel/Cargo.toml\n--- 17_kernel_symbols/kernel/Cargo.toml\n+++ 18_backtrace/kernel/Cargo.toml\n@@ -1,6 +1,6 @@\n [package]\n name = \"mingo\"\n-version = \"0.17.0\"\n+version = \"0.18.0\"\n authors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\n edition = \"2021\"\n\n@@ -56,3 +56,15 @@\n [[test]]\n name = \"03_exception_restore_sanity\"\n harness = false\n+\n+[[test]]\n+name = \"05_backtrace_sanity\"\n+harness = false\n+\n+[[test]]\n+name = \"06_backtrace_invalid_frame\"\n+harness = false\n+\n+[[test]]\n+name = \"07_backtrace_invalid_link\"\n+harness = false\n\ndiff -uNr 17_kernel_symbols/kernel/src/_arch/aarch64/backtrace.rs 18_backtrace/kernel/src/_arch/aarch64/backtrace.rs\n--- 17_kernel_symbols/kernel/src/_arch/aarch64/backtrace.rs\n+++ 18_backtrace/kernel/src/_arch/aarch64/backtrace.rs\n@@ -0,0 +1,136 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Architectural backtracing support.\n+//!\n+//! # Orientation\n+//!\n+//! Since arch modules are imported into generic modules using the path attribute, the path of this\n+//! file is:\n+//!\n+//! crate::backtrace::arch_backtrace\n+\n+use crate::{\n+    backtrace::BacktraceItem,\n+    memory::{Address, Virtual},\n+};\n+use aarch64_cpu::registers::*;\n+use tock_registers::interfaces::Readable;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// A Stack frame record.\n+///\n+/// # Note\n+///\n+/// The convention is that `previous_record` is valid as long as it contains a non-null value.\n+/// Therefore, it is possible to type the member as `Option<&StackFrameRecord>` because of Rust's\n+/// `null-pointer optimization`.\n+#[repr(C)]\n+struct StackFrameRecord<'a> {\n+    previous_record: Option<&'a StackFrameRecord<'a>>,\n+    link: Address<Virtual>,\n+}\n+\n+struct StackFrameRecordIterator<'a> {\n+    cur: &'a StackFrameRecord<'a>,\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Code\n+//--------------------------------------------------------------------------------------------------\n+\n+impl<'a> Iterator for StackFrameRecordIterator<'a> {\n+    type Item = BacktraceItem;\n+\n+    fn next(&mut self) -> Option<Self::Item> {\n+        static ABORT_FRAME: StackFrameRecord = StackFrameRecord {\n+            previous_record: None,\n+            link: Address::new(0),\n+        };\n+\n+        // If previous is None, this is the root frame, so iteration will stop here.\n+        let previous = self.cur.previous_record?;\n+\n+        // Need to abort if the pointer to the previous frame record is invalid.\n+        let prev_addr = Address::<Virtual>::new(previous as *const _ as usize);\n+        if !prev_addr.is_valid_stack_addr() {\n+            // This allows to return the error and then stop on the next iteration.\n+            self.cur = &ABORT_FRAME;\n+            return Some(BacktraceItem::InvalidFramePointer(prev_addr));\n+        }\n+\n+        let ret = if !self.cur.link.is_valid_code_addr() {\n+            Some(BacktraceItem::InvalidLink(self.cur.link))\n+        } else {\n+            // The link points to the instruction to be executed _after_ returning from a branch.\n+            // However, we want to show the instruction that caused the branch, so subtract by one\n+            // instruction.\n+            //\n+            // This might be called from panic!, so it must not panic itself on the subtraction.\n+            let link = if self.cur.link >= Address::new(4) {\n+                self.cur.link - 4\n+            } else {\n+                self.cur.link\n+            };\n+\n+            Some(BacktraceItem::Link(link))\n+        };\n+\n+        // Advance the iterator.\n+        self.cur = previous;\n+\n+        ret\n+    }\n+}\n+\n+fn stack_frame_record_iterator<'a>() -> Option<StackFrameRecordIterator<'a>> {\n+    let fp = Address::<Virtual>::new(FP.get() as usize);\n+    if !fp.is_valid_stack_addr() {\n+        return None;\n+    }\n+\n+    Some(StackFrameRecordIterator {\n+        cur: unsafe { &*(fp.as_usize() as *const _) },\n+    })\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Architectural implementation of the backtrace.\n+pub fn backtrace(f: impl FnOnce(Option<&mut dyn Iterator<Item = BacktraceItem>>)) {\n+    f(stack_frame_record_iterator().as_mut().map(|s| s as _))\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Testing\n+//--------------------------------------------------------------------------------------------------\n+\n+#[cfg(feature = \"test_build\")]\n+#[inline(always)]\n+/// Hack for corrupting the previous frame address in the current stack frame.\n+///\n+/// # Safety\n+///\n+/// - To be used only by testing code.\n+pub unsafe fn corrupt_previous_frame_addr() {\n+    let sf = FP.get() as *mut usize;\n+    *sf = 0x123;\n+}\n+\n+#[cfg(feature = \"test_build\")]\n+#[inline(always)]\n+/// Hack for corrupting the link in the current stack frame.\n+///\n+/// # Safety\n+///\n+/// - To be used only by testing code.\n+pub unsafe fn corrupt_link() {\n+    let sf = FP.get() as *mut StackFrameRecord;\n+    (*sf).link = Address::new(0x456);\n+}\n\ndiff -uNr 17_kernel_symbols/kernel/src/_arch/aarch64/cpu/boot.rs 18_backtrace/kernel/src/_arch/aarch64/cpu/boot.rs\n--- 17_kernel_symbols/kernel/src/_arch/aarch64/cpu/boot.rs\n+++ 18_backtrace/kernel/src/_arch/aarch64/cpu/boot.rs\n@@ -13,7 +13,10 @@\n\n use crate::{memory, memory::Address};\n use aarch64_cpu::{asm, registers::*};\n-use core::arch::global_asm;\n+use core::{\n+    arch::global_asm,\n+    sync::atomic::{compiler_fence, Ordering},\n+};\n use tock_registers::interfaces::Writeable;\n\n // Assembly counterpart to this file.\n@@ -67,6 +70,18 @@\n     SP_EL1.set(virt_boot_core_stack_end_exclusive_addr);\n }\n\n+/// Reset the backtrace by setting link register and frame pointer to zero.\n+///\n+/// # Safety\n+///\n+/// - This function must only be used immediately before entering EL1.\n+#[inline(always)]\n+unsafe fn prepare_backtrace_reset() {\n+    compiler_fence(Ordering::SeqCst);\n+    FP.set(0);\n+    LR.set(0);\n+}\n+\n //--------------------------------------------------------------------------------------------------\n // Public Code\n //--------------------------------------------------------------------------------------------------\n@@ -93,6 +108,9 @@\n     let addr = Address::new(phys_kernel_tables_base_addr as usize);\n     memory::mmu::enable_mmu_and_caching(addr).unwrap();\n\n+    // Make the function we return to the root of a backtrace.\n+    prepare_backtrace_reset();\n+\n     // Use `eret` to \"return\" to EL1. Since virtual memory will already be enabled, this results in\n     // execution of kernel_init() in EL1 from its _virtual address_.\n     asm::eret()\n\ndiff -uNr 17_kernel_symbols/kernel/src/_arch/aarch64/exception.rs 18_backtrace/kernel/src/_arch/aarch64/exception.rs\n--- 17_kernel_symbols/kernel/src/_arch/aarch64/exception.rs\n+++ 18_backtrace/kernel/src/_arch/aarch64/exception.rs\n@@ -20,7 +20,11 @@\n };\n\n // Assembly counterpart to this file.\n-global_asm!(include_str!(\"exception.s\"));\n+global_asm!(\n+    include_str!(\"exception.s\"),\n+    CONST_ESR_EL1_EC_SHIFT = const 26,\n+    CONST_ESR_EL1_EC_VALUE_SVC64 = const 0x15\n+);\n\n //--------------------------------------------------------------------------------------------------\n // Private Definitions\n\ndiff -uNr 17_kernel_symbols/kernel/src/_arch/aarch64/exception.s 18_backtrace/kernel/src/_arch/aarch64/exception.s\n--- 17_kernel_symbols/kernel/src/_arch/aarch64/exception.s\n+++ 18_backtrace/kernel/src/_arch/aarch64/exception.s\n@@ -8,10 +8,10 @@\n\n /// Call the function provided by parameter `\\handler` after saving the exception context. Provide\n /// the context as the first parameter to '\\handler'.\n-.macro CALL_WITH_CONTEXT handler\n+.macro CALL_WITH_CONTEXT handler is_lower_el is_sync\n __vector_\\handler:\n \t// Make room on the stack for the exception context.\n-\tsub\tsp,  sp,  #16 * 17\n+\tsub\tsp,  sp,  #16 * 18\n\n \t// Store all general purpose registers on the stack.\n \tstp\tx0,  x1,  [sp, #16 * 0]\n@@ -39,6 +39,42 @@\n \tstp\tlr,  x1,  [sp, #16 * 15]\n \tstp\tx2,  x3,  [sp, #16 * 16]\n\n+\t// Build a stack frame for backtracing.\n+.if \\is_lower_el == 1\n+\t// If we came from a lower EL, make it a root frame (by storing zero) so that the kernel\n+\t// does not attempt to trace into userspace.\n+\tstp\txzr, xzr, [sp, #16 * 17]\n+.else\n+\t// For normal branches, the link address points to the instruction to be executed _after_\n+\t// returning from a branch. In a backtrace, we want to show the instruction that caused the\n+\t// branch, though. That is why code in backtrace.rs subtracts 4 (length of one instruction)\n+\t// from the link address.\n+\t//\n+\t// Here we have a special case, though, because ELR_EL1 is used instead of LR to build the\n+\t// stack frame, so that it becomes possible to trace beyond an exception. Hence, it must be\n+\t// considered that semantics for ELR_EL1 differ from case to case.\n+\t//\n+\t// Unless an \"exception generating instruction\" was executed, ELR_EL1 already points to the\n+\t// the correct instruction, and hence the subtraction by 4 in backtrace.rs would yield wrong\n+\t// results. To cover for this, 4 is added to ELR_EL1 below unless the cause of exception was\n+\t// an SVC instruction. BRK and HLT are \"exception generating instructions\" as well, but they\n+\t// are not expected and therefore left out for now.\n+\t//\n+\t// For reference: Search for \"preferred exception return address\" in the Architecture\n+\t// Reference Manual for ARMv8-A.\n+.if \\is_sync == 1\n+\tlsr\tw3,  w3, {CONST_ESR_EL1_EC_SHIFT}   // w3 = ESR_EL1.EC\n+\tcmp\tw3,  {CONST_ESR_EL1_EC_VALUE_SVC64} // w3 == SVC64 ?\n+\tb.eq\t1f\n+.endif\n+\tadd\tx1,  x1, #4\n+1:\n+\tstp\tx29, x1, [sp, #16 * 17]\n+.endif\n+\n+\t// Set the frame pointer to the stack frame record.\n+\tadd\tx29, sp, #16 * 17\n+\n \t// x0 is the first argument for the function called through `\\handler`.\n \tmov\tx0,  sp\n\n@@ -81,43 +117,43 @@\n //\n // - It must be ensured that `CALL_WITH_CONTEXT` <= 0x80 bytes.\n .org 0x000\n-\tCALL_WITH_CONTEXT current_el0_synchronous\n+\tCALL_WITH_CONTEXT current_el0_synchronous, 0, 1\n .org 0x080\n-\tCALL_WITH_CONTEXT current_el0_irq\n+\tCALL_WITH_CONTEXT current_el0_irq, 0, 0\n .org 0x100\n \tFIQ_SUSPEND\n .org 0x180\n-\tCALL_WITH_CONTEXT current_el0_serror\n+\tCALL_WITH_CONTEXT current_el0_serror, 0, 0\n\n // Current exception level with SP_ELx, x > 0.\n .org 0x200\n-\tCALL_WITH_CONTEXT current_elx_synchronous\n+\tCALL_WITH_CONTEXT current_elx_synchronous, 0, 1\n .org 0x280\n-\tCALL_WITH_CONTEXT current_elx_irq\n+\tCALL_WITH_CONTEXT current_elx_irq, 0, 0\n .org 0x300\n \tFIQ_SUSPEND\n .org 0x380\n-\tCALL_WITH_CONTEXT current_elx_serror\n+\tCALL_WITH_CONTEXT current_elx_serror, 0, 0\n\n // Lower exception level, AArch64\n .org 0x400\n-\tCALL_WITH_CONTEXT lower_aarch64_synchronous\n+\tCALL_WITH_CONTEXT lower_aarch64_synchronous, 1, 1\n .org 0x480\n-\tCALL_WITH_CONTEXT lower_aarch64_irq\n+\tCALL_WITH_CONTEXT lower_aarch64_irq, 1, 0\n .org 0x500\n \tFIQ_SUSPEND\n .org 0x580\n-\tCALL_WITH_CONTEXT lower_aarch64_serror\n+\tCALL_WITH_CONTEXT lower_aarch64_serror, 1, 0\n\n // Lower exception level, AArch32\n .org 0x600\n-\tCALL_WITH_CONTEXT lower_aarch32_synchronous\n+\tCALL_WITH_CONTEXT lower_aarch32_synchronous, 1, 0\n .org 0x680\n-\tCALL_WITH_CONTEXT lower_aarch32_irq\n+\tCALL_WITH_CONTEXT lower_aarch32_irq, 1, 0\n .org 0x700\n \tFIQ_SUSPEND\n .org 0x780\n-\tCALL_WITH_CONTEXT lower_aarch32_serror\n+\tCALL_WITH_CONTEXT lower_aarch32_serror, 1, 0\n .org 0x800\n\n //------------------------------------------------------------------------------\n@@ -146,7 +182,7 @@\n \tldp\tx26, x27, [sp, #16 * 13]\n \tldp\tx28, x29, [sp, #16 * 14]\n\n-\tadd\tsp,  sp,  #16 * 17\n+\tadd\tsp,  sp,  #16 * 18\n\n \teret\n\n\ndiff -uNr 17_kernel_symbols/kernel/src/backtrace.rs 18_backtrace/kernel/src/backtrace.rs\n--- 17_kernel_symbols/kernel/src/backtrace.rs\n+++ 18_backtrace/kernel/src/backtrace.rs\n@@ -0,0 +1,114 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Backtracing support.\n+\n+#[cfg(target_arch = \"aarch64\")]\n+#[path = \"_arch/aarch64/backtrace.rs\"]\n+mod arch_backtrace;\n+\n+use crate::{\n+    memory::{Address, Virtual},\n+    symbols,\n+};\n+use core::fmt;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Architectural Public Reexports\n+//--------------------------------------------------------------------------------------------------\n+#[cfg(feature = \"test_build\")]\n+pub use arch_backtrace::{corrupt_link, corrupt_previous_frame_addr};\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// A backtrace item.\n+#[allow(missing_docs)]\n+pub enum BacktraceItem {\n+    InvalidFramePointer(Address<Virtual>),\n+    InvalidLink(Address<Virtual>),\n+    Link(Address<Virtual>),\n+}\n+\n+/// Pseudo-struct for printing a backtrace using its fmt::Display implementation.\n+pub struct Backtrace;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+impl fmt::Display for Backtrace {\n+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n+        writeln!(f, \"Backtrace:\")?;\n+        writeln!(\n+            f,\n+            \"      ----------------------------------------------------------------------------------------------\"\n+        )?;\n+        writeln!(\n+            f,\n+            \"          Address            Function containing address\"\n+        )?;\n+        writeln!(\n+            f,\n+            \"      ----------------------------------------------------------------------------------------------\"\n+        )?;\n+\n+        let mut fmt_res: fmt::Result = Ok(());\n+        let trace_formatter =\n+            |maybe_iter: Option<&mut dyn Iterator<Item = BacktraceItem>>| match maybe_iter {\n+                None => fmt_res = writeln!(f, \"ERROR! No valid stack frame found\"),\n+                Some(iter) => {\n+                    // Since the backtrace is printed, the first function is always\n+                    // core::fmt::write. Skip 1 so it is excluded and doesn't bloat the output.\n+                    for (i, backtrace_res) in iter.skip(1).enumerate() {\n+                        match backtrace_res {\n+                            BacktraceItem::InvalidFramePointer(addr) => {\n+                                fmt_res = writeln!(\n+                                    f,\n+                                    \"      {:>2}. ERROR! \\\n+                                    Encountered invalid frame pointer ({}) during backtrace\",\n+                                    i + 1,\n+                                    addr\n+                                );\n+                            }\n+                            BacktraceItem::InvalidLink(addr) => {\n+                                fmt_res = writeln!(\n+                                    f,\n+                                    \"      {:>2}. ERROR! \\\n+                                    Link address ({}) is not contained in kernel .text section\",\n+                                    i + 1,\n+                                    addr\n+                                );\n+                            }\n+                            BacktraceItem::Link(addr) => {\n+                                fmt_res = writeln!(\n+                                    f,\n+                                    \"      {:>2}. {:016x} | {:<50}\",\n+                                    i + 1,\n+                                    addr.as_usize(),\n+                                    match symbols::lookup_symbol(addr) {\n+                                        Some(sym) => sym.name(),\n+                                        _ => \"Symbol not found\",\n+                                    }\n+                                )\n+                            }\n+                        };\n+\n+                        if fmt_res.is_err() {\n+                            break;\n+                        }\n+                    }\n+                }\n+            };\n+\n+        arch_backtrace::backtrace(trace_formatter);\n+        fmt_res?;\n+\n+        writeln!(\n+            f,\n+            \"      ----------------------------------------------------------------------------------------------\"\n+        )\n+    }\n+}\n\ndiff -uNr 17_kernel_symbols/kernel/src/bsp/raspberrypi/memory/mmu.rs 18_backtrace/kernel/src/bsp/raspberrypi/memory/mmu.rs\n--- 17_kernel_symbols/kernel/src/bsp/raspberrypi/memory/mmu.rs\n+++ 18_backtrace/kernel/src/bsp/raspberrypi/memory/mmu.rs\n@@ -80,16 +80,6 @@\n     size >> KernelGranule::SHIFT\n }\n\n-/// The code pages of the kernel binary.\n-fn virt_code_region() -> MemoryRegion<Virtual> {\n-    let num_pages = size_to_num_pages(super::code_size());\n-\n-    let start_page_addr = super::virt_code_start();\n-    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n-\n-    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n-}\n-\n /// The data pages of the kernel binary.\n fn virt_data_region() -> MemoryRegion<Virtual> {\n     let num_pages = size_to_num_pages(super::data_size());\n@@ -100,16 +90,6 @@\n     MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n }\n\n-/// The boot core stack pages.\n-fn virt_boot_core_stack_region() -> MemoryRegion<Virtual> {\n-    let num_pages = size_to_num_pages(super::boot_core_stack_size());\n-\n-    let start_page_addr = super::virt_boot_core_stack_start();\n-    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n-\n-    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n-}\n-\n // There is no reason to expect the following conversions to fail, since they were generated offline\n // by the `translation table tool`. If it doesn't work, a panic due to the unwraps is justified.\n fn kernel_virt_to_phys_region(virt_region: MemoryRegion<Virtual>) -> MemoryRegion<Physical> {\n@@ -132,6 +112,26 @@\n // Public Code\n //--------------------------------------------------------------------------------------------------\n\n+/// The code pages of the kernel binary.\n+pub fn virt_code_region() -> MemoryRegion<Virtual> {\n+    let num_pages = size_to_num_pages(super::code_size());\n+\n+    let start_page_addr = super::virt_code_start();\n+    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n+\n+    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n+}\n+\n+/// The boot core stack pages.\n+pub fn virt_boot_core_stack_region() -> MemoryRegion<Virtual> {\n+    let num_pages = size_to_num_pages(super::boot_core_stack_size());\n+\n+    let start_page_addr = super::virt_boot_core_stack_start();\n+    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n+\n+    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n+}\n+\n /// Return a reference to the kernel's translation tables.\n pub fn kernel_translation_tables() -> &'static InitStateLock<KernelTranslationTable> {\n     &KERNEL_TABLES\n\ndiff -uNr 17_kernel_symbols/kernel/src/lib.rs 18_backtrace/kernel/src/lib.rs\n--- 17_kernel_symbols/kernel/src/lib.rs\n+++ 18_backtrace/kernel/src/lib.rs\n@@ -133,6 +133,7 @@\n mod panic_wait;\n mod synchronization;\n\n+pub mod backtrace;\n pub mod bsp;\n pub mod common;\n pub mod console;\n\ndiff -uNr 17_kernel_symbols/kernel/src/memory.rs 18_backtrace/kernel/src/memory.rs\n--- 17_kernel_symbols/kernel/src/memory.rs\n+++ 18_backtrace/kernel/src/memory.rs\n@@ -95,6 +95,18 @@\n     }\n }\n\n+impl<ATYPE: AddressType> Sub<usize> for Address<ATYPE> {\n+    type Output = Self;\n+\n+    #[inline(always)]\n+    fn sub(self, rhs: usize) -> Self::Output {\n+        match self.value.checked_sub(rhs) {\n+            None => panic!(\"Overflow on Address::sub\"),\n+            Some(x) => Self::new(x),\n+        }\n+    }\n+}\n+\n impl<ATYPE: AddressType> Sub<Address<ATYPE>> for Address<ATYPE> {\n     type Output = Self;\n\n@@ -107,6 +119,18 @@\n     }\n }\n\n+impl Address<Virtual> {\n+    /// Checks if the address is part of the boot core stack region.\n+    pub fn is_valid_stack_addr(&self) -> bool {\n+        bsp::memory::mmu::virt_boot_core_stack_region().contains(*self)\n+    }\n+\n+    /// Checks if the address is part of the kernel code region.\n+    pub fn is_valid_code_addr(&self) -> bool {\n+        bsp::memory::mmu::virt_code_region().contains(*self)\n+    }\n+}\n+\n impl fmt::Display for Address<Physical> {\n     // Don't expect to see physical addresses greater than 40 bit.\n     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n\ndiff -uNr 17_kernel_symbols/kernel/src/panic_wait.rs 18_backtrace/kernel/src/panic_wait.rs\n--- 17_kernel_symbols/kernel/src/panic_wait.rs\n+++ 18_backtrace/kernel/src/panic_wait.rs\n@@ -4,7 +4,7 @@\n\n //! A panic handler that infinitely waits.\n\n-use crate::{cpu, exception, println};\n+use crate::{backtrace, cpu, exception, println};\n use core::panic::PanicInfo;\n\n //--------------------------------------------------------------------------------------------------\n@@ -73,6 +73,7 @@\n     println!(\n         \"[  {:>3}.{:06}] Kernel panic!\\n\\n\\\n         Panic location:\\n      File '{}', line {}, column {}\\n\\n\\\n+        {}\\n\\n\\\n         {}\",\n         timestamp.as_secs(),\n         timestamp.subsec_micros(),\n@@ -80,6 +81,7 @@\n         line,\n         column,\n         info.message().unwrap_or(&format_args!(\"\")),\n+        backtrace::Backtrace\n     );\n\n     _panic_exit()\n\ndiff -uNr 17_kernel_symbols/kernel/src/state.rs 18_backtrace/kernel/src/state.rs\n--- 17_kernel_symbols/kernel/src/state.rs\n+++ 18_backtrace/kernel/src/state.rs\n@@ -52,7 +52,7 @@\n     const SINGLE_CORE_MAIN: u8 = 1;\n     const MULTI_CORE_MAIN: u8 = 2;\n\n-    /// Create a new instance.\n+    /// Create an instance.\n     pub const fn new() -> Self {\n         Self(AtomicU8::new(Self::INIT))\n     }\n\ndiff -uNr 17_kernel_symbols/kernel/tests/05_backtrace_sanity.rb 18_backtrace/kernel/tests/05_backtrace_sanity.rb\n--- 17_kernel_symbols/kernel/tests/05_backtrace_sanity.rb\n+++ 18_backtrace/kernel/tests/05_backtrace_sanity.rb\n@@ -0,0 +1,39 @@\n+# frozen_string_literal: true\n+\n+# SPDX-License-Identifier: MIT OR Apache-2.0\n+#\n+# Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+require 'console_io_test'\n+\n+# Verify that panic produces a backtrace.\n+class PanicBacktraceTest < SubtestBase\n+    def name\n+        'Panic produces backtrace'\n+    end\n+\n+    def run(qemu_out, _qemu_in)\n+        expect_or_raise(qemu_out, 'Kernel panic!')\n+        expect_or_raise(qemu_out, 'Backtrace:')\n+    end\n+end\n+\n+# Verify backtrace correctness.\n+class BacktraceCorrectnessTest < SubtestBase\n+    def name\n+        'Backtrace is correct'\n+    end\n+\n+    def run(qemu_out, _qemu_in)\n+        expect_or_raise(qemu_out, '| core::panicking::panic')\n+        expect_or_raise(qemu_out, '| _05_backtrace_sanity::nested')\n+        expect_or_raise(qemu_out, '| kernel_init')\n+    end\n+end\n+\n+##--------------------------------------------------------------------------------------------------\n+## Test registration\n+##--------------------------------------------------------------------------------------------------\n+def subtest_collection\n+    [PanicBacktraceTest.new, BacktraceCorrectnessTest.new]\n+end\n\ndiff -uNr 17_kernel_symbols/kernel/tests/05_backtrace_sanity.rs 18_backtrace/kernel/tests/05_backtrace_sanity.rs\n--- 17_kernel_symbols/kernel/tests/05_backtrace_sanity.rs\n+++ 18_backtrace/kernel/tests/05_backtrace_sanity.rs\n@@ -0,0 +1,31 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Test if backtracing code detects an invalid frame pointer.\n+\n+#![feature(format_args_nl)]\n+#![no_main]\n+#![no_std]\n+\n+/// Console tests should time out on the I/O harness in case of panic.\n+mod panic_wait_forever;\n+\n+use libkernel::{bsp, cpu, exception, memory};\n+\n+#[inline(never)]\n+fn nested() {\n+    panic!()\n+}\n+\n+#[no_mangle]\n+unsafe fn kernel_init() -> ! {\n+    exception::handling_init();\n+    memory::init();\n+    bsp::driver::qemu_bring_up_console();\n+\n+    nested();\n+\n+    // The QEMU process running this test will be closed by the I/O test harness.\n+    cpu::wait_forever()\n+}\n\ndiff -uNr 17_kernel_symbols/kernel/tests/06_backtrace_invalid_frame.rb 18_backtrace/kernel/tests/06_backtrace_invalid_frame.rb\n--- 17_kernel_symbols/kernel/tests/06_backtrace_invalid_frame.rb\n+++ 18_backtrace/kernel/tests/06_backtrace_invalid_frame.rb\n@@ -0,0 +1,26 @@\n+# frozen_string_literal: true\n+\n+# SPDX-License-Identifier: MIT OR Apache-2.0\n+#\n+# Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+require 'console_io_test'\n+\n+# Test detection of invalid frame pointers.\n+class InvalidFramePointerTest < SubtestBase\n+    def name\n+        'Detect invalid frame pointer'\n+    end\n+\n+    def run(qemu_out, _qemu_in)\n+        expect_or_raise(qemu_out,\n+                        /Encountered invalid frame pointer \\(.*\\) during backtrace/)\n+    end\n+end\n+\n+##--------------------------------------------------------------------------------------------------\n+## Test registration\n+##--------------------------------------------------------------------------------------------------\n+def subtest_collection\n+    [InvalidFramePointerTest.new]\n+end\n\ndiff -uNr 17_kernel_symbols/kernel/tests/06_backtrace_invalid_frame.rs 18_backtrace/kernel/tests/06_backtrace_invalid_frame.rs\n--- 17_kernel_symbols/kernel/tests/06_backtrace_invalid_frame.rs\n+++ 18_backtrace/kernel/tests/06_backtrace_invalid_frame.rs\n@@ -0,0 +1,33 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Test if backtracing code detects an invalid frame pointer.\n+\n+#![feature(format_args_nl)]\n+#![no_main]\n+#![no_std]\n+\n+/// Console tests should time out on the I/O harness in case of panic.\n+mod panic_wait_forever;\n+\n+use libkernel::{backtrace, bsp, cpu, exception, memory};\n+\n+#[inline(never)]\n+fn nested() {\n+    unsafe { backtrace::corrupt_previous_frame_addr() };\n+\n+    panic!()\n+}\n+\n+#[no_mangle]\n+unsafe fn kernel_init() -> ! {\n+    exception::handling_init();\n+    memory::init();\n+    bsp::driver::qemu_bring_up_console();\n+\n+    nested();\n+\n+    // The QEMU process running this test will be closed by the I/O test harness.\n+    cpu::wait_forever()\n+}\n\ndiff -uNr 17_kernel_symbols/kernel/tests/07_backtrace_invalid_link.rb 18_backtrace/kernel/tests/07_backtrace_invalid_link.rb\n--- 17_kernel_symbols/kernel/tests/07_backtrace_invalid_link.rb\n+++ 18_backtrace/kernel/tests/07_backtrace_invalid_link.rb\n@@ -0,0 +1,25 @@\n+# frozen_string_literal: true\n+\n+# SPDX-License-Identifier: MIT OR Apache-2.0\n+#\n+# Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+require 'console_io_test'\n+\n+# Test detection of invalid link.\n+class InvalidLinkTest < SubtestBase\n+    def name\n+        'Detect invalid link'\n+    end\n+\n+    def run(qemu_out, _qemu_in)\n+        expect_or_raise(qemu_out, /Link address \\(.*\\) is not contained in kernel .text section/)\n+    end\n+end\n+\n+##--------------------------------------------------------------------------------------------------\n+## Test registration\n+##--------------------------------------------------------------------------------------------------\n+def subtest_collection\n+    [InvalidLinkTest.new]\n+end\n\ndiff -uNr 17_kernel_symbols/kernel/tests/07_backtrace_invalid_link.rs 18_backtrace/kernel/tests/07_backtrace_invalid_link.rs\n--- 17_kernel_symbols/kernel/tests/07_backtrace_invalid_link.rs\n+++ 18_backtrace/kernel/tests/07_backtrace_invalid_link.rs\n@@ -0,0 +1,38 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Test if backtracing code detects an invalid link.\n+\n+#![feature(format_args_nl)]\n+#![no_main]\n+#![no_std]\n+\n+/// Console tests should time out on the I/O harness in case of panic.\n+mod panic_wait_forever;\n+\n+use libkernel::{backtrace, bsp, cpu, exception, memory};\n+\n+#[inline(never)]\n+fn nested_2() -> &'static str {\n+    unsafe { backtrace::corrupt_link() };\n+    libkernel::println!(\"{}\", libkernel::backtrace::Backtrace);\n+    \"foo\"\n+}\n+\n+#[inline(never)]\n+fn nested_1() {\n+    libkernel::println!(\"{}\", nested_2())\n+}\n+\n+#[no_mangle]\n+unsafe fn kernel_init() -> ! {\n+    exception::handling_init();\n+    memory::init();\n+    bsp::driver::qemu_bring_up_console();\n+\n+    nested_1();\n+\n+    // The QEMU process running this test will be closed by the I/O test harness.\n+    cpu::wait_forever()\n+}\n\ndiff -uNr 17_kernel_symbols/Makefile 18_backtrace/Makefile\n--- 17_kernel_symbols/Makefile\n+++ 18_backtrace/Makefile\n@@ -43,7 +43,7 @@\n     OPENOCD_ARG       = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi3.cfg\n     JTAG_BOOT_IMAGE   = ../X1_JTAG_boot/jtag_boot_rpi3.img\n     LD_SCRIPT_PATH    = $(shell pwd)/kernel/src/bsp/raspberrypi\n-    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a53\n+    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a53 -C force-frame-pointers\n else ifeq ($(BSP),rpi4)\n     TARGET            = aarch64-unknown-none-softfloat\n     KERNEL_BIN        = kernel8.img\n@@ -57,7 +57,7 @@\n     OPENOCD_ARG       = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi4.cfg\n     JTAG_BOOT_IMAGE   = ../X1_JTAG_boot/jtag_boot_rpi4.img\n     LD_SCRIPT_PATH    = $(shell pwd)/kernel/src/bsp/raspberrypi\n-    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a72\n+    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a72 -C force-frame-pointers\n endif\n\n # Export for build.rs.\n@@ -122,10 +122,12 @@\n     $(FEATURES)                    \\\n     --release\n\n-RUSTC_CMD   = cargo rustc $(COMPILER_ARGS) --manifest-path $(KERNEL_MANIFEST)\n+# build-std can be skipped for helper commands that do not rely on correct stack frames and other\n+# custom compiler options. This results in a huge speedup.\n+RUSTC_CMD   = cargo rustc $(COMPILER_ARGS) -Z build-std=core --manifest-path $(KERNEL_MANIFEST)\n DOC_CMD     = cargo doc $(COMPILER_ARGS)\n CLIPPY_CMD  = cargo clippy $(COMPILER_ARGS)\n-TEST_CMD    = cargo test $(COMPILER_ARGS) --manifest-path $(KERNEL_MANIFEST)\n+TEST_CMD    = cargo test $(COMPILER_ARGS) -Z build-std=core --manifest-path $(KERNEL_MANIFEST)\n OBJCOPY_CMD = rust-objcopy \\\n     --strip-all            \\\n     -O binary\n@@ -302,8 +304,7 @@\n ##------------------------------------------------------------------------------\n ## Start GDB session\n ##------------------------------------------------------------------------------\n-gdb: RUSTC_MISC_ARGS += -C debuginfo=2\n-gdb-opt0: RUSTC_MISC_ARGS += -C debuginfo=2 -C opt-level=0\n+gdb-opt0: RUSTC_MISC_ARGS += -C opt-level=0\n gdb gdb-opt0: $(KERNEL_ELF)\n \t$(call color_header, \"Launching GDB\")\n \t@$(DOCKER_GDB) gdb-multiarch -q $(KERNEL_ELF)\n\ndiff -uNr 17_kernel_symbols/tools/translation_table_tool/bsp.rb 18_backtrace/tools/translation_table_tool/bsp.rb\n--- 17_kernel_symbols/tools/translation_table_tool/bsp.rb\n+++ 18_backtrace/tools/translation_table_tool/bsp.rb\n@@ -45,6 +45,10 @@\n                 raise\n             end\n\n-        x.scan(/\\d+/).join.to_i(16)\n+        # Extract the hex literal with underscores like 0x0123_abcd.\n+        x = x.scan(/0x[\\h_]*/)[0]\n+\n+        # Further remove x and _ and convert to int.\n+        x.scan(/\\h+/).join.to_i(16)\n     end\n end\n\ndiff -uNr 17_kernel_symbols/tools/translation_table_tool/generic.rb 18_backtrace/tools/translation_table_tool/generic.rb\n--- 17_kernel_symbols/tools/translation_table_tool/generic.rb\n+++ 18_backtrace/tools/translation_table_tool/generic.rb\n@@ -109,13 +109,23 @@\n         @attributes = attributes\n     end\n\n+    def size_human_readable(size)\n+        if size >= (1024 * 1024)\n+            \"#{(size / (1024 * 1024)).to_s.rjust(3)} MiB\"\n+        elsif size >= 1024\n+            \"#{(size / 1024).to_s.rjust(3)} KiB\"\n+        else\n+            raise\n+        end\n+    end\n+\n     def to_s\n         name = @name.ljust(self.class.max_section_name_length)\n         virt_start = @virt_region.first.to_hex_underscore(with_leading_zeros: true)\n         phys_start = @phys_region.first.to_hex_underscore(with_leading_zeros: true)\n-        size = ((@virt_region.size * 65_536) / 1024).to_s.rjust(3)\n+        size = size_human_readable(@virt_region.size * 65_536)\n\n-        \"#{name} | #{virt_start} | #{phys_start} | #{size} KiB | #{@attributes}\"\n+        \"#{name} | #{virt_start} | #{phys_start} | #{size} | #{@attributes}\"\n     end\n\n     def self.print_divider\n\n```\n"
  },
  {
    "path": "18_backtrace/kernel/Cargo.toml",
    "content": "[package]\nname = \"mingo\"\nversion = \"0.18.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n\n[features]\ndefault = []\nbsp_rpi3 = [\"tock-registers\"]\nbsp_rpi4 = [\"tock-registers\"]\ntest_build = [\"qemu-exit\"]\n\n##--------------------------------------------------------------------------------------------------\n## Dependencies\n##--------------------------------------------------------------------------------------------------\n\n[dependencies]\ntest-types = { path = \"../libraries/test-types\" }\ndebug-symbol-types = { path = \"../libraries/debug-symbol-types\" }\n\n# Optional dependencies\ntock-registers = { version = \"0.8.x\", default-features = false, features = [\"register_types\"], optional = true }\nqemu-exit = { version = \"3.x.x\", optional = true }\n\n# Platform specific dependencies\n[target.'cfg(target_arch = \"aarch64\")'.dependencies]\naarch64-cpu = { version = \"9.x.x\" }\n\n##--------------------------------------------------------------------------------------------------\n## Testing\n##--------------------------------------------------------------------------------------------------\n\n[dev-dependencies]\ntest-macros = { path = \"../libraries/test-macros\" }\n\n# Unit tests are done in the library part of the kernel.\n[lib]\nname = \"libkernel\"\ntest = true\n\n# Disable unit tests for the kernel binary.\n[[bin]]\nname = \"kernel\"\npath = \"src/main.rs\"\ntest = false\n\n# List of tests without harness.\n[[test]]\nname = \"00_console_sanity\"\nharness = false\n\n[[test]]\nname = \"02_exception_sync_page_fault\"\nharness = false\n\n[[test]]\nname = \"03_exception_restore_sanity\"\nharness = false\n\n[[test]]\nname = \"05_backtrace_sanity\"\nharness = false\n\n[[test]]\nname = \"06_backtrace_invalid_frame\"\nharness = false\n\n[[test]]\nname = \"07_backtrace_invalid_link\"\nharness = false\n"
  },
  {
    "path": "18_backtrace/kernel/build.rs",
    "content": "use std::{env, fs, process};\n\nfn main() {\n    let ld_script_path = match env::var(\"LD_SCRIPT_PATH\") {\n        Ok(var) => var,\n        _ => process::exit(0),\n    };\n\n    let files = fs::read_dir(ld_script_path).unwrap();\n    files\n        .filter_map(Result::ok)\n        .filter(|d| {\n            if let Some(e) = d.path().extension() {\n                e == \"ld\"\n            } else {\n                false\n            }\n        })\n        .for_each(|f| println!(\"cargo:rerun-if-changed={}\", f.path().display()));\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/_arch/aarch64/backtrace.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural backtracing support.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::backtrace::arch_backtrace\n\nuse crate::{\n    backtrace::BacktraceItem,\n    memory::{Address, Virtual},\n};\nuse aarch64_cpu::registers::*;\nuse tock_registers::interfaces::Readable;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// A Stack frame record.\n///\n/// # Note\n///\n/// The convention is that `previous_record` is valid as long as it contains a non-null value.\n/// Therefore, it is possible to type the member as `Option<&StackFrameRecord>` because of Rust's\n/// `null-pointer optimization`.\n#[repr(C)]\nstruct StackFrameRecord<'a> {\n    previous_record: Option<&'a StackFrameRecord<'a>>,\n    link: Address<Virtual>,\n}\n\nstruct StackFrameRecordIterator<'a> {\n    cur: &'a StackFrameRecord<'a>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<'a> Iterator for StackFrameRecordIterator<'a> {\n    type Item = BacktraceItem;\n\n    fn next(&mut self) -> Option<Self::Item> {\n        static ABORT_FRAME: StackFrameRecord = StackFrameRecord {\n            previous_record: None,\n            link: Address::new(0),\n        };\n\n        // If previous is None, this is the root frame, so iteration will stop here.\n        let previous = self.cur.previous_record?;\n\n        // Need to abort if the pointer to the previous frame record is invalid.\n        let prev_addr = Address::<Virtual>::new(previous as *const _ as usize);\n        if !prev_addr.is_valid_stack_addr() {\n            // This allows to return the error and then stop on the next iteration.\n            self.cur = &ABORT_FRAME;\n            return Some(BacktraceItem::InvalidFramePointer(prev_addr));\n        }\n\n        let ret = if !self.cur.link.is_valid_code_addr() {\n            Some(BacktraceItem::InvalidLink(self.cur.link))\n        } else {\n            // The link points to the instruction to be executed _after_ returning from a branch.\n            // However, we want to show the instruction that caused the branch, so subtract by one\n            // instruction.\n            //\n            // This might be called from panic!, so it must not panic itself on the subtraction.\n            let link = if self.cur.link >= Address::new(4) {\n                self.cur.link - 4\n            } else {\n                self.cur.link\n            };\n\n            Some(BacktraceItem::Link(link))\n        };\n\n        // Advance the iterator.\n        self.cur = previous;\n\n        ret\n    }\n}\n\nfn stack_frame_record_iterator<'a>() -> Option<StackFrameRecordIterator<'a>> {\n    let fp = Address::<Virtual>::new(FP.get() as usize);\n    if !fp.is_valid_stack_addr() {\n        return None;\n    }\n\n    Some(StackFrameRecordIterator {\n        cur: unsafe { &*(fp.as_usize() as *const _) },\n    })\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Architectural implementation of the backtrace.\npub fn backtrace(f: impl FnOnce(Option<&mut dyn Iterator<Item = BacktraceItem>>)) {\n    f(stack_frame_record_iterator().as_mut().map(|s| s as _))\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(feature = \"test_build\")]\n#[inline(always)]\n/// Hack for corrupting the previous frame address in the current stack frame.\n///\n/// # Safety\n///\n/// - To be used only by testing code.\npub unsafe fn corrupt_previous_frame_addr() {\n    let sf = FP.get() as *mut usize;\n    *sf = 0x123;\n}\n\n#[cfg(feature = \"test_build\")]\n#[inline(always)]\n/// Hack for corrupting the link in the current stack frame.\n///\n/// # Safety\n///\n/// - To be used only by testing code.\npub unsafe fn corrupt_link() {\n    let sf = FP.get() as *mut StackFrameRecord;\n    (*sf).link = Address::new(0x456);\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/_arch/aarch64/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural boot code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::boot::arch_boot\n\nuse crate::{memory, memory::Address};\nuse aarch64_cpu::{asm, registers::*};\nuse core::{\n    arch::global_asm,\n    sync::atomic::{compiler_fence, Ordering},\n};\nuse tock_registers::interfaces::Writeable;\n\n// Assembly counterpart to this file.\nglobal_asm!(\n    include_str!(\"boot.s\"),\n    CONST_CURRENTEL_EL2 = const 0x8,\n    CONST_CORE_ID_MASK = const 0b11\n);\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Prepares the transition from EL2 to EL1.\n///\n/// # Safety\n///\n/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.\n/// - The HW state of EL1 must be prepared in a sound way.\n#[inline(always)]\nunsafe fn prepare_el2_to_el1_transition(\n    virt_boot_core_stack_end_exclusive_addr: u64,\n    virt_kernel_init_addr: u64,\n) {\n    // Enable timer counter registers for EL1.\n    CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET);\n\n    // No offset for reading the counters.\n    CNTVOFF_EL2.set(0);\n\n    // Set EL1 execution state to AArch64.\n    HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64);\n\n    // Set up a simulated exception return.\n    //\n    // First, fake a saved program status where all interrupts were masked and SP_EL1 was used as a\n    // stack pointer.\n    SPSR_EL2.write(\n        SPSR_EL2::D::Masked\n            + SPSR_EL2::A::Masked\n            + SPSR_EL2::I::Masked\n            + SPSR_EL2::F::Masked\n            + SPSR_EL2::M::EL1h,\n    );\n\n    // Second, let the link register point to kernel_init().\n    ELR_EL2.set(virt_kernel_init_addr);\n\n    // Set up SP_EL1 (stack pointer), which will be used by EL1 once we \"return\" to it. Since there\n    // are no plans to ever return to EL2, just re-use the same stack.\n    SP_EL1.set(virt_boot_core_stack_end_exclusive_addr);\n}\n\n/// Reset the backtrace by setting link register and frame pointer to zero.\n///\n/// # Safety\n///\n/// - This function must only be used immediately before entering EL1.\n#[inline(always)]\nunsafe fn prepare_backtrace_reset() {\n    compiler_fence(Ordering::SeqCst);\n    FP.set(0);\n    LR.set(0);\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The Rust entry of the `kernel` binary.\n///\n/// The function is called from the assembly `_start` function.\n///\n/// # Safety\n///\n/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`.\n#[no_mangle]\npub unsafe extern \"C\" fn _start_rust(\n    phys_kernel_tables_base_addr: u64,\n    virt_boot_core_stack_end_exclusive_addr: u64,\n    virt_kernel_init_addr: u64,\n) -> ! {\n    prepare_el2_to_el1_transition(\n        virt_boot_core_stack_end_exclusive_addr,\n        virt_kernel_init_addr,\n    );\n\n    // Turn on the MMU for EL1.\n    let addr = Address::new(phys_kernel_tables_base_addr as usize);\n    memory::mmu::enable_mmu_and_caching(addr).unwrap();\n\n    // Make the function we return to the root of a backtrace.\n    prepare_backtrace_reset();\n\n    // Use `eret` to \"return\" to EL1. Since virtual memory will already be enabled, this results in\n    // execution of kernel_init() in EL1 from its _virtual address_.\n    asm::eret()\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/_arch/aarch64/cpu/boot.s",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//--------------------------------------------------------------------------------------------------\n// Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Load the address of a symbol into a register, PC-relative.\n//\n// The symbol must lie within +/- 4 GiB of the Program Counter.\n//\n// # Resources\n//\n// - https://sourceware.org/binutils/docs-2.36/as/AArch64_002dRelocations.html\n.macro ADR_REL register, symbol\n\tadrp\t\\register, \\symbol\n\tadd\t\\register, \\register, #:lo12:\\symbol\n.endm\n\n// Load the address of a symbol into a register, absolute.\n//\n// # Resources\n//\n// - https://sourceware.org/binutils/docs-2.36/as/AArch64_002dRelocations.html\n.macro ADR_ABS register, symbol\n\tmovz\t\\register, #:abs_g3:\\symbol\n\tmovk\t\\register, #:abs_g2_nc:\\symbol\n\tmovk\t\\register, #:abs_g1_nc:\\symbol\n\tmovk\t\\register, #:abs_g0_nc:\\symbol\n.endm\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n.section .text._start\n\n//------------------------------------------------------------------------------\n// fn _start()\n//------------------------------------------------------------------------------\n_start:\n\t// Only proceed if the core executes in EL2. Park it otherwise.\n\tmrs\tx0, CurrentEL\n\tcmp\tx0, {CONST_CURRENTEL_EL2}\n\tb.ne\t.L_parking_loop\n\n\t// Only proceed on the boot core. Park it otherwise.\n\tmrs\tx1, MPIDR_EL1\n\tand\tx1, x1, {CONST_CORE_ID_MASK}\n\tldr\tx2, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs\n\tcmp\tx1, x2\n\tb.ne\t.L_parking_loop\n\n\t// If execution reaches here, it is the boot core.\n\n\t// Initialize DRAM.\n\tADR_REL\tx0, __bss_start\n\tADR_REL x1, __bss_end_exclusive\n\n.L_bss_init_loop:\n\tcmp\tx0, x1\n\tb.eq\t.L_prepare_rust\n\tstp\txzr, xzr, [x0], #16\n\tb\t.L_bss_init_loop\n\n\t// Prepare the jump to Rust code.\n.L_prepare_rust:\n\t// Load the base address of the kernel's translation tables.\n\tldr\tx0, PHYS_KERNEL_TABLES_BASE_ADDR // provided by bsp/__board_name__/memory/mmu.rs\n\n\t// Load the _absolute_ addresses of the following symbols. Since the kernel is linked at\n\t// the top of the 64 bit address space, these are effectively virtual addresses.\n\tADR_ABS\tx1, __boot_core_stack_end_exclusive\n\tADR_ABS\tx2, kernel_init\n\n\t// Load the PC-relative address of the stack and set the stack pointer.\n\t//\n\t// Since _start() is the first function that runs after the firmware has loaded the kernel\n\t// into memory, retrieving this symbol PC-relative returns the \"physical\" address.\n\t//\n\t// Setting the stack pointer to this value ensures that anything that still runs in EL2,\n\t// until the kernel returns to EL1 with the MMU enabled, works as well. After the return to\n\t// EL1, the virtual address of the stack retrieved above will be used.\n\tADR_REL\tx3, __boot_core_stack_end_exclusive\n\tmov\tsp, x3\n\n\t// Read the CPU's timer counter frequency and store it in ARCH_TIMER_COUNTER_FREQUENCY.\n\t// Abort if the frequency read back as 0.\n\tADR_REL\tx4, ARCH_TIMER_COUNTER_FREQUENCY // provided by aarch64/time.rs\n\tmrs\tx5, CNTFRQ_EL0\n\tcmp\tx5, xzr\n\tb.eq\t.L_parking_loop\n\tstr\tw5, [x4]\n\n\t// Jump to Rust code. x0, x1 and x2 hold the function arguments provided to _start_rust().\n\tb\t_start_rust\n\n\t// Infinitely wait for events (aka \"park the core\").\n.L_parking_loop:\n\twfe\n\tb\t.L_parking_loop\n\n.size\t_start, . - _start\n.type\t_start, function\n.global\t_start\n"
  },
  {
    "path": "18_backtrace/kernel/src/_arch/aarch64/cpu/smp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural symmetric multiprocessing.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::smp::arch_smp\n\nuse aarch64_cpu::registers::*;\nuse tock_registers::interfaces::Readable;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return the executing core's id.\n#[inline(always)]\npub fn core_id<T>() -> T\nwhere\n    T: From<u8>,\n{\n    const CORE_MASK: u64 = 0b11;\n\n    T::from((MPIDR_EL1.get() & CORE_MASK) as u8)\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/_arch/aarch64/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural processor code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::arch_cpu\n\nuse aarch64_cpu::asm;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\npub use asm::nop;\n\n/// Pause execution on the core.\n#[inline(always)]\npub fn wait_forever() -> ! {\n    loop {\n        asm::wfe()\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n#[cfg(feature = \"test_build\")]\nuse qemu_exit::QEMUExit;\n\n#[cfg(feature = \"test_build\")]\nconst QEMU_EXIT_HANDLE: qemu_exit::AArch64 = qemu_exit::AArch64::new();\n\n/// Make the host QEMU binary execute `exit(1)`.\n#[cfg(feature = \"test_build\")]\npub fn qemu_exit_failure() -> ! {\n    QEMU_EXIT_HANDLE.exit_failure()\n}\n\n/// Make the host QEMU binary execute `exit(0)`.\n#[cfg(feature = \"test_build\")]\npub fn qemu_exit_success() -> ! {\n    QEMU_EXIT_HANDLE.exit_success()\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/_arch/aarch64/exception/asynchronous.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural asynchronous exception handling.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::exception::asynchronous::arch_asynchronous\n\nuse aarch64_cpu::registers::*;\nuse core::arch::asm;\nuse tock_registers::interfaces::{Readable, Writeable};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nmod daif_bits {\n    pub const IRQ: u8 = 0b0010;\n}\n\ntrait DaifField {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register>;\n}\n\nstruct Debug;\nstruct SError;\nstruct IRQ;\nstruct FIQ;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl DaifField for Debug {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::D\n    }\n}\n\nimpl DaifField for SError {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::A\n    }\n}\n\nimpl DaifField for IRQ {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::I\n    }\n}\n\nimpl DaifField for FIQ {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::F\n    }\n}\n\nfn is_masked<T>() -> bool\nwhere\n    T: DaifField,\n{\n    DAIF.is_set(T::daif_field())\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Returns whether IRQs are masked on the executing core.\npub fn is_local_irq_masked() -> bool {\n    !is_masked::<IRQ>()\n}\n\n/// Unmask IRQs on the executing core.\n///\n/// It is not needed to place an explicit instruction synchronization barrier after the `msr`.\n/// Quoting the Architecture Reference Manual for ARMv8-A, section C5.1.3:\n///\n/// \"Writes to PSTATE.{PAN, D, A, I, F} occur in program order without the need for additional\n/// synchronization.\"\n#[inline(always)]\npub fn local_irq_unmask() {\n    unsafe {\n        asm!(\n            \"msr DAIFClr, {arg}\",\n            arg = const daif_bits::IRQ,\n            options(nomem, nostack, preserves_flags)\n        );\n    }\n}\n\n/// Mask IRQs on the executing core.\n#[inline(always)]\npub fn local_irq_mask() {\n    unsafe {\n        asm!(\n            \"msr DAIFSet, {arg}\",\n            arg = const daif_bits::IRQ,\n            options(nomem, nostack, preserves_flags)\n        );\n    }\n}\n\n/// Mask IRQs on the executing core and return the previously saved interrupt mask bits (DAIF).\n#[inline(always)]\npub fn local_irq_mask_save() -> u64 {\n    let saved = DAIF.get();\n    local_irq_mask();\n\n    saved\n}\n\n/// Restore the interrupt mask bits (DAIF) using the callee's argument.\n///\n/// # Invariant\n///\n/// - No sanity checks on the input.\n#[inline(always)]\npub fn local_irq_restore(saved: u64) {\n    DAIF.set(saved);\n}\n\n/// Print the AArch64 exceptions status.\n#[rustfmt::skip]\npub fn print_state() {\n    use crate::info;\n\n    let to_mask_str = |x| -> _ {\n        if x { \"Masked\" } else { \"Unmasked\" }\n    };\n\n    info!(\"      Debug:  {}\", to_mask_str(is_masked::<Debug>()));\n    info!(\"      SError: {}\", to_mask_str(is_masked::<SError>()));\n    info!(\"      IRQ:    {}\", to_mask_str(is_masked::<IRQ>()));\n    info!(\"      FIQ:    {}\", to_mask_str(is_masked::<FIQ>()));\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/_arch/aarch64/exception.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural synchronous and asynchronous exception handling.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::exception::arch_exception\n\nuse crate::{exception, memory, symbols};\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::{arch::global_asm, cell::UnsafeCell, fmt};\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    registers::InMemoryRegister,\n};\n\n// Assembly counterpart to this file.\nglobal_asm!(\n    include_str!(\"exception.s\"),\n    CONST_ESR_EL1_EC_SHIFT = const 26,\n    CONST_ESR_EL1_EC_VALUE_SVC64 = const 0x15\n);\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Wrapper structs for memory copies of registers.\n#[repr(transparent)]\nstruct SpsrEL1(InMemoryRegister<u64, SPSR_EL1::Register>);\nstruct EsrEL1(InMemoryRegister<u64, ESR_EL1::Register>);\n\n/// The exception context as it is stored on the stack on exception entry.\n#[repr(C)]\nstruct ExceptionContext {\n    /// General Purpose Registers.\n    gpr: [u64; 30],\n\n    /// The link register, aka x30.\n    lr: u64,\n\n    /// Exception link register. The program counter at the time the exception happened.\n    elr_el1: u64,\n\n    /// Saved program status.\n    spsr_el1: SpsrEL1,\n\n    /// Exception syndrome register.\n    esr_el1: EsrEL1,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Prints verbose information about the exception and then panics.\nfn default_exception_handler(exc: &ExceptionContext) {\n    panic!(\n        \"CPU Exception!\\n\\n\\\n        {}\",\n        exc\n    );\n}\n\n//------------------------------------------------------------------------------\n// Current, EL0\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn current_el0_synchronous(_e: &mut ExceptionContext) {\n    panic!(\"Should not be here. Use of SP_EL0 in EL1 is not supported.\")\n}\n\n#[no_mangle]\nextern \"C\" fn current_el0_irq(_e: &mut ExceptionContext) {\n    panic!(\"Should not be here. Use of SP_EL0 in EL1 is not supported.\")\n}\n\n#[no_mangle]\nextern \"C\" fn current_el0_serror(_e: &mut ExceptionContext) {\n    panic!(\"Should not be here. Use of SP_EL0 in EL1 is not supported.\")\n}\n\n//------------------------------------------------------------------------------\n// Current, ELx\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn current_elx_synchronous(e: &mut ExceptionContext) {\n    #[cfg(feature = \"test_build\")]\n    {\n        const TEST_SVC_ID: u64 = 0x1337;\n\n        if let Some(ESR_EL1::EC::Value::SVC64) = e.esr_el1.exception_class() {\n            if e.esr_el1.iss() == TEST_SVC_ID {\n                return;\n            }\n        }\n    }\n\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn current_elx_irq(_e: &mut ExceptionContext) {\n    let token = unsafe { &exception::asynchronous::IRQContext::new() };\n    exception::asynchronous::irq_manager().handle_pending_irqs(token);\n}\n\n#[no_mangle]\nextern \"C\" fn current_elx_serror(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n//------------------------------------------------------------------------------\n// Lower, AArch64\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn lower_aarch64_synchronous(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch64_irq(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch64_serror(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n//------------------------------------------------------------------------------\n// Lower, AArch32\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn lower_aarch32_synchronous(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch32_irq(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch32_serror(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n//------------------------------------------------------------------------------\n// Misc\n//------------------------------------------------------------------------------\n\n/// Human readable SPSR_EL1.\n#[rustfmt::skip]\nimpl fmt::Display for SpsrEL1 {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        // Raw value.\n        writeln!(f, \"SPSR_EL1: {:#010x}\", self.0.get())?;\n\n        let to_flag_str = |x| -> _ {\n            if x { \"Set\" } else { \"Not set\" }\n         };\n\n        writeln!(f, \"      Flags:\")?;\n        writeln!(f, \"            Negative (N): {}\", to_flag_str(self.0.is_set(SPSR_EL1::N)))?;\n        writeln!(f, \"            Zero     (Z): {}\", to_flag_str(self.0.is_set(SPSR_EL1::Z)))?;\n        writeln!(f, \"            Carry    (C): {}\", to_flag_str(self.0.is_set(SPSR_EL1::C)))?;\n        writeln!(f, \"            Overflow (V): {}\", to_flag_str(self.0.is_set(SPSR_EL1::V)))?;\n\n        let to_mask_str = |x| -> _ {\n            if x { \"Masked\" } else { \"Unmasked\" }\n        };\n\n        writeln!(f, \"      Exception handling state:\")?;\n        writeln!(f, \"            Debug  (D): {}\", to_mask_str(self.0.is_set(SPSR_EL1::D)))?;\n        writeln!(f, \"            SError (A): {}\", to_mask_str(self.0.is_set(SPSR_EL1::A)))?;\n        writeln!(f, \"            IRQ    (I): {}\", to_mask_str(self.0.is_set(SPSR_EL1::I)))?;\n        writeln!(f, \"            FIQ    (F): {}\", to_mask_str(self.0.is_set(SPSR_EL1::F)))?;\n\n        write!(f, \"      Illegal Execution State (IL): {}\",\n            to_flag_str(self.0.is_set(SPSR_EL1::IL))\n        )\n    }\n}\n\nimpl EsrEL1 {\n    #[inline(always)]\n    fn exception_class(&self) -> Option<ESR_EL1::EC::Value> {\n        self.0.read_as_enum(ESR_EL1::EC)\n    }\n\n    #[cfg(feature = \"test_build\")]\n    #[inline(always)]\n    fn iss(&self) -> u64 {\n        self.0.read(ESR_EL1::ISS)\n    }\n}\n\n/// Human readable ESR_EL1.\n#[rustfmt::skip]\nimpl fmt::Display for EsrEL1 {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        // Raw print of whole register.\n        writeln!(f, \"ESR_EL1: {:#010x}\", self.0.get())?;\n\n        // Raw print of exception class.\n        write!(f, \"      Exception Class         (EC) : {:#x}\", self.0.read(ESR_EL1::EC))?;\n\n        // Exception class.\n        let ec_translation = match self.exception_class() {\n            Some(ESR_EL1::EC::Value::DataAbortCurrentEL) => \"Data Abort, current EL\",\n            _ => \"N/A\",\n        };\n        writeln!(f, \" - {}\", ec_translation)?;\n\n        // Raw print of instruction specific syndrome.\n        write!(f, \"      Instr Specific Syndrome (ISS): {:#x}\", self.0.read(ESR_EL1::ISS))\n    }\n}\n\nimpl ExceptionContext {\n    #[inline(always)]\n    fn exception_class(&self) -> Option<ESR_EL1::EC::Value> {\n        self.esr_el1.exception_class()\n    }\n\n    #[inline(always)]\n    fn fault_address_valid(&self) -> bool {\n        use ESR_EL1::EC::Value::*;\n\n        match self.exception_class() {\n            None => false,\n            Some(ec) => matches!(\n                ec,\n                InstrAbortLowerEL\n                    | InstrAbortCurrentEL\n                    | PCAlignmentFault\n                    | DataAbortLowerEL\n                    | DataAbortCurrentEL\n                    | WatchpointLowerEL\n                    | WatchpointCurrentEL\n            ),\n        }\n    }\n}\n\n/// Human readable print of the exception context.\nimpl fmt::Display for ExceptionContext {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        writeln!(f, \"{}\", self.esr_el1)?;\n\n        if self.fault_address_valid() {\n            writeln!(f, \"FAR_EL1: {:#018x}\", FAR_EL1.get() as usize)?;\n        }\n\n        writeln!(f, \"{}\", self.spsr_el1)?;\n        writeln!(f, \"ELR_EL1: {:#018x}\", self.elr_el1)?;\n        writeln!(\n            f,\n            \"      Symbol: {}\",\n            match symbols::lookup_symbol(memory::Address::new(self.elr_el1 as usize)) {\n                Some(sym) => sym.name(),\n                _ => \"Symbol not found\",\n            }\n        )?;\n        writeln!(f)?;\n        writeln!(f, \"General purpose register:\")?;\n\n        #[rustfmt::skip]\n        let alternating = |x| -> _ {\n            if x % 2 == 0 { \"   \" } else { \"\\n\" }\n        };\n\n        // Print two registers per line.\n        for (i, reg) in self.gpr.iter().enumerate() {\n            write!(f, \"      x{: <2}: {: >#018x}{}\", i, reg, alternating(i))?;\n        }\n        write!(f, \"      lr : {:#018x}\", self.lr)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse crate::exception::PrivilegeLevel;\n\n/// The processing element's current privilege level.\npub fn current_privilege_level() -> (PrivilegeLevel, &'static str) {\n    let el = CurrentEL.read_as_enum(CurrentEL::EL);\n    match el {\n        Some(CurrentEL::EL::Value::EL2) => (PrivilegeLevel::Hypervisor, \"EL2\"),\n        Some(CurrentEL::EL::Value::EL1) => (PrivilegeLevel::Kernel, \"EL1\"),\n        Some(CurrentEL::EL::Value::EL0) => (PrivilegeLevel::User, \"EL0\"),\n        _ => (PrivilegeLevel::Unknown, \"Unknown\"),\n    }\n}\n\n/// Init exception handling by setting the exception vector base address register.\n///\n/// # Safety\n///\n/// - Changes the HW state of the executing core.\n/// - The vector table and the symbol `__exception_vector_table_start` from the linker script must\n///   adhere to the alignment and size constraints demanded by the ARMv8-A Architecture Reference\n///   Manual.\npub unsafe fn handling_init() {\n    // Provided by exception.S.\n    extern \"Rust\" {\n        static __exception_vector_start: UnsafeCell<()>;\n    }\n\n    VBAR_EL1.set(__exception_vector_start.get() as u64);\n\n    // Force VBAR update to complete before next instruction.\n    barrier::isb(barrier::SY);\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/_arch/aarch64/exception.s",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2022 Andre Richter <andre.o.richter@gmail.com>\n\n//--------------------------------------------------------------------------------------------------\n// Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Call the function provided by parameter `\\handler` after saving the exception context. Provide\n/// the context as the first parameter to '\\handler'.\n.macro CALL_WITH_CONTEXT handler is_lower_el is_sync\n__vector_\\handler:\n\t// Make room on the stack for the exception context.\n\tsub\tsp,  sp,  #16 * 18\n\n\t// Store all general purpose registers on the stack.\n\tstp\tx0,  x1,  [sp, #16 * 0]\n\tstp\tx2,  x3,  [sp, #16 * 1]\n\tstp\tx4,  x5,  [sp, #16 * 2]\n\tstp\tx6,  x7,  [sp, #16 * 3]\n\tstp\tx8,  x9,  [sp, #16 * 4]\n\tstp\tx10, x11, [sp, #16 * 5]\n\tstp\tx12, x13, [sp, #16 * 6]\n\tstp\tx14, x15, [sp, #16 * 7]\n\tstp\tx16, x17, [sp, #16 * 8]\n\tstp\tx18, x19, [sp, #16 * 9]\n\tstp\tx20, x21, [sp, #16 * 10]\n\tstp\tx22, x23, [sp, #16 * 11]\n\tstp\tx24, x25, [sp, #16 * 12]\n\tstp\tx26, x27, [sp, #16 * 13]\n\tstp\tx28, x29, [sp, #16 * 14]\n\n\t// Add the exception link register (ELR_EL1), saved program status (SPSR_EL1) and exception\n\t// syndrome register (ESR_EL1).\n\tmrs\tx1,  ELR_EL1\n\tmrs\tx2,  SPSR_EL1\n\tmrs\tx3,  ESR_EL1\n\n\tstp\tlr,  x1,  [sp, #16 * 15]\n\tstp\tx2,  x3,  [sp, #16 * 16]\n\n\t// Build a stack frame for backtracing.\n.if \\is_lower_el == 1\n\t// If we came from a lower EL, make it a root frame (by storing zero) so that the kernel\n\t// does not attempt to trace into userspace.\n\tstp\txzr, xzr, [sp, #16 * 17]\n.else\n\t// For normal branches, the link address points to the instruction to be executed _after_\n\t// returning from a branch. In a backtrace, we want to show the instruction that caused the\n\t// branch, though. That is why code in backtrace.rs subtracts 4 (length of one instruction)\n\t// from the link address.\n\t//\n\t// Here we have a special case, though, because ELR_EL1 is used instead of LR to build the\n\t// stack frame, so that it becomes possible to trace beyond an exception. Hence, it must be\n\t// considered that semantics for ELR_EL1 differ from case to case.\n\t//\n\t// Unless an \"exception generating instruction\" was executed, ELR_EL1 already points to the\n\t// the correct instruction, and hence the subtraction by 4 in backtrace.rs would yield wrong\n\t// results. To cover for this, 4 is added to ELR_EL1 below unless the cause of exception was\n\t// an SVC instruction. BRK and HLT are \"exception generating instructions\" as well, but they\n\t// are not expected and therefore left out for now.\n\t//\n\t// For reference: Search for \"preferred exception return address\" in the Architecture\n\t// Reference Manual for ARMv8-A.\n.if \\is_sync == 1\n\tlsr\tw3,  w3, {CONST_ESR_EL1_EC_SHIFT}   // w3 = ESR_EL1.EC\n\tcmp\tw3,  {CONST_ESR_EL1_EC_VALUE_SVC64} // w3 == SVC64 ?\n\tb.eq\t1f\n.endif\n\tadd\tx1,  x1, #4\n1:\n\tstp\tx29, x1, [sp, #16 * 17]\n.endif\n\n\t// Set the frame pointer to the stack frame record.\n\tadd\tx29, sp, #16 * 17\n\n\t// x0 is the first argument for the function called through `\\handler`.\n\tmov\tx0,  sp\n\n\t// Call `\\handler`.\n\tbl\t\\handler\n\n\t// After returning from exception handling code, replay the saved context and return via\n\t// `eret`.\n\tb\t__exception_restore_context\n\n.size\t__vector_\\handler, . - __vector_\\handler\n.type\t__vector_\\handler, function\n.endm\n\n.macro FIQ_SUSPEND\n1:\twfe\n\tb\t1b\n.endm\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n.section .text\n\n//------------------------------------------------------------------------------\n// The exception vector table.\n//------------------------------------------------------------------------------\n\n// Align by 2^11 bytes, as demanded by ARMv8-A. Same as ALIGN(2048) in an ld script.\n.align 11\n\n// Export a symbol for the Rust code to use.\n__exception_vector_start:\n\n// Current exception level with SP_EL0.\n//\n// .org sets the offset relative to section start.\n//\n// # Safety\n//\n// - It must be ensured that `CALL_WITH_CONTEXT` <= 0x80 bytes.\n.org 0x000\n\tCALL_WITH_CONTEXT current_el0_synchronous, 0, 1\n.org 0x080\n\tCALL_WITH_CONTEXT current_el0_irq, 0, 0\n.org 0x100\n\tFIQ_SUSPEND\n.org 0x180\n\tCALL_WITH_CONTEXT current_el0_serror, 0, 0\n\n// Current exception level with SP_ELx, x > 0.\n.org 0x200\n\tCALL_WITH_CONTEXT current_elx_synchronous, 0, 1\n.org 0x280\n\tCALL_WITH_CONTEXT current_elx_irq, 0, 0\n.org 0x300\n\tFIQ_SUSPEND\n.org 0x380\n\tCALL_WITH_CONTEXT current_elx_serror, 0, 0\n\n// Lower exception level, AArch64\n.org 0x400\n\tCALL_WITH_CONTEXT lower_aarch64_synchronous, 1, 1\n.org 0x480\n\tCALL_WITH_CONTEXT lower_aarch64_irq, 1, 0\n.org 0x500\n\tFIQ_SUSPEND\n.org 0x580\n\tCALL_WITH_CONTEXT lower_aarch64_serror, 1, 0\n\n// Lower exception level, AArch32\n.org 0x600\n\tCALL_WITH_CONTEXT lower_aarch32_synchronous, 1, 0\n.org 0x680\n\tCALL_WITH_CONTEXT lower_aarch32_irq, 1, 0\n.org 0x700\n\tFIQ_SUSPEND\n.org 0x780\n\tCALL_WITH_CONTEXT lower_aarch32_serror, 1, 0\n.org 0x800\n\n//------------------------------------------------------------------------------\n// fn __exception_restore_context()\n//------------------------------------------------------------------------------\n__exception_restore_context:\n\tldr\tw19,      [sp, #16 * 16]\n\tldp\tlr,  x20, [sp, #16 * 15]\n\n\tmsr\tSPSR_EL1, x19\n\tmsr\tELR_EL1,  x20\n\n\tldp\tx0,  x1,  [sp, #16 * 0]\n\tldp\tx2,  x3,  [sp, #16 * 1]\n\tldp\tx4,  x5,  [sp, #16 * 2]\n\tldp\tx6,  x7,  [sp, #16 * 3]\n\tldp\tx8,  x9,  [sp, #16 * 4]\n\tldp\tx10, x11, [sp, #16 * 5]\n\tldp\tx12, x13, [sp, #16 * 6]\n\tldp\tx14, x15, [sp, #16 * 7]\n\tldp\tx16, x17, [sp, #16 * 8]\n\tldp\tx18, x19, [sp, #16 * 9]\n\tldp\tx20, x21, [sp, #16 * 10]\n\tldp\tx22, x23, [sp, #16 * 11]\n\tldp\tx24, x25, [sp, #16 * 12]\n\tldp\tx26, x27, [sp, #16 * 13]\n\tldp\tx28, x29, [sp, #16 * 14]\n\n\tadd\tsp,  sp,  #16 * 18\n\n\teret\n\n.size\t__exception_restore_context, . - __exception_restore_context\n.type\t__exception_restore_context, function\n"
  },
  {
    "path": "18_backtrace/kernel/src/_arch/aarch64/memory/mmu/translation_table.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural translation table.\n//!\n//! Only 64 KiB granule is supported.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::memory::mmu::translation_table::arch_translation_table\n\nuse crate::{\n    bsp,\n    memory::{\n        self,\n        mmu::{\n            arch_mmu::{Granule512MiB, Granule64KiB},\n            AccessPermissions, AttributeFields, MemAttributes, MemoryRegion, PageAddress,\n        },\n        Address, Physical, Virtual,\n    },\n};\nuse core::convert;\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields,\n    registers::InMemoryRegister,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// A table descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-15.\nregister_bitfields! {u64,\n    STAGE1_TABLE_DESCRIPTOR [\n        /// Physical address of the next descriptor.\n        NEXT_LEVEL_TABLE_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16]\n\n        TYPE  OFFSET(1) NUMBITS(1) [\n            Block = 0,\n            Table = 1\n        ],\n\n        VALID OFFSET(0) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ]\n    ]\n}\n\n// A level 3 page descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-17.\nregister_bitfields! {u64,\n    STAGE1_PAGE_DESCRIPTOR [\n        /// Unprivileged execute-never.\n        UXN      OFFSET(54) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ],\n\n        /// Privileged execute-never.\n        PXN      OFFSET(53) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ],\n\n        /// Physical address of the next table descriptor (lvl2) or the page descriptor (lvl3).\n        OUTPUT_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16]\n\n        /// Access flag.\n        AF       OFFSET(10) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ],\n\n        /// Shareability field.\n        SH       OFFSET(8) NUMBITS(2) [\n            OuterShareable = 0b10,\n            InnerShareable = 0b11\n        ],\n\n        /// Access Permissions.\n        AP       OFFSET(6) NUMBITS(2) [\n            RW_EL1 = 0b00,\n            RW_EL1_EL0 = 0b01,\n            RO_EL1 = 0b10,\n            RO_EL1_EL0 = 0b11\n        ],\n\n        /// Memory attributes index into the MAIR_EL1 register.\n        AttrIndx OFFSET(2) NUMBITS(3) [],\n\n        TYPE     OFFSET(1) NUMBITS(1) [\n            Reserved_Invalid = 0,\n            Page = 1\n        ],\n\n        VALID    OFFSET(0) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ]\n    ]\n}\n\n/// A table descriptor for 64 KiB aperture.\n///\n/// The output points to the next table.\n#[derive(Copy, Clone)]\n#[repr(C)]\nstruct TableDescriptor {\n    value: u64,\n}\n\n/// A page descriptor with 64 KiB aperture.\n///\n/// The output points to physical memory.\n#[derive(Copy, Clone)]\n#[repr(C)]\nstruct PageDescriptor {\n    value: u64,\n}\n\ntrait StartAddr {\n    fn virt_start_addr(&self) -> Address<Virtual>;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB\n/// aligned, so the lvl3 is put first.\n#[repr(C)]\n#[repr(align(65536))]\npub struct FixedSizeTranslationTable<const NUM_TABLES: usize, const START_FROM_TOP: bool> {\n    /// Page descriptors, covering 64 KiB windows per entry.\n    lvl3: [[PageDescriptor; 8192]; NUM_TABLES],\n\n    /// Table descriptors, covering 512 MiB windows.\n    lvl2: [TableDescriptor; NUM_TABLES],\n\n    /// Have the tables been initialized?\n    initialized: bool,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T, const N: usize> StartAddr for [T; N] {\n    fn virt_start_addr(&self) -> Address<Virtual> {\n        Address::new(self as *const _ as usize)\n    }\n}\n\nimpl TableDescriptor {\n    /// Create an instance.\n    ///\n    /// Descriptor is invalid by default.\n    pub const fn new_zeroed() -> Self {\n        Self { value: 0 }\n    }\n\n    /// Create an instance pointing to the supplied address.\n    pub fn from_next_lvl_table_addr(phys_next_lvl_table_addr: Address<Physical>) -> Self {\n        let val = InMemoryRegister::<u64, STAGE1_TABLE_DESCRIPTOR::Register>::new(0);\n\n        let shifted = phys_next_lvl_table_addr.as_usize() >> Granule64KiB::SHIFT;\n        val.write(\n            STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64)\n                + STAGE1_TABLE_DESCRIPTOR::TYPE::Table\n                + STAGE1_TABLE_DESCRIPTOR::VALID::True,\n        );\n\n        TableDescriptor { value: val.get() }\n    }\n}\n\n/// Convert the kernel's generic memory attributes to HW-specific attributes of the MMU.\nimpl convert::From<AttributeFields>\n    for tock_registers::fields::FieldValue<u64, STAGE1_PAGE_DESCRIPTOR::Register>\n{\n    fn from(attribute_fields: AttributeFields) -> Self {\n        // Memory attributes.\n        let mut desc = match attribute_fields.mem_attributes {\n            MemAttributes::CacheableDRAM => {\n                STAGE1_PAGE_DESCRIPTOR::SH::InnerShareable\n                    + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::NORMAL)\n            }\n            MemAttributes::Device => {\n                STAGE1_PAGE_DESCRIPTOR::SH::OuterShareable\n                    + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::DEVICE)\n            }\n        };\n\n        // Access Permissions.\n        desc += match attribute_fields.acc_perms {\n            AccessPermissions::ReadOnly => STAGE1_PAGE_DESCRIPTOR::AP::RO_EL1,\n            AccessPermissions::ReadWrite => STAGE1_PAGE_DESCRIPTOR::AP::RW_EL1,\n        };\n\n        // The execute-never attribute is mapped to PXN in AArch64.\n        desc += if attribute_fields.execute_never {\n            STAGE1_PAGE_DESCRIPTOR::PXN::True\n        } else {\n            STAGE1_PAGE_DESCRIPTOR::PXN::False\n        };\n\n        // Always set unprivileged exectue-never as long as userspace is not implemented yet.\n        desc += STAGE1_PAGE_DESCRIPTOR::UXN::True;\n\n        desc\n    }\n}\n\n/// Convert the HW-specific attributes of the MMU to kernel's generic memory attributes.\nimpl convert::TryFrom<InMemoryRegister<u64, STAGE1_PAGE_DESCRIPTOR::Register>> for AttributeFields {\n    type Error = &'static str;\n\n    fn try_from(\n        desc: InMemoryRegister<u64, STAGE1_PAGE_DESCRIPTOR::Register>,\n    ) -> Result<AttributeFields, Self::Error> {\n        let mem_attributes = match desc.read(STAGE1_PAGE_DESCRIPTOR::AttrIndx) {\n            memory::mmu::arch_mmu::mair::NORMAL => MemAttributes::CacheableDRAM,\n            memory::mmu::arch_mmu::mair::DEVICE => MemAttributes::Device,\n            _ => return Err(\"Unexpected memory attribute\"),\n        };\n\n        let acc_perms = match desc.read_as_enum(STAGE1_PAGE_DESCRIPTOR::AP) {\n            Some(STAGE1_PAGE_DESCRIPTOR::AP::Value::RO_EL1) => AccessPermissions::ReadOnly,\n            Some(STAGE1_PAGE_DESCRIPTOR::AP::Value::RW_EL1) => AccessPermissions::ReadWrite,\n            _ => return Err(\"Unexpected access permission\"),\n        };\n\n        let execute_never = desc.read(STAGE1_PAGE_DESCRIPTOR::PXN) > 0;\n\n        Ok(AttributeFields {\n            mem_attributes,\n            acc_perms,\n            execute_never,\n        })\n    }\n}\n\nimpl PageDescriptor {\n    /// Create an instance.\n    ///\n    /// Descriptor is invalid by default.\n    pub const fn new_zeroed() -> Self {\n        Self { value: 0 }\n    }\n\n    /// Create an instance.\n    pub fn from_output_page_addr(\n        phys_output_page_addr: PageAddress<Physical>,\n        attribute_fields: &AttributeFields,\n    ) -> Self {\n        let val = InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(0);\n\n        let shifted = phys_output_page_addr.into_inner().as_usize() >> Granule64KiB::SHIFT;\n        val.write(\n            STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted as u64)\n                + STAGE1_PAGE_DESCRIPTOR::AF::True\n                + STAGE1_PAGE_DESCRIPTOR::TYPE::Page\n                + STAGE1_PAGE_DESCRIPTOR::VALID::True\n                + (*attribute_fields).into(),\n        );\n\n        Self { value: val.get() }\n    }\n\n    /// Returns the valid bit.\n    fn is_valid(&self) -> bool {\n        InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(self.value)\n            .is_set(STAGE1_PAGE_DESCRIPTOR::VALID)\n    }\n\n    /// Returns the output page.\n    fn output_page_addr(&self) -> PageAddress<Physical> {\n        let shifted = InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(self.value)\n            .read(STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB) as usize;\n\n        PageAddress::from(shifted << Granule64KiB::SHIFT)\n    }\n\n    /// Returns the attributes.\n    fn try_attributes(&self) -> Result<AttributeFields, &'static str> {\n        InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(self.value).try_into()\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<const AS_SIZE: usize> memory::mmu::AssociatedTranslationTable\n    for memory::mmu::AddressSpace<AS_SIZE>\nwhere\n    [u8; Self::SIZE >> Granule512MiB::SHIFT]: Sized,\n{\n    type TableStartFromTop =\n        FixedSizeTranslationTable<{ Self::SIZE >> Granule512MiB::SHIFT }, true>;\n\n    type TableStartFromBottom =\n        FixedSizeTranslationTable<{ Self::SIZE >> Granule512MiB::SHIFT }, false>;\n}\n\nimpl<const NUM_TABLES: usize, const START_FROM_TOP: bool>\n    FixedSizeTranslationTable<NUM_TABLES, START_FROM_TOP>\n{\n    const START_FROM_TOP_OFFSET: Address<Virtual> =\n        Address::new((usize::MAX - (Granule512MiB::SIZE * NUM_TABLES)) + 1);\n\n    /// Create an instance.\n    #[allow(clippy::assertions_on_constants)]\n    const fn _new(for_precompute: bool) -> Self {\n        assert!(bsp::memory::mmu::KernelGranule::SIZE == Granule64KiB::SIZE);\n\n        // Can't have a zero-sized address space.\n        assert!(NUM_TABLES > 0);\n\n        Self {\n            lvl3: [[PageDescriptor::new_zeroed(); 8192]; NUM_TABLES],\n            lvl2: [TableDescriptor::new_zeroed(); NUM_TABLES],\n            initialized: for_precompute,\n        }\n    }\n\n    pub const fn new_for_precompute() -> Self {\n        Self::_new(true)\n    }\n\n    #[cfg(test)]\n    pub fn new_for_runtime() -> Self {\n        Self::_new(false)\n    }\n\n    /// Helper to calculate the lvl2 and lvl3 indices from an address.\n    #[inline(always)]\n    fn lvl2_lvl3_index_from_page_addr(\n        &self,\n        virt_page_addr: PageAddress<Virtual>,\n    ) -> Result<(usize, usize), &'static str> {\n        let mut addr = virt_page_addr.into_inner();\n\n        if START_FROM_TOP {\n            addr = addr - Self::START_FROM_TOP_OFFSET;\n        }\n\n        let lvl2_index = addr.as_usize() >> Granule512MiB::SHIFT;\n        let lvl3_index = (addr.as_usize() & Granule512MiB::MASK) >> Granule64KiB::SHIFT;\n\n        if lvl2_index > (NUM_TABLES - 1) {\n            return Err(\"Virtual page is out of bounds of translation table\");\n        }\n\n        Ok((lvl2_index, lvl3_index))\n    }\n\n    /// Returns the PageDescriptor corresponding to the supplied page address.\n    #[inline(always)]\n    fn page_descriptor_from_page_addr(\n        &self,\n        virt_page_addr: PageAddress<Virtual>,\n    ) -> Result<&PageDescriptor, &'static str> {\n        let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from_page_addr(virt_page_addr)?;\n        let desc = &self.lvl3[lvl2_index][lvl3_index];\n\n        Ok(desc)\n    }\n\n    /// Sets the PageDescriptor corresponding to the supplied page address.\n    ///\n    /// Doesn't allow overriding an already valid page.\n    #[inline(always)]\n    fn set_page_descriptor_from_page_addr(\n        &mut self,\n        virt_page_addr: PageAddress<Virtual>,\n        new_desc: &PageDescriptor,\n    ) -> Result<(), &'static str> {\n        let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from_page_addr(virt_page_addr)?;\n        let desc = &mut self.lvl3[lvl2_index][lvl3_index];\n\n        if desc.is_valid() {\n            return Err(\"Virtual page is already mapped\");\n        }\n\n        *desc = *new_desc;\n        Ok(())\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\n\nimpl<const NUM_TABLES: usize, const START_FROM_TOP: bool>\n    memory::mmu::translation_table::interface::TranslationTable\n    for FixedSizeTranslationTable<NUM_TABLES, START_FROM_TOP>\n{\n    fn init(&mut self) -> Result<(), &'static str> {\n        if self.initialized {\n            return Ok(());\n        }\n\n        // Populate the l2 entries.\n        for (lvl2_nr, lvl2_entry) in self.lvl2.iter_mut().enumerate() {\n            let virt_table_addr = self.lvl3[lvl2_nr].virt_start_addr();\n            let phys_table_addr = memory::mmu::try_kernel_virt_addr_to_phys_addr(virt_table_addr)?;\n\n            let new_desc = TableDescriptor::from_next_lvl_table_addr(phys_table_addr);\n            *lvl2_entry = new_desc;\n        }\n\n        self.initialized = true;\n\n        Ok(())\n    }\n\n    unsafe fn map_at(\n        &mut self,\n        virt_region: &MemoryRegion<Virtual>,\n        phys_region: &MemoryRegion<Physical>,\n        attr: &AttributeFields,\n    ) -> Result<(), &'static str> {\n        assert!(self.initialized, \"Translation tables not initialized\");\n\n        if virt_region.size() != phys_region.size() {\n            return Err(\"Tried to map memory regions with unequal sizes\");\n        }\n\n        if phys_region.end_exclusive_page_addr() > bsp::memory::phys_addr_space_end_exclusive_addr()\n        {\n            return Err(\"Tried to map outside of physical address space\");\n        }\n\n        let iter = phys_region.into_iter().zip(virt_region.into_iter());\n        for (phys_page_addr, virt_page_addr) in iter {\n            let new_desc = PageDescriptor::from_output_page_addr(phys_page_addr, attr);\n            let virt_page = virt_page_addr;\n\n            self.set_page_descriptor_from_page_addr(virt_page, &new_desc)?;\n        }\n\n        Ok(())\n    }\n\n    fn try_virt_page_addr_to_phys_page_addr(\n        &self,\n        virt_page_addr: PageAddress<Virtual>,\n    ) -> Result<PageAddress<Physical>, &'static str> {\n        let page_desc = self.page_descriptor_from_page_addr(virt_page_addr)?;\n\n        if !page_desc.is_valid() {\n            return Err(\"Page marked invalid\");\n        }\n\n        Ok(page_desc.output_page_addr())\n    }\n\n    fn try_page_attributes(\n        &self,\n        virt_page_addr: PageAddress<Virtual>,\n    ) -> Result<AttributeFields, &'static str> {\n        let page_desc = self.page_descriptor_from_page_addr(virt_page_addr)?;\n\n        if !page_desc.is_valid() {\n            return Err(\"Page marked invalid\");\n        }\n\n        page_desc.try_attributes()\n    }\n\n    /// Try to translate a virtual address to a physical address.\n    ///\n    /// Will only succeed if there exists a valid mapping for the input address.\n    fn try_virt_addr_to_phys_addr(\n        &self,\n        virt_addr: Address<Virtual>,\n    ) -> Result<Address<Physical>, &'static str> {\n        let virt_page = PageAddress::from(virt_addr.align_down_page());\n        let phys_page = self.try_virt_page_addr_to_phys_page_addr(virt_page)?;\n\n        Ok(phys_page.into_inner() + virt_addr.offset_into_page())\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\npub type MinSizeTranslationTable = FixedSizeTranslationTable<1, true>;\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// Check if the size of `struct TableDescriptor` is as expected.\n    #[kernel_test]\n    fn size_of_tabledescriptor_equals_64_bit() {\n        assert_eq!(\n            core::mem::size_of::<TableDescriptor>(),\n            core::mem::size_of::<u64>()\n        );\n    }\n\n    /// Check if the size of `struct PageDescriptor` is as expected.\n    #[kernel_test]\n    fn size_of_pagedescriptor_equals_64_bit() {\n        assert_eq!(\n            core::mem::size_of::<PageDescriptor>(),\n            core::mem::size_of::<u64>()\n        );\n    }\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/_arch/aarch64/memory/mmu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management Unit Driver.\n//!\n//! Only 64 KiB granule is supported.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::memory::mmu::arch_mmu\n\nuse crate::{\n    bsp, memory,\n    memory::{mmu::TranslationGranule, Address, Physical},\n};\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::intrinsics::unlikely;\nuse tock_registers::interfaces::{ReadWriteable, Readable, Writeable};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Memory Management Unit type.\nstruct MemoryManagementUnit;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub type Granule512MiB = TranslationGranule<{ 512 * 1024 * 1024 }>;\npub type Granule64KiB = TranslationGranule<{ 64 * 1024 }>;\n\n/// Constants for indexing the MAIR_EL1.\n#[allow(dead_code)]\npub mod mair {\n    pub const DEVICE: u64 = 0;\n    pub const NORMAL: u64 = 1;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic MMU: MemoryManagementUnit = MemoryManagementUnit;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<const AS_SIZE: usize> memory::mmu::AddressSpace<AS_SIZE> {\n    /// Checks for architectural restrictions.\n    pub const fn arch_address_space_size_sanity_checks() {\n        // Size must be at least one full 512 MiB table.\n        assert!((AS_SIZE % Granule512MiB::SIZE) == 0);\n\n        // Check for 48 bit virtual address size as maximum, which is supported by any ARMv8\n        // version.\n        assert!(AS_SIZE <= (1 << 48));\n    }\n}\n\nimpl MemoryManagementUnit {\n    /// Setup function for the MAIR_EL1 register.\n    #[inline(always)]\n    fn set_up_mair(&self) {\n        // Define the memory types being mapped.\n        MAIR_EL1.write(\n            // Attribute 1 - Cacheable normal DRAM.\n            MAIR_EL1::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc +\n        MAIR_EL1::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc +\n\n        // Attribute 0 - Device.\n        MAIR_EL1::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck,\n        );\n    }\n\n    /// Configure various settings of stage 1 of the EL1 translation regime.\n    #[inline(always)]\n    fn configure_translation_control(&self) {\n        let t1sz = (64 - bsp::memory::mmu::KernelVirtAddrSpace::SIZE_SHIFT) as u64;\n\n        TCR_EL1.write(\n            TCR_EL1::TBI1::Used\n                + TCR_EL1::IPS::Bits_40\n                + TCR_EL1::TG1::KiB_64\n                + TCR_EL1::SH1::Inner\n                + TCR_EL1::ORGN1::WriteBack_ReadAlloc_WriteAlloc_Cacheable\n                + TCR_EL1::IRGN1::WriteBack_ReadAlloc_WriteAlloc_Cacheable\n                + TCR_EL1::EPD1::EnableTTBR1Walks\n                + TCR_EL1::A1::TTBR1\n                + TCR_EL1::T1SZ.val(t1sz)\n                + TCR_EL1::EPD0::DisableTTBR0Walks,\n        );\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the MMU instance.\npub fn mmu() -> &'static impl memory::mmu::interface::MMU {\n    &MMU\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse memory::mmu::MMUEnableError;\n\nimpl memory::mmu::interface::MMU for MemoryManagementUnit {\n    unsafe fn enable_mmu_and_caching(\n        &self,\n        phys_tables_base_addr: Address<Physical>,\n    ) -> Result<(), MMUEnableError> {\n        if unlikely(self.is_enabled()) {\n            return Err(MMUEnableError::AlreadyEnabled);\n        }\n\n        // Fail early if translation granule is not supported.\n        if unlikely(!ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran64::Supported)) {\n            return Err(MMUEnableError::Other(\n                \"Translation granule not supported in HW\",\n            ));\n        }\n\n        // Prepare the memory attribute indirection register.\n        self.set_up_mair();\n\n        // Set the \"Translation Table Base Register\".\n        TTBR1_EL1.set_baddr(phys_tables_base_addr.as_usize() as u64);\n\n        self.configure_translation_control();\n\n        // Switch the MMU on.\n        //\n        // First, force all previous changes to be seen before the MMU is enabled.\n        barrier::isb(barrier::SY);\n\n        // Enable the MMU and turn on data and instruction caching.\n        SCTLR_EL1.modify(SCTLR_EL1::M::Enable + SCTLR_EL1::C::Cacheable + SCTLR_EL1::I::Cacheable);\n\n        // Force MMU init to complete before next instruction.\n        barrier::isb(barrier::SY);\n\n        Ok(())\n    }\n\n    #[inline(always)]\n    fn is_enabled(&self) -> bool {\n        SCTLR_EL1.matches_all(SCTLR_EL1::M::Enable)\n    }\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/_arch/aarch64/time.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural timer primitives.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::time::arch_time\n\nuse crate::warn;\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::{\n    num::{NonZeroU128, NonZeroU32, NonZeroU64},\n    ops::{Add, Div},\n    time::Duration,\n};\nuse tock_registers::interfaces::Readable;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nconst NANOSEC_PER_SEC: NonZeroU64 = NonZeroU64::new(1_000_000_000).unwrap();\n\n#[derive(Copy, Clone, PartialOrd, PartialEq)]\nstruct GenericTimerCounterValue(u64);\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n/// Boot assembly code overwrites this value with the value of CNTFRQ_EL0 before any Rust code is\n/// executed. This given value here is just a (safe) dummy.\n#[no_mangle]\nstatic ARCH_TIMER_COUNTER_FREQUENCY: NonZeroU32 = NonZeroU32::MIN;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nfn arch_timer_counter_frequency() -> NonZeroU32 {\n    // Read volatile is needed here to prevent the compiler from optimizing\n    // ARCH_TIMER_COUNTER_FREQUENCY away.\n    //\n    // This is safe, because all the safety requirements as stated in read_volatile()'s\n    // documentation are fulfilled.\n    unsafe { core::ptr::read_volatile(&ARCH_TIMER_COUNTER_FREQUENCY) }\n}\n\nimpl GenericTimerCounterValue {\n    pub const MAX: Self = GenericTimerCounterValue(u64::MAX);\n}\n\nimpl Add for GenericTimerCounterValue {\n    type Output = Self;\n\n    fn add(self, other: Self) -> Self {\n        GenericTimerCounterValue(self.0.wrapping_add(other.0))\n    }\n}\n\nimpl From<GenericTimerCounterValue> for Duration {\n    fn from(counter_value: GenericTimerCounterValue) -> Self {\n        if counter_value.0 == 0 {\n            return Duration::ZERO;\n        }\n\n        let frequency: NonZeroU64 = arch_timer_counter_frequency().into();\n\n        // Div<NonZeroU64> implementation for u64 cannot panic.\n        let secs = counter_value.0.div(frequency);\n\n        // This is safe, because frequency can never be greater than u32::MAX, which means the\n        // largest theoretical value for sub_second_counter_value is (u32::MAX - 1). Therefore,\n        // (sub_second_counter_value * NANOSEC_PER_SEC) cannot overflow an u64.\n        //\n        // The subsequent division ensures the result fits into u32, since the max result is smaller\n        // than NANOSEC_PER_SEC. Therefore, just cast it to u32 using `as`.\n        let sub_second_counter_value = counter_value.0 % frequency;\n        let nanos = unsafe { sub_second_counter_value.unchecked_mul(u64::from(NANOSEC_PER_SEC)) }\n            .div(frequency) as u32;\n\n        Duration::new(secs, nanos)\n    }\n}\n\nfn max_duration() -> Duration {\n    Duration::from(GenericTimerCounterValue::MAX)\n}\n\nimpl TryFrom<Duration> for GenericTimerCounterValue {\n    type Error = &'static str;\n\n    fn try_from(duration: Duration) -> Result<Self, Self::Error> {\n        if duration < resolution() {\n            return Ok(GenericTimerCounterValue(0));\n        }\n\n        if duration > max_duration() {\n            return Err(\"Conversion error. Duration too big\");\n        }\n\n        let frequency: u128 = u32::from(arch_timer_counter_frequency()) as u128;\n        let duration: u128 = duration.as_nanos();\n\n        // This is safe, because frequency can never be greater than u32::MAX, and\n        // (Duration::MAX.as_nanos() * u32::MAX) < u128::MAX.\n        let counter_value =\n            unsafe { duration.unchecked_mul(frequency) }.div(NonZeroU128::from(NANOSEC_PER_SEC));\n\n        // Since we checked above that we are <= max_duration(), just cast to u64.\n        Ok(GenericTimerCounterValue(counter_value as u64))\n    }\n}\n\n#[inline(always)]\nfn read_cntpct() -> GenericTimerCounterValue {\n    // Prevent that the counter is read ahead of time due to out-of-order execution.\n    barrier::isb(barrier::SY);\n    let cnt = CNTPCT_EL0.get();\n\n    GenericTimerCounterValue(cnt)\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The timer's resolution.\npub fn resolution() -> Duration {\n    Duration::from(GenericTimerCounterValue(1))\n}\n\n/// The uptime since power-on of the device.\n///\n/// This includes time consumed by firmware and bootloaders.\npub fn uptime() -> Duration {\n    read_cntpct().into()\n}\n\n/// Spin for a given duration.\npub fn spin_for(duration: Duration) {\n    let curr_counter_value = read_cntpct();\n\n    let counter_value_delta: GenericTimerCounterValue = match duration.try_into() {\n        Err(msg) => {\n            warn!(\"spin_for: {}. Skipping\", msg);\n            return;\n        }\n        Ok(val) => val,\n    };\n    let counter_value_target = curr_counter_value + counter_value_delta;\n\n    // Busy wait.\n    //\n    // Read CNTPCT_EL0 directly to avoid the ISB that is part of [`read_cntpct`].\n    while GenericTimerCounterValue(CNTPCT_EL0.get()) < counter_value_target {}\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/backtrace.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Backtracing support.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/backtrace.rs\"]\nmod arch_backtrace;\n\nuse crate::{\n    memory::{Address, Virtual},\n    symbols,\n};\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\n#[cfg(feature = \"test_build\")]\npub use arch_backtrace::{corrupt_link, corrupt_previous_frame_addr};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// A backtrace item.\n#[allow(missing_docs)]\npub enum BacktraceItem {\n    InvalidFramePointer(Address<Virtual>),\n    InvalidLink(Address<Virtual>),\n    Link(Address<Virtual>),\n}\n\n/// Pseudo-struct for printing a backtrace using its fmt::Display implementation.\npub struct Backtrace;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl fmt::Display for Backtrace {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        writeln!(f, \"Backtrace:\")?;\n        writeln!(\n            f,\n            \"      ----------------------------------------------------------------------------------------------\"\n        )?;\n        writeln!(\n            f,\n            \"          Address            Function containing address\"\n        )?;\n        writeln!(\n            f,\n            \"      ----------------------------------------------------------------------------------------------\"\n        )?;\n\n        let mut fmt_res: fmt::Result = Ok(());\n        let trace_formatter =\n            |maybe_iter: Option<&mut dyn Iterator<Item = BacktraceItem>>| match maybe_iter {\n                None => fmt_res = writeln!(f, \"ERROR! No valid stack frame found\"),\n                Some(iter) => {\n                    // Since the backtrace is printed, the first function is always\n                    // core::fmt::write. Skip 1 so it is excluded and doesn't bloat the output.\n                    for (i, backtrace_res) in iter.skip(1).enumerate() {\n                        match backtrace_res {\n                            BacktraceItem::InvalidFramePointer(addr) => {\n                                fmt_res = writeln!(\n                                    f,\n                                    \"      {:>2}. ERROR! \\\n                                    Encountered invalid frame pointer ({}) during backtrace\",\n                                    i + 1,\n                                    addr\n                                );\n                            }\n                            BacktraceItem::InvalidLink(addr) => {\n                                fmt_res = writeln!(\n                                    f,\n                                    \"      {:>2}. ERROR! \\\n                                    Link address ({}) is not contained in kernel .text section\",\n                                    i + 1,\n                                    addr\n                                );\n                            }\n                            BacktraceItem::Link(addr) => {\n                                fmt_res = writeln!(\n                                    f,\n                                    \"      {:>2}. {:016x} | {:<50}\",\n                                    i + 1,\n                                    addr.as_usize(),\n                                    match symbols::lookup_symbol(addr) {\n                                        Some(sym) => sym.name(),\n                                        _ => \"Symbol not found\",\n                                    }\n                                )\n                            }\n                        };\n\n                        if fmt_res.is_err() {\n                            break;\n                        }\n                    }\n                }\n            };\n\n        arch_backtrace::backtrace(trace_formatter);\n        fmt_res?;\n\n        writeln!(\n            f,\n            \"      ----------------------------------------------------------------------------------------------\"\n        )\n    }\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/bsp/device_driver/arm/gicv2/gicc.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GICC Driver - GIC CPU interface.\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    exception,\n    memory::{Address, Virtual},\n};\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields, register_structs,\n    registers::ReadWrite,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nregister_bitfields! {\n    u32,\n\n    /// CPU Interface Control Register\n    CTLR [\n        Enable OFFSET(0) NUMBITS(1) []\n    ],\n\n    /// Interrupt Priority Mask Register\n    PMR [\n        Priority OFFSET(0) NUMBITS(8) []\n    ],\n\n    /// Interrupt Acknowledge Register\n    IAR [\n        InterruptID OFFSET(0) NUMBITS(10) []\n    ],\n\n    /// End of Interrupt Register\n    EOIR [\n        EOIINTID OFFSET(0) NUMBITS(10) []\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    pub RegisterBlock {\n        (0x000 => CTLR: ReadWrite<u32, CTLR::Register>),\n        (0x004 => PMR: ReadWrite<u32, PMR::Register>),\n        (0x008 => _reserved1),\n        (0x00C => IAR: ReadWrite<u32, IAR::Register>),\n        (0x010 => EOIR: ReadWrite<u32, EOIR::Register>),\n        (0x014  => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the GIC CPU interface.\npub struct GICC {\n    registers: Registers,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GICC {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n        }\n    }\n\n    /// Accept interrupts of any priority.\n    ///\n    /// Quoting the GICv2 Architecture Specification:\n    ///\n    ///   \"Writing 255 to the GICC_PMR always sets it to the largest supported priority field\n    ///    value.\"\n    ///\n    /// # Safety\n    ///\n    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n    ///   of `&mut self`.\n    pub fn priority_accept_all(&self) {\n        self.registers.PMR.write(PMR::Priority.val(255)); // Comment in arch spec.\n    }\n\n    /// Enable the interface - start accepting IRQs.\n    ///\n    /// # Safety\n    ///\n    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n    ///   of `&mut self`.\n    pub fn enable(&self) {\n        self.registers.CTLR.write(CTLR::Enable::SET);\n    }\n\n    /// Extract the number of the highest-priority pending IRQ.\n    ///\n    /// Can only be called from IRQ context, which is ensured by taking an `IRQContext` token.\n    ///\n    /// # Safety\n    ///\n    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n    ///   of `&mut self`.\n    #[allow(clippy::trivially_copy_pass_by_ref)]\n    pub fn pending_irq_number<'irq_context>(\n        &self,\n        _ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) -> usize {\n        self.registers.IAR.read(IAR::InterruptID) as usize\n    }\n\n    /// Complete handling of the currently active IRQ.\n    ///\n    /// Can only be called from IRQ context, which is ensured by taking an `IRQContext` token.\n    ///\n    /// To be called after `pending_irq_number()`.\n    ///\n    /// # Safety\n    ///\n    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n    ///   of `&mut self`.\n    #[allow(clippy::trivially_copy_pass_by_ref)]\n    pub fn mark_comleted<'irq_context>(\n        &self,\n        irq_number: u32,\n        _ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) {\n        self.registers.EOIR.write(EOIR::EOIINTID.val(irq_number));\n    }\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/bsp/device_driver/arm/gicv2/gicd.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GICD Driver - GIC Distributor.\n//!\n//! # Glossary\n//!   - SPI - Shared Peripheral Interrupt.\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    memory::{Address, Virtual},\n    state, synchronization,\n    synchronization::IRQSafeNullLock,\n};\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields, register_structs,\n    registers::{ReadOnly, ReadWrite},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nregister_bitfields! {\n    u32,\n\n    /// Distributor Control Register\n    CTLR [\n        Enable OFFSET(0) NUMBITS(1) []\n    ],\n\n    /// Interrupt Controller Type Register\n    TYPER [\n        ITLinesNumber OFFSET(0)  NUMBITS(5) []\n    ],\n\n    /// Interrupt Processor Targets Registers\n    ITARGETSR [\n        Offset3 OFFSET(24) NUMBITS(8) [],\n        Offset2 OFFSET(16) NUMBITS(8) [],\n        Offset1 OFFSET(8)  NUMBITS(8) [],\n        Offset0 OFFSET(0)  NUMBITS(8) []\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    SharedRegisterBlock {\n        (0x000 => CTLR: ReadWrite<u32, CTLR::Register>),\n        (0x004 => TYPER: ReadOnly<u32, TYPER::Register>),\n        (0x008 => _reserved1),\n        (0x104 => ISENABLER: [ReadWrite<u32>; 31]),\n        (0x180 => _reserved2),\n        (0x820 => ITARGETSR: [ReadWrite<u32, ITARGETSR::Register>; 248]),\n        (0xC00 => @END),\n    }\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    BankedRegisterBlock {\n        (0x000 => _reserved1),\n        (0x100 => ISENABLER: ReadWrite<u32>),\n        (0x104 => _reserved2),\n        (0x800 => ITARGETSR: [ReadOnly<u32, ITARGETSR::Register>; 8]),\n        (0x820 => @END),\n    }\n}\n\n/// Abstraction for the non-banked parts of the associated MMIO registers.\ntype SharedRegisters = MMIODerefWrapper<SharedRegisterBlock>;\n\n/// Abstraction for the banked parts of the associated MMIO registers.\ntype BankedRegisters = MMIODerefWrapper<BankedRegisterBlock>;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the GIC Distributor.\npub struct GICD {\n    /// Access to shared registers is guarded with a lock.\n    shared_registers: IRQSafeNullLock<SharedRegisters>,\n\n    /// Access to banked registers is unguarded.\n    banked_registers: BankedRegisters,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl SharedRegisters {\n    /// Return the number of IRQs that this HW implements.\n    #[inline(always)]\n    fn num_irqs(&mut self) -> usize {\n        // Query number of implemented IRQs.\n        //\n        // Refer to GICv2 Architecture Specification, Section 4.3.2.\n        ((self.TYPER.read(TYPER::ITLinesNumber) as usize) + 1) * 32\n    }\n\n    /// Return a slice of the implemented ITARGETSR.\n    #[inline(always)]\n    fn implemented_itargets_slice(&mut self) -> &[ReadWrite<u32, ITARGETSR::Register>] {\n        assert!(self.num_irqs() >= 36);\n\n        // Calculate the max index of the shared ITARGETSR array.\n        //\n        // The first 32 IRQs are private, so not included in `shared_registers`. Each ITARGETS\n        // register has four entries, so shift right by two. Subtract one because we start\n        // counting at zero.\n        let spi_itargetsr_max_index = ((self.num_irqs() - 32) >> 2) - 1;\n\n        // Rust automatically inserts slice range sanity check, i.e. max >= min.\n        &self.ITARGETSR[0..spi_itargetsr_max_index]\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl GICD {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            shared_registers: IRQSafeNullLock::new(SharedRegisters::new(mmio_start_addr)),\n            banked_registers: BankedRegisters::new(mmio_start_addr),\n        }\n    }\n\n    /// Use a banked ITARGETSR to retrieve the executing core's GIC target mask.\n    ///\n    /// Quoting the GICv2 Architecture Specification:\n    ///\n    ///   \"GICD_ITARGETSR0 to GICD_ITARGETSR7 are read-only, and each field returns a value that\n    ///    corresponds only to the processor reading the register.\"\n    fn local_gic_target_mask(&self) -> u32 {\n        self.banked_registers.ITARGETSR[0].read(ITARGETSR::Offset0)\n    }\n\n    /// Route all SPIs to the boot core and enable the distributor.\n    pub fn boot_core_init(&self) {\n        assert!(\n            state::state_manager().is_init(),\n            \"Only allowed during kernel init phase\"\n        );\n\n        // Target all SPIs to the boot core only.\n        let mask = self.local_gic_target_mask();\n\n        self.shared_registers.lock(|regs| {\n            for i in regs.implemented_itargets_slice().iter() {\n                i.write(\n                    ITARGETSR::Offset3.val(mask)\n                        + ITARGETSR::Offset2.val(mask)\n                        + ITARGETSR::Offset1.val(mask)\n                        + ITARGETSR::Offset0.val(mask),\n                );\n            }\n\n            regs.CTLR.write(CTLR::Enable::SET);\n        });\n    }\n\n    /// Enable an interrupt.\n    pub fn enable(&self, irq_num: &super::IRQNumber) {\n        let irq_num = irq_num.get();\n\n        // Each bit in the u32 enable register corresponds to one IRQ number. Shift right by 5\n        // (division by 32) and arrive at the index for the respective ISENABLER[i].\n        let enable_reg_index = irq_num >> 5;\n        let enable_bit: u32 = 1u32 << (irq_num % 32);\n\n        // Check if we are handling a private or shared IRQ.\n        match irq_num {\n            // Private.\n            0..=31 => {\n                let enable_reg = &self.banked_registers.ISENABLER;\n                enable_reg.set(enable_reg.get() | enable_bit);\n            }\n            // Shared.\n            _ => {\n                let enable_reg_index_shared = enable_reg_index - 1;\n\n                self.shared_registers.lock(|regs| {\n                    let enable_reg = &regs.ISENABLER[enable_reg_index_shared];\n                    enable_reg.set(enable_reg.get() | enable_bit);\n                });\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/bsp/device_driver/arm/gicv2.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GICv2 Driver - ARM Generic Interrupt Controller v2.\n//!\n//! The following is a collection of excerpts with useful information from\n//!   - `Programmer's Guide for ARMv8-A`\n//!   - `ARM Generic Interrupt Controller Architecture Specification`\n//!\n//! # Programmer's Guide - 10.6.1 Configuration\n//!\n//! The GIC is accessed as a memory-mapped peripheral.\n//!\n//! All cores can access the common Distributor, but the CPU interface is banked, that is, each core\n//! uses the same address to access its own private CPU interface.\n//!\n//! It is not possible for a core to access the CPU interface of another core.\n//!\n//! # Architecture Specification - 10.6.2 Initialization\n//!\n//! Both the Distributor and the CPU interfaces are disabled at reset. The GIC must be initialized\n//! after reset before it can deliver interrupts to the core.\n//!\n//! In the Distributor, software must configure the priority, target, security and enable individual\n//! interrupts. The Distributor must subsequently be enabled through its control register\n//! (GICD_CTLR). For each CPU interface, software must program the priority mask and preemption\n//! settings.\n//!\n//! Each CPU interface block itself must be enabled through its control register (GICD_CTLR). This\n//! prepares the GIC to deliver interrupts to the core.\n//!\n//! Before interrupts are expected in the core, software prepares the core to take interrupts by\n//! setting a valid interrupt vector in the vector table, and clearing interrupt mask bits in\n//! PSTATE, and setting the routing controls.\n//!\n//! The entire interrupt mechanism in the system can be disabled by disabling the Distributor.\n//! Interrupt delivery to an individual core can be disabled by disabling its CPU interface.\n//! Individual interrupts can also be disabled (or enabled) in the distributor.\n//!\n//! For an interrupt to reach the core, the individual interrupt, Distributor and CPU interface must\n//! all be enabled. The interrupt also needs to be of sufficient priority, that is, higher than the\n//! core's priority mask.\n//!\n//! # Architecture Specification - 1.4.2 Interrupt types\n//!\n//! - Peripheral interrupt\n//!     - Private Peripheral Interrupt (PPI)\n//!         - This is a peripheral interrupt that is specific to a single processor.\n//!     - Shared Peripheral Interrupt (SPI)\n//!         - This is a peripheral interrupt that the Distributor can route to any of a specified\n//!           combination of processors.\n//!\n//! - Software-generated interrupt (SGI)\n//!     - This is an interrupt generated by software writing to a GICD_SGIR register in the GIC. The\n//!       system uses SGIs for interprocessor communication.\n//!     - An SGI has edge-triggered properties. The software triggering of the interrupt is\n//!       equivalent to the edge transition of the interrupt request signal.\n//!     - When an SGI occurs in a multiprocessor implementation, the CPUID field in the Interrupt\n//!       Acknowledge Register, GICC_IAR, or the Aliased Interrupt Acknowledge Register, GICC_AIAR,\n//!       identifies the processor that requested the interrupt.\n//!\n//! # Architecture Specification - 2.2.1 Interrupt IDs\n//!\n//! Interrupts from sources are identified using ID numbers. Each CPU interface can see up to 1020\n//! interrupts. The banking of SPIs and PPIs increases the total number of interrupts supported by\n//! the Distributor.\n//!\n//! The GIC assigns interrupt ID numbers ID0-ID1019 as follows:\n//!   - Interrupt numbers 32..1019 are used for SPIs.\n//!   - Interrupt numbers 0..31 are used for interrupts that are private to a CPU interface. These\n//!     interrupts are banked in the Distributor.\n//!       - A banked interrupt is one where the Distributor can have multiple interrupts with the\n//!         same ID. A banked interrupt is identified uniquely by its ID number and its associated\n//!         CPU interface number. Of the banked interrupt IDs:\n//!           - 00..15 SGIs\n//!           - 16..31 PPIs\n\nmod gicc;\nmod gicd;\n\nuse crate::{\n    bsp::{self, device_driver::common::BoundedUsize},\n    cpu, driver, exception,\n    memory::{Address, Virtual},\n    synchronization,\n    synchronization::InitStateLock,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\ntype HandlerTable = [Option<exception::asynchronous::IRQHandlerDescriptor<IRQNumber>>;\n    IRQNumber::MAX_INCLUSIVE + 1];\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Used for the associated type of trait [`exception::asynchronous::interface::IRQManager`].\npub type IRQNumber = BoundedUsize<{ GICv2::MAX_IRQ_NUMBER }>;\n\n/// Representation of the GIC.\npub struct GICv2 {\n    /// The Distributor.\n    gicd: gicd::GICD,\n\n    /// The CPU Interface.\n    gicc: gicc::GICC,\n\n    /// Stores registered IRQ handlers. Writable only during kernel init. RO afterwards.\n    handler_table: InitStateLock<HandlerTable>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GICv2 {\n    const MAX_IRQ_NUMBER: usize = 300; // Normally 1019, but keep it lower to save some space.\n\n    pub const COMPATIBLE: &'static str = \"GICv2 (ARM Generic Interrupt Controller v2)\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(\n        gicd_mmio_start_addr: Address<Virtual>,\n        gicc_mmio_start_addr: Address<Virtual>,\n    ) -> Self {\n        Self {\n            gicd: gicd::GICD::new(gicd_mmio_start_addr),\n            gicc: gicc::GICC::new(gicc_mmio_start_addr),\n            handler_table: InitStateLock::new([None; IRQNumber::MAX_INCLUSIVE + 1]),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::ReadWriteEx;\n\nimpl driver::interface::DeviceDriver for GICv2 {\n    type IRQNumberType = IRQNumber;\n\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n\n    unsafe fn init(&self) -> Result<(), &'static str> {\n        if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() {\n            self.gicd.boot_core_init();\n        }\n\n        self.gicc.priority_accept_all();\n        self.gicc.enable();\n\n        Ok(())\n    }\n}\n\nimpl exception::asynchronous::interface::IRQManager for GICv2 {\n    type IRQNumberType = IRQNumber;\n\n    fn register_handler(\n        &self,\n        irq_handler_descriptor: exception::asynchronous::IRQHandlerDescriptor<Self::IRQNumberType>,\n    ) -> Result<(), &'static str> {\n        self.handler_table.write(|table| {\n            let irq_number = irq_handler_descriptor.number().get();\n\n            if table[irq_number].is_some() {\n                return Err(\"IRQ handler already registered\");\n            }\n\n            table[irq_number] = Some(irq_handler_descriptor);\n\n            Ok(())\n        })\n    }\n\n    fn enable(&self, irq_number: &Self::IRQNumberType) {\n        self.gicd.enable(irq_number);\n    }\n\n    fn handle_pending_irqs<'irq_context>(\n        &'irq_context self,\n        ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) {\n        // Extract the highest priority pending IRQ number from the Interrupt Acknowledge Register\n        // (IAR).\n        let irq_number = self.gicc.pending_irq_number(ic);\n\n        // Guard against spurious interrupts.\n        if irq_number > GICv2::MAX_IRQ_NUMBER {\n            return;\n        }\n\n        // Call the IRQ handler. Panic if there is none.\n        self.handler_table.read(|table| {\n            match table[irq_number] {\n                None => panic!(\"No handler registered for IRQ {}\", irq_number),\n                Some(descriptor) => {\n                    // Call the IRQ handler. Panics on failure.\n                    descriptor.handler().handle().expect(\"Error handling IRQ\");\n                }\n            }\n        });\n\n        // Signal completion of handling.\n        self.gicc.mark_comleted(irq_number as u32, ic);\n    }\n\n    fn print_handler(&self) {\n        use crate::info;\n\n        info!(\"      Peripheral handler:\");\n\n        self.handler_table.read(|table| {\n            for (i, opt) in table.iter().skip(32).enumerate() {\n                if let Some(handler) = opt {\n                    info!(\"            {: >3}. {}\", i + 32, handler.name());\n                }\n            }\n        });\n    }\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/bsp/device_driver/arm.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! ARM driver top level.\n\npub mod gicv2;\n\npub use gicv2::*;\n"
  },
  {
    "path": "18_backtrace/kernel/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GPIO Driver.\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    driver,\n    exception::asynchronous::IRQNumber,\n    memory::{Address, Virtual},\n    synchronization,\n    synchronization::IRQSafeNullLock,\n};\nuse tock_registers::{\n    interfaces::{ReadWriteable, Writeable},\n    register_bitfields, register_structs,\n    registers::ReadWrite,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// GPIO registers.\n//\n// Descriptions taken from\n// - https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf\n// - https://datasheets.raspberrypi.org/bcm2711/bcm2711-peripherals.pdf\nregister_bitfields! {\n    u32,\n\n    /// GPIO Function Select 1\n    GPFSEL1 [\n        /// Pin 15\n        FSEL15 OFFSET(15) NUMBITS(3) [\n            Input = 0b000,\n            Output = 0b001,\n            AltFunc0 = 0b100  // PL011 UART RX\n\n        ],\n\n        /// Pin 14\n        FSEL14 OFFSET(12) NUMBITS(3) [\n            Input = 0b000,\n            Output = 0b001,\n            AltFunc0 = 0b100  // PL011 UART TX\n        ]\n    ],\n\n    /// GPIO Pull-up/down Register\n    ///\n    /// BCM2837 only.\n    GPPUD [\n        /// Controls the actuation of the internal pull-up/down control line to ALL the GPIO pins.\n        PUD OFFSET(0) NUMBITS(2) [\n            Off = 0b00,\n            PullDown = 0b01,\n            PullUp = 0b10\n        ]\n    ],\n\n    /// GPIO Pull-up/down Clock Register 0\n    ///\n    /// BCM2837 only.\n    GPPUDCLK0 [\n        /// Pin 15\n        PUDCLK15 OFFSET(15) NUMBITS(1) [\n            NoEffect = 0,\n            AssertClock = 1\n        ],\n\n        /// Pin 14\n        PUDCLK14 OFFSET(14) NUMBITS(1) [\n            NoEffect = 0,\n            AssertClock = 1\n        ]\n    ],\n\n    /// GPIO Pull-up / Pull-down Register 0\n    ///\n    /// BCM2711 only.\n    GPIO_PUP_PDN_CNTRL_REG0 [\n        /// Pin 15\n        GPIO_PUP_PDN_CNTRL15 OFFSET(30) NUMBITS(2) [\n            NoResistor = 0b00,\n            PullUp = 0b01\n        ],\n\n        /// Pin 14\n        GPIO_PUP_PDN_CNTRL14 OFFSET(28) NUMBITS(2) [\n            NoResistor = 0b00,\n            PullUp = 0b01\n        ]\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    RegisterBlock {\n        (0x00 => _reserved1),\n        (0x04 => GPFSEL1: ReadWrite<u32, GPFSEL1::Register>),\n        (0x08 => _reserved2),\n        (0x94 => GPPUD: ReadWrite<u32, GPPUD::Register>),\n        (0x98 => GPPUDCLK0: ReadWrite<u32, GPPUDCLK0::Register>),\n        (0x9C => _reserved3),\n        (0xE4 => GPIO_PUP_PDN_CNTRL_REG0: ReadWrite<u32, GPIO_PUP_PDN_CNTRL_REG0::Register>),\n        (0xE8 => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\nstruct GPIOInner {\n    registers: Registers,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the GPIO HW.\npub struct GPIO {\n    inner: IRQSafeNullLock<GPIOInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GPIOInner {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n        }\n    }\n\n    /// Disable pull-up/down on pins 14 and 15.\n    #[cfg(feature = \"bsp_rpi3\")]\n    fn disable_pud_14_15_bcm2837(&mut self) {\n        use crate::time;\n        use core::time::Duration;\n\n        // The Linux 2837 GPIO driver waits 1 µs between the steps.\n        const DELAY: Duration = Duration::from_micros(1);\n\n        self.registers.GPPUD.write(GPPUD::PUD::Off);\n        time::time_manager().spin_for(DELAY);\n\n        self.registers\n            .GPPUDCLK0\n            .write(GPPUDCLK0::PUDCLK15::AssertClock + GPPUDCLK0::PUDCLK14::AssertClock);\n        time::time_manager().spin_for(DELAY);\n\n        self.registers.GPPUD.write(GPPUD::PUD::Off);\n        self.registers.GPPUDCLK0.set(0);\n    }\n\n    /// Disable pull-up/down on pins 14 and 15.\n    #[cfg(feature = \"bsp_rpi4\")]\n    fn disable_pud_14_15_bcm2711(&mut self) {\n        self.registers.GPIO_PUP_PDN_CNTRL_REG0.write(\n            GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL15::PullUp\n                + GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL14::PullUp,\n        );\n    }\n\n    /// Map PL011 UART as standard output.\n    ///\n    /// TX to pin 14\n    /// RX to pin 15\n    pub fn map_pl011_uart(&mut self) {\n        // Select the UART on pins 14 and 15.\n        self.registers\n            .GPFSEL1\n            .modify(GPFSEL1::FSEL15::AltFunc0 + GPFSEL1::FSEL14::AltFunc0);\n\n        // Disable pull-up/down on pins 14 and 15.\n        #[cfg(feature = \"bsp_rpi3\")]\n        self.disable_pud_14_15_bcm2837();\n\n        #[cfg(feature = \"bsp_rpi4\")]\n        self.disable_pud_14_15_bcm2711();\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GPIO {\n    pub const COMPATIBLE: &'static str = \"BCM GPIO\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            inner: IRQSafeNullLock::new(GPIOInner::new(mmio_start_addr)),\n        }\n    }\n\n    /// Concurrency safe version of `GPIOInner.map_pl011_uart()`\n    pub fn map_pl011_uart(&self) {\n        self.inner.lock(|inner| inner.map_pl011_uart())\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl driver::interface::DeviceDriver for GPIO {\n    type IRQNumberType = IRQNumber;\n\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Peripheral Interrupt Controller Driver.\n//!\n//! # Resources\n//!\n//! - <https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf>\n\nuse super::{PendingIRQs, PeripheralIRQ};\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    exception,\n    memory::{Address, Virtual},\n    synchronization,\n    synchronization::{IRQSafeNullLock, InitStateLock},\n};\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_structs,\n    registers::{ReadOnly, WriteOnly},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    WORegisterBlock {\n        (0x00 => _reserved1),\n        (0x10 => ENABLE_1: WriteOnly<u32>),\n        (0x14 => ENABLE_2: WriteOnly<u32>),\n        (0x18 => @END),\n    }\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    RORegisterBlock {\n        (0x00 => _reserved1),\n        (0x04 => PENDING_1: ReadOnly<u32>),\n        (0x08 => PENDING_2: ReadOnly<u32>),\n        (0x0c => @END),\n    }\n}\n\n/// Abstraction for the WriteOnly parts of the associated MMIO registers.\ntype WriteOnlyRegisters = MMIODerefWrapper<WORegisterBlock>;\n\n/// Abstraction for the ReadOnly parts of the associated MMIO registers.\ntype ReadOnlyRegisters = MMIODerefWrapper<RORegisterBlock>;\n\ntype HandlerTable = [Option<exception::asynchronous::IRQHandlerDescriptor<PeripheralIRQ>>;\n    PeripheralIRQ::MAX_INCLUSIVE + 1];\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the peripheral interrupt controller.\npub struct PeripheralIC {\n    /// Access to write registers is guarded with a lock.\n    wo_registers: IRQSafeNullLock<WriteOnlyRegisters>,\n\n    /// Register read access is unguarded.\n    ro_registers: ReadOnlyRegisters,\n\n    /// Stores registered IRQ handlers. Writable only during kernel init. RO afterwards.\n    handler_table: InitStateLock<HandlerTable>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PeripheralIC {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            wo_registers: IRQSafeNullLock::new(WriteOnlyRegisters::new(mmio_start_addr)),\n            ro_registers: ReadOnlyRegisters::new(mmio_start_addr),\n            handler_table: InitStateLock::new([None; PeripheralIRQ::MAX_INCLUSIVE + 1]),\n        }\n    }\n\n    /// Query the list of pending IRQs.\n    fn pending_irqs(&self) -> PendingIRQs {\n        let pending_mask: u64 = (u64::from(self.ro_registers.PENDING_2.get()) << 32)\n            | u64::from(self.ro_registers.PENDING_1.get());\n\n        PendingIRQs::new(pending_mask)\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::{Mutex, ReadWriteEx};\n\nimpl exception::asynchronous::interface::IRQManager for PeripheralIC {\n    type IRQNumberType = PeripheralIRQ;\n\n    fn register_handler(\n        &self,\n        irq_handler_descriptor: exception::asynchronous::IRQHandlerDescriptor<Self::IRQNumberType>,\n    ) -> Result<(), &'static str> {\n        self.handler_table.write(|table| {\n            let irq_number = irq_handler_descriptor.number().get();\n\n            if table[irq_number].is_some() {\n                return Err(\"IRQ handler already registered\");\n            }\n\n            table[irq_number] = Some(irq_handler_descriptor);\n\n            Ok(())\n        })\n    }\n\n    fn enable(&self, irq: &Self::IRQNumberType) {\n        self.wo_registers.lock(|regs| {\n            let enable_reg = if irq.get() <= 31 {\n                &regs.ENABLE_1\n            } else {\n                &regs.ENABLE_2\n            };\n\n            let enable_bit: u32 = 1 << (irq.get() % 32);\n\n            // Writing a 1 to a bit will set the corresponding IRQ enable bit. All other IRQ enable\n            // bits are unaffected. So we don't need read and OR'ing here.\n            enable_reg.set(enable_bit);\n        });\n    }\n\n    fn handle_pending_irqs<'irq_context>(\n        &'irq_context self,\n        _ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) {\n        self.handler_table.read(|table| {\n            for irq_number in self.pending_irqs() {\n                match table[irq_number] {\n                    None => panic!(\"No handler registered for IRQ {}\", irq_number),\n                    Some(descriptor) => {\n                        // Call the IRQ handler. Panics on failure.\n                        descriptor.handler().handle().expect(\"Error handling IRQ\");\n                    }\n                }\n            }\n        })\n    }\n\n    fn print_handler(&self) {\n        use crate::info;\n\n        info!(\"      Peripheral handler:\");\n\n        self.handler_table.read(|table| {\n            for (i, opt) in table.iter().enumerate() {\n                if let Some(handler) = opt {\n                    info!(\"            {: >3}. {}\", i, handler.name());\n                }\n            }\n        });\n    }\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Interrupt Controller Driver.\n\nmod peripheral_ic;\n\nuse crate::{\n    bsp::device_driver::common::BoundedUsize,\n    driver,\n    exception::{self, asynchronous::IRQHandlerDescriptor},\n    memory::{Address, Virtual},\n};\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Wrapper struct for a bitmask indicating pending IRQ numbers.\nstruct PendingIRQs {\n    bitmask: u64,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub type LocalIRQ = BoundedUsize<{ InterruptController::MAX_LOCAL_IRQ_NUMBER }>;\npub type PeripheralIRQ = BoundedUsize<{ InterruptController::MAX_PERIPHERAL_IRQ_NUMBER }>;\n\n/// Used for the associated type of trait [`exception::asynchronous::interface::IRQManager`].\n#[derive(Copy, Clone)]\n#[allow(missing_docs)]\npub enum IRQNumber {\n    Local(LocalIRQ),\n    Peripheral(PeripheralIRQ),\n}\n\n/// Representation of the Interrupt Controller.\npub struct InterruptController {\n    periph: peripheral_ic::PeripheralIC,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PendingIRQs {\n    pub fn new(bitmask: u64) -> Self {\n        Self { bitmask }\n    }\n}\n\nimpl Iterator for PendingIRQs {\n    type Item = usize;\n\n    fn next(&mut self) -> Option<Self::Item> {\n        if self.bitmask == 0 {\n            return None;\n        }\n\n        let next = self.bitmask.trailing_zeros() as usize;\n        self.bitmask &= self.bitmask.wrapping_sub(1);\n        Some(next)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl fmt::Display for IRQNumber {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        match self {\n            Self::Local(number) => write!(f, \"Local({})\", number),\n            Self::Peripheral(number) => write!(f, \"Peripheral({})\", number),\n        }\n    }\n}\n\nimpl InterruptController {\n    // Restrict to 3 for now. This makes future code for local_ic.rs more straight forward.\n    const MAX_LOCAL_IRQ_NUMBER: usize = 3;\n    const MAX_PERIPHERAL_IRQ_NUMBER: usize = 63;\n\n    pub const COMPATIBLE: &'static str = \"BCM Interrupt Controller\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(periph_mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            periph: peripheral_ic::PeripheralIC::new(periph_mmio_start_addr),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\n\nimpl driver::interface::DeviceDriver for InterruptController {\n    type IRQNumberType = IRQNumber;\n\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n}\n\nimpl exception::asynchronous::interface::IRQManager for InterruptController {\n    type IRQNumberType = IRQNumber;\n\n    fn register_handler(\n        &self,\n        irq_handler_descriptor: exception::asynchronous::IRQHandlerDescriptor<Self::IRQNumberType>,\n    ) -> Result<(), &'static str> {\n        match irq_handler_descriptor.number() {\n            IRQNumber::Local(_) => unimplemented!(\"Local IRQ controller not implemented.\"),\n            IRQNumber::Peripheral(pirq) => {\n                let periph_descriptor = IRQHandlerDescriptor::new(\n                    pirq,\n                    irq_handler_descriptor.name(),\n                    irq_handler_descriptor.handler(),\n                );\n\n                self.periph.register_handler(periph_descriptor)\n            }\n        }\n    }\n\n    fn enable(&self, irq: &Self::IRQNumberType) {\n        match irq {\n            IRQNumber::Local(_) => unimplemented!(\"Local IRQ controller not implemented.\"),\n            IRQNumber::Peripheral(pirq) => self.periph.enable(pirq),\n        }\n    }\n\n    fn handle_pending_irqs<'irq_context>(\n        &'irq_context self,\n        ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) {\n        // It can only be a peripheral IRQ pending because enable() does not support local IRQs yet.\n        self.periph.handle_pending_irqs(ic)\n    }\n\n    fn print_handler(&self) {\n        self.periph.print_handler();\n    }\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! PL011 UART driver.\n//!\n//! # Resources\n//!\n//! - <https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf>\n//! - <https://developer.arm.com/documentation/ddi0183/latest>\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    console, cpu, driver,\n    exception::{self, asynchronous::IRQNumber},\n    memory::{Address, Virtual},\n    synchronization,\n    synchronization::IRQSafeNullLock,\n};\nuse core::fmt;\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields, register_structs,\n    registers::{ReadOnly, ReadWrite, WriteOnly},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// PL011 UART registers.\n//\n// Descriptions taken from \"PrimeCell UART (PL011) Technical Reference Manual\" r1p5.\nregister_bitfields! {\n    u32,\n\n    /// Flag Register.\n    FR [\n        /// Transmit FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n        /// Line Control Register, LCR_H.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the transmit holding register is empty.\n        /// - If the FIFO is enabled, the TXFE bit is set when the transmit FIFO is empty.\n        /// - This bit does not indicate if there is data in the transmit shift register.\n        TXFE OFFSET(7) NUMBITS(1) [],\n\n        /// Transmit FIFO full. The meaning of this bit depends on the state of the FEN bit in the\n        /// LCR_H Register.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the transmit holding register is full.\n        /// - If the FIFO is enabled, the TXFF bit is set when the transmit FIFO is full.\n        TXFF OFFSET(5) NUMBITS(1) [],\n\n        /// Receive FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n        /// LCR_H Register.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the receive holding register is empty.\n        /// - If the FIFO is enabled, the RXFE bit is set when the receive FIFO is empty.\n        RXFE OFFSET(4) NUMBITS(1) [],\n\n        /// UART busy. If this bit is set to 1, the UART is busy transmitting data. This bit remains\n        /// set until the complete byte, including all the stop bits, has been sent from the shift\n        /// register.\n        ///\n        /// This bit is set as soon as the transmit FIFO becomes non-empty, regardless of whether\n        /// the UART is enabled or not.\n        BUSY OFFSET(3) NUMBITS(1) []\n    ],\n\n    /// Integer Baud Rate Divisor.\n    IBRD [\n        /// The integer baud rate divisor.\n        BAUD_DIVINT OFFSET(0) NUMBITS(16) []\n    ],\n\n    /// Fractional Baud Rate Divisor.\n    FBRD [\n        ///  The fractional baud rate divisor.\n        BAUD_DIVFRAC OFFSET(0) NUMBITS(6) []\n    ],\n\n    /// Line Control Register.\n    LCR_H [\n        /// Word length. These bits indicate the number of data bits transmitted or received in a\n        /// frame.\n        #[allow(clippy::enum_variant_names)]\n        WLEN OFFSET(5) NUMBITS(2) [\n            FiveBit = 0b00,\n            SixBit = 0b01,\n            SevenBit = 0b10,\n            EightBit = 0b11\n        ],\n\n        /// Enable FIFOs:\n        ///\n        /// 0 = FIFOs are disabled (character mode) that is, the FIFOs become 1-byte-deep holding\n        /// registers.\n        ///\n        /// 1 = Transmit and receive FIFO buffers are enabled (FIFO mode).\n        FEN  OFFSET(4) NUMBITS(1) [\n            FifosDisabled = 0,\n            FifosEnabled = 1\n        ]\n    ],\n\n    /// Control Register.\n    CR [\n        /// Receive enable. If this bit is set to 1, the receive section of the UART is enabled.\n        /// Data reception occurs for either UART signals or SIR signals depending on the setting of\n        /// the SIREN bit. When the UART is disabled in the middle of reception, it completes the\n        /// current character before stopping.\n        RXE OFFSET(9) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// Transmit enable. If this bit is set to 1, the transmit section of the UART is enabled.\n        /// Data transmission occurs for either UART signals, or SIR signals depending on the\n        /// setting of the SIREN bit. When the UART is disabled in the middle of transmission, it\n        /// completes the current character before stopping.\n        TXE OFFSET(8) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// UART enable:\n        ///\n        /// 0 = UART is disabled. If the UART is disabled in the middle of transmission or\n        /// reception, it completes the current character before stopping.\n        ///\n        /// 1 = The UART is enabled. Data transmission and reception occurs for either UART signals\n        /// or SIR signals depending on the setting of the SIREN bit\n        UARTEN OFFSET(0) NUMBITS(1) [\n            /// If the UART is disabled in the middle of transmission or reception, it completes the\n            /// current character before stopping.\n            Disabled = 0,\n            Enabled = 1\n        ]\n    ],\n\n    /// Interrupt FIFO Level Select Register.\n    IFLS [\n        /// Receive interrupt FIFO level select. The trigger points for the receive interrupt are as\n        /// follows.\n        RXIFLSEL OFFSET(3) NUMBITS(5) [\n            OneEigth = 0b000,\n            OneQuarter = 0b001,\n            OneHalf = 0b010,\n            ThreeQuarters = 0b011,\n            SevenEights = 0b100\n        ]\n    ],\n\n    /// Interrupt Mask Set/Clear Register.\n    IMSC [\n        /// Receive timeout interrupt mask. A read returns the current mask for the UARTRTINTR\n        /// interrupt.\n        ///\n        /// - On a write of 1, the mask of the UARTRTINTR interrupt is set.\n        /// - A write of 0 clears the mask.\n        RTIM OFFSET(6) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// Receive interrupt mask. A read returns the current mask for the UARTRXINTR interrupt.\n        ///\n        /// - On a write of 1, the mask of the UARTRXINTR interrupt is set.\n        /// - A write of 0 clears the mask.\n        RXIM OFFSET(4) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ]\n    ],\n\n    /// Masked Interrupt Status Register.\n    MIS [\n        /// Receive timeout masked interrupt status. Returns the masked interrupt state of the\n        /// UARTRTINTR interrupt.\n        RTMIS OFFSET(6) NUMBITS(1) [],\n\n        /// Receive masked interrupt status. Returns the masked interrupt state of the UARTRXINTR\n        /// interrupt.\n        RXMIS OFFSET(4) NUMBITS(1) []\n    ],\n\n    /// Interrupt Clear Register.\n    ICR [\n        /// Meta field for all pending interrupts.\n        ALL OFFSET(0) NUMBITS(11) []\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    pub RegisterBlock {\n        (0x00 => DR: ReadWrite<u32>),\n        (0x04 => _reserved1),\n        (0x18 => FR: ReadOnly<u32, FR::Register>),\n        (0x1c => _reserved2),\n        (0x24 => IBRD: WriteOnly<u32, IBRD::Register>),\n        (0x28 => FBRD: WriteOnly<u32, FBRD::Register>),\n        (0x2c => LCR_H: WriteOnly<u32, LCR_H::Register>),\n        (0x30 => CR: WriteOnly<u32, CR::Register>),\n        (0x34 => IFLS: ReadWrite<u32, IFLS::Register>),\n        (0x38 => IMSC: ReadWrite<u32, IMSC::Register>),\n        (0x3C => _reserved3),\n        (0x40 => MIS: ReadOnly<u32, MIS::Register>),\n        (0x44 => ICR: WriteOnly<u32, ICR::Register>),\n        (0x48 => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\n#[derive(PartialEq)]\nenum BlockingMode {\n    Blocking,\n    NonBlocking,\n}\n\nstruct PL011UartInner {\n    registers: Registers,\n    chars_written: usize,\n    chars_read: usize,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the UART.\npub struct PL011Uart {\n    inner: IRQSafeNullLock<PL011UartInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PL011UartInner {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n            chars_written: 0,\n            chars_read: 0,\n        }\n    }\n\n    /// Set up baud rate and characteristics.\n    ///\n    /// This results in 8N1 and 921_600 baud.\n    ///\n    /// The calculation for the BRD is (we set the clock to 48 MHz in config.txt):\n    /// `(48_000_000 / 16) / 921_600 = 3.2552083`.\n    ///\n    /// This means the integer part is `3` and goes into the `IBRD`.\n    /// The fractional part is `0.2552083`.\n    ///\n    /// `FBRD` calculation according to the PL011 Technical Reference Manual:\n    /// `INTEGER((0.2552083 * 64) + 0.5) = 16`.\n    ///\n    /// Therefore, the generated baud rate divider is: `3 + 16/64 = 3.25`. Which results in a\n    /// genrated baud rate of `48_000_000 / (16 * 3.25) = 923_077`.\n    ///\n    /// Error = `((923_077 - 921_600) / 921_600) * 100 = 0.16%`.\n    pub fn init(&mut self) {\n        // Execution can arrive here while there are still characters queued in the TX FIFO and\n        // actively being sent out by the UART hardware. If the UART is turned off in this case,\n        // those queued characters would be lost.\n        //\n        // For example, this can happen during runtime on a call to panic!(), because panic!()\n        // initializes its own UART instance and calls init().\n        //\n        // Hence, flush first to ensure all pending characters are transmitted.\n        self.flush();\n\n        // Turn the UART off temporarily.\n        self.registers.CR.set(0);\n\n        // Clear all pending interrupts.\n        self.registers.ICR.write(ICR::ALL::CLEAR);\n\n        // From the PL011 Technical Reference Manual:\n        //\n        // The LCR_H, IBRD, and FBRD registers form the single 30-bit wide LCR Register that is\n        // updated on a single write strobe generated by a LCR_H write. So, to internally update the\n        // contents of IBRD or FBRD, a LCR_H write must always be performed at the end.\n        //\n        // Set the baud rate, 8N1 and FIFO enabled.\n        self.registers.IBRD.write(IBRD::BAUD_DIVINT.val(3));\n        self.registers.FBRD.write(FBRD::BAUD_DIVFRAC.val(16));\n        self.registers\n            .LCR_H\n            .write(LCR_H::WLEN::EightBit + LCR_H::FEN::FifosEnabled);\n\n        // Set RX FIFO fill level at 1/8.\n        self.registers.IFLS.write(IFLS::RXIFLSEL::OneEigth);\n\n        // Enable RX IRQ + RX timeout IRQ.\n        self.registers\n            .IMSC\n            .write(IMSC::RXIM::Enabled + IMSC::RTIM::Enabled);\n\n        // Turn the UART on.\n        self.registers\n            .CR\n            .write(CR::UARTEN::Enabled + CR::TXE::Enabled + CR::RXE::Enabled);\n    }\n\n    /// Send a character.\n    fn write_char(&mut self, c: char) {\n        // Spin while TX FIFO full is set, waiting for an empty slot.\n        while self.registers.FR.matches_all(FR::TXFF::SET) {\n            cpu::nop();\n        }\n\n        // Write the character to the buffer.\n        self.registers.DR.set(c as u32);\n\n        self.chars_written += 1;\n    }\n\n    /// Block execution until the last buffered character has been physically put on the TX wire.\n    fn flush(&self) {\n        // Spin until the busy bit is cleared.\n        while self.registers.FR.matches_all(FR::BUSY::SET) {\n            cpu::nop();\n        }\n    }\n\n    /// Retrieve a character.\n    fn read_char_converting(&mut self, blocking_mode: BlockingMode) -> Option<char> {\n        // If RX FIFO is empty,\n        if self.registers.FR.matches_all(FR::RXFE::SET) {\n            // immediately return in non-blocking mode.\n            if blocking_mode == BlockingMode::NonBlocking {\n                return None;\n            }\n\n            // Otherwise, wait until a char was received.\n            while self.registers.FR.matches_all(FR::RXFE::SET) {\n                cpu::nop();\n            }\n        }\n\n        // Read one character.\n        let mut ret = self.registers.DR.get() as u8 as char;\n\n        // Convert carrige return to newline.\n        if ret == '\\r' {\n            ret = '\\n'\n        }\n\n        // Update statistics.\n        self.chars_read += 1;\n\n        Some(ret)\n    }\n}\n\n/// Implementing `core::fmt::Write` enables usage of the `format_args!` macros, which in turn are\n/// used to implement the `kernel`'s `print!` and `println!` macros. By implementing `write_str()`,\n/// we get `write_fmt()` automatically.\n///\n/// The function takes an `&mut self`, so it must be implemented for the inner struct.\n///\n/// See [`src/print.rs`].\n///\n/// [`src/print.rs`]: ../../print/index.html\nimpl fmt::Write for PL011UartInner {\n    fn write_str(&mut self, s: &str) -> fmt::Result {\n        for c in s.chars() {\n            self.write_char(c);\n        }\n\n        Ok(())\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PL011Uart {\n    pub const COMPATIBLE: &'static str = \"BCM PL011 UART\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            inner: IRQSafeNullLock::new(PL011UartInner::new(mmio_start_addr)),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl driver::interface::DeviceDriver for PL011Uart {\n    type IRQNumberType = IRQNumber;\n\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n\n    unsafe fn init(&self) -> Result<(), &'static str> {\n        self.inner.lock(|inner| inner.init());\n\n        Ok(())\n    }\n\n    fn register_and_enable_irq_handler(\n        &'static self,\n        irq_number: &Self::IRQNumberType,\n    ) -> Result<(), &'static str> {\n        use exception::asynchronous::{irq_manager, IRQHandlerDescriptor};\n\n        let descriptor = IRQHandlerDescriptor::new(*irq_number, Self::COMPATIBLE, self);\n\n        irq_manager().register_handler(descriptor)?;\n        irq_manager().enable(irq_number);\n\n        Ok(())\n    }\n}\n\nimpl console::interface::Write for PL011Uart {\n    /// Passthrough of `args` to the `core::fmt::Write` implementation, but guarded by a Mutex to\n    /// serialize access.\n    fn write_char(&self, c: char) {\n        self.inner.lock(|inner| inner.write_char(c));\n    }\n\n    fn write_fmt(&self, args: core::fmt::Arguments) -> fmt::Result {\n        // Fully qualified syntax for the call to `core::fmt::Write::write_fmt()` to increase\n        // readability.\n        self.inner.lock(|inner| fmt::Write::write_fmt(inner, args))\n    }\n\n    fn flush(&self) {\n        // Spin until TX FIFO empty is set.\n        self.inner.lock(|inner| inner.flush());\n    }\n}\n\nimpl console::interface::Read for PL011Uart {\n    fn read_char(&self) -> char {\n        self.inner\n            .lock(|inner| inner.read_char_converting(BlockingMode::Blocking).unwrap())\n    }\n\n    fn clear_rx(&self) {\n        // Read from the RX FIFO until it is indicating empty.\n        while self\n            .inner\n            .lock(|inner| inner.read_char_converting(BlockingMode::NonBlocking))\n            .is_some()\n        {}\n    }\n}\n\nimpl console::interface::Statistics for PL011Uart {\n    fn chars_written(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_written)\n    }\n\n    fn chars_read(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_read)\n    }\n}\n\nimpl console::interface::All for PL011Uart {}\n\nimpl exception::asynchronous::interface::IRQHandler for PL011Uart {\n    fn handle(&self) -> Result<(), &'static str> {\n        self.inner.lock(|inner| {\n            let pending = inner.registers.MIS.extract();\n\n            // Clear all pending IRQs.\n            inner.registers.ICR.write(ICR::ALL::CLEAR);\n\n            // Check for any kind of RX interrupt.\n            if pending.matches_any(MIS::RXMIS::SET + MIS::RTMIS::SET) {\n                // Echo any received characters.\n                while let Some(c) = inner.read_char_converting(BlockingMode::NonBlocking) {\n                    inner.write_char(c)\n                }\n            }\n        });\n\n        Ok(())\n    }\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/bsp/device_driver/bcm.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BCM driver top level.\n\nmod bcm2xxx_gpio;\n#[cfg(feature = \"bsp_rpi3\")]\nmod bcm2xxx_interrupt_controller;\nmod bcm2xxx_pl011_uart;\n\npub use bcm2xxx_gpio::*;\n#[cfg(feature = \"bsp_rpi3\")]\npub use bcm2xxx_interrupt_controller::*;\npub use bcm2xxx_pl011_uart::*;\n"
  },
  {
    "path": "18_backtrace/kernel/src/bsp/device_driver/common.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Common device driver code.\n\nuse crate::memory::{Address, Virtual};\nuse core::{fmt, marker::PhantomData, ops};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct MMIODerefWrapper<T> {\n    start_addr: Address<Virtual>,\n    phantom: PhantomData<fn() -> T>,\n}\n\n/// A wrapper type for usize with integrated range bound check.\n#[derive(Copy, Clone)]\npub struct BoundedUsize<const MAX_INCLUSIVE: usize>(usize);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T> MMIODerefWrapper<T> {\n    /// Create an instance.\n    pub const unsafe fn new(start_addr: Address<Virtual>) -> Self {\n        Self {\n            start_addr,\n            phantom: PhantomData,\n        }\n    }\n}\n\nimpl<T> ops::Deref for MMIODerefWrapper<T> {\n    type Target = T;\n\n    fn deref(&self) -> &Self::Target {\n        unsafe { &*(self.start_addr.as_usize() as *const _) }\n    }\n}\n\nimpl<const MAX_INCLUSIVE: usize> BoundedUsize<{ MAX_INCLUSIVE }> {\n    pub const MAX_INCLUSIVE: usize = MAX_INCLUSIVE;\n\n    /// Creates a new instance if number <= MAX_INCLUSIVE.\n    pub const fn new(number: usize) -> Self {\n        assert!(number <= MAX_INCLUSIVE);\n\n        Self(number)\n    }\n\n    /// Return the wrapped number.\n    pub const fn get(self) -> usize {\n        self.0\n    }\n}\n\nimpl<const MAX_INCLUSIVE: usize> fmt::Display for BoundedUsize<{ MAX_INCLUSIVE }> {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        write!(f, \"{}\", self.0)\n    }\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/bsp/device_driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Device driver.\n\n#[cfg(feature = \"bsp_rpi4\")]\nmod arm;\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod bcm;\nmod common;\n\n#[cfg(feature = \"bsp_rpi4\")]\npub use arm::*;\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use bcm::*;\n"
  },
  {
    "path": "18_backtrace/kernel/src/bsp/raspberrypi/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Processor code.\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Used by `arch` code to find the early boot core.\n#[no_mangle]\n#[link_section = \".text._start_arguments\"]\npub static BOOT_CORE_ID: u64 = 0;\n"
  },
  {
    "path": "18_backtrace/kernel/src/bsp/raspberrypi/driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP driver support.\n\nuse super::{exception, memory::map::mmio};\nuse crate::{\n    bsp::device_driver,\n    console, driver as generic_driver,\n    exception::{self as generic_exception},\n    memory,\n    memory::mmu::MMIODescriptor,\n};\nuse core::{\n    mem::MaybeUninit,\n    sync::atomic::{AtomicBool, Ordering},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic mut PL011_UART: MaybeUninit<device_driver::PL011Uart> = MaybeUninit::uninit();\nstatic mut GPIO: MaybeUninit<device_driver::GPIO> = MaybeUninit::uninit();\n\n#[cfg(feature = \"bsp_rpi3\")]\nstatic mut INTERRUPT_CONTROLLER: MaybeUninit<device_driver::InterruptController> =\n    MaybeUninit::uninit();\n\n#[cfg(feature = \"bsp_rpi4\")]\nstatic mut INTERRUPT_CONTROLLER: MaybeUninit<device_driver::GICv2> = MaybeUninit::uninit();\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// This must be called only after successful init of the memory subsystem.\nunsafe fn instantiate_uart() -> Result<(), &'static str> {\n    let mmio_descriptor = MMIODescriptor::new(mmio::PL011_UART_START, mmio::PL011_UART_SIZE);\n    let virt_addr =\n        memory::mmu::kernel_map_mmio(device_driver::PL011Uart::COMPATIBLE, &mmio_descriptor)?;\n\n    PL011_UART.write(device_driver::PL011Uart::new(virt_addr));\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the UART driver.\nunsafe fn post_init_uart() -> Result<(), &'static str> {\n    console::register_console(PL011_UART.assume_init_ref());\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the memory subsystem.\nunsafe fn instantiate_gpio() -> Result<(), &'static str> {\n    let mmio_descriptor = MMIODescriptor::new(mmio::GPIO_START, mmio::GPIO_SIZE);\n    let virt_addr =\n        memory::mmu::kernel_map_mmio(device_driver::GPIO::COMPATIBLE, &mmio_descriptor)?;\n\n    GPIO.write(device_driver::GPIO::new(virt_addr));\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the GPIO driver.\nunsafe fn post_init_gpio() -> Result<(), &'static str> {\n    GPIO.assume_init_ref().map_pl011_uart();\n    Ok(())\n}\n\n/// This must be called only after successful init of the memory subsystem.\n#[cfg(feature = \"bsp_rpi3\")]\nunsafe fn instantiate_interrupt_controller() -> Result<(), &'static str> {\n    let periph_mmio_descriptor =\n        MMIODescriptor::new(mmio::PERIPHERAL_IC_START, mmio::PERIPHERAL_IC_SIZE);\n    let periph_virt_addr = memory::mmu::kernel_map_mmio(\n        device_driver::InterruptController::COMPATIBLE,\n        &periph_mmio_descriptor,\n    )?;\n\n    INTERRUPT_CONTROLLER.write(device_driver::InterruptController::new(periph_virt_addr));\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the memory subsystem.\n#[cfg(feature = \"bsp_rpi4\")]\nunsafe fn instantiate_interrupt_controller() -> Result<(), &'static str> {\n    let gicd_mmio_descriptor = MMIODescriptor::new(mmio::GICD_START, mmio::GICD_SIZE);\n    let gicd_virt_addr = memory::mmu::kernel_map_mmio(\"GICv2 GICD\", &gicd_mmio_descriptor)?;\n\n    let gicc_mmio_descriptor = MMIODescriptor::new(mmio::GICC_START, mmio::GICC_SIZE);\n    let gicc_virt_addr = memory::mmu::kernel_map_mmio(\"GICV2 GICC\", &gicc_mmio_descriptor)?;\n\n    INTERRUPT_CONTROLLER.write(device_driver::GICv2::new(gicd_virt_addr, gicc_virt_addr));\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the interrupt controller driver.\nunsafe fn post_init_interrupt_controller() -> Result<(), &'static str> {\n    generic_exception::asynchronous::register_irq_manager(INTERRUPT_CONTROLLER.assume_init_ref());\n\n    Ok(())\n}\n\n/// Function needs to ensure that driver registration happens only after correct instantiation.\nunsafe fn driver_uart() -> Result<(), &'static str> {\n    instantiate_uart()?;\n\n    let uart_descriptor = generic_driver::DeviceDriverDescriptor::new(\n        PL011_UART.assume_init_ref(),\n        Some(post_init_uart),\n        Some(exception::asynchronous::irq_map::PL011_UART),\n    );\n    generic_driver::driver_manager().register_driver(uart_descriptor);\n\n    Ok(())\n}\n\n/// Function needs to ensure that driver registration happens only after correct instantiation.\nunsafe fn driver_gpio() -> Result<(), &'static str> {\n    instantiate_gpio()?;\n\n    let gpio_descriptor = generic_driver::DeviceDriverDescriptor::new(\n        GPIO.assume_init_ref(),\n        Some(post_init_gpio),\n        None,\n    );\n    generic_driver::driver_manager().register_driver(gpio_descriptor);\n\n    Ok(())\n}\n\n/// Function needs to ensure that driver registration happens only after correct instantiation.\nunsafe fn driver_interrupt_controller() -> Result<(), &'static str> {\n    instantiate_interrupt_controller()?;\n\n    let interrupt_controller_descriptor = generic_driver::DeviceDriverDescriptor::new(\n        INTERRUPT_CONTROLLER.assume_init_ref(),\n        Some(post_init_interrupt_controller),\n        None,\n    );\n    generic_driver::driver_manager().register_driver(interrupt_controller_descriptor);\n\n    Ok(())\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Initialize the driver subsystem.\n///\n/// # Safety\n///\n/// See child function calls.\npub unsafe fn init() -> Result<(), &'static str> {\n    static INIT_DONE: AtomicBool = AtomicBool::new(false);\n    if INIT_DONE.load(Ordering::Relaxed) {\n        return Err(\"Init already done\");\n    }\n\n    driver_uart()?;\n    driver_gpio()?;\n    driver_interrupt_controller()?;\n\n    INIT_DONE.store(true, Ordering::Relaxed);\n    Ok(())\n}\n\n/// Minimal code needed to bring up the console in QEMU (for testing only). This is often less steps\n/// than on real hardware due to QEMU's abstractions.\n#[cfg(feature = \"test_build\")]\npub fn qemu_bring_up_console() {\n    use crate::cpu;\n\n    unsafe {\n        instantiate_uart().unwrap_or_else(|_| cpu::qemu_exit_failure());\n        console::register_console(PL011_UART.assume_init_ref());\n    };\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/bsp/raspberrypi/exception/asynchronous.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP asynchronous exception handling.\n\nuse crate::bsp;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Export for reuse in generic asynchronous.rs.\npub use bsp::device_driver::IRQNumber;\n\n#[cfg(feature = \"bsp_rpi3\")]\npub(in crate::bsp) mod irq_map {\n    use super::bsp::device_driver::{IRQNumber, PeripheralIRQ};\n\n    pub const PL011_UART: IRQNumber = IRQNumber::Peripheral(PeripheralIRQ::new(57));\n}\n\n#[cfg(feature = \"bsp_rpi4\")]\npub(in crate::bsp) mod irq_map {\n    use super::bsp::device_driver::IRQNumber;\n\n    pub const PL011_UART: IRQNumber = IRQNumber::new(153);\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/bsp/raspberrypi/exception.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP synchronous and asynchronous exception handling.\n\npub mod asynchronous;\n"
  },
  {
    "path": "18_backtrace/kernel/src/bsp/raspberrypi/kernel.ld",
    "content": "/* SPDX-License-Identifier: MIT OR Apache-2.0\n *\n * Copyright (c) 2018-2022 Andre Richter <andre.o.richter@gmail.com>\n */\n\nINCLUDE kernel_virt_addr_space_size.ld;\n\nPAGE_SIZE = 64K;\nPAGE_MASK = PAGE_SIZE - 1;\n\n/* The kernel's virtual address range will be:\n *\n * [END_ADDRESS_INCLUSIVE, START_ADDRESS]\n * [u64::MAX             , (u64::MAX - __kernel_virt_addr_space_size) + 1]\n */\n__kernel_virt_start_addr = ((0xffffffffffffffff - __kernel_virt_addr_space_size) + 1);\n\n__rpi_phys_dram_start_addr = 0;\n\n/* The physical address at which the the kernel binary will be loaded by the Raspberry's firmware */\n__rpi_phys_binary_load_addr = 0x80000;\n\n\nENTRY(__rpi_phys_binary_load_addr)\n\n/* Flags:\n *     4 == R\n *     5 == RX\n *     6 == RW\n *\n * Segments are marked PT_LOAD below so that the ELF file provides virtual and physical addresses.\n * It doesn't mean all of them need actually be loaded.\n */\nPHDRS\n{\n    segment_code            PT_LOAD FLAGS(5);\n    segment_data            PT_LOAD FLAGS(6);\n    segment_boot_core_stack PT_LOAD FLAGS(6);\n}\n\nSECTIONS\n{\n    . =  __kernel_virt_start_addr;\n\n    ASSERT((. & PAGE_MASK) == 0, \"Start of address space is not page aligned\")\n\n    /***********************************************************************************************\n    * Code + RO Data + Global Offset Table\n    ***********************************************************************************************/\n    __code_start = .;\n    .text : AT(__rpi_phys_binary_load_addr)\n    {\n        KEEP(*(.text._start))\n        *(.text._start_arguments) /* Constants (or statics in Rust speak) read by _start(). */\n        *(.text._start_rust)      /* The Rust entry point */\n        *(.text*)                 /* Everything else */\n    } :segment_code\n\n    .rodata         : ALIGN(8) { *(.rodata*) } :segment_code\n    .kernel_symbols : ALIGN(8) {\n        __kernel_symbols_start = .;\n        . += 32 * 1024;\n    } :segment_code\n\n    . = ALIGN(PAGE_SIZE);\n    __code_end_exclusive = .;\n\n    /***********************************************************************************************\n    * Data + BSS\n    ***********************************************************************************************/\n    __data_start = .;\n    .data : { *(.data*) } :segment_data\n\n    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */\n    .bss (NOLOAD) : ALIGN(16)\n    {\n        __bss_start = .;\n        *(.bss*);\n        . = ALIGN(16);\n        __bss_end_exclusive = .;\n    } :segment_data\n\n    . = ALIGN(PAGE_SIZE);\n    __data_end_exclusive = .;\n\n    /***********************************************************************************************\n    * MMIO Remap Reserved\n    ***********************************************************************************************/\n    __mmio_remap_start = .;\n    . += 8 * 1024 * 1024;\n    __mmio_remap_end_exclusive = .;\n\n    ASSERT((. & PAGE_MASK) == 0, \"MMIO remap reservation is not page aligned\")\n\n    /***********************************************************************************************\n    * Guard Page\n    ***********************************************************************************************/\n    . += PAGE_SIZE;\n\n    /***********************************************************************************************\n    * Boot Core Stack\n    ***********************************************************************************************/\n    .boot_core_stack (NOLOAD) : AT(__rpi_phys_dram_start_addr)\n    {\n        __boot_core_stack_start = .;         /*   ^             */\n                                             /*   | stack       */\n        . += __rpi_phys_binary_load_addr;    /*   | growth      */\n                                             /*   | direction   */\n        __boot_core_stack_end_exclusive = .; /*   |             */\n    } :segment_boot_core_stack\n\n    ASSERT((. & PAGE_MASK) == 0, \"End of boot core stack is not page aligned\")\n\n    /***********************************************************************************************\n    * Misc\n    ***********************************************************************************************/\n    .got : { *(.got*) }\n    ASSERT(SIZEOF(.got) == 0, \"Relocation support not expected\")\n\n    /DISCARD/ : { *(.comment*) }\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/bsp/raspberrypi/kernel_virt_addr_space_size.ld",
    "content": "__kernel_virt_addr_space_size = 1024 * 1024 * 1024\n"
  },
  {
    "path": "18_backtrace/kernel/src/bsp/raspberrypi/memory/mmu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Memory Management Unit.\n\nuse crate::{\n    memory::{\n        mmu::{\n            self as generic_mmu, AddressSpace, AssociatedTranslationTable, AttributeFields,\n            MemoryRegion, PageAddress, TranslationGranule,\n        },\n        Physical, Virtual,\n    },\n    synchronization::InitStateLock,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\ntype KernelTranslationTable =\n    <KernelVirtAddrSpace as AssociatedTranslationTable>::TableStartFromTop;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// The translation granule chosen by this BSP. This will be used everywhere else in the kernel to\n/// derive respective data structures and their sizes. For example, the `crate::memory::mmu::Page`.\npub type KernelGranule = TranslationGranule<{ 64 * 1024 }>;\n\n/// The kernel's virtual address space defined by this BSP.\npub type KernelVirtAddrSpace = AddressSpace<{ kernel_virt_addr_space_size() }>;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n/// The kernel translation tables.\n///\n/// It is mandatory that InitStateLock is transparent.\n///\n/// That is, `size_of(InitStateLock<KernelTranslationTable>) == size_of(KernelTranslationTable)`.\n/// There is a unit tests that checks this porperty.\n#[link_section = \".data\"]\n#[no_mangle]\nstatic KERNEL_TABLES: InitStateLock<KernelTranslationTable> =\n    InitStateLock::new(KernelTranslationTable::new_for_precompute());\n\n/// This value is needed during early boot for MMU setup.\n///\n/// This will be patched to the correct value by the \"translation table tool\" after linking. This\n/// given value here is just a dummy.\n#[link_section = \".text._start_arguments\"]\n#[no_mangle]\nstatic PHYS_KERNEL_TABLES_BASE_ADDR: u64 = 0xCCCCAAAAFFFFEEEE;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// This is a hack for retrieving the value for the kernel's virtual address space size as a\n/// constant from a common place, since it is needed as a compile-time/link-time constant in both,\n/// the linker script and the Rust sources.\n#[allow(clippy::needless_late_init)]\nconst fn kernel_virt_addr_space_size() -> usize {\n    let __kernel_virt_addr_space_size;\n\n    include!(\"../kernel_virt_addr_space_size.ld\");\n\n    __kernel_virt_addr_space_size\n}\n\n/// Helper function for calculating the number of pages the given parameter spans.\nconst fn size_to_num_pages(size: usize) -> usize {\n    assert!(size > 0);\n    assert!(size % KernelGranule::SIZE == 0);\n\n    size >> KernelGranule::SHIFT\n}\n\n/// The data pages of the kernel binary.\nfn virt_data_region() -> MemoryRegion<Virtual> {\n    let num_pages = size_to_num_pages(super::data_size());\n\n    let start_page_addr = super::virt_data_start();\n    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n\n    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n}\n\n// There is no reason to expect the following conversions to fail, since they were generated offline\n// by the `translation table tool`. If it doesn't work, a panic due to the unwraps is justified.\nfn kernel_virt_to_phys_region(virt_region: MemoryRegion<Virtual>) -> MemoryRegion<Physical> {\n    let phys_start_page_addr =\n        generic_mmu::try_kernel_virt_page_addr_to_phys_page_addr(virt_region.start_page_addr())\n            .unwrap();\n\n    let phys_end_exclusive_page_addr = phys_start_page_addr\n        .checked_offset(virt_region.num_pages() as isize)\n        .unwrap();\n\n    MemoryRegion::new(phys_start_page_addr, phys_end_exclusive_page_addr)\n}\n\nfn kernel_page_attributes(virt_page_addr: PageAddress<Virtual>) -> AttributeFields {\n    generic_mmu::try_kernel_page_attributes(virt_page_addr).unwrap()\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The code pages of the kernel binary.\npub fn virt_code_region() -> MemoryRegion<Virtual> {\n    let num_pages = size_to_num_pages(super::code_size());\n\n    let start_page_addr = super::virt_code_start();\n    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n\n    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n}\n\n/// The boot core stack pages.\npub fn virt_boot_core_stack_region() -> MemoryRegion<Virtual> {\n    let num_pages = size_to_num_pages(super::boot_core_stack_size());\n\n    let start_page_addr = super::virt_boot_core_stack_start();\n    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n\n    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n}\n\n/// Return a reference to the kernel's translation tables.\npub fn kernel_translation_tables() -> &'static InitStateLock<KernelTranslationTable> {\n    &KERNEL_TABLES\n}\n\n/// The MMIO remap pages.\npub fn virt_mmio_remap_region() -> MemoryRegion<Virtual> {\n    let num_pages = size_to_num_pages(super::mmio_remap_size());\n\n    let start_page_addr = super::virt_mmio_remap_start();\n    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n\n    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n}\n\n/// Add mapping records for the kernel binary.\n///\n/// The actual translation table entries for the kernel binary are generated using the offline\n/// `translation table tool` and patched into the kernel binary. This function just adds the mapping\n/// record entries.\npub fn kernel_add_mapping_records_for_precomputed() {\n    let virt_code_region = virt_code_region();\n    generic_mmu::kernel_add_mapping_record(\n        \"Kernel code and RO data\",\n        &virt_code_region,\n        &kernel_virt_to_phys_region(virt_code_region),\n        &kernel_page_attributes(virt_code_region.start_page_addr()),\n    );\n\n    let virt_data_region = virt_data_region();\n    generic_mmu::kernel_add_mapping_record(\n        \"Kernel data and bss\",\n        &virt_data_region,\n        &kernel_virt_to_phys_region(virt_data_region),\n        &kernel_page_attributes(virt_data_region.start_page_addr()),\n    );\n\n    let virt_boot_core_stack_region = virt_boot_core_stack_region();\n    generic_mmu::kernel_add_mapping_record(\n        \"Kernel boot-core stack\",\n        &virt_boot_core_stack_region,\n        &kernel_virt_to_phys_region(virt_boot_core_stack_region),\n        &kernel_page_attributes(virt_boot_core_stack_region.start_page_addr()),\n    );\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/bsp/raspberrypi/memory.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Memory Management.\n//!\n//! The physical memory layout.\n//!\n//! The Raspberry's firmware copies the kernel binary to 0x8_0000. The preceding region will be used\n//! as the boot core's stack.\n//!\n//! +---------------------------------------+\n//! |                                       | boot_core_stack_start @ 0x0\n//! |                                       |                                ^\n//! | Boot-core Stack                       |                                | stack\n//! |                                       |                                | growth\n//! |                                       |                                | direction\n//! +---------------------------------------+\n//! |                                       | code_start @ 0x8_0000 == boot_core_stack_end_exclusive\n//! | .text                                 |\n//! | .rodata                               |\n//! | .got                                  |\n//! | .kernel_symbols                       |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       | data_start == code_end_exclusive\n//! | .data                                 |\n//! | .bss                                  |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       | data_end_exclusive\n//! |                                       |\n//!\n//!\n//!\n//!\n//!\n//! The virtual memory layout is as follows:\n//!\n//! +---------------------------------------+\n//! |                                       | code_start @ __kernel_virt_start_addr\n//! | .text                                 |\n//! | .rodata                               |\n//! | .got                                  |\n//! | .kernel_symbols                       |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       | data_start == code_end_exclusive\n//! | .data                                 |\n//! | .bss                                  |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       |  mmio_remap_start == data_end_exclusive\n//! | VA region for MMIO remapping          |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       |  mmio_remap_end_exclusive\n//! | Unmapped guard page                   |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       | boot_core_stack_start\n//! |                                       |                                ^\n//! | Boot-core Stack                       |                                | stack\n//! |                                       |                                | growth\n//! |                                       |                                | direction\n//! +---------------------------------------+\n//! |                                       | boot_core_stack_end_exclusive\n//! |                                       |\npub mod mmu;\n\nuse crate::memory::{mmu::PageAddress, Address, Physical, Virtual};\nuse core::cell::UnsafeCell;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Symbols from the linker script.\nextern \"Rust\" {\n    static __code_start: UnsafeCell<()>;\n    static __code_end_exclusive: UnsafeCell<()>;\n\n    static __data_start: UnsafeCell<()>;\n    static __data_end_exclusive: UnsafeCell<()>;\n\n    static __mmio_remap_start: UnsafeCell<()>;\n    static __mmio_remap_end_exclusive: UnsafeCell<()>;\n\n    static __boot_core_stack_start: UnsafeCell<()>;\n    static __boot_core_stack_end_exclusive: UnsafeCell<()>;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// The board's physical memory map.\n#[rustfmt::skip]\npub(super) mod map {\n    use super::*;\n\n    /// Physical devices.\n    #[cfg(feature = \"bsp_rpi3\")]\n    pub mod mmio {\n        use super::*;\n\n        pub const PERIPHERAL_IC_START: Address<Physical> = Address::new(0x3F00_B200);\n        pub const PERIPHERAL_IC_SIZE:  usize             =              0x24;\n\n        pub const GPIO_START:          Address<Physical> = Address::new(0x3F20_0000);\n        pub const GPIO_SIZE:           usize             =              0xA0;\n\n        pub const PL011_UART_START:    Address<Physical> = Address::new(0x3F20_1000);\n        pub const PL011_UART_SIZE:     usize             =              0x48;\n\n        pub const END:                 Address<Physical> = Address::new(0x4001_0000);\n    }\n\n    /// Physical devices.\n    #[cfg(feature = \"bsp_rpi4\")]\n    pub mod mmio {\n        use super::*;\n\n        pub const GPIO_START:       Address<Physical> = Address::new(0xFE20_0000);\n        pub const GPIO_SIZE:        usize             =              0xA0;\n\n        pub const PL011_UART_START: Address<Physical> = Address::new(0xFE20_1000);\n        pub const PL011_UART_SIZE:  usize             =              0x48;\n\n        pub const GICD_START:       Address<Physical> = Address::new(0xFF84_1000);\n        pub const GICD_SIZE:        usize             =              0x824;\n\n        pub const GICC_START:       Address<Physical> = Address::new(0xFF84_2000);\n        pub const GICC_SIZE:        usize             =              0x14;\n\n        pub const END:              Address<Physical> = Address::new(0xFF85_0000);\n    }\n\n    pub const END: Address<Physical> = mmio::END;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Start page address of the code segment.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn virt_code_start() -> PageAddress<Virtual> {\n    PageAddress::from(unsafe { __code_start.get() as usize })\n}\n\n/// Size of the code segment.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn code_size() -> usize {\n    unsafe { (__code_end_exclusive.get() as usize) - (__code_start.get() as usize) }\n}\n\n/// Start page address of the data segment.\n#[inline(always)]\nfn virt_data_start() -> PageAddress<Virtual> {\n    PageAddress::from(unsafe { __data_start.get() as usize })\n}\n\n/// Size of the data segment.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn data_size() -> usize {\n    unsafe { (__data_end_exclusive.get() as usize) - (__data_start.get() as usize) }\n}\n\n/// Start page address of the MMIO remap reservation.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn virt_mmio_remap_start() -> PageAddress<Virtual> {\n    PageAddress::from(unsafe { __mmio_remap_start.get() as usize })\n}\n\n/// Size of the MMIO remap reservation.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn mmio_remap_size() -> usize {\n    unsafe { (__mmio_remap_end_exclusive.get() as usize) - (__mmio_remap_start.get() as usize) }\n}\n\n/// Start page address of the boot core's stack.\n#[inline(always)]\nfn virt_boot_core_stack_start() -> PageAddress<Virtual> {\n    PageAddress::from(unsafe { __boot_core_stack_start.get() as usize })\n}\n\n/// Size of the boot core's stack.\n#[inline(always)]\nfn boot_core_stack_size() -> usize {\n    unsafe {\n        (__boot_core_stack_end_exclusive.get() as usize) - (__boot_core_stack_start.get() as usize)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Exclusive end address of the physical address space.\n#[inline(always)]\npub fn phys_addr_space_end_exclusive_addr() -> PageAddress<Physical> {\n    PageAddress::from(map::END)\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/bsp/raspberrypi.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Top-level BSP file for the Raspberry Pi 3 and 4.\n\npub mod cpu;\npub mod driver;\npub mod exception;\npub mod memory;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Board identification.\npub fn board_name() -> &'static str {\n    #[cfg(feature = \"bsp_rpi3\")]\n    {\n        \"Raspberry Pi 3\"\n    }\n\n    #[cfg(feature = \"bsp_rpi4\")]\n    {\n        \"Raspberry Pi 4\"\n    }\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/bsp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Conditional reexporting of Board Support Packages.\n\nmod device_driver;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod raspberrypi;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use raspberrypi::*;\n"
  },
  {
    "path": "18_backtrace/kernel/src/common.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! General purpose code.\n\n/// Check if a value is aligned to a given size.\n#[inline(always)]\npub const fn is_aligned(value: usize, alignment: usize) -> bool {\n    assert!(alignment.is_power_of_two());\n\n    (value & (alignment - 1)) == 0\n}\n\n/// Align down.\n#[inline(always)]\npub const fn align_down(value: usize, alignment: usize) -> usize {\n    assert!(alignment.is_power_of_two());\n\n    value & !(alignment - 1)\n}\n\n/// Align up.\n#[inline(always)]\npub const fn align_up(value: usize, alignment: usize) -> usize {\n    assert!(alignment.is_power_of_two());\n\n    (value + alignment - 1) & !(alignment - 1)\n}\n\n/// Convert a size into human readable format.\npub const fn size_human_readable_ceil(size: usize) -> (usize, &'static str) {\n    const KIB: usize = 1024;\n    const MIB: usize = 1024 * 1024;\n    const GIB: usize = 1024 * 1024 * 1024;\n\n    if (size / GIB) > 0 {\n        (size.div_ceil(GIB), \"GiB\")\n    } else if (size / MIB) > 0 {\n        (size.div_ceil(MIB), \"MiB\")\n    } else if (size / KIB) > 0 {\n        (size.div_ceil(KIB), \"KiB\")\n    } else {\n        (size, \"Byte\")\n    }\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/console/null_console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Null console.\n\nuse super::interface;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct NullConsole;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\npub static NULL_CONSOLE: NullConsole = NullConsole {};\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl interface::Write for NullConsole {\n    fn write_char(&self, _c: char) {}\n\n    fn write_fmt(&self, _args: fmt::Arguments) -> fmt::Result {\n        fmt::Result::Ok(())\n    }\n\n    fn flush(&self) {}\n}\n\nimpl interface::Read for NullConsole {\n    fn clear_rx(&self) {}\n}\n\nimpl interface::Statistics for NullConsole {}\nimpl interface::All for NullConsole {}\n"
  },
  {
    "path": "18_backtrace/kernel/src/console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! System console.\n\nmod null_console;\n\nuse crate::synchronization;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Console interfaces.\npub mod interface {\n    use core::fmt;\n\n    /// Console write functions.\n    pub trait Write {\n        /// Write a single character.\n        fn write_char(&self, c: char);\n\n        /// Write a Rust format string.\n        fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result;\n\n        /// Block until the last buffered character has been physically put on the TX wire.\n        fn flush(&self);\n    }\n\n    /// Console read functions.\n    pub trait Read {\n        /// Read a single character.\n        fn read_char(&self) -> char {\n            ' '\n        }\n\n        /// Clear RX buffers, if any.\n        fn clear_rx(&self);\n    }\n\n    /// Console statistics.\n    pub trait Statistics {\n        /// Return the number of characters written.\n        fn chars_written(&self) -> usize {\n            0\n        }\n\n        /// Return the number of characters read.\n        fn chars_read(&self) -> usize {\n            0\n        }\n    }\n\n    /// Trait alias for a full-fledged console.\n    pub trait All: Write + Read + Statistics {}\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic CUR_CONSOLE: InitStateLock<&'static (dyn interface::All + Sync)> =\n    InitStateLock::new(&null_console::NULL_CONSOLE);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::{interface::ReadWriteEx, InitStateLock};\n\n/// Register a new console.\npub fn register_console(new_console: &'static (dyn interface::All + Sync)) {\n    CUR_CONSOLE.write(|con| *con = new_console);\n}\n\n/// Return a reference to the currently registered console.\n///\n/// This is the global console used by all printing macros.\npub fn console() -> &'static dyn interface::All {\n    CUR_CONSOLE.read(|con| *con)\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Boot code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/cpu/boot.rs\"]\nmod arch_boot;\n"
  },
  {
    "path": "18_backtrace/kernel/src/cpu/smp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Symmetric multiprocessing.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/cpu/smp.rs\"]\nmod arch_smp;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_smp::core_id;\n"
  },
  {
    "path": "18_backtrace/kernel/src/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Processor code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/cpu.rs\"]\nmod arch_cpu;\n\nmod boot;\n\npub mod smp;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_cpu::{nop, wait_forever};\n\n#[cfg(feature = \"test_build\")]\npub use arch_cpu::{qemu_exit_failure, qemu_exit_success};\n"
  },
  {
    "path": "18_backtrace/kernel/src/driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Driver support.\n\nuse crate::{\n    exception, info,\n    synchronization::{interface::ReadWriteEx, InitStateLock},\n};\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nconst NUM_DRIVERS: usize = 5;\n\nstruct DriverManagerInner<T>\nwhere\n    T: 'static,\n{\n    next_index: usize,\n    descriptors: [Option<DeviceDriverDescriptor<T>>; NUM_DRIVERS],\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Driver interfaces.\npub mod interface {\n    /// Device Driver functions.\n    pub trait DeviceDriver {\n        /// Different interrupt controllers might use different types for IRQ number.\n        type IRQNumberType: super::fmt::Display;\n\n        /// Return a compatibility string for identifying the driver.\n        fn compatible(&self) -> &'static str;\n\n        /// Called by the kernel to bring up the device.\n        ///\n        /// # Safety\n        ///\n        /// - During init, drivers might do stuff with system-wide impact.\n        unsafe fn init(&self) -> Result<(), &'static str> {\n            Ok(())\n        }\n\n        /// Called by the kernel to register and enable the device's IRQ handler.\n        ///\n        /// Rust's type system will prevent a call to this function unless the calling instance\n        /// itself has static lifetime.\n        fn register_and_enable_irq_handler(\n            &'static self,\n            irq_number: &Self::IRQNumberType,\n        ) -> Result<(), &'static str> {\n            panic!(\n                \"Attempt to enable IRQ {} for device {}, but driver does not support this\",\n                irq_number,\n                self.compatible()\n            )\n        }\n    }\n}\n\n/// Tpye to be used as an optional callback after a driver's init() has run.\npub type DeviceDriverPostInitCallback = unsafe fn() -> Result<(), &'static str>;\n\n/// A descriptor for device drivers.\n#[derive(Copy, Clone)]\npub struct DeviceDriverDescriptor<T>\nwhere\n    T: 'static,\n{\n    device_driver: &'static (dyn interface::DeviceDriver<IRQNumberType = T> + Sync),\n    post_init_callback: Option<DeviceDriverPostInitCallback>,\n    irq_number: Option<T>,\n}\n\n/// Provides device driver management functions.\npub struct DriverManager<T>\nwhere\n    T: 'static,\n{\n    inner: InitStateLock<DriverManagerInner<T>>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic DRIVER_MANAGER: DriverManager<exception::asynchronous::IRQNumber> = DriverManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T> DriverManagerInner<T>\nwhere\n    T: 'static + Copy,\n{\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            next_index: 0,\n            descriptors: [None; NUM_DRIVERS],\n        }\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T> DeviceDriverDescriptor<T> {\n    /// Create an instance.\n    pub fn new(\n        device_driver: &'static (dyn interface::DeviceDriver<IRQNumberType = T> + Sync),\n        post_init_callback: Option<DeviceDriverPostInitCallback>,\n        irq_number: Option<T>,\n    ) -> Self {\n        Self {\n            device_driver,\n            post_init_callback,\n            irq_number,\n        }\n    }\n}\n\n/// Return a reference to the global DriverManager.\npub fn driver_manager() -> &'static DriverManager<exception::asynchronous::IRQNumber> {\n    &DRIVER_MANAGER\n}\n\nimpl<T> DriverManager<T>\nwhere\n    T: fmt::Display + Copy,\n{\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            inner: InitStateLock::new(DriverManagerInner::new()),\n        }\n    }\n\n    /// Register a device driver with the kernel.\n    pub fn register_driver(&self, descriptor: DeviceDriverDescriptor<T>) {\n        self.inner.write(|inner| {\n            inner.descriptors[inner.next_index] = Some(descriptor);\n            inner.next_index += 1;\n        })\n    }\n\n    /// Helper for iterating over registered drivers.\n    fn for_each_descriptor<'a>(&'a self, f: impl FnMut(&'a DeviceDriverDescriptor<T>)) {\n        self.inner.read(|inner| {\n            inner\n                .descriptors\n                .iter()\n                .filter_map(|x| x.as_ref())\n                .for_each(f)\n        })\n    }\n\n    /// Fully initialize all drivers and their interrupts handlers.\n    ///\n    /// # Safety\n    ///\n    /// - During init, drivers might do stuff with system-wide impact.\n    pub unsafe fn init_drivers_and_irqs(&self) {\n        self.for_each_descriptor(|descriptor| {\n            // 1. Initialize driver.\n            if let Err(x) = descriptor.device_driver.init() {\n                panic!(\n                    \"Error initializing driver: {}: {}\",\n                    descriptor.device_driver.compatible(),\n                    x\n                );\n            }\n\n            // 2. Call corresponding post init callback.\n            if let Some(callback) = &descriptor.post_init_callback {\n                if let Err(x) = callback() {\n                    panic!(\n                        \"Error during driver post-init callback: {}: {}\",\n                        descriptor.device_driver.compatible(),\n                        x\n                    );\n                }\n            }\n        });\n\n        // 3. After all post-init callbacks were done, the interrupt controller should be\n        //    registered and functional. So let drivers register with it now.\n        self.for_each_descriptor(|descriptor| {\n            if let Some(irq_number) = &descriptor.irq_number {\n                if let Err(x) = descriptor\n                    .device_driver\n                    .register_and_enable_irq_handler(irq_number)\n                {\n                    panic!(\n                        \"Error during driver interrupt handler registration: {}: {}\",\n                        descriptor.device_driver.compatible(),\n                        x\n                    );\n                }\n            }\n        });\n    }\n\n    /// Enumerate all registered device drivers.\n    pub fn enumerate(&self) {\n        let mut i: usize = 1;\n        self.for_each_descriptor(|descriptor| {\n            info!(\"      {}. {}\", i, descriptor.device_driver.compatible());\n\n            i += 1;\n        });\n    }\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/exception/asynchronous/null_irq_manager.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Null IRQ Manager.\n\nuse super::{interface, IRQContext, IRQHandlerDescriptor};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct NullIRQManager;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\npub static NULL_IRQ_MANAGER: NullIRQManager = NullIRQManager {};\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl interface::IRQManager for NullIRQManager {\n    type IRQNumberType = super::IRQNumber;\n\n    fn register_handler(\n        &self,\n        _descriptor: IRQHandlerDescriptor<Self::IRQNumberType>,\n    ) -> Result<(), &'static str> {\n        panic!(\"No IRQ Manager registered yet\");\n    }\n\n    fn enable(&self, _irq_number: &Self::IRQNumberType) {\n        panic!(\"No IRQ Manager registered yet\");\n    }\n\n    fn handle_pending_irqs<'irq_context>(&'irq_context self, _ic: &IRQContext<'irq_context>) {\n        panic!(\"No IRQ Manager registered yet\");\n    }\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/exception/asynchronous.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Asynchronous exception handling.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/exception/asynchronous.rs\"]\nmod arch_asynchronous;\nmod null_irq_manager;\n\nuse crate::{bsp, synchronization};\nuse core::marker::PhantomData;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_asynchronous::{\n    is_local_irq_masked, local_irq_mask, local_irq_mask_save, local_irq_restore, local_irq_unmask,\n    print_state,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Interrupt number as defined by the BSP.\npub type IRQNumber = bsp::exception::asynchronous::IRQNumber;\n\n/// Interrupt descriptor.\n#[derive(Copy, Clone)]\npub struct IRQHandlerDescriptor<T>\nwhere\n    T: Copy,\n{\n    /// The IRQ number.\n    number: T,\n\n    /// Descriptive name.\n    name: &'static str,\n\n    /// Reference to handler trait object.\n    handler: &'static (dyn interface::IRQHandler + Sync),\n}\n\n/// IRQContext token.\n///\n/// An instance of this type indicates that the local core is currently executing in IRQ\n/// context, aka executing an interrupt vector or subcalls of it.\n///\n/// Concept and implementation derived from the `CriticalSection` introduced in\n/// <https://github.com/rust-embedded/bare-metal>\n#[derive(Clone, Copy)]\npub struct IRQContext<'irq_context> {\n    _0: PhantomData<&'irq_context ()>,\n}\n\n/// Asynchronous exception handling interfaces.\npub mod interface {\n\n    /// Implemented by types that handle IRQs.\n    pub trait IRQHandler {\n        /// Called when the corresponding interrupt is asserted.\n        fn handle(&self) -> Result<(), &'static str>;\n    }\n\n    /// IRQ management functions.\n    ///\n    /// The `BSP` is supposed to supply one global instance. Typically implemented by the\n    /// platform's interrupt controller.\n    pub trait IRQManager {\n        /// The IRQ number type depends on the implementation.\n        type IRQNumberType: Copy;\n\n        /// Register a handler.\n        fn register_handler(\n            &self,\n            irq_handler_descriptor: super::IRQHandlerDescriptor<Self::IRQNumberType>,\n        ) -> Result<(), &'static str>;\n\n        /// Enable an interrupt in the controller.\n        fn enable(&self, irq_number: &Self::IRQNumberType);\n\n        /// Handle pending interrupts.\n        ///\n        /// This function is called directly from the CPU's IRQ exception vector. On AArch64,\n        /// this means that the respective CPU core has disabled exception handling.\n        /// This function can therefore not be preempted and runs start to finish.\n        ///\n        /// Takes an IRQContext token to ensure it can only be called from IRQ context.\n        #[allow(clippy::trivially_copy_pass_by_ref)]\n        fn handle_pending_irqs<'irq_context>(\n            &'irq_context self,\n            ic: &super::IRQContext<'irq_context>,\n        );\n\n        /// Print list of registered handlers.\n        fn print_handler(&self) {}\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic CUR_IRQ_MANAGER: InitStateLock<\n    &'static (dyn interface::IRQManager<IRQNumberType = IRQNumber> + Sync),\n> = InitStateLock::new(&null_irq_manager::NULL_IRQ_MANAGER);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::{interface::ReadWriteEx, InitStateLock};\n\nimpl<T> IRQHandlerDescriptor<T>\nwhere\n    T: Copy,\n{\n    /// Create an instance.\n    pub const fn new(\n        number: T,\n        name: &'static str,\n        handler: &'static (dyn interface::IRQHandler + Sync),\n    ) -> Self {\n        Self {\n            number,\n            name,\n            handler,\n        }\n    }\n\n    /// Return the number.\n    pub const fn number(&self) -> T {\n        self.number\n    }\n\n    /// Return the name.\n    pub const fn name(&self) -> &'static str {\n        self.name\n    }\n\n    /// Return the handler.\n    pub const fn handler(&self) -> &'static (dyn interface::IRQHandler + Sync) {\n        self.handler\n    }\n}\n\nimpl<'irq_context> IRQContext<'irq_context> {\n    /// Creates an IRQContext token.\n    ///\n    /// # Safety\n    ///\n    /// - This must only be called when the current core is in an interrupt context and will not\n    ///   live beyond the end of it. That is, creation is allowed in interrupt vector functions. For\n    ///   example, in the ARMv8-A case, in `extern \"C\" fn current_elx_irq()`.\n    /// - Note that the lifetime `'irq_context` of the returned instance is unconstrained. User code\n    ///   must not be able to influence the lifetime picked for this type, since that might cause it\n    ///   to be inferred to `'static`.\n    #[inline(always)]\n    pub unsafe fn new() -> Self {\n        IRQContext { _0: PhantomData }\n    }\n}\n\n/// Executes the provided closure while IRQs are masked on the executing core.\n///\n/// While the function temporarily changes the HW state of the executing core, it restores it to the\n/// previous state before returning, so this is deemed safe.\n#[inline(always)]\npub fn exec_with_irq_masked<T>(f: impl FnOnce() -> T) -> T {\n    let saved = local_irq_mask_save();\n    let ret = f();\n    local_irq_restore(saved);\n\n    ret\n}\n\n/// Register a new IRQ manager.\npub fn register_irq_manager(\n    new_manager: &'static (dyn interface::IRQManager<IRQNumberType = IRQNumber> + Sync),\n) {\n    CUR_IRQ_MANAGER.write(|manager| *manager = new_manager);\n}\n\n/// Return a reference to the currently registered IRQ manager.\n///\n/// This is the IRQ manager used by the architectural interrupt handling code.\npub fn irq_manager() -> &'static dyn interface::IRQManager<IRQNumberType = IRQNumber> {\n    CUR_IRQ_MANAGER.read(|manager| *manager)\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/exception.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Synchronous and asynchronous exception handling.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/exception.rs\"]\nmod arch_exception;\n\npub mod asynchronous;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_exception::{current_privilege_level, handling_init};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Kernel privilege levels.\n#[allow(missing_docs)]\n#[derive(Eq, PartialEq)]\npub enum PrivilegeLevel {\n    User,\n    Kernel,\n    Hypervisor,\n    Unknown,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// Libkernel unit tests must execute in kernel mode.\n    #[kernel_test]\n    fn test_runner_executes_in_kernel_mode() {\n        let (level, _) = current_privilege_level();\n\n        assert!(level == PrivilegeLevel::Kernel)\n    }\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/lib.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n// Rust embedded logo for `make doc`.\n#![doc(\n    html_logo_url = \"https://raw.githubusercontent.com/rust-embedded/wg/master/assets/logo/ewg-logo-blue-white-on-transparent.png\"\n)]\n\n//! The `kernel` library.\n//!\n//! Used to compose the final kernel binary.\n//!\n//! # Code organization and architecture\n//!\n//! The code is divided into different *modules*, each representing a typical **subsystem** of the\n//! `kernel`. Top-level module files of subsystems reside directly in the `src` folder. For example,\n//! `src/memory.rs` contains code that is concerned with all things memory management.\n//!\n//! ## Visibility of processor architecture code\n//!\n//! Some of the `kernel`'s subsystems depend on low-level code that is specific to the target\n//! processor architecture. For each supported processor architecture, there exists a subfolder in\n//! `src/_arch`, for example, `src/_arch/aarch64`.\n//!\n//! The architecture folders mirror the subsystem modules laid out in `src`. For example,\n//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go\n//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in\n//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic\n//! module's name prefixed with `arch_`.\n//!\n//! For example, this is the top of `src/memory/mmu.rs`:\n//!\n//! ```\n//! #[cfg(target_arch = \"aarch64\")]\n//! #[path = \"../_arch/aarch64/memory/mmu.rs\"]\n//! mod arch_mmu;\n//! ```\n//!\n//! Often times, items from the `arch_ module` will be publicly reexported by the parent module.\n//! This way, each architecture specific module can provide its implementation of an item, while the\n//! caller must not be concerned which architecture has been conditionally compiled.\n//!\n//! ## BSP code\n//!\n//! `BSP` stands for Board Support Package. `BSP` code is organized under `src/bsp.rs` and contains\n//! target board specific definitions and functions. These are things such as the board's memory map\n//! or instances of drivers for devices that are featured on the respective board.\n//!\n//! Just like processor architecture code, the `BSP` code's module structure tries to mirror the\n//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is\n//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`.\n//!\n//! ## Kernel interfaces\n//!\n//! Both `arch` and `bsp` contain code that is conditionally compiled depending on the actual target\n//! and board for which the kernel is compiled. For example, the `interrupt controller` hardware of\n//! the `Raspberry Pi 3` and the `Raspberry Pi 4` is different, but we want the rest of the `kernel`\n//! code to play nicely with any of the two without much hassle.\n//!\n//! In order to provide a clean abstraction between `arch`, `bsp` and `generic kernel code`,\n//! `interface` traits are provided *whenever possible* and *where it makes sense*. They are defined\n//! in the respective subsystem module and help to enforce the idiom of *program to an interface,\n//! not an implementation*. For example, there will be a common IRQ handling interface which the two\n//! different interrupt controller `drivers` of both Raspberrys will implement, and only export the\n//! interface to the rest of the `kernel`.\n//!\n//! ```\n//!         +-------------------+\n//!         | Interface (Trait) |\n//!         |                   |\n//!         +--+-------------+--+\n//!            ^             ^\n//!            |             |\n//!            |             |\n//! +----------+--+       +--+----------+\n//! | kernel code |       |  bsp code   |\n//! |             |       |  arch code  |\n//! +-------------+       +-------------+\n//! ```\n//!\n//! # Summary\n//!\n//! For a logical `kernel` subsystem, corresponding code can be distributed over several physical\n//! locations. Here is an example for the **memory** subsystem:\n//!\n//! - `src/memory.rs` and `src/memory/**/*`\n//!   - Common code that is agnostic of target processor architecture and `BSP` characteristics.\n//!     - Example: A function to zero a chunk of memory.\n//!   - Interfaces for the memory subsystem that are implemented by `arch` or `BSP` code.\n//!     - Example: An `MMU` interface that defines `MMU` function prototypes.\n//! - `src/bsp/__board_name__/memory.rs` and `src/bsp/__board_name__/memory/**/*`\n//!   - `BSP` specific code.\n//!   - Example: The board's memory map (physical addresses of DRAM and MMIO devices).\n//! - `src/_arch/__arch_name__/memory.rs` and `src/_arch/__arch_name__/memory/**/*`\n//!   - Processor architecture specific code.\n//!   - Example: Implementation of the `MMU` interface for the `__arch_name__` processor\n//!     architecture.\n//!\n//! From a namespace perspective, **memory** subsystem code lives in:\n//!\n//! - `crate::memory::*`\n//! - `crate::bsp::memory::*`\n//!\n//! # Boot flow\n//!\n//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.\n//!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.\n//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.\n\n#![allow(clippy::upper_case_acronyms)]\n#![allow(incomplete_features)]\n#![feature(asm_const)]\n#![feature(const_option)]\n#![feature(core_intrinsics)]\n#![feature(format_args_nl)]\n#![feature(generic_const_exprs)]\n#![feature(int_roundings)]\n#![feature(is_sorted)]\n#![feature(linkage)]\n#![feature(nonzero_min_max)]\n#![feature(panic_info_message)]\n#![feature(step_trait)]\n#![feature(trait_alias)]\n#![feature(unchecked_math)]\n#![no_std]\n// Testing\n#![cfg_attr(test, no_main)]\n#![feature(custom_test_frameworks)]\n#![reexport_test_harness_main = \"test_main\"]\n#![test_runner(crate::test_runner)]\n\nmod panic_wait;\nmod synchronization;\n\npub mod backtrace;\npub mod bsp;\npub mod common;\npub mod console;\npub mod cpu;\npub mod driver;\npub mod exception;\npub mod memory;\npub mod print;\npub mod state;\npub mod symbols;\npub mod time;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Version string.\npub fn version() -> &'static str {\n    concat!(\n        env!(\"CARGO_PKG_NAME\"),\n        \" version \",\n        env!(\"CARGO_PKG_VERSION\")\n    )\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n/// The default runner for unit tests.\npub fn test_runner(tests: &[&test_types::UnitTest]) {\n    // This line will be printed as the test header.\n    println!(\"Running {} tests\", tests.len());\n\n    for (i, test) in tests.iter().enumerate() {\n        print!(\"{:>3}. {:.<58}\", i + 1, test.name);\n\n        // Run the actual test.\n        (test.test_func)();\n\n        // Failed tests call panic!(). Execution reaches here only if the test has passed.\n        println!(\"[ok]\")\n    }\n}\n\n/// The `kernel_init()` for unit tests.\n#[cfg(test)]\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    test_main();\n\n    cpu::qemu_exit_success()\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/main.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n// Rust embedded logo for `make doc`.\n#![doc(\n    html_logo_url = \"https://raw.githubusercontent.com/rust-embedded/wg/master/assets/logo/ewg-logo-blue-white-on-transparent.png\"\n)]\n\n//! The `kernel` binary.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\nuse libkernel::{bsp, cpu, driver, exception, info, memory, state, time};\n\n/// Early init code.\n///\n/// When this code runs, virtual memory is already enabled.\n///\n/// # Safety\n///\n/// - Only a single core must be active and running this function.\n/// - Printing will not work until the respective driver's MMIO is remapped.\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n\n    // Initialize the BSP driver subsystem.\n    if let Err(x) = bsp::driver::init() {\n        panic!(\"Error initializing BSP driver subsystem: {}\", x);\n    }\n\n    // Initialize all device drivers.\n    driver::driver_manager().init_drivers_and_irqs();\n\n    bsp::memory::mmu::kernel_add_mapping_records_for_precomputed();\n\n    // Unmask interrupts on the boot CPU core.\n    exception::asynchronous::local_irq_unmask();\n\n    // Announce conclusion of the kernel_init() phase.\n    state::state_manager().transition_to_single_core_main();\n\n    // Transition from unsafe to safe.\n    kernel_main()\n}\n\n/// The main function running after the early init.\nfn kernel_main() -> ! {\n    info!(\"{}\", libkernel::version());\n    info!(\"Booting on: {}\", bsp::board_name());\n\n    info!(\"MMU online:\");\n    memory::mmu::kernel_print_mappings();\n\n    let (_, privilege_level) = exception::current_privilege_level();\n    info!(\"Current privilege level: {}\", privilege_level);\n\n    info!(\"Exception handling state:\");\n    exception::asynchronous::print_state();\n\n    info!(\n        \"Architectural timer resolution: {} ns\",\n        time::time_manager().resolution().as_nanos()\n    );\n\n    info!(\"Drivers loaded:\");\n    driver::driver_manager().enumerate();\n\n    info!(\"Registered IRQ handlers:\");\n    exception::asynchronous::irq_manager().print_handler();\n\n    info!(\"Echoing input now\");\n    cpu::wait_forever();\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/memory/mmu/mapping_record.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A record of mapped pages.\n\nuse super::{\n    AccessPermissions, Address, AttributeFields, MMIODescriptor, MemAttributes, MemoryRegion,\n    Physical, Virtual,\n};\nuse crate::{bsp, common, info, synchronization, synchronization::InitStateLock, warn};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Type describing a virtual memory mapping.\n#[allow(missing_docs)]\n#[derive(Copy, Clone)]\nstruct MappingRecordEntry {\n    pub users: [Option<&'static str>; 5],\n    pub phys_start_addr: Address<Physical>,\n    pub virt_start_addr: Address<Virtual>,\n    pub num_pages: usize,\n    pub attribute_fields: AttributeFields,\n}\n\nstruct MappingRecord {\n    inner: [Option<MappingRecordEntry>; 12],\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic KERNEL_MAPPING_RECORD: InitStateLock<MappingRecord> =\n    InitStateLock::new(MappingRecord::new());\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl MappingRecordEntry {\n    pub fn new(\n        name: &'static str,\n        virt_region: &MemoryRegion<Virtual>,\n        phys_region: &MemoryRegion<Physical>,\n        attr: &AttributeFields,\n    ) -> Self {\n        Self {\n            users: [Some(name), None, None, None, None],\n            phys_start_addr: phys_region.start_addr(),\n            virt_start_addr: virt_region.start_addr(),\n            num_pages: phys_region.num_pages(),\n            attribute_fields: *attr,\n        }\n    }\n\n    fn find_next_free_user(&mut self) -> Result<&mut Option<&'static str>, &'static str> {\n        if let Some(x) = self.users.iter_mut().find(|x| x.is_none()) {\n            return Ok(x);\n        };\n\n        Err(\"Storage for user info exhausted\")\n    }\n\n    pub fn add_user(&mut self, user: &'static str) -> Result<(), &'static str> {\n        let x = self.find_next_free_user()?;\n        *x = Some(user);\n        Ok(())\n    }\n}\n\nimpl MappingRecord {\n    pub const fn new() -> Self {\n        Self { inner: [None; 12] }\n    }\n\n    fn size(&self) -> usize {\n        self.inner.iter().filter(|x| x.is_some()).count()\n    }\n\n    fn sort(&mut self) {\n        let upper_bound_exclusive = self.size();\n        let entries = &mut self.inner[0..upper_bound_exclusive];\n\n        if !entries.is_sorted_by_key(|item| item.unwrap().virt_start_addr) {\n            entries.sort_unstable_by_key(|item| item.unwrap().virt_start_addr)\n        }\n    }\n\n    fn find_next_free(&mut self) -> Result<&mut Option<MappingRecordEntry>, &'static str> {\n        if let Some(x) = self.inner.iter_mut().find(|x| x.is_none()) {\n            return Ok(x);\n        }\n\n        Err(\"Storage for mapping info exhausted\")\n    }\n\n    fn find_duplicate(\n        &mut self,\n        phys_region: &MemoryRegion<Physical>,\n    ) -> Option<&mut MappingRecordEntry> {\n        self.inner\n            .iter_mut()\n            .filter_map(|x| x.as_mut())\n            .filter(|x| x.attribute_fields.mem_attributes == MemAttributes::Device)\n            .find(|x| {\n                if x.phys_start_addr != phys_region.start_addr() {\n                    return false;\n                }\n\n                if x.num_pages != phys_region.num_pages() {\n                    return false;\n                }\n\n                true\n            })\n    }\n\n    pub fn add(\n        &mut self,\n        name: &'static str,\n        virt_region: &MemoryRegion<Virtual>,\n        phys_region: &MemoryRegion<Physical>,\n        attr: &AttributeFields,\n    ) -> Result<(), &'static str> {\n        let x = self.find_next_free()?;\n\n        *x = Some(MappingRecordEntry::new(\n            name,\n            virt_region,\n            phys_region,\n            attr,\n        ));\n\n        self.sort();\n\n        Ok(())\n    }\n\n    pub fn print(&self) {\n        info!(\"      -------------------------------------------------------------------------------------------------------------------------------------------\");\n        info!(\n            \"      {:^44}     {:^30}   {:^7}   {:^9}   {:^35}\",\n            \"Virtual\", \"Physical\", \"Size\", \"Attr\", \"Entity\"\n        );\n        info!(\"      -------------------------------------------------------------------------------------------------------------------------------------------\");\n\n        for i in self.inner.iter().flatten() {\n            let size = i.num_pages * bsp::memory::mmu::KernelGranule::SIZE;\n            let virt_start = i.virt_start_addr;\n            let virt_end_inclusive = virt_start + (size - 1);\n            let phys_start = i.phys_start_addr;\n            let phys_end_inclusive = phys_start + (size - 1);\n\n            let (size, unit) = common::size_human_readable_ceil(size);\n\n            let attr = match i.attribute_fields.mem_attributes {\n                MemAttributes::CacheableDRAM => \"C\",\n                MemAttributes::Device => \"Dev\",\n            };\n\n            let acc_p = match i.attribute_fields.acc_perms {\n                AccessPermissions::ReadOnly => \"RO\",\n                AccessPermissions::ReadWrite => \"RW\",\n            };\n\n            let xn = if i.attribute_fields.execute_never {\n                \"XN\"\n            } else {\n                \"X\"\n            };\n\n            info!(\n                \"      {}..{} --> {}..{} | {:>3} {} | {:<3} {} {:<2} | {}\",\n                virt_start,\n                virt_end_inclusive,\n                phys_start,\n                phys_end_inclusive,\n                size,\n                unit,\n                attr,\n                acc_p,\n                xn,\n                i.users[0].unwrap()\n            );\n\n            for k in i.users[1..].iter() {\n                if let Some(additional_user) = *k {\n                    info!(\n                        \"                                                                                                            | {}\",\n                        additional_user\n                    );\n                }\n            }\n        }\n\n        info!(\"      -------------------------------------------------------------------------------------------------------------------------------------------\");\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::interface::ReadWriteEx;\n\n/// Add an entry to the mapping info record.\npub fn kernel_add(\n    name: &'static str,\n    virt_region: &MemoryRegion<Virtual>,\n    phys_region: &MemoryRegion<Physical>,\n    attr: &AttributeFields,\n) -> Result<(), &'static str> {\n    KERNEL_MAPPING_RECORD.write(|mr| mr.add(name, virt_region, phys_region, attr))\n}\n\npub fn kernel_find_and_insert_mmio_duplicate(\n    mmio_descriptor: &MMIODescriptor,\n    new_user: &'static str,\n) -> Option<Address<Virtual>> {\n    let phys_region: MemoryRegion<Physical> = (*mmio_descriptor).into();\n\n    KERNEL_MAPPING_RECORD.write(|mr| {\n        let dup = mr.find_duplicate(&phys_region)?;\n\n        if let Err(x) = dup.add_user(new_user) {\n            warn!(\"{}\", x);\n        }\n\n        Some(dup.virt_start_addr)\n    })\n}\n\n/// Human-readable print of all recorded kernel mappings.\npub fn kernel_print() {\n    KERNEL_MAPPING_RECORD.read(|mr| mr.print());\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/memory/mmu/page_alloc.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Page allocation.\n\nuse super::MemoryRegion;\nuse crate::{\n    memory::{AddressType, Virtual},\n    synchronization::IRQSafeNullLock,\n    warn,\n};\nuse core::num::NonZeroUsize;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// A page allocator that can be lazyily initialized.\npub struct PageAllocator<ATYPE: AddressType> {\n    pool: Option<MemoryRegion<ATYPE>>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic KERNEL_MMIO_VA_ALLOCATOR: IRQSafeNullLock<PageAllocator<Virtual>> =\n    IRQSafeNullLock::new(PageAllocator::new());\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the kernel's MMIO virtual address allocator.\npub fn kernel_mmio_va_allocator() -> &'static IRQSafeNullLock<PageAllocator<Virtual>> {\n    &KERNEL_MMIO_VA_ALLOCATOR\n}\n\nimpl<ATYPE: AddressType> PageAllocator<ATYPE> {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self { pool: None }\n    }\n\n    /// Initialize the allocator.\n    pub fn init(&mut self, pool: MemoryRegion<ATYPE>) {\n        if self.pool.is_some() {\n            warn!(\"Already initialized\");\n            return;\n        }\n\n        self.pool = Some(pool);\n    }\n\n    /// Allocate a number of pages.\n    pub fn alloc(\n        &mut self,\n        num_requested_pages: NonZeroUsize,\n    ) -> Result<MemoryRegion<ATYPE>, &'static str> {\n        if self.pool.is_none() {\n            return Err(\"Allocator not initialized\");\n        }\n\n        self.pool\n            .as_mut()\n            .unwrap()\n            .take_first_n_pages(num_requested_pages)\n    }\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/memory/mmu/translation_table.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Translation table.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../../_arch/aarch64/memory/mmu/translation_table.rs\"]\nmod arch_translation_table;\n\nuse super::{AttributeFields, MemoryRegion};\nuse crate::memory::{Address, Physical, Virtual};\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\n#[cfg(target_arch = \"aarch64\")]\npub use arch_translation_table::FixedSizeTranslationTable;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Translation table interfaces.\npub mod interface {\n    use crate::memory::mmu::PageAddress;\n\n    use super::*;\n\n    /// Translation table operations.\n    pub trait TranslationTable {\n        /// Anything that needs to run before any of the other provided functions can be used.\n        ///\n        /// # Safety\n        ///\n        /// - Implementor must ensure that this function can run only once or is harmless if invoked\n        ///   multiple times.\n        fn init(&mut self) -> Result<(), &'static str>;\n\n        /// Map the given virtual memory region to the given physical memory region.\n        ///\n        /// # Safety\n        ///\n        /// - Using wrong attributes can cause multiple issues of different nature in the system.\n        /// - It is not required that the architectural implementation prevents aliasing. That is,\n        ///   mapping to the same physical memory using multiple virtual addresses, which would\n        ///   break Rust's ownership assumptions. This should be protected against in the kernel's\n        ///   generic MMU code.\n        unsafe fn map_at(\n            &mut self,\n            virt_region: &MemoryRegion<Virtual>,\n            phys_region: &MemoryRegion<Physical>,\n            attr: &AttributeFields,\n        ) -> Result<(), &'static str>;\n\n        /// Try to translate a virtual page address to a physical page address.\n        ///\n        /// Will only succeed if there exists a valid mapping for the input page.\n        fn try_virt_page_addr_to_phys_page_addr(\n            &self,\n            virt_page_addr: PageAddress<Virtual>,\n        ) -> Result<PageAddress<Physical>, &'static str>;\n\n        /// Try to get the attributes of a page.\n        ///\n        /// Will only succeed if there exists a valid mapping for the input page.\n        fn try_page_attributes(\n            &self,\n            virt_page_addr: PageAddress<Virtual>,\n        ) -> Result<AttributeFields, &'static str>;\n\n        /// Try to translate a virtual address to a physical address.\n        ///\n        /// Will only succeed if there exists a valid mapping for the input address.\n        fn try_virt_addr_to_phys_addr(\n            &self,\n            virt_addr: Address<Virtual>,\n        ) -> Result<Address<Physical>, &'static str>;\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use crate::memory::mmu::{AccessPermissions, MemAttributes, PageAddress};\n    use arch_translation_table::MinSizeTranslationTable;\n    use interface::TranslationTable;\n    use test_macros::kernel_test;\n\n    /// Sanity checks for the TranslationTable implementation.\n    #[kernel_test]\n    fn translationtable_implementation_sanity() {\n        // This will occupy a lot of space on the stack.\n        let mut tables = MinSizeTranslationTable::new_for_runtime();\n\n        assert_eq!(tables.init(), Ok(()));\n\n        let virt_end_exclusive_page_addr: PageAddress<Virtual> = PageAddress::MAX;\n        let virt_start_page_addr: PageAddress<Virtual> =\n            virt_end_exclusive_page_addr.checked_offset(-5).unwrap();\n\n        let phys_start_page_addr: PageAddress<Physical> = PageAddress::from(0);\n        let phys_end_exclusive_page_addr: PageAddress<Physical> =\n            phys_start_page_addr.checked_offset(5).unwrap();\n\n        let virt_region = MemoryRegion::new(virt_start_page_addr, virt_end_exclusive_page_addr);\n        let phys_region = MemoryRegion::new(phys_start_page_addr, phys_end_exclusive_page_addr);\n\n        let attr = AttributeFields {\n            mem_attributes: MemAttributes::CacheableDRAM,\n            acc_perms: AccessPermissions::ReadWrite,\n            execute_never: true,\n        };\n\n        unsafe { assert_eq!(tables.map_at(&virt_region, &phys_region, &attr), Ok(())) };\n\n        assert_eq!(\n            tables.try_virt_page_addr_to_phys_page_addr(virt_start_page_addr),\n            Ok(phys_start_page_addr)\n        );\n\n        assert_eq!(\n            tables.try_page_attributes(virt_start_page_addr.checked_offset(-1).unwrap()),\n            Err(\"Page marked invalid\")\n        );\n\n        assert_eq!(tables.try_page_attributes(virt_start_page_addr), Ok(attr));\n\n        let virt_addr = virt_start_page_addr.into_inner() + 0x100;\n        let phys_addr = phys_start_page_addr.into_inner() + 0x100;\n        assert_eq!(tables.try_virt_addr_to_phys_addr(virt_addr), Ok(phys_addr));\n    }\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/memory/mmu/types.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management Unit types.\n\nuse crate::{\n    bsp, common,\n    memory::{Address, AddressType, Physical},\n};\nuse core::{convert::From, iter::Step, num::NonZeroUsize, ops::Range};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// A wrapper type around [Address] that ensures page alignment.\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub struct PageAddress<ATYPE: AddressType> {\n    inner: Address<ATYPE>,\n}\n\n/// A type that describes a region of memory in quantities of pages.\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub struct MemoryRegion<ATYPE: AddressType> {\n    start: PageAddress<ATYPE>,\n    end_exclusive: PageAddress<ATYPE>,\n}\n\n/// Architecture agnostic memory attributes.\n#[allow(missing_docs)]\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub enum MemAttributes {\n    CacheableDRAM,\n    Device,\n}\n\n/// Architecture agnostic access permissions.\n#[allow(missing_docs)]\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub enum AccessPermissions {\n    ReadOnly,\n    ReadWrite,\n}\n\n/// Collection of memory attributes.\n#[allow(missing_docs)]\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub struct AttributeFields {\n    pub mem_attributes: MemAttributes,\n    pub acc_perms: AccessPermissions,\n    pub execute_never: bool,\n}\n\n/// An MMIO descriptor for use in device drivers.\n#[derive(Copy, Clone)]\npub struct MMIODescriptor {\n    start_addr: Address<Physical>,\n    end_addr_exclusive: Address<Physical>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n//------------------------------------------------------------------------------\n// PageAddress\n//------------------------------------------------------------------------------\nimpl<ATYPE: AddressType> PageAddress<ATYPE> {\n    /// The largest value that can be represented by this type.\n    pub const MAX: Self = PageAddress {\n        inner: Address::new(usize::MAX).align_down_page(),\n    };\n\n    /// Unwraps the value.\n    pub fn into_inner(self) -> Address<ATYPE> {\n        self.inner\n    }\n\n    /// Calculates the offset from the page address.\n    ///\n    /// `count` is in units of [PageAddress]. For example, a count of 2 means `result = self + 2 *\n    /// page_size`.\n    pub fn checked_offset(self, count: isize) -> Option<Self> {\n        if count == 0 {\n            return Some(self);\n        }\n\n        let delta = count\n            .unsigned_abs()\n            .checked_mul(bsp::memory::mmu::KernelGranule::SIZE)?;\n        let result = if count.is_positive() {\n            self.inner.as_usize().checked_add(delta)?\n        } else {\n            self.inner.as_usize().checked_sub(delta)?\n        };\n\n        Some(Self {\n            inner: Address::new(result),\n        })\n    }\n}\n\nimpl<ATYPE: AddressType> From<usize> for PageAddress<ATYPE> {\n    fn from(addr: usize) -> Self {\n        assert!(\n            common::is_aligned(addr, bsp::memory::mmu::KernelGranule::SIZE),\n            \"Input usize not page aligned\"\n        );\n\n        Self {\n            inner: Address::new(addr),\n        }\n    }\n}\n\nimpl<ATYPE: AddressType> From<Address<ATYPE>> for PageAddress<ATYPE> {\n    fn from(addr: Address<ATYPE>) -> Self {\n        assert!(addr.is_page_aligned(), \"Input Address not page aligned\");\n\n        Self { inner: addr }\n    }\n}\n\nimpl<ATYPE: AddressType> Step for PageAddress<ATYPE> {\n    fn steps_between(start: &Self, end: &Self) -> Option<usize> {\n        if start > end {\n            return None;\n        }\n\n        // Since start <= end, do unchecked arithmetic.\n        Some(\n            (end.inner.as_usize() - start.inner.as_usize())\n                >> bsp::memory::mmu::KernelGranule::SHIFT,\n        )\n    }\n\n    fn forward_checked(start: Self, count: usize) -> Option<Self> {\n        start.checked_offset(count as isize)\n    }\n\n    fn backward_checked(start: Self, count: usize) -> Option<Self> {\n        start.checked_offset(-(count as isize))\n    }\n}\n\n//------------------------------------------------------------------------------\n// MemoryRegion\n//------------------------------------------------------------------------------\nimpl<ATYPE: AddressType> MemoryRegion<ATYPE> {\n    /// Create an instance.\n    pub fn new(start: PageAddress<ATYPE>, end_exclusive: PageAddress<ATYPE>) -> Self {\n        assert!(start <= end_exclusive);\n\n        Self {\n            start,\n            end_exclusive,\n        }\n    }\n\n    fn as_range(&self) -> Range<PageAddress<ATYPE>> {\n        self.into_iter()\n    }\n\n    /// Returns the start page address.\n    pub fn start_page_addr(&self) -> PageAddress<ATYPE> {\n        self.start\n    }\n\n    /// Returns the start address.\n    pub fn start_addr(&self) -> Address<ATYPE> {\n        self.start.into_inner()\n    }\n\n    /// Returns the exclusive end page address.\n    pub fn end_exclusive_page_addr(&self) -> PageAddress<ATYPE> {\n        self.end_exclusive\n    }\n\n    /// Returns the exclusive end page address.\n    pub fn end_inclusive_page_addr(&self) -> PageAddress<ATYPE> {\n        self.end_exclusive.checked_offset(-1).unwrap()\n    }\n\n    /// Checks if self contains an address.\n    pub fn contains(&self, addr: Address<ATYPE>) -> bool {\n        let page_addr = PageAddress::from(addr.align_down_page());\n        self.as_range().contains(&page_addr)\n    }\n\n    /// Checks if there is an overlap with another memory region.\n    pub fn overlaps(&self, other_region: &Self) -> bool {\n        let self_range = self.as_range();\n\n        self_range.contains(&other_region.start_page_addr())\n            || self_range.contains(&other_region.end_inclusive_page_addr())\n    }\n\n    /// Returns the number of pages contained in this region.\n    pub fn num_pages(&self) -> usize {\n        PageAddress::steps_between(&self.start, &self.end_exclusive).unwrap()\n    }\n\n    /// Returns the size in bytes of this region.\n    pub fn size(&self) -> usize {\n        // Invariant: start <= end_exclusive, so do unchecked arithmetic.\n        let end_exclusive = self.end_exclusive.into_inner().as_usize();\n        let start = self.start.into_inner().as_usize();\n\n        end_exclusive - start\n    }\n\n    /// Splits the MemoryRegion like:\n    ///\n    /// --------------------------------------------------------------------------------\n    /// |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |\n    /// --------------------------------------------------------------------------------\n    ///   ^                               ^                                       ^\n    ///   |                               |                                       |\n    ///   left_start     left_end_exclusive                                       |\n    ///                                                                           |\n    ///                                   ^                                       |\n    ///                                   |                                       |\n    ///                                   right_start           right_end_exclusive\n    ///\n    /// Left region is returned to the caller. Right region is the new region for this struct.\n    pub fn take_first_n_pages(&mut self, num_pages: NonZeroUsize) -> Result<Self, &'static str> {\n        let count: usize = num_pages.into();\n\n        let left_end_exclusive = self.start.checked_offset(count as isize);\n        let left_end_exclusive = match left_end_exclusive {\n            None => return Err(\"Overflow while calculating left_end_exclusive\"),\n            Some(x) => x,\n        };\n\n        if left_end_exclusive > self.end_exclusive {\n            return Err(\"Not enough free pages\");\n        }\n\n        let allocation = Self {\n            start: self.start,\n            end_exclusive: left_end_exclusive,\n        };\n        self.start = left_end_exclusive;\n\n        Ok(allocation)\n    }\n}\n\nimpl<ATYPE: AddressType> IntoIterator for MemoryRegion<ATYPE> {\n    type Item = PageAddress<ATYPE>;\n    type IntoIter = Range<Self::Item>;\n\n    fn into_iter(self) -> Self::IntoIter {\n        Range {\n            start: self.start,\n            end: self.end_exclusive,\n        }\n    }\n}\n\nimpl From<MMIODescriptor> for MemoryRegion<Physical> {\n    fn from(desc: MMIODescriptor) -> Self {\n        let start = PageAddress::from(desc.start_addr.align_down_page());\n        let end_exclusive = PageAddress::from(desc.end_addr_exclusive().align_up_page());\n\n        Self {\n            start,\n            end_exclusive,\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// MMIODescriptor\n//------------------------------------------------------------------------------\n\nimpl MMIODescriptor {\n    /// Create an instance.\n    pub const fn new(start_addr: Address<Physical>, size: usize) -> Self {\n        assert!(size > 0);\n        let end_addr_exclusive = Address::new(start_addr.as_usize() + size);\n\n        Self {\n            start_addr,\n            end_addr_exclusive,\n        }\n    }\n\n    /// Return the start address.\n    pub const fn start_addr(&self) -> Address<Physical> {\n        self.start_addr\n    }\n\n    /// Return the exclusive end address.\n    pub fn end_addr_exclusive(&self) -> Address<Physical> {\n        self.end_addr_exclusive\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use crate::memory::Virtual;\n    use test_macros::kernel_test;\n\n    /// Sanity of [PageAddress] methods.\n    #[kernel_test]\n    fn pageaddress_type_method_sanity() {\n        let page_addr: PageAddress<Virtual> =\n            PageAddress::from(bsp::memory::mmu::KernelGranule::SIZE * 2);\n\n        assert_eq!(\n            page_addr.checked_offset(-2),\n            Some(PageAddress::<Virtual>::from(0))\n        );\n\n        assert_eq!(\n            page_addr.checked_offset(2),\n            Some(PageAddress::<Virtual>::from(\n                bsp::memory::mmu::KernelGranule::SIZE * 4\n            ))\n        );\n\n        assert_eq!(\n            PageAddress::<Virtual>::from(0).checked_offset(0),\n            Some(PageAddress::<Virtual>::from(0))\n        );\n        assert_eq!(PageAddress::<Virtual>::from(0).checked_offset(-1), None);\n\n        let max_page_addr = Address::<Virtual>::new(usize::MAX).align_down_page();\n        assert_eq!(\n            PageAddress::<Virtual>::from(max_page_addr).checked_offset(1),\n            None\n        );\n\n        let zero = PageAddress::<Virtual>::from(0);\n        let three = PageAddress::<Virtual>::from(bsp::memory::mmu::KernelGranule::SIZE * 3);\n        assert_eq!(PageAddress::steps_between(&zero, &three), Some(3));\n    }\n\n    /// Sanity of [MemoryRegion] methods.\n    #[kernel_test]\n    fn memoryregion_type_method_sanity() {\n        let zero = PageAddress::<Virtual>::from(0);\n        let zero_region = MemoryRegion::new(zero, zero);\n        assert_eq!(zero_region.num_pages(), 0);\n        assert_eq!(zero_region.size(), 0);\n\n        let one = PageAddress::<Virtual>::from(bsp::memory::mmu::KernelGranule::SIZE);\n        let one_region = MemoryRegion::new(zero, one);\n        assert_eq!(one_region.num_pages(), 1);\n        assert_eq!(one_region.size(), bsp::memory::mmu::KernelGranule::SIZE);\n\n        let three = PageAddress::<Virtual>::from(bsp::memory::mmu::KernelGranule::SIZE * 3);\n        let mut three_region = MemoryRegion::new(zero, three);\n        assert!(three_region.contains(zero.into_inner()));\n        assert!(!three_region.contains(three.into_inner()));\n        assert!(three_region.overlaps(&one_region));\n\n        let allocation = three_region\n            .take_first_n_pages(NonZeroUsize::new(2).unwrap())\n            .unwrap();\n        assert_eq!(allocation.num_pages(), 2);\n        assert_eq!(three_region.num_pages(), 1);\n\n        for (i, alloc) in allocation.into_iter().enumerate() {\n            assert_eq!(\n                alloc.into_inner().as_usize(),\n                i * bsp::memory::mmu::KernelGranule::SIZE\n            );\n        }\n    }\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/memory/mmu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management Unit.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/memory/mmu.rs\"]\nmod arch_mmu;\n\nmod mapping_record;\nmod page_alloc;\nmod translation_table;\nmod types;\n\nuse crate::{\n    bsp,\n    memory::{Address, Physical, Virtual},\n    synchronization::{self, interface::Mutex},\n    warn,\n};\nuse core::{fmt, num::NonZeroUsize};\n\npub use types::*;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// MMU enable errors variants.\n#[allow(missing_docs)]\n#[derive(Debug)]\npub enum MMUEnableError {\n    AlreadyEnabled,\n    Other(&'static str),\n}\n\n/// Memory Management interfaces.\npub mod interface {\n    use super::*;\n\n    /// MMU functions.\n    pub trait MMU {\n        /// Turns on the MMU for the first time and enables data and instruction caching.\n        ///\n        /// # Safety\n        ///\n        /// - Changes the HW's global state.\n        unsafe fn enable_mmu_and_caching(\n            &self,\n            phys_tables_base_addr: Address<Physical>,\n        ) -> Result<(), MMUEnableError>;\n\n        /// Returns true if the MMU is enabled, false otherwise.\n        fn is_enabled(&self) -> bool;\n    }\n}\n\n/// Describes the characteristics of a translation granule.\npub struct TranslationGranule<const GRANULE_SIZE: usize>;\n\n/// Describes properties of an address space.\npub struct AddressSpace<const AS_SIZE: usize>;\n\n/// Intended to be implemented for [`AddressSpace`].\npub trait AssociatedTranslationTable {\n    /// A translation table whose address range is:\n    ///\n    /// [u64::MAX, (u64::MAX - AS_SIZE) + 1]\n    type TableStartFromTop;\n\n    /// A translation table whose address range is:\n    ///\n    /// [AS_SIZE - 1, 0]\n    type TableStartFromBottom;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\nuse interface::MMU;\nuse synchronization::interface::ReadWriteEx;\nuse translation_table::interface::TranslationTable;\n\n/// Map a region in the kernel's translation tables.\n///\n/// No input checks done, input is passed through to the architectural implementation.\n///\n/// # Safety\n///\n/// - See `map_at()`.\n/// - Does not prevent aliasing.\nunsafe fn kernel_map_at_unchecked(\n    name: &'static str,\n    virt_region: &MemoryRegion<Virtual>,\n    phys_region: &MemoryRegion<Physical>,\n    attr: &AttributeFields,\n) -> Result<(), &'static str> {\n    bsp::memory::mmu::kernel_translation_tables()\n        .write(|tables| tables.map_at(virt_region, phys_region, attr))?;\n\n    kernel_add_mapping_record(name, virt_region, phys_region, attr);\n\n    Ok(())\n}\n\n/// Try to translate a kernel virtual address to a physical address.\n///\n/// Will only succeed if there exists a valid mapping for the input address.\nfn try_kernel_virt_addr_to_phys_addr(\n    virt_addr: Address<Virtual>,\n) -> Result<Address<Physical>, &'static str> {\n    bsp::memory::mmu::kernel_translation_tables()\n        .read(|tables| tables.try_virt_addr_to_phys_addr(virt_addr))\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl fmt::Display for MMUEnableError {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        match self {\n            MMUEnableError::AlreadyEnabled => write!(f, \"MMU is already enabled\"),\n            MMUEnableError::Other(x) => write!(f, \"{}\", x),\n        }\n    }\n}\n\nimpl<const GRANULE_SIZE: usize> TranslationGranule<GRANULE_SIZE> {\n    /// The granule's size.\n    pub const SIZE: usize = Self::size_checked();\n\n    /// The granule's mask.\n    pub const MASK: usize = Self::SIZE - 1;\n\n    /// The granule's shift, aka log2(size).\n    pub const SHIFT: usize = Self::SIZE.trailing_zeros() as usize;\n\n    const fn size_checked() -> usize {\n        assert!(GRANULE_SIZE.is_power_of_two());\n\n        GRANULE_SIZE\n    }\n}\n\nimpl<const AS_SIZE: usize> AddressSpace<AS_SIZE> {\n    /// The address space size.\n    pub const SIZE: usize = Self::size_checked();\n\n    /// The address space shift, aka log2(size).\n    pub const SIZE_SHIFT: usize = Self::SIZE.trailing_zeros() as usize;\n\n    const fn size_checked() -> usize {\n        assert!(AS_SIZE.is_power_of_two());\n\n        // Check for architectural restrictions as well.\n        Self::arch_address_space_size_sanity_checks();\n\n        AS_SIZE\n    }\n}\n\n/// Query the BSP for the reserved virtual addresses for MMIO remapping and initialize the kernel's\n/// MMIO VA allocator with it.\npub fn kernel_init_mmio_va_allocator() {\n    let region = bsp::memory::mmu::virt_mmio_remap_region();\n\n    page_alloc::kernel_mmio_va_allocator().lock(|allocator| allocator.init(region));\n}\n\n/// Add an entry to the mapping info record.\npub fn kernel_add_mapping_record(\n    name: &'static str,\n    virt_region: &MemoryRegion<Virtual>,\n    phys_region: &MemoryRegion<Physical>,\n    attr: &AttributeFields,\n) {\n    if let Err(x) = mapping_record::kernel_add(name, virt_region, phys_region, attr) {\n        warn!(\"{}\", x);\n    }\n}\n\n/// MMIO remapping in the kernel translation tables.\n///\n/// Typically used by device drivers.\n///\n/// # Safety\n///\n/// - Same as `kernel_map_at_unchecked()`, minus the aliasing part.\npub unsafe fn kernel_map_mmio(\n    name: &'static str,\n    mmio_descriptor: &MMIODescriptor,\n) -> Result<Address<Virtual>, &'static str> {\n    let phys_region = MemoryRegion::from(*mmio_descriptor);\n    let offset_into_start_page = mmio_descriptor.start_addr().offset_into_page();\n\n    // Check if an identical region has been mapped for another driver. If so, reuse it.\n    let virt_addr = if let Some(addr) =\n        mapping_record::kernel_find_and_insert_mmio_duplicate(mmio_descriptor, name)\n    {\n        addr\n    // Otherwise, allocate a new region and map it.\n    } else {\n        let num_pages = match NonZeroUsize::new(phys_region.num_pages()) {\n            None => return Err(\"Requested 0 pages\"),\n            Some(x) => x,\n        };\n\n        let virt_region =\n            page_alloc::kernel_mmio_va_allocator().lock(|allocator| allocator.alloc(num_pages))?;\n\n        kernel_map_at_unchecked(\n            name,\n            &virt_region,\n            &phys_region,\n            &AttributeFields {\n                mem_attributes: MemAttributes::Device,\n                acc_perms: AccessPermissions::ReadWrite,\n                execute_never: true,\n            },\n        )?;\n\n        virt_region.start_addr()\n    };\n\n    Ok(virt_addr + offset_into_start_page)\n}\n\n/// Try to translate a kernel virtual page address to a physical page address.\n///\n/// Will only succeed if there exists a valid mapping for the input page.\npub fn try_kernel_virt_page_addr_to_phys_page_addr(\n    virt_page_addr: PageAddress<Virtual>,\n) -> Result<PageAddress<Physical>, &'static str> {\n    bsp::memory::mmu::kernel_translation_tables()\n        .read(|tables| tables.try_virt_page_addr_to_phys_page_addr(virt_page_addr))\n}\n\n/// Try to get the attributes of a kernel page.\n///\n/// Will only succeed if there exists a valid mapping for the input page.\npub fn try_kernel_page_attributes(\n    virt_page_addr: PageAddress<Virtual>,\n) -> Result<AttributeFields, &'static str> {\n    bsp::memory::mmu::kernel_translation_tables()\n        .read(|tables| tables.try_page_attributes(virt_page_addr))\n}\n\n/// Human-readable print of all recorded kernel mappings.\npub fn kernel_print_mappings() {\n    mapping_record::kernel_print()\n}\n\n/// Enable the MMU and data + instruction caching.\n///\n/// # Safety\n///\n/// - Crucial function during kernel init. Changes the the complete memory view of the processor.\n#[inline(always)]\npub unsafe fn enable_mmu_and_caching(\n    phys_tables_base_addr: Address<Physical>,\n) -> Result<(), MMUEnableError> {\n    arch_mmu::mmu().enable_mmu_and_caching(phys_tables_base_addr)\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/memory.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management.\n\npub mod mmu;\n\nuse crate::{bsp, common};\nuse core::{\n    fmt,\n    marker::PhantomData,\n    ops::{Add, Sub},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Metadata trait for marking the type of an address.\npub trait AddressType: Copy + Clone + PartialOrd + PartialEq + Ord + Eq {}\n\n/// Zero-sized type to mark a physical address.\n#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq)]\npub enum Physical {}\n\n/// Zero-sized type to mark a virtual address.\n#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq)]\npub enum Virtual {}\n\n/// Generic address type.\n#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq)]\npub struct Address<ATYPE: AddressType> {\n    value: usize,\n    _address_type: PhantomData<fn() -> ATYPE>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl AddressType for Physical {}\nimpl AddressType for Virtual {}\n\nimpl<ATYPE: AddressType> Address<ATYPE> {\n    /// Create an instance.\n    pub const fn new(value: usize) -> Self {\n        Self {\n            value,\n            _address_type: PhantomData,\n        }\n    }\n\n    /// Convert to usize.\n    pub const fn as_usize(self) -> usize {\n        self.value\n    }\n\n    /// Align down to page size.\n    #[must_use]\n    pub const fn align_down_page(self) -> Self {\n        let aligned = common::align_down(self.value, bsp::memory::mmu::KernelGranule::SIZE);\n\n        Self::new(aligned)\n    }\n\n    /// Align up to page size.\n    #[must_use]\n    pub const fn align_up_page(self) -> Self {\n        let aligned = common::align_up(self.value, bsp::memory::mmu::KernelGranule::SIZE);\n\n        Self::new(aligned)\n    }\n\n    /// Checks if the address is page aligned.\n    pub const fn is_page_aligned(&self) -> bool {\n        common::is_aligned(self.value, bsp::memory::mmu::KernelGranule::SIZE)\n    }\n\n    /// Return the address' offset into the corresponding page.\n    pub const fn offset_into_page(&self) -> usize {\n        self.value & bsp::memory::mmu::KernelGranule::MASK\n    }\n}\n\nimpl<ATYPE: AddressType> Add<usize> for Address<ATYPE> {\n    type Output = Self;\n\n    #[inline(always)]\n    fn add(self, rhs: usize) -> Self::Output {\n        match self.value.checked_add(rhs) {\n            None => panic!(\"Overflow on Address::add\"),\n            Some(x) => Self::new(x),\n        }\n    }\n}\n\nimpl<ATYPE: AddressType> Sub<usize> for Address<ATYPE> {\n    type Output = Self;\n\n    #[inline(always)]\n    fn sub(self, rhs: usize) -> Self::Output {\n        match self.value.checked_sub(rhs) {\n            None => panic!(\"Overflow on Address::sub\"),\n            Some(x) => Self::new(x),\n        }\n    }\n}\n\nimpl<ATYPE: AddressType> Sub<Address<ATYPE>> for Address<ATYPE> {\n    type Output = Self;\n\n    #[inline(always)]\n    fn sub(self, rhs: Address<ATYPE>) -> Self::Output {\n        match self.value.checked_sub(rhs.value) {\n            None => panic!(\"Overflow on Address::sub\"),\n            Some(x) => Self::new(x),\n        }\n    }\n}\n\nimpl Address<Virtual> {\n    /// Checks if the address is part of the boot core stack region.\n    pub fn is_valid_stack_addr(&self) -> bool {\n        bsp::memory::mmu::virt_boot_core_stack_region().contains(*self)\n    }\n\n    /// Checks if the address is part of the kernel code region.\n    pub fn is_valid_code_addr(&self) -> bool {\n        bsp::memory::mmu::virt_code_region().contains(*self)\n    }\n}\n\nimpl fmt::Display for Address<Physical> {\n    // Don't expect to see physical addresses greater than 40 bit.\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        let q3: u8 = ((self.value >> 32) & 0xff) as u8;\n        let q2: u16 = ((self.value >> 16) & 0xffff) as u16;\n        let q1: u16 = (self.value & 0xffff) as u16;\n\n        write!(f, \"0x\")?;\n        write!(f, \"{:02x}_\", q3)?;\n        write!(f, \"{:04x}_\", q2)?;\n        write!(f, \"{:04x}\", q1)\n    }\n}\n\nimpl fmt::Display for Address<Virtual> {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        let q4: u16 = ((self.value >> 48) & 0xffff) as u16;\n        let q3: u16 = ((self.value >> 32) & 0xffff) as u16;\n        let q2: u16 = ((self.value >> 16) & 0xffff) as u16;\n        let q1: u16 = (self.value & 0xffff) as u16;\n\n        write!(f, \"0x\")?;\n        write!(f, \"{:04x}_\", q4)?;\n        write!(f, \"{:04x}_\", q3)?;\n        write!(f, \"{:04x}_\", q2)?;\n        write!(f, \"{:04x}\", q1)\n    }\n}\n\n/// Initialize the memory subsystem.\npub fn init() {\n    mmu::kernel_init_mmio_va_allocator();\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// Sanity of [Address] methods.\n    #[kernel_test]\n    fn address_type_method_sanity() {\n        let addr = Address::<Virtual>::new(bsp::memory::mmu::KernelGranule::SIZE + 100);\n\n        assert_eq!(\n            addr.align_down_page().as_usize(),\n            bsp::memory::mmu::KernelGranule::SIZE\n        );\n\n        assert_eq!(\n            addr.align_up_page().as_usize(),\n            bsp::memory::mmu::KernelGranule::SIZE * 2\n        );\n\n        assert!(!addr.is_page_aligned());\n\n        assert_eq!(addr.offset_into_page(), 100);\n    }\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/panic_wait.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A panic handler that infinitely waits.\n\nuse crate::{backtrace, cpu, exception, println};\nuse core::panic::PanicInfo;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// The point of exit for `libkernel`.\n///\n/// It is linked weakly, so that the integration tests can overload its standard behavior.\n#[linkage = \"weak\"]\n#[no_mangle]\nfn _panic_exit() -> ! {\n    #[cfg(not(feature = \"test_build\"))]\n    {\n        cpu::wait_forever()\n    }\n\n    #[cfg(feature = \"test_build\")]\n    {\n        cpu::qemu_exit_failure()\n    }\n}\n\n/// Stop immediately if called a second time.\n///\n/// # Note\n///\n/// Using atomics here relieves us from needing to use `unsafe` for the static variable.\n///\n/// On `AArch64`, which is the only implemented architecture at the time of writing this,\n/// [`AtomicBool::load`] and [`AtomicBool::store`] are lowered to ordinary load and store\n/// instructions. They are therefore safe to use even with MMU + caching deactivated.\n///\n/// [`AtomicBool::load`]: core::sync::atomic::AtomicBool::load\n/// [`AtomicBool::store`]: core::sync::atomic::AtomicBool::store\nfn panic_prevent_reenter() {\n    use core::sync::atomic::{AtomicBool, Ordering};\n\n    #[cfg(not(target_arch = \"aarch64\"))]\n    compile_error!(\"Add the target_arch to above's check if the following code is safe to use\");\n\n    static PANIC_IN_PROGRESS: AtomicBool = AtomicBool::new(false);\n\n    if !PANIC_IN_PROGRESS.load(Ordering::Relaxed) {\n        PANIC_IN_PROGRESS.store(true, Ordering::Relaxed);\n\n        return;\n    }\n\n    _panic_exit()\n}\n\n#[panic_handler]\nfn panic(info: &PanicInfo) -> ! {\n    exception::asynchronous::local_irq_mask();\n\n    // Protect against panic infinite loops if any of the following code panics itself.\n    panic_prevent_reenter();\n\n    let timestamp = crate::time::time_manager().uptime();\n    let (location, line, column) = match info.location() {\n        Some(loc) => (loc.file(), loc.line(), loc.column()),\n        _ => (\"???\", 0, 0),\n    };\n\n    println!(\n        \"[  {:>3}.{:06}] Kernel panic!\\n\\n\\\n        Panic location:\\n      File '{}', line {}, column {}\\n\\n\\\n        {}\\n\\n\\\n        {}\",\n        timestamp.as_secs(),\n        timestamp.subsec_micros(),\n        location,\n        line,\n        column,\n        info.message().unwrap_or(&format_args!(\"\")),\n        backtrace::Backtrace\n    );\n\n    _panic_exit()\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/print.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Printing.\n\nuse crate::console;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n#[doc(hidden)]\npub fn _print(args: fmt::Arguments) {\n    console::console().write_fmt(args).unwrap();\n}\n\n/// Prints without a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! print {\n    ($($arg:tt)*) => ($crate::print::_print(format_args!($($arg)*)));\n}\n\n/// Prints with a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! println {\n    () => ($crate::print!(\"\\n\"));\n    ($($arg:tt)*) => ({\n        $crate::print::_print(format_args_nl!($($arg)*));\n    })\n}\n\n/// Prints an info, with a newline.\n#[macro_export]\nmacro_rules! info {\n    ($string:expr) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[  {:>3}.{:06}] \", $string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n        ));\n    });\n    ($format_string:expr, $($arg:tt)*) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[  {:>3}.{:06}] \", $format_string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n            $($arg)*\n        ));\n    })\n}\n\n/// Prints a warning, with a newline.\n#[macro_export]\nmacro_rules! warn {\n    ($string:expr) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[W {:>3}.{:06}] \", $string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n        ));\n    });\n    ($format_string:expr, $($arg:tt)*) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[W {:>3}.{:06}] \", $format_string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n            $($arg)*\n        ));\n    })\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/state.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! State information about the kernel itself.\n\nuse core::sync::atomic::{AtomicU8, Ordering};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Different stages in the kernel execution.\n#[derive(Copy, Clone, Eq, PartialEq)]\nenum State {\n    /// The kernel starts booting in this state.\n    Init,\n\n    /// The kernel transitions to this state when jumping to `kernel_main()` (at the end of\n    /// `kernel_init()`, after all init calls are done).\n    SingleCoreMain,\n\n    /// The kernel transitions to this state when it boots the secondary cores, aka switches\n    /// exectution mode to symmetric multiprocessing (SMP).\n    MultiCoreMain,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Maintains the kernel state and state transitions.\npub struct StateManager(AtomicU8);\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic STATE_MANAGER: StateManager = StateManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the global StateManager.\npub fn state_manager() -> &'static StateManager {\n    &STATE_MANAGER\n}\n\nimpl StateManager {\n    const INIT: u8 = 0;\n    const SINGLE_CORE_MAIN: u8 = 1;\n    const MULTI_CORE_MAIN: u8 = 2;\n\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self(AtomicU8::new(Self::INIT))\n    }\n\n    /// Return the current state.\n    fn state(&self) -> State {\n        let state = self.0.load(Ordering::Acquire);\n\n        match state {\n            Self::INIT => State::Init,\n            Self::SINGLE_CORE_MAIN => State::SingleCoreMain,\n            Self::MULTI_CORE_MAIN => State::MultiCoreMain,\n            _ => panic!(\"Invalid KERNEL_STATE\"),\n        }\n    }\n\n    /// Return if the kernel is init state.\n    pub fn is_init(&self) -> bool {\n        self.state() == State::Init\n    }\n\n    /// Transition from Init to SingleCoreMain.\n    pub fn transition_to_single_core_main(&self) {\n        if self\n            .0\n            .compare_exchange(\n                Self::INIT,\n                Self::SINGLE_CORE_MAIN,\n                Ordering::Acquire,\n                Ordering::Relaxed,\n            )\n            .is_err()\n        {\n            panic!(\"transition_to_single_core_main() called while state != Init\");\n        }\n    }\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/symbols.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Debug symbol support.\n\nuse crate::memory::{Address, Virtual};\nuse core::{cell::UnsafeCell, slice};\nuse debug_symbol_types::Symbol;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Symbol from the linker script.\nextern \"Rust\" {\n    static __kernel_symbols_start: UnsafeCell<()>;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n/// This will be patched to the correct value by the \"kernel symbols tool\" after linking. This given\n/// value here is just a (safe) dummy.\n#[no_mangle]\nstatic NUM_KERNEL_SYMBOLS: u64 = 0;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nfn kernel_symbol_section_virt_start_addr() -> Address<Virtual> {\n    Address::new(unsafe { __kernel_symbols_start.get() as usize })\n}\n\nfn num_kernel_symbols() -> usize {\n    unsafe {\n        // Read volatile is needed here to prevent the compiler from optimizing NUM_KERNEL_SYMBOLS\n        // away.\n        core::ptr::read_volatile(&NUM_KERNEL_SYMBOLS as *const u64) as usize\n    }\n}\n\nfn kernel_symbols_slice() -> &'static [Symbol] {\n    let ptr = kernel_symbol_section_virt_start_addr().as_usize() as *const Symbol;\n\n    unsafe { slice::from_raw_parts(ptr, num_kernel_symbols()) }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Retrieve the symbol corresponding to a virtual address, if any.\npub fn lookup_symbol(addr: Address<Virtual>) -> Option<&'static Symbol> {\n    kernel_symbols_slice()\n        .iter()\n        .find(|&i| i.contains(addr.as_usize()))\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// Sanity of symbols module.\n    #[kernel_test]\n    fn symbols_sanity() {\n        let first_sym = lookup_symbol(Address::new(\n            crate::common::is_aligned as *const usize as usize,\n        ))\n        .unwrap()\n        .name();\n\n        assert_eq!(first_sym, \"libkernel::common::is_aligned\");\n\n        let second_sym = lookup_symbol(Address::new(crate::version as *const usize as usize))\n            .unwrap()\n            .name();\n\n        assert_eq!(second_sym, \"libkernel::version\");\n    }\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/synchronization.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Synchronization primitives.\n//!\n//! # Resources\n//!\n//!   - <https://doc.rust-lang.org/book/ch16-04-extensible-concurrency-sync-and-send.html>\n//!   - <https://stackoverflow.com/questions/59428096/understanding-the-send-trait>\n//!   - <https://doc.rust-lang.org/std/cell/index.html>\n\nuse core::cell::UnsafeCell;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Synchronization interfaces.\npub mod interface {\n\n    /// Any object implementing this trait guarantees exclusive access to the data wrapped within\n    /// the Mutex for the duration of the provided closure.\n    pub trait Mutex {\n        /// The type of the data that is wrapped by this mutex.\n        type Data;\n\n        /// Locks the mutex and grants the closure temporary mutable access to the wrapped data.\n        fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R;\n    }\n\n    /// A reader-writer exclusion type.\n    ///\n    /// The implementing object allows either a number of readers or at most one writer at any point\n    /// in time.\n    pub trait ReadWriteEx {\n        /// The type of encapsulated data.\n        type Data;\n\n        /// Grants temporary mutable access to the encapsulated data.\n        fn write<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R;\n\n        /// Grants temporary immutable access to the encapsulated data.\n        fn read<'a, R>(&'a self, f: impl FnOnce(&'a Self::Data) -> R) -> R;\n    }\n}\n\n/// A pseudo-lock for teaching purposes.\n///\n/// In contrast to a real Mutex implementation, does not protect against concurrent access from\n/// other cores to the contained data. This part is preserved for later lessons.\n///\n/// The lock will only be used as long as it is safe to do so, i.e. as long as the kernel is\n/// executing on a single core.\npub struct IRQSafeNullLock<T>\nwhere\n    T: ?Sized,\n{\n    data: UnsafeCell<T>,\n}\n\n/// A pseudo-lock that is RW during the single-core kernel init phase and RO afterwards.\n///\n/// Intended to encapsulate data that is populated during kernel init when no concurrency exists.\npub struct InitStateLock<T>\nwhere\n    T: ?Sized,\n{\n    data: UnsafeCell<T>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nunsafe impl<T> Send for IRQSafeNullLock<T> where T: ?Sized + Send {}\nunsafe impl<T> Sync for IRQSafeNullLock<T> where T: ?Sized + Send {}\n\nimpl<T> IRQSafeNullLock<T> {\n    /// Create an instance.\n    pub const fn new(data: T) -> Self {\n        Self {\n            data: UnsafeCell::new(data),\n        }\n    }\n}\n\nunsafe impl<T> Send for InitStateLock<T> where T: ?Sized + Send {}\nunsafe impl<T> Sync for InitStateLock<T> where T: ?Sized + Send {}\n\nimpl<T> InitStateLock<T> {\n    /// Create an instance.\n    pub const fn new(data: T) -> Self {\n        Self {\n            data: UnsafeCell::new(data),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse crate::{exception, state};\n\nimpl<T> interface::Mutex for IRQSafeNullLock<T> {\n    type Data = T;\n\n    fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R {\n        // In a real lock, there would be code encapsulating this line that ensures that this\n        // mutable reference will ever only be given out once at a time.\n        let data = unsafe { &mut *self.data.get() };\n\n        // Execute the closure while IRQs are masked.\n        exception::asynchronous::exec_with_irq_masked(|| f(data))\n    }\n}\n\nimpl<T> interface::ReadWriteEx for InitStateLock<T> {\n    type Data = T;\n\n    fn write<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R {\n        assert!(\n            state::state_manager().is_init(),\n            \"InitStateLock::write called after kernel init phase\"\n        );\n        assert!(\n            !exception::asynchronous::is_local_irq_masked(),\n            \"InitStateLock::write called with IRQs unmasked\"\n        );\n\n        let data = unsafe { &mut *self.data.get() };\n\n        f(data)\n    }\n\n    fn read<'a, R>(&'a self, f: impl FnOnce(&'a Self::Data) -> R) -> R {\n        let data = unsafe { &*self.data.get() };\n\n        f(data)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// InitStateLock must be transparent.\n    #[kernel_test]\n    fn init_state_lock_is_transparent() {\n        use core::mem::size_of;\n\n        assert_eq!(size_of::<InitStateLock<u64>>(), size_of::<u64>());\n    }\n}\n"
  },
  {
    "path": "18_backtrace/kernel/src/time.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Timer primitives.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/time.rs\"]\nmod arch_time;\n\nuse core::time::Duration;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Provides time management functions.\npub struct TimeManager;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic TIME_MANAGER: TimeManager = TimeManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the global TimeManager.\npub fn time_manager() -> &'static TimeManager {\n    &TIME_MANAGER\n}\n\nimpl TimeManager {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self\n    }\n\n    /// The timer's resolution.\n    pub fn resolution(&self) -> Duration {\n        arch_time::resolution()\n    }\n\n    /// The uptime since power-on of the device.\n    ///\n    /// This includes time consumed by firmware and bootloaders.\n    pub fn uptime(&self) -> Duration {\n        arch_time::uptime()\n    }\n\n    /// Spin for a given duration.\n    pub fn spin_for(&self, duration: Duration) {\n        arch_time::spin_for(duration)\n    }\n}\n"
  },
  {
    "path": "18_backtrace/kernel/tests/00_console_sanity.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'console_io_test'\n\n# Verify sending and receiving works as expected.\nclass TxRxHandshakeTest < SubtestBase\n    def name\n        'Transmit and Receive handshake'\n    end\n\n    def run(qemu_out, qemu_in)\n        qemu_in.write_nonblock('ABC')\n        expect_or_raise(qemu_out, 'OK1234')\n    end\nend\n\n# Check for correct TX statistics implementation. Depends on test 1 being run first.\nclass TxStatisticsTest < SubtestBase\n    def name\n        'Transmit statistics'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, '6')\n    end\nend\n\n# Check for correct RX statistics implementation. Depends on test 1 being run first.\nclass RxStatisticsTest < SubtestBase\n    def name\n        'Receive statistics'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, '3')\n    end\nend\n\n## -------------------------------------------------------------------------------------------------\n## Test registration\n## -------------------------------------------------------------------------------------------------\ndef subtest_collection\n    [TxRxHandshakeTest.new, TxStatisticsTest.new, RxStatisticsTest.new]\nend\n"
  },
  {
    "path": "18_backtrace/kernel/tests/00_console_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Console sanity tests - RX, TX and statistics.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Console tests should time out on the I/O harness in case of panic.\nmod panic_wait_forever;\n\nuse libkernel::{bsp, console, cpu, exception, memory, print};\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    use console::console;\n\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    // Handshake\n    assert_eq!(console().read_char(), 'A');\n    assert_eq!(console().read_char(), 'B');\n    assert_eq!(console().read_char(), 'C');\n    print!(\"OK1234\");\n\n    // 6\n    print!(\"{}\", console().chars_written());\n\n    // 3\n    print!(\"{}\", console().chars_read());\n\n    // The QEMU process running this test will be closed by the I/O test harness.\n    cpu::wait_forever();\n}\n"
  },
  {
    "path": "18_backtrace/kernel/tests/01_timer_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Timer sanity tests.\n\n#![feature(custom_test_frameworks)]\n#![no_main]\n#![no_std]\n#![reexport_test_harness_main = \"test_main\"]\n#![test_runner(libkernel::test_runner)]\n\nuse core::time::Duration;\nuse libkernel::{bsp, cpu, exception, memory, time};\nuse test_macros::kernel_test;\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    // Depending on CPU arch, some timer bring-up code could go here. Not needed for the RPi.\n\n    test_main();\n\n    cpu::qemu_exit_success()\n}\n\n/// Simple check that the timer is running.\n#[kernel_test]\nfn timer_is_counting() {\n    assert!(time::time_manager().uptime().as_nanos() > 0)\n}\n\n/// Timer resolution must be sufficient.\n#[kernel_test]\nfn timer_resolution_is_sufficient() {\n    assert!(time::time_manager().resolution().as_nanos() > 0);\n    assert!(time::time_manager().resolution().as_nanos() < 100)\n}\n\n/// Sanity check spin_for() implementation.\n#[kernel_test]\nfn spin_accuracy_check_1_second() {\n    let t1 = time::time_manager().uptime();\n    time::time_manager().spin_for(Duration::from_secs(1));\n    let t2 = time::time_manager().uptime();\n\n    assert_eq!((t2 - t1).as_secs(), 1)\n}\n"
  },
  {
    "path": "18_backtrace/kernel/tests/02_exception_sync_page_fault.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Page faults must result in synchronous exceptions.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Overwrites libkernel's `panic_wait::_panic_exit()` so that it returns a \"success\" code.\n///\n/// In this test, reaching the panic is a success, because it is called from the synchronous\n/// exception handler, which is what this test wants to achieve.\n///\n/// It also means that this integration test can not use any other code that calls panic!() directly\n/// or indirectly.\nmod panic_exit_success;\n\nuse libkernel::{bsp, cpu, exception, info, memory, println};\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    // This line will be printed as the test header.\n    println!(\"Testing synchronous exception handling by causing a page fault\");\n\n    info!(\"Writing to bottom of address space to address 1 GiB...\");\n    let big_addr: u64 = 1024 * 1024 * 1024;\n    core::ptr::read_volatile(big_addr as *mut u64);\n\n    // If execution reaches here, the memory access above did not cause a page fault exception.\n    cpu::qemu_exit_failure()\n}\n"
  },
  {
    "path": "18_backtrace/kernel/tests/03_exception_restore_sanity.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'console_io_test'\n\n# Verify that exception restore works.\nclass ExceptionRestoreTest < SubtestBase\n    def name\n        'Exception restore'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, 'Back from system call!')\n    end\nend\n\n## -------------------------------------------------------------------------------------------------\n## Test registration\n## -------------------------------------------------------------------------------------------------\ndef subtest_collection\n    [ExceptionRestoreTest.new]\nend\n"
  },
  {
    "path": "18_backtrace/kernel/tests/03_exception_restore_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A simple sanity test to see if exception restore code works.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Console tests should time out on the I/O harness in case of panic.\nmod panic_wait_forever;\n\nuse core::arch::asm;\nuse libkernel::{bsp, cpu, exception, info, memory, println};\n\n#[inline(never)]\nfn nested_system_call() {\n    #[cfg(target_arch = \"aarch64\")]\n    unsafe {\n        asm!(\"svc #0x1337\", options(nomem, nostack, preserves_flags));\n    }\n\n    #[cfg(not(target_arch = \"aarch64\"))]\n    {\n        info!(\"Not supported yet\");\n        cpu::wait_forever();\n    }\n}\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    // This line will be printed as the test header.\n    println!(\"Testing exception restore\");\n\n    info!(\"Making a dummy system call\");\n\n    // Calling this inside a function indirectly tests if the link register is restored properly.\n    nested_system_call();\n\n    info!(\"Back from system call!\");\n\n    // The QEMU process running this test will be closed by the I/O test harness.\n    cpu::wait_forever();\n}\n"
  },
  {
    "path": "18_backtrace/kernel/tests/04_exception_irq_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! IRQ handling sanity tests.\n\n#![feature(custom_test_frameworks)]\n#![no_main]\n#![no_std]\n#![reexport_test_harness_main = \"test_main\"]\n#![test_runner(libkernel::test_runner)]\n\nuse libkernel::{bsp, cpu, exception, memory};\nuse test_macros::kernel_test;\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    exception::handling_init();\n    exception::asynchronous::local_irq_unmask();\n\n    test_main();\n\n    cpu::qemu_exit_success()\n}\n\n/// Check that IRQ masking works.\n#[kernel_test]\nfn local_irq_mask_works() {\n    // Precondition: IRQs are unmasked.\n    assert!(exception::asynchronous::is_local_irq_masked());\n\n    exception::asynchronous::local_irq_mask();\n    assert!(!exception::asynchronous::is_local_irq_masked());\n\n    // Restore earlier state.\n    exception::asynchronous::local_irq_unmask();\n}\n\n/// Check that IRQ unmasking works.\n#[kernel_test]\nfn local_irq_unmask_works() {\n    // Precondition: IRQs are masked.\n    exception::asynchronous::local_irq_mask();\n    assert!(!exception::asynchronous::is_local_irq_masked());\n\n    exception::asynchronous::local_irq_unmask();\n    assert!(exception::asynchronous::is_local_irq_masked());\n}\n\n/// Check that IRQ mask save is saving \"something\".\n#[kernel_test]\nfn local_irq_mask_save_works() {\n    // Precondition: IRQs are unmasked.\n    assert!(exception::asynchronous::is_local_irq_masked());\n\n    let first = exception::asynchronous::local_irq_mask_save();\n    assert!(!exception::asynchronous::is_local_irq_masked());\n\n    let second = exception::asynchronous::local_irq_mask_save();\n    assert_ne!(first, second);\n\n    exception::asynchronous::local_irq_restore(first);\n    assert!(exception::asynchronous::is_local_irq_masked());\n}\n"
  },
  {
    "path": "18_backtrace/kernel/tests/05_backtrace_sanity.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'console_io_test'\n\n# Verify that panic produces a backtrace.\nclass PanicBacktraceTest < SubtestBase\n    def name\n        'Panic produces backtrace'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, 'Kernel panic!')\n        expect_or_raise(qemu_out, 'Backtrace:')\n    end\nend\n\n# Verify backtrace correctness.\nclass BacktraceCorrectnessTest < SubtestBase\n    def name\n        'Backtrace is correct'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, '| core::panicking::panic')\n        expect_or_raise(qemu_out, '| _05_backtrace_sanity::nested')\n        expect_or_raise(qemu_out, '| kernel_init')\n    end\nend\n\n## -------------------------------------------------------------------------------------------------\n## Test registration\n## -------------------------------------------------------------------------------------------------\ndef subtest_collection\n    [PanicBacktraceTest.new, BacktraceCorrectnessTest.new]\nend\n"
  },
  {
    "path": "18_backtrace/kernel/tests/05_backtrace_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Test if backtracing code detects an invalid frame pointer.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Console tests should time out on the I/O harness in case of panic.\nmod panic_wait_forever;\n\nuse libkernel::{bsp, cpu, exception, memory};\n\n#[inline(never)]\nfn nested() {\n    panic!()\n}\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    nested();\n\n    // The QEMU process running this test will be closed by the I/O test harness.\n    cpu::wait_forever()\n}\n"
  },
  {
    "path": "18_backtrace/kernel/tests/06_backtrace_invalid_frame.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'console_io_test'\n\n# Test detection of invalid frame pointers.\nclass InvalidFramePointerTest < SubtestBase\n    def name\n        'Detect invalid frame pointer'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out,\n                        /Encountered invalid frame pointer \\(.*\\) during backtrace/)\n    end\nend\n\n## -------------------------------------------------------------------------------------------------\n## Test registration\n## -------------------------------------------------------------------------------------------------\ndef subtest_collection\n    [InvalidFramePointerTest.new]\nend\n"
  },
  {
    "path": "18_backtrace/kernel/tests/06_backtrace_invalid_frame.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Test if backtracing code detects an invalid frame pointer.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Console tests should time out on the I/O harness in case of panic.\nmod panic_wait_forever;\n\nuse libkernel::{backtrace, bsp, cpu, exception, memory};\n\n#[inline(never)]\nfn nested() {\n    unsafe { backtrace::corrupt_previous_frame_addr() };\n\n    panic!()\n}\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    nested();\n\n    // The QEMU process running this test will be closed by the I/O test harness.\n    cpu::wait_forever()\n}\n"
  },
  {
    "path": "18_backtrace/kernel/tests/07_backtrace_invalid_link.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'console_io_test'\n\n# Test detection of invalid link.\nclass InvalidLinkTest < SubtestBase\n    def name\n        'Detect invalid link'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, /Link address \\(.*\\) is not contained in kernel .text section/)\n    end\nend\n\n## -------------------------------------------------------------------------------------------------\n## Test registration\n## -------------------------------------------------------------------------------------------------\ndef subtest_collection\n    [InvalidLinkTest.new]\nend\n"
  },
  {
    "path": "18_backtrace/kernel/tests/07_backtrace_invalid_link.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Test if backtracing code detects an invalid link.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Console tests should time out on the I/O harness in case of panic.\nmod panic_wait_forever;\n\nuse libkernel::{backtrace, bsp, cpu, exception, memory};\n\n#[inline(never)]\nfn nested_2() -> &'static str {\n    unsafe { backtrace::corrupt_link() };\n    libkernel::println!(\"{}\", libkernel::backtrace::Backtrace);\n    \"foo\"\n}\n\n#[inline(never)]\nfn nested_1() {\n    libkernel::println!(\"{}\", nested_2())\n}\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    nested_1();\n\n    // The QEMU process running this test will be closed by the I/O test harness.\n    cpu::wait_forever()\n}\n"
  },
  {
    "path": "18_backtrace/kernel/tests/boot_test_string.rb",
    "content": "# frozen_string_literal: true\n\nEXPECTED_PRINT = 'Echoing input now'\n"
  },
  {
    "path": "18_backtrace/kernel/tests/panic_exit_success/mod.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n/// Overwrites libkernel's `panic_wait::_panic_exit()` with the QEMU-exit version.\n#[no_mangle]\nfn _panic_exit() -> ! {\n    libkernel::cpu::qemu_exit_success()\n}\n"
  },
  {
    "path": "18_backtrace/kernel/tests/panic_wait_forever/mod.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n/// Overwrites libkernel's `panic_wait::_panic_exit()` with wait_forever.\n#[no_mangle]\nfn _panic_exit() -> ! {\n    libkernel::cpu::wait_forever()\n}\n"
  },
  {
    "path": "18_backtrace/kernel_symbols/Cargo.toml",
    "content": "[package]\nname = \"kernel_symbols\"\nversion = \"0.1.0\"\nedition = \"2021\"\n\n[features]\ndefault = []\ngenerated_symbols_available = []\n\n##--------------------------------------------------------------------------------------------------\n## Dependencies\n##--------------------------------------------------------------------------------------------------\n\n[dependencies]\ndebug-symbol-types = { path = \"../libraries/debug-symbol-types\" }\n"
  },
  {
    "path": "18_backtrace/kernel_symbols/build.rs",
    "content": "use std::{env, path::Path};\n\nfn main() {\n    if let Ok(path) = env::var(\"KERNEL_SYMBOLS_DEMANGLED_RS\") {\n        if Path::new(&path).exists() {\n            println!(\"cargo:rustc-cfg=feature=\\\"generated_symbols_available\\\"\")\n        }\n    }\n\n    println!(\n        \"cargo:rerun-if-changed={}\",\n        Path::new(\"kernel_symbols.ld\").display()\n    );\n}\n"
  },
  {
    "path": "18_backtrace/kernel_symbols/kernel_symbols.ld",
    "content": "/* SPDX-License-Identifier: MIT OR Apache-2.0\n *\n * Copyright (c) 2022 Andre Richter <andre.o.richter@gmail.com>\n */\n\nSECTIONS\n{\n    .rodata : {\n        ASSERT(. > 0xffffffff00000000, \"Expected higher half address\")\n\n        KEEP(*(.rodata.symbol_desc*))\n        . = ALIGN(8);\n        *(.rodata*)\n    }\n}\n"
  },
  {
    "path": "18_backtrace/kernel_symbols/src/main.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Generation of kernel symbols.\n\n#![no_std]\n#![no_main]\n\n#[cfg(feature = \"generated_symbols_available\")]\ninclude!(env!(\"KERNEL_SYMBOLS_DEMANGLED_RS\"));\n\n#[panic_handler]\nfn panic(_info: &core::panic::PanicInfo) -> ! {\n    unimplemented!()\n}\n"
  },
  {
    "path": "18_backtrace/kernel_symbols.mk",
    "content": "## SPDX-License-Identifier: MIT OR Apache-2.0\n##\n## Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\ninclude ../common/format.mk\ninclude ../common/docker.mk\n\n##--------------------------------------------------------------------------------------------------\n## Check for input variables that need be exported by the calling Makefile\n##--------------------------------------------------------------------------------------------------\nifndef KERNEL_SYMBOLS_TOOL_PATH\n$(error KERNEL_SYMBOLS_TOOL_PATH is not set)\nendif\n\nifndef TARGET\n$(error TARGET is not set)\nendif\n\nifndef KERNEL_SYMBOLS_INPUT_ELF\n$(error KERNEL_SYMBOLS_INPUT_ELF is not set)\nendif\n\nifndef KERNEL_SYMBOLS_OUTPUT_ELF\n$(error KERNEL_SYMBOLS_OUTPUT_ELF is not set)\nendif\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets and Prerequisites\n##--------------------------------------------------------------------------------------------------\nKERNEL_SYMBOLS_MANIFEST      = kernel_symbols/Cargo.toml\nKERNEL_SYMBOLS_LINKER_SCRIPT = kernel_symbols/kernel_symbols.ld\n\nKERNEL_SYMBOLS_RS           = $(KERNEL_SYMBOLS_INPUT_ELF)_symbols.rs\nKERNEL_SYMBOLS_DEMANGLED_RS = $(shell pwd)/$(KERNEL_SYMBOLS_INPUT_ELF)_symbols_demangled.rs\n\nKERNEL_SYMBOLS_ELF      = target/$(TARGET)/release/kernel_symbols\nKERNEL_SYMBOLS_STRIPPED = target/$(TARGET)/release/kernel_symbols_stripped\n\n# Export for build.rs of kernel_symbols crate.\nexport KERNEL_SYMBOLS_DEMANGLED_RS\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Command building blocks\n##--------------------------------------------------------------------------------------------------\nGET_SYMBOLS_SECTION_VIRT_ADDR = $(DOCKER_TOOLS) $(EXEC_SYMBOLS_TOOL) \\\n    --get_symbols_section_virt_addr $(KERNEL_SYMBOLS_OUTPUT_ELF)\n\nRUSTFLAGS = -C link-arg=--script=$(KERNEL_SYMBOLS_LINKER_SCRIPT) \\\n    -C link-arg=--section-start=.rodata=$$($(GET_SYMBOLS_SECTION_VIRT_ADDR))\n\nRUSTFLAGS_PEDANTIC = $(RUSTFLAGS) \\\n    -D warnings                   \\\n    -D missing_docs\n\nCOMPILER_ARGS = --target=$(TARGET) \\\n    --release\n\nRUSTC_CMD   = cargo rustc $(COMPILER_ARGS) --manifest-path $(KERNEL_SYMBOLS_MANIFEST)\nOBJCOPY_CMD = rust-objcopy \\\n    --strip-all            \\\n    -O binary\n\nEXEC_SYMBOLS_TOOL  = ruby $(KERNEL_SYMBOLS_TOOL_PATH)/main.rb\n\n##------------------------------------------------------------------------------\n## Dockerization\n##------------------------------------------------------------------------------\nDOCKER_CMD = docker run -t --rm -v $(shell pwd):/work/tutorial -w /work/tutorial\n\n# DOCKER_IMAGE defined in include file (see top of this file).\nDOCKER_TOOLS = $(DOCKER_CMD) $(DOCKER_IMAGE)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: all symbols measure_time_start measure_time_finish\n\nall: measure_time_start symbols measure_time_finish\n\nsymbols:\n\t@cp $(KERNEL_SYMBOLS_INPUT_ELF) $(KERNEL_SYMBOLS_OUTPUT_ELF)\n\n\t@$(DOCKER_TOOLS) $(EXEC_SYMBOLS_TOOL) --gen_symbols $(KERNEL_SYMBOLS_OUTPUT_ELF) \\\n                $(KERNEL_SYMBOLS_RS)\n\n\t$(call color_progress_prefix, \"Demangling\")\n\t@echo Symbol names\n\t@cat $(KERNEL_SYMBOLS_RS) | rustfilt > $(KERNEL_SYMBOLS_DEMANGLED_RS)\n\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(RUSTC_CMD)\n\n\t$(call color_progress_prefix, \"Stripping\")\n\t@echo Symbols ELF file\n\t@$(OBJCOPY_CMD) $(KERNEL_SYMBOLS_ELF) $(KERNEL_SYMBOLS_STRIPPED)\n\n\t@$(DOCKER_TOOLS) $(EXEC_SYMBOLS_TOOL) --patch_data $(KERNEL_SYMBOLS_OUTPUT_ELF) \\\n                $(KERNEL_SYMBOLS_STRIPPED)\n\n# Note: The following is the only _trivial_ way I could think of that works out of the box on both\n# Linux and macOS. Since macOS does not have the %N nanosecond format string option, the\n# resolution is restricted to whole seconds.\nmeasure_time_start:\n\t@date +%s > /tmp/kernel_symbols_start.date\n\nmeasure_time_finish:\n\t@date +%s > /tmp/kernel_symbols_end.date\n\n\t$(call color_progress_prefix, \"Finished\")\n\t@echo \"in $$((`cat /tmp/kernel_symbols_end.date` - `cat /tmp/kernel_symbols_start.date`)).0s\"\n\n\t@rm /tmp/kernel_symbols_end.date /tmp/kernel_symbols_start.date\n"
  },
  {
    "path": "18_backtrace/libraries/debug-symbol-types/Cargo.toml",
    "content": "[package]\nname = \"debug-symbol-types\"\nversion = \"0.1.0\"\nedition = \"2021\"\n"
  },
  {
    "path": "18_backtrace/libraries/debug-symbol-types/src/lib.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Types for implementing debug symbol support.\n\n#![no_std]\n\nuse core::ops::Range;\n\n/// A symbol containing a size.\n#[repr(C)]\n#[derive(Clone)]\npub struct Symbol {\n    addr_range: Range<usize>,\n    name: &'static str,\n}\n\nimpl Symbol {\n    /// Create an instance.\n    pub const fn new(start: usize, size: usize, name: &'static str) -> Symbol {\n        Symbol {\n            addr_range: Range {\n                start,\n                end: start + size,\n            },\n            name,\n        }\n    }\n\n    /// Returns true if addr is contained in the range.\n    pub fn contains(&self, addr: usize) -> bool {\n        self.addr_range.contains(&addr)\n    }\n\n    /// Returns the symbol's name.\n    pub fn name(&self) -> &'static str {\n        self.name\n    }\n\n    /// Returns the symbol's size.\n    pub fn size(&self) -> usize {\n        self.addr_range.end - self.addr_range.start\n    }\n}\n"
  },
  {
    "path": "18_backtrace/libraries/test-macros/Cargo.toml",
    "content": "[package]\nname = \"test-macros\"\nversion = \"0.1.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n\n[lib]\nproc-macro = true\n\n[dependencies]\nproc-macro2 = \"1.x\"\nquote = \"1.x\"\nsyn = { version = \"1.x\", features = [\"full\"] }\ntest-types = { path = \"../test-types\" }\n"
  },
  {
    "path": "18_backtrace/libraries/test-macros/src/lib.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\nuse proc_macro::TokenStream;\nuse proc_macro2::Span;\nuse quote::quote;\nuse syn::{parse_macro_input, Ident, ItemFn};\n\n#[proc_macro_attribute]\npub fn kernel_test(_attr: TokenStream, input: TokenStream) -> TokenStream {\n    let f = parse_macro_input!(input as ItemFn);\n\n    let test_name = &format!(\"{}\", f.sig.ident);\n    let test_ident = Ident::new(\n        &format!(\"{}_TEST_CONTAINER\", f.sig.ident.to_string().to_uppercase()),\n        Span::call_site(),\n    );\n    let test_code_block = f.block;\n\n    quote!(\n        #[test_case]\n        const #test_ident: test_types::UnitTest = test_types::UnitTest {\n            name: #test_name,\n            test_func: || #test_code_block,\n        };\n    )\n    .into()\n}\n"
  },
  {
    "path": "18_backtrace/libraries/test-types/Cargo.toml",
    "content": "[package]\nname = \"test-types\"\nversion = \"0.1.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n"
  },
  {
    "path": "18_backtrace/libraries/test-types/src/lib.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Types for the `custom_test_frameworks` implementation.\n\n#![no_std]\n\n/// Unit test container.\npub struct UnitTest {\n    /// Name of the test.\n    pub name: &'static str,\n\n    /// Function pointer to the test.\n    pub test_func: fn(),\n}\n"
  },
  {
    "path": "18_backtrace/tools/kernel_symbols_tool/cmds.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\ndef generate_symbols(kernel_elf, output_file)\n    File.open(output_file, 'w') do |file|\n        header = <<~HEREDOC\n            use debug_symbol_types::Symbol;\n\n            # [no_mangle]\n            # [link_section = \".rodata.symbol_desc\"]\n            static KERNEL_SYMBOLS: [Symbol; #{kernel_elf.num_symbols}] = [\n        HEREDOC\n\n        file.write(header)\n        kernel_elf.symbols.each do |sym|\n            value = sym.header.st_value\n            size = sym.header.st_size\n            name = sym.name\n\n            file.write(\"    Symbol::new(#{value}, #{size}, \\\"#{name}\\\"),\\n\")\n        end\n        file.write(\"];\\n\")\n    end\nend\n\ndef get_symbols_section_virt_addr(kernel_elf)\n    kernel_elf.kernel_symbols_section_virt_addr\nend\n\ndef patch_symbol_data(kernel_elf, symbols_blob_path)\n    symbols_blob = File.binread(symbols_blob_path)\n\n    raise if symbols_blob.size > kernel_elf.kernel_symbols_section_size\n\n    File.binwrite(kernel_elf.path, File.binread(symbols_blob_path),\n                  kernel_elf.kernel_symbols_section_offset_in_file)\nend\n\ndef patch_num_symbols(kernel_elf)\n    num_packed = [kernel_elf.num_symbols].pack('Q<*') # \"Q\" == uint64_t, \"<\" == little endian\n    File.binwrite(kernel_elf.path, num_packed, kernel_elf.num_kernel_symbols_offset_in_file)\nend\n"
  },
  {
    "path": "18_backtrace/tools/kernel_symbols_tool/kernel_elf.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n# KernelELF\nclass KernelELF\n    attr_reader :path\n\n    def initialize(kernel_elf_path, kernel_symbols_section, num_kernel_symbols)\n        @elf = ELFTools::ELFFile.new(File.open(kernel_elf_path))\n        @symtab_section = @elf.section_by_name('.symtab')\n\n        @path = kernel_elf_path\n        fetch_values(kernel_symbols_section, num_kernel_symbols)\n    end\n\n    private\n\n    def fetch_values(kernel_symbols_section, num_kernel_symbols)\n        sym = @symtab_section.symbol_by_name(num_kernel_symbols)\n        raise \"Symbol \\\"#{num_kernel_symbols}\\\" not found\" if sym.nil?\n\n        @num_kernel_symbols = sym\n\n        section = @elf.section_by_name(kernel_symbols_section)\n        raise \"Section \\\"#{kernel_symbols_section}\\\" not found\" if section.nil?\n\n        @kernel_symbols_section = section\n    end\n\n    def num_kernel_symbols_virt_addr\n        @num_kernel_symbols.header.st_value\n    end\n\n    def segment_containing_virt_addr(virt_addr)\n        @elf.each_segments do |segment|\n            return segment if segment.vma_in?(virt_addr)\n        end\n    end\n\n    def virt_addr_to_file_offset(virt_addr)\n        segment = segment_containing_virt_addr(virt_addr)\n        segment.vma_to_offset(virt_addr)\n    end\n\n    public\n\n    def symbols\n        non_zero_symbols = @symtab_section.symbols.reject { |sym| sym.header.st_size.zero? }\n        non_zero_symbols.sort_by { |sym| sym.header.st_value }\n    end\n\n    def num_symbols\n        symbols.size\n    end\n\n    def kernel_symbols_section_virt_addr\n        @kernel_symbols_section.header.sh_addr.to_i\n    end\n\n    def kernel_symbols_section_size\n        @kernel_symbols_section.header.sh_size.to_i\n    end\n\n    def kernel_symbols_section_offset_in_file\n        virt_addr_to_file_offset(kernel_symbols_section_virt_addr)\n    end\n\n    def num_kernel_symbols_offset_in_file\n        virt_addr_to_file_offset(num_kernel_symbols_virt_addr)\n    end\nend\n"
  },
  {
    "path": "18_backtrace/tools/kernel_symbols_tool/main.rb",
    "content": "#!/usr/bin/env ruby\n# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'rubygems'\nrequire 'bundler/setup'\nrequire 'colorize'\nrequire 'elftools'\n\nrequire_relative 'kernel_elf'\nrequire_relative 'cmds'\n\nKERNEL_SYMBOLS_SECTION = '.kernel_symbols'\nNUM_KERNEL_SYMBOLS = 'NUM_KERNEL_SYMBOLS'\n\ncmd = ARGV[0]\n\nkernel_elf_path = ARGV[1]\nkernel_elf = KernelELF.new(kernel_elf_path, KERNEL_SYMBOLS_SECTION, NUM_KERNEL_SYMBOLS)\n\ncase cmd\nwhen '--gen_symbols'\n    output_file = ARGV[2]\n\n    print 'Generating'.rjust(12).green.bold\n    puts ' Symbols source file'\n\n    generate_symbols(kernel_elf, output_file)\nwhen '--get_symbols_section_virt_addr'\n    addr = get_symbols_section_virt_addr(kernel_elf)\n\n    puts \"0x#{addr.to_s(16)}\"\nwhen '--patch_data'\n    symbols_blob_path = ARGV[2]\n    num_symbols = kernel_elf.num_symbols\n\n    print 'Patching'.rjust(12).green.bold\n    puts \" Symbols blob and number of symbols (#{num_symbols}) into ELF\"\n\n    patch_symbol_data(kernel_elf, symbols_blob_path)\n    patch_num_symbols(kernel_elf)\nelse\n    raise\nend\n"
  },
  {
    "path": "18_backtrace/tools/translation_table_tool/arch.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n# Bitfield manipulation.\nclass BitField\n    def initialize\n        @value = 0\n    end\n\n    def self.attr_bitfield(name, offset, num_bits)\n        define_method(\"#{name}=\") do |bits|\n            mask = (2**num_bits) - 1\n\n            raise \"Input out of range: #{name} = 0x#{bits.to_s(16)}\" if (bits & ~mask).positive?\n\n            # Clear bitfield\n            @value &= ~(mask << offset)\n\n            # Set it\n            @value |= (bits << offset)\n        end\n    end\n\n    def to_i\n        @value\n    end\n\n    def size_in_byte\n        8\n    end\nend\n\n# An array class that knows its memory location.\nclass CArray < Array\n    attr_reader :phys_start_addr\n\n    def initialize(phys_start_addr, size, &block)\n        @phys_start_addr = phys_start_addr\n\n        super(size, &block)\n    end\n\n    def size_in_byte\n        inject(0) { |sum, n| sum + n.size_in_byte }\n    end\nend\n\n#---------------------------------------------------------------------------------------------------\n# Arch::\n#---------------------------------------------------------------------------------------------------\nmodule Arch\n#---------------------------------------------------------------------------------------------------\n# Arch::ARMv8\n#---------------------------------------------------------------------------------------------------\nmodule ARMv8\n# ARMv8 Table Descriptor.\nclass Stage1TableDescriptor < BitField\n    module NextLevelTableAddr\n        OFFSET = 16\n        NUMBITS = 32\n    end\n\n    module Type\n        OFFSET = 1\n        NUMBITS = 1\n\n        BLOCK = 0\n        TABLE = 1\n    end\n\n    module Valid\n        OFFSET = 0\n        NUMBITS = 1\n\n        FALSE = 0\n        TRUE = 1\n    end\n\n    attr_bitfield(:__next_level_table_addr, NextLevelTableAddr::OFFSET, NextLevelTableAddr::NUMBITS)\n    attr_bitfield(:type, Type::OFFSET, Type::NUMBITS)\n    attr_bitfield(:valid, Valid::OFFSET, Valid::NUMBITS)\n\n    def next_level_table_addr=(addr)\n        addr >>= Granule64KiB::SHIFT\n\n        self.__next_level_table_addr = addr\n    end\n\n    private :__next_level_table_addr=\nend\n\n# ARMv8 level 3 page descriptor.\nclass Stage1PageDescriptor < BitField\n    module UXN\n        OFFSET = 54\n        NUMBITS = 1\n\n        FALSE = 0\n        TRUE = 1\n    end\n\n    module PXN\n        OFFSET = 53\n        NUMBITS = 1\n\n        FALSE = 0\n        TRUE = 1\n    end\n\n    module OutputAddr\n        OFFSET = 16\n        NUMBITS = 32\n    end\n\n    module AF\n        OFFSET = 10\n        NUMBITS = 1\n\n        FALSE = 0\n        TRUE = 1\n    end\n\n    module SH\n        OFFSET = 8\n        NUMBITS = 2\n\n        INNER_SHAREABLE = 0b11\n    end\n\n    module AP\n        OFFSET = 6\n        NUMBITS = 2\n\n        RW_EL1 = 0b00\n        RO_EL1 = 0b10\n    end\n\n    module AttrIndx\n        OFFSET = 2\n        NUMBITS = 3\n    end\n\n    module Type\n        OFFSET = 1\n        NUMBITS = 1\n\n        RESERVED_INVALID = 0\n        PAGE = 1\n    end\n\n    module Valid\n        OFFSET = 0\n        NUMBITS = 1\n\n        FALSE = 0\n        TRUE = 1\n    end\n\n    attr_bitfield(:uxn, UXN::OFFSET, UXN::NUMBITS)\n    attr_bitfield(:pxn, PXN::OFFSET, PXN::NUMBITS)\n    attr_bitfield(:__output_addr, OutputAddr::OFFSET, OutputAddr::NUMBITS)\n    attr_bitfield(:af, AF::OFFSET, AF::NUMBITS)\n    attr_bitfield(:sh, SH::OFFSET, SH::NUMBITS)\n    attr_bitfield(:ap, AP::OFFSET, AP::NUMBITS)\n    attr_bitfield(:attr_indx, AttrIndx::OFFSET, AttrIndx::NUMBITS)\n    attr_bitfield(:type, Type::OFFSET, Type::NUMBITS)\n    attr_bitfield(:valid, Valid::OFFSET, Valid::NUMBITS)\n\n    def output_addr=(addr)\n        addr >>= Granule64KiB::SHIFT\n\n        self.__output_addr = addr\n    end\n\n    private :__output_addr=\nend\n\n# Translation table representing the structure defined in translation_table.rs.\nclass TranslationTable\n    module MAIR\n        NORMAL = 1\n    end\n\n    def initialize\n        do_sanity_checks\n\n        num_lvl2_tables = BSP.kernel_virt_addr_space_size >> Granule512MiB::SHIFT\n\n        @lvl3 = new_lvl3(num_lvl2_tables, BSP.phys_addr_of_kernel_tables)\n\n        @lvl2_phys_start_addr = @lvl3.phys_start_addr + @lvl3.size_in_byte\n        @lvl2 = new_lvl2(num_lvl2_tables, @lvl2_phys_start_addr)\n\n        populate_lvl2_entries\n    end\n\n    def map_at(virt_region, phys_region, attributes)\n        return if virt_region.empty?\n\n        raise if virt_region.size != phys_region.size\n        raise if phys_region.last > BSP.phys_addr_space_end_page\n\n        virt_region.zip(phys_region).each do |virt_page, phys_page|\n            desc = page_descriptor_from(virt_page)\n            set_lvl3_entry(desc, phys_page, attributes)\n        end\n    end\n\n    def to_binary\n        data = @lvl3.flatten.map(&:to_i) + @lvl2.map(&:to_i)\n        data.pack('Q<*') # \"Q\" == uint64_t, \"<\" == little endian\n    end\n\n    def phys_tables_base_addr_binary\n        [@lvl2_phys_start_addr].pack('Q<*') # \"Q\" == uint64_t, \"<\" == little endian\n    end\n\n    def phys_tables_base_addr\n        @lvl2_phys_start_addr\n    end\n\n    private\n\n    def do_sanity_checks\n        raise unless BSP.kernel_granule::SIZE == Granule64KiB::SIZE\n        raise unless (BSP.kernel_virt_addr_space_size % Granule512MiB::SIZE).zero?\n    end\n\n    def new_lvl3(num_lvl2_tables, start_addr)\n        CArray.new(start_addr, num_lvl2_tables) do\n            temp = CArray.new(start_addr, 8192) do\n                Stage1PageDescriptor.new\n            end\n            start_addr += temp.size_in_byte\n\n            temp\n        end\n    end\n\n    def new_lvl2(num_lvl2_tables, start_addr)\n        CArray.new(start_addr, num_lvl2_tables) do\n            Stage1TableDescriptor.new\n        end\n    end\n\n    def populate_lvl2_entries\n        @lvl2.each_with_index do |descriptor, i|\n            descriptor.next_level_table_addr = @lvl3[i].phys_start_addr\n            descriptor.type = Stage1TableDescriptor::Type::TABLE\n            descriptor.valid = Stage1TableDescriptor::Valid::TRUE\n        end\n    end\n\n    def lvl2_lvl3_index_from(addr)\n        addr -= BSP.kernel_virt_start_addr\n\n        lvl2_index = addr >> Granule512MiB::SHIFT\n        lvl3_index = (addr & Granule512MiB::MASK) >> Granule64KiB::SHIFT\n\n        raise unless lvl2_index < @lvl2.size\n\n        [lvl2_index, lvl3_index]\n    end\n\n    def page_descriptor_from(virt_addr)\n        lvl2_index, lvl3_index = lvl2_lvl3_index_from(virt_addr)\n\n        @lvl3[lvl2_index][lvl3_index]\n    end\n\n    # rubocop:disable Metrics/MethodLength\n    def set_attributes(desc, attributes)\n        case attributes.mem_attributes\n        when :CacheableDRAM\n            desc.sh = Stage1PageDescriptor::SH::INNER_SHAREABLE\n            desc.attr_indx = MAIR::NORMAL\n        else\n            raise 'Invalid input'\n        end\n\n        desc.ap = case attributes.acc_perms\n                  when :ReadOnly\n                      Stage1PageDescriptor::AP::RO_EL1\n                  when :ReadWrite\n                      Stage1PageDescriptor::AP::RW_EL1\n                  else\n                      raise 'Invalid input'\n\n                  end\n\n        desc.pxn = if attributes.execute_never\n                       Stage1PageDescriptor::PXN::TRUE\n                   else\n                       Stage1PageDescriptor::PXN::FALSE\n                   end\n\n        desc.uxn = Stage1PageDescriptor::UXN::TRUE\n    end\n    # rubocop:enable Metrics/MethodLength\n\n    def set_lvl3_entry(desc, output_addr, attributes)\n        desc.output_addr = output_addr\n        desc.af = Stage1PageDescriptor::AF::TRUE\n        desc.type = Stage1PageDescriptor::Type::PAGE\n        desc.valid = Stage1PageDescriptor::Valid::TRUE\n\n        set_attributes(desc, attributes)\n    end\nend\nend\nend\n"
  },
  {
    "path": "18_backtrace/tools/translation_table_tool/bsp.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n# Raspberry Pi 3 + 4\nclass RaspberryPi\n    attr_reader :kernel_granule, :kernel_virt_addr_space_size, :kernel_virt_start_addr\n\n    MEMORY_SRC = File.read('kernel/src/bsp/raspberrypi/memory.rs').split(\"\\n\")\n\n    def initialize\n        @kernel_granule = Granule64KiB\n\n        @kernel_virt_addr_space_size = KERNEL_ELF.symbol_value('__kernel_virt_addr_space_size')\n        @kernel_virt_start_addr = KERNEL_ELF.symbol_value('__kernel_virt_start_addr')\n\n        @virt_addr_of_kernel_tables = KERNEL_ELF.symbol_value('KERNEL_TABLES')\n        @virt_addr_of_phys_kernel_tables_base_addr = KERNEL_ELF.symbol_value(\n            'PHYS_KERNEL_TABLES_BASE_ADDR'\n        )\n    end\n\n    def phys_addr_of_kernel_tables\n        KERNEL_ELF.virt_to_phys(@virt_addr_of_kernel_tables)\n    end\n\n    def kernel_tables_offset_in_file\n        KERNEL_ELF.virt_addr_to_file_offset(@virt_addr_of_kernel_tables)\n    end\n\n    def phys_kernel_tables_base_addr_offset_in_file\n        KERNEL_ELF.virt_addr_to_file_offset(@virt_addr_of_phys_kernel_tables_base_addr)\n    end\n\n    def phys_addr_space_end_page\n        x = MEMORY_SRC.grep(/pub const END/)\n        x = case BSP_TYPE\n            when :rpi3\n                x[0]\n            when :rpi4\n                x[1]\n            else\n                raise\n            end\n\n        # Extract the hex literal with underscores like 0x0123_abcd.\n        x = x.scan(/0x[\\h_]*/)[0]\n\n        # Further remove x and _ and convert to int.\n        x.scan(/\\h+/).join.to_i(16)\n    end\nend\n"
  },
  {
    "path": "18_backtrace/tools/translation_table_tool/generic.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\nmodule Granule64KiB\n    SIZE = 64 * 1024\n    SHIFT = Math.log2(SIZE).to_i\nend\n\nmodule Granule512MiB\n    SIZE = 512 * 1024 * 1024\n    SHIFT = Math.log2(SIZE).to_i\n    MASK = SIZE - 1\nend\n\n# Monkey-patch Integer with some helper functions.\nclass Integer\n    def power_of_two?\n        self[0].zero?\n    end\n\n    def aligned?(alignment)\n        raise unless alignment.power_of_two?\n\n        (self & (alignment - 1)).zero?\n    end\n\n    def align_up(alignment)\n        raise unless alignment.power_of_two?\n\n        (self + alignment - 1) & ~(alignment - 1)\n    end\n\n    def to_hex_underscore(with_leading_zeros: false)\n        fmt = with_leading_zeros ? '%016x' : '%x'\n        value = format(fmt, self).to_s.reverse.scan(/.{4}|.+/).join('_').reverse\n\n        format('0x%s', value)\n    end\nend\n\n# An array where each value is the start address of a Page.\nclass MemoryRegion < Array\n    def initialize(start_addr, size, granule_size)\n        raise unless start_addr.aligned?(granule_size)\n        raise unless size.positive?\n        raise unless (size % granule_size).zero?\n\n        num_pages = size / granule_size\n        super(num_pages) do |i|\n            (i * granule_size) + start_addr\n        end\n    end\nend\n\n# Collection of memory attributes.\nclass AttributeFields\n    attr_reader :mem_attributes, :acc_perms, :execute_never\n\n    def initialize(mem_attributes, acc_perms, execute_never)\n        @mem_attributes = mem_attributes\n        @acc_perms = acc_perms\n        @execute_never = execute_never\n    end\n\n    def to_s\n        x = case @mem_attributes\n            when :CacheableDRAM\n                'C'\n            else\n                '?'\n            end\n\n        y = case @acc_perms\n            when :ReadWrite\n                'RW'\n            when :ReadOnly\n                'RO'\n            else\n                '??'\n            end\n\n        z = @execute_never ? 'XN' : 'X '\n\n        \"#{x} #{y} #{z}\"\n    end\nend\n\n# A container that describes a virt-to-phys region mapping.\nclass MappingDescriptor\n    @max_section_name_length = 'Sections'.length\n\n    class << self\n        attr_accessor :max_section_name_length\n\n        def update_max_section_name_length(length)\n            @max_section_name_length = [@max_section_name_length, length].max\n        end\n    end\n\n    attr_reader :name, :virt_region, :phys_region, :attributes\n\n    def initialize(name, virt_region, phys_region, attributes)\n        @name = name\n        @virt_region = virt_region\n        @phys_region = phys_region\n        @attributes = attributes\n    end\n\n    def size_human_readable(size)\n        if size >= (1024 * 1024)\n            \"#{(size / (1024 * 1024)).to_s.rjust(3)} MiB\"\n        elsif size >= 1024\n            \"#{(size / 1024).to_s.rjust(3)} KiB\"\n        else\n            raise\n        end\n    end\n\n    def to_s\n        name = @name.ljust(self.class.max_section_name_length)\n        virt_start = @virt_region.first.to_hex_underscore(with_leading_zeros: true)\n        phys_start = @phys_region.first.to_hex_underscore(with_leading_zeros: true)\n        size = size_human_readable(@virt_region.size * 65_536)\n\n        \"#{name} | #{virt_start} | #{phys_start} | #{size} | #{@attributes}\"\n    end\n\n    def self.print_divider\n        print '             '\n        print '-' * max_section_name_length\n        puts '--------------------------------------------------------------------'\n    end\n\n    def self.print_header\n        print_divider\n        print '             '\n        print 'Sections'.center(max_section_name_length)\n        print '   '\n        print 'Virt Start Addr'.center(21)\n        print '   '\n        print 'Phys Start Addr'.center(21)\n        print '   '\n        print 'Size'.center(7)\n        print '   '\n        print 'Attr'.center(7)\n        puts\n        print_divider\n    end\nend\n\ndef kernel_map_binary\n    mapping_descriptors = KERNEL_ELF.generate_mapping_descriptors\n\n    # Generate_mapping_descriptors updates the header being printed with this call. So it must come\n    # afterwards.\n    MappingDescriptor.print_header\n\n    mapping_descriptors.each do |i|\n        print 'Generating'.rjust(12).green.bold\n        print ' '\n        puts i\n\n        TRANSLATION_TABLES.map_at(i.virt_region, i.phys_region, i.attributes)\n    end\n\n    MappingDescriptor.print_divider\nend\n\ndef kernel_patch_tables(kernel_elf_path)\n    print 'Patching'.rjust(12).green.bold\n    print ' Kernel table struct at ELF file offset '\n    puts BSP.kernel_tables_offset_in_file.to_hex_underscore\n\n    File.binwrite(kernel_elf_path, TRANSLATION_TABLES.to_binary, BSP.kernel_tables_offset_in_file)\nend\n\ndef kernel_patch_base_addr(kernel_elf_path)\n    print 'Patching'.rjust(12).green.bold\n    print ' Kernel tables physical base address start argument to value '\n    print TRANSLATION_TABLES.phys_tables_base_addr.to_hex_underscore\n    print ' at ELF file offset '\n    puts BSP.phys_kernel_tables_base_addr_offset_in_file.to_hex_underscore\n\n    File.binwrite(kernel_elf_path, TRANSLATION_TABLES.phys_tables_base_addr_binary,\n                  BSP.phys_kernel_tables_base_addr_offset_in_file)\nend\n"
  },
  {
    "path": "18_backtrace/tools/translation_table_tool/kernel_elf.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n# KernelELF\nclass KernelELF\n    SECTION_FLAG_ALLOC = 2\n\n    def initialize(kernel_elf_path)\n        @elf = ELFTools::ELFFile.new(File.open(kernel_elf_path))\n        @symtab_section = @elf.section_by_name('.symtab')\n    end\n\n    def machine\n        @elf.machine.to_sym\n    end\n\n    def symbol_value(symbol_name)\n        @symtab_section.symbol_by_name(symbol_name).header.st_value\n    end\n\n    def segment_containing_virt_addr(virt_addr)\n        @elf.each_segments do |segment|\n            return segment if segment.vma_in?(virt_addr)\n        end\n    end\n\n    def virt_to_phys(virt_addr)\n        segment = segment_containing_virt_addr(virt_addr)\n        translation_offset = segment.header.p_vaddr - segment.header.p_paddr\n\n        virt_addr - translation_offset\n    end\n\n    def virt_addr_to_file_offset(virt_addr)\n        segment = segment_containing_virt_addr(virt_addr)\n        segment.vma_to_offset(virt_addr)\n    end\n\n    def sections_in_segment(segment)\n        head = segment.mem_head\n        tail = segment.mem_tail\n\n        sections = @elf.each_sections.select do |section|\n            file_offset = section.header.sh_addr\n            flags = section.header.sh_flags\n\n            file_offset >= head && file_offset < tail && (flags & SECTION_FLAG_ALLOC != 0)\n        end\n\n        sections.map(&:name).join(' ')\n    end\n\n    def select_load_segments\n        @elf.each_segments.select do |segment|\n            segment.instance_of?(ELFTools::Segments::LoadSegment)\n        end\n    end\n\n    def segment_get_acc_perms(segment)\n        if segment.readable? && segment.writable?\n            :ReadWrite\n        elsif segment.readable?\n            :ReadOnly\n        else\n            :Invalid\n        end\n    end\n\n    def update_max_section_name_length(descriptors)\n        MappingDescriptor.update_max_section_name_length(descriptors.map { |i| i.name.size }.max)\n    end\n\n    def generate_mapping_descriptors\n        descriptors = select_load_segments.map do |segment|\n            # Assume each segment is page aligned.\n            size = segment.mem_size.align_up(BSP.kernel_granule::SIZE)\n            virt_start_addr = segment.header.p_vaddr\n            phys_start_addr = segment.header.p_paddr\n            acc_perms = segment_get_acc_perms(segment)\n            execute_never = !segment.executable?\n            section_names = sections_in_segment(segment)\n\n            virt_region = MemoryRegion.new(virt_start_addr, size, BSP.kernel_granule::SIZE)\n            phys_region = MemoryRegion.new(phys_start_addr, size, BSP.kernel_granule::SIZE)\n            attributes = AttributeFields.new(:CacheableDRAM, acc_perms, execute_never)\n\n            MappingDescriptor.new(section_names, virt_region, phys_region, attributes)\n        end\n\n        update_max_section_name_length(descriptors)\n        descriptors\n    end\nend\n"
  },
  {
    "path": "18_backtrace/tools/translation_table_tool/main.rb",
    "content": "#!/usr/bin/env ruby\n# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'rubygems'\nrequire 'bundler/setup'\nrequire 'colorize'\nrequire 'elftools'\n\nrequire_relative 'generic'\nrequire_relative 'kernel_elf'\nrequire_relative 'bsp'\nrequire_relative 'arch'\n\nBSP_TYPE = ARGV[0].to_sym\nkernel_elf_path = ARGV[1]\n\nstart = Time.now\n\nKERNEL_ELF = KernelELF.new(kernel_elf_path)\n\nBSP = case BSP_TYPE\n      when :rpi3, :rpi4\n          RaspberryPi.new\n      else\n          raise\n      end\n\nTRANSLATION_TABLES = case KERNEL_ELF.machine\n                     when :AArch64\n                         Arch::ARMv8::TranslationTable.new\n                     else\n                         raise\n                     end\n\nkernel_map_binary\nkernel_patch_tables(kernel_elf_path)\nkernel_patch_base_addr(kernel_elf_path)\n\nelapsed = Time.now - start\n\nprint 'Finished'.rjust(12).green.bold\nputs \" in #{elapsed.round(2)}s\"\n"
  },
  {
    "path": "19_kernel_heap/.cargo/config.toml",
    "content": "[target.'cfg(target_os = \"none\")']\nrunner = \"target/kernel_test_runner.sh\"\n"
  },
  {
    "path": "19_kernel_heap/.vscode/settings.json",
    "content": "{\n    \"editor.formatOnSave\": true,\n    \"editor.rulers\": [100],\n    \"rust-analyzer.cargo.target\": \"aarch64-unknown-none-softfloat\",\n    \"rust-analyzer.cargo.features\": [\"bsp_rpi3\"],\n    \"rust-analyzer.checkOnSave.allTargets\": false,\n    \"rust-analyzer.checkOnSave.extraArgs\": [\"--lib\", \"--bins\"],\n    \"rust-analyzer.lens.debug\": false,\n    \"rust-analyzer.lens.run\": false\n}\n"
  },
  {
    "path": "19_kernel_heap/Cargo.toml",
    "content": "[workspace]\n\nmembers = [\n        \"libraries/*\",\n        \"kernel\",\n        \"kernel_symbols\"\n]\n\n[profile.release]\nlto = true\ndebug = true\n"
  },
  {
    "path": "19_kernel_heap/Makefile",
    "content": "## SPDX-License-Identifier: MIT OR Apache-2.0\n##\n## Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\ninclude ../common/docker.mk\ninclude ../common/format.mk\ninclude ../common/operating_system.mk\n\n##--------------------------------------------------------------------------------------------------\n## Optional, user-provided configuration values\n##--------------------------------------------------------------------------------------------------\n\n# Default to the RPi3.\nBSP ?= rpi3\n\n# Default to a serial device name that is common in Linux.\nDEV_SERIAL ?= /dev/ttyUSB0\n\n# Optional debug prints.\nifdef DEBUG_PRINTS\n    FEATURES = --features debug_prints\nendif\n\n# Optional integration test name.\nifdef TEST\n    TEST_ARG = --test $(TEST)\nelse\n    TEST_ARG = --test '*'\nendif\n\n\n\n##--------------------------------------------------------------------------------------------------\n## BSP-specific configuration values\n##--------------------------------------------------------------------------------------------------\nQEMU_MISSING_STRING = \"This board is not yet supported for QEMU.\"\n\nifeq ($(BSP),rpi3)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE = raspi3\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    QEMU_TEST_ARGS    = $(QEMU_RELEASE_ARGS) -semihosting\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    OPENOCD_ARG       = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi3.cfg\n    JTAG_BOOT_IMAGE   = ../X1_JTAG_boot/jtag_boot_rpi3.img\n    LD_SCRIPT_PATH    = $(shell pwd)/kernel/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a53 -C force-frame-pointers\nelse ifeq ($(BSP),rpi4)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE =\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    QEMU_TEST_ARGS    = $(QEMU_RELEASE_ARGS) -semihosting\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    OPENOCD_ARG       = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi4.cfg\n    JTAG_BOOT_IMAGE   = ../X1_JTAG_boot/jtag_boot_rpi4.img\n    LD_SCRIPT_PATH    = $(shell pwd)/kernel/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a72 -C force-frame-pointers\nendif\n\n# Export for build.rs.\nexport LD_SCRIPT_PATH\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets and Prerequisites\n##--------------------------------------------------------------------------------------------------\nKERNEL_MANIFEST      = kernel/Cargo.toml\nKERNEL_LINKER_SCRIPT = kernel.ld\nLAST_BUILD_CONFIG    = target/$(BSP)_$(DEBUG_PRINTS).build_config\n\nKERNEL_ELF_RAW      = target/$(TARGET)/release/kernel\n# This parses cargo's dep-info file.\n# https://doc.rust-lang.org/cargo/guide/build-cache.html#dep-info-files\nKERNEL_ELF_RAW_DEPS = $(filter-out %: ,$(file < $(KERNEL_ELF_RAW).d)) $(KERNEL_MANIFEST) $(LAST_BUILD_CONFIG)\n\n##------------------------------------------------------------------------------\n## Translation tables\n##------------------------------------------------------------------------------\nTT_TOOL_PATH = tools/translation_table_tool\n\nKERNEL_ELF_TTABLES      = target/$(TARGET)/release/kernel+ttables\nKERNEL_ELF_TTABLES_DEPS = $(KERNEL_ELF_RAW) $(wildcard $(TT_TOOL_PATH)/*)\n\n##------------------------------------------------------------------------------\n## Kernel symbols\n##------------------------------------------------------------------------------\nexport KERNEL_SYMBOLS_TOOL_PATH = tools/kernel_symbols_tool\n\nKERNEL_ELF_TTABLES_SYMS = target/$(TARGET)/release/kernel+ttables+symbols\n\n# Unlike with KERNEL_ELF_RAW, we are not relying on dep-info here. One of the reasons being that the\n# name of the generated symbols file varies between runs, which can cause confusion.\nKERNEL_ELF_TTABLES_SYMS_DEPS = $(KERNEL_ELF_TTABLES) \\\n    $(wildcard kernel_symbols/*)                     \\\n    $(wildcard $(KERNEL_SYMBOLS_TOOL_PATH)/*)\n\nexport TARGET\nexport KERNEL_SYMBOLS_INPUT_ELF  = $(KERNEL_ELF_TTABLES)\nexport KERNEL_SYMBOLS_OUTPUT_ELF = $(KERNEL_ELF_TTABLES_SYMS)\n\nKERNEL_ELF = $(KERNEL_ELF_TTABLES_SYMS)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Command building blocks\n##--------------------------------------------------------------------------------------------------\nRUSTFLAGS = $(RUSTC_MISC_ARGS)                   \\\n    -C link-arg=--library-path=$(LD_SCRIPT_PATH) \\\n    -C link-arg=--script=$(KERNEL_LINKER_SCRIPT)\n\nRUSTFLAGS_PEDANTIC = $(RUSTFLAGS) \\\n    -D warnings                   \\\n    -D missing_docs\n\nFEATURES     += --features bsp_$(BSP)\nCOMPILER_ARGS = --target=$(TARGET) \\\n    $(FEATURES)                    \\\n    --release\n\n# build-std can be skipped for helper commands that do not rely on correct stack frames and other\n# custom compiler options. This results in a huge speedup.\nRUSTC_CMD   = cargo rustc $(COMPILER_ARGS) -Z build-std=core,alloc --manifest-path $(KERNEL_MANIFEST)\nDOC_CMD     = cargo doc $(COMPILER_ARGS)\nCLIPPY_CMD  = cargo clippy $(COMPILER_ARGS)\nTEST_CMD    = cargo test $(COMPILER_ARGS) -Z build-std=core,alloc --manifest-path $(KERNEL_MANIFEST)\nOBJCOPY_CMD = rust-objcopy \\\n    --strip-all            \\\n    -O binary\n\nEXEC_QEMU          = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE)\nEXEC_TT_TOOL       = ruby $(TT_TOOL_PATH)/main.rb\nEXEC_TEST_DISPATCH = ruby ../common/tests/dispatch.rb\nEXEC_MINIPUSH      = ruby ../common/serial/minipush.rb\n\n##------------------------------------------------------------------------------\n## Dockerization\n##------------------------------------------------------------------------------\nDOCKER_CMD            = docker run -t --rm -v $(shell pwd):/work/tutorial -w /work/tutorial\nDOCKER_CMD_INTERACT   = $(DOCKER_CMD) -i\nDOCKER_ARG_DIR_COMMON = -v $(shell pwd)/../common:/work/common\nDOCKER_ARG_DIR_JTAG   = -v $(shell pwd)/../X1_JTAG_boot:/work/X1_JTAG_boot\nDOCKER_ARG_DEV        = --privileged -v /dev:/dev\nDOCKER_ARG_NET        = --network host\n\n# DOCKER_IMAGE defined in include file (see top of this file).\nDOCKER_QEMU  = $(DOCKER_CMD_INTERACT) $(DOCKER_IMAGE)\nDOCKER_TOOLS = $(DOCKER_CMD) $(DOCKER_IMAGE)\nDOCKER_TEST  = $(DOCKER_CMD) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\nDOCKER_GDB   = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_NET) $(DOCKER_IMAGE)\n\n# Dockerize commands, which require USB device passthrough, only on Linux.\nifeq ($(shell uname -s),Linux)\n    DOCKER_CMD_DEV = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_DEV)\n\n    DOCKER_CHAINBOOT = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\n    DOCKER_JTAGBOOT  = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_ARG_DIR_JTAG) $(DOCKER_IMAGE)\n    DOCKER_OPENOCD   = $(DOCKER_CMD_DEV) $(DOCKER_ARG_NET) $(DOCKER_IMAGE)\nelse\n    DOCKER_OPENOCD   = echo \"Not yet supported on non-Linux systems.\"; \\#\nendif\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: all doc qemu chainboot clippy clean readelf objdump nm check\n\nall: $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Save the configuration as a file, so make understands if it changed.\n##------------------------------------------------------------------------------\n$(LAST_BUILD_CONFIG):\n\t@rm -f target/*.build_config\n\t@mkdir -p target\n\t@touch $(LAST_BUILD_CONFIG)\n\n##------------------------------------------------------------------------------\n## Compile the kernel ELF\n##------------------------------------------------------------------------------\n$(KERNEL_ELF_RAW): $(KERNEL_ELF_RAW_DEPS)\n\t$(call color_header, \"Compiling kernel ELF - $(BSP)\")\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(RUSTC_CMD)\n\n##------------------------------------------------------------------------------\n## Precompute the kernel translation tables and patch them into the kernel ELF\n##------------------------------------------------------------------------------\n$(KERNEL_ELF_TTABLES): $(KERNEL_ELF_TTABLES_DEPS)\n\t$(call color_header, \"Precomputing kernel translation tables and patching kernel ELF\")\n\t@cp $(KERNEL_ELF_RAW) $(KERNEL_ELF_TTABLES)\n\t@$(DOCKER_TOOLS) $(EXEC_TT_TOOL) $(BSP) $(KERNEL_ELF_TTABLES)\n\n##------------------------------------------------------------------------------\n## Generate kernel symbols and patch them into the kernel ELF\n##------------------------------------------------------------------------------\n$(KERNEL_ELF_TTABLES_SYMS): $(KERNEL_ELF_TTABLES_SYMS_DEPS)\n\t$(call color_header, \"Generating kernel symbols and patching kernel ELF\")\n\t@$(MAKE) --no-print-directory -f kernel_symbols.mk\n\n##------------------------------------------------------------------------------\n## Generate the stripped kernel binary\n##------------------------------------------------------------------------------\n$(KERNEL_BIN): $(KERNEL_ELF_TTABLES_SYMS)\n\t$(call color_header, \"Generating stripped binary\")\n\t@$(OBJCOPY_CMD) $(KERNEL_ELF_TTABLES_SYMS) $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Name\")\n\t@echo $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Size\")\n\t$(call disk_usage_KiB, $(KERNEL_BIN))\n\n##------------------------------------------------------------------------------\n## Generate the documentation\n##------------------------------------------------------------------------------\ndoc: clean\n\t$(call color_header, \"Generating docs\")\n\t@$(DOC_CMD) --document-private-items --open\n\n##------------------------------------------------------------------------------\n## Run the kernel in QEMU\n##------------------------------------------------------------------------------\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\nqemu:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\nqemu: $(KERNEL_BIN)\n\t$(call color_header, \"Launching QEMU\")\n\t@$(DOCKER_QEMU) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\nendif\n\n##------------------------------------------------------------------------------\n## Push the kernel to the real HW target\n##------------------------------------------------------------------------------\nchainboot: $(KERNEL_BIN)\n\t@$(DOCKER_CHAINBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run clippy\n##------------------------------------------------------------------------------\nclippy:\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(CLIPPY_CMD)\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(CLIPPY_CMD) --features test_build --tests \\\n                --manifest-path $(KERNEL_MANIFEST)\n\n##------------------------------------------------------------------------------\n## Clean\n##------------------------------------------------------------------------------\nclean:\n\trm -rf target $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run readelf\n##------------------------------------------------------------------------------\nreadelf: $(KERNEL_ELF)\n\t$(call color_header, \"Launching readelf\")\n\t@$(DOCKER_TOOLS) $(READELF_BINARY) --headers $(KERNEL_ELF)\n\n##------------------------------------------------------------------------------\n## Run objdump\n##------------------------------------------------------------------------------\nobjdump: $(KERNEL_ELF)\n\t$(call color_header, \"Launching objdump\")\n\t@$(DOCKER_TOOLS) $(OBJDUMP_BINARY) --disassemble --demangle \\\n                --section .text   \\\n                --section .rodata \\\n                $(KERNEL_ELF) | rustfilt\n\n##------------------------------------------------------------------------------\n## Run nm\n##------------------------------------------------------------------------------\nnm: $(KERNEL_ELF)\n\t$(call color_header, \"Launching nm\")\n\t@$(DOCKER_TOOLS) $(NM_BINARY) --demangle --print-size $(KERNEL_ELF) | sort | rustfilt\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Debugging targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: jtagboot openocd gdb gdb-opt0\n\n##------------------------------------------------------------------------------\n## Push the JTAG boot image to the real HW target\n##------------------------------------------------------------------------------\njtagboot:\n\t@$(DOCKER_JTAGBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(JTAG_BOOT_IMAGE)\n\n##------------------------------------------------------------------------------\n## Start OpenOCD session\n##------------------------------------------------------------------------------\nopenocd:\n\t$(call color_header, \"Launching OpenOCD\")\n\t@$(DOCKER_OPENOCD) openocd $(OPENOCD_ARG)\n\n##------------------------------------------------------------------------------\n## Start GDB session\n##------------------------------------------------------------------------------\ngdb-opt0: RUSTC_MISC_ARGS += -C opt-level=0\ngdb gdb-opt0: $(KERNEL_ELF)\n\t$(call color_header, \"Launching GDB\")\n\t@$(DOCKER_GDB) gdb-multiarch -q $(KERNEL_ELF)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Testing targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: test test_boot test_unit test_integration\n\ntest_unit test_integration: FEATURES += --features test_build\n\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\ntest_boot test_unit test_integration test:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\n##------------------------------------------------------------------------------\n## Run boot test\n##------------------------------------------------------------------------------\ntest_boot: $(KERNEL_BIN)\n\t$(call color_header, \"Boot test - $(BSP)\")\n\t@$(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Helpers for unit and integration test targets\n##------------------------------------------------------------------------------\ndefine KERNEL_TEST_RUNNER\n#!/usr/bin/env bash\n\n    # The cargo test runner seems to change into the crate under test's directory. Therefore, ensure\n    # this script executes from the root.\n    cd $(shell pwd)\n\n    TEST_ELF=$$(echo $$1 | sed -e 's/.*target/target/g')\n    TEST_ELF_SYMS=\"$${TEST_ELF}_syms\"\n    TEST_BINARY=$$(echo $$1.img | sed -e 's/.*target/target/g')\n\n    $(DOCKER_TOOLS) $(EXEC_TT_TOOL) $(BSP) $$TEST_ELF > /dev/null\n\n    # This overrides the two ENV variables. The other ENV variables that are required as input for\n    # the .mk file are set already because they are exported by this Makefile and this script is\n    # started by the same.\n    KERNEL_SYMBOLS_INPUT_ELF=$$TEST_ELF           \\\n        KERNEL_SYMBOLS_OUTPUT_ELF=$$TEST_ELF_SYMS \\\n        $(MAKE) --no-print-directory -f kernel_symbols.mk > /dev/null 2>&1\n\n    $(OBJCOPY_CMD) $$TEST_ELF_SYMS $$TEST_BINARY\n    $(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_TEST_ARGS) -kernel $$TEST_BINARY\nendef\n\nexport KERNEL_TEST_RUNNER\n\ndefine test_prepare\n    @mkdir -p target\n    @echo \"$$KERNEL_TEST_RUNNER\" > target/kernel_test_runner.sh\n    @chmod +x target/kernel_test_runner.sh\nendef\n\n##------------------------------------------------------------------------------\n## Run unit test(s)\n##------------------------------------------------------------------------------\ntest_unit:\n\t$(call color_header, \"Compiling unit test(s) - $(BSP)\")\n\t$(call test_prepare)\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(TEST_CMD) --lib\n\n##------------------------------------------------------------------------------\n## Run integration test(s)\n##------------------------------------------------------------------------------\ntest_integration:\n\t$(call color_header, \"Compiling integration test(s) - $(BSP)\")\n\t$(call test_prepare)\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(TEST_CMD) $(TEST_ARG)\n\ntest: test_boot test_unit test_integration\n\nendif\n"
  },
  {
    "path": "19_kernel_heap/README.md",
    "content": "# Tutorial 19 - Kernel Heap\n\n## tl;dr\n\n- A global heap for the kernel is added, which enables runtime dynamic memory allocation (`Box`,\n  `Vec`, etc.).\n- Heap memory management is using a `linked list allocator`.\n- A `debug!` printing macro is added that is only effective when `make` is invoked with\n  `DEBUG_PRINTS=y`.\n\n## Table of Contents\n\n- [Introduction](#introduction)\n- [Implementation](#implementation)\n  - [Debug Prints](#debug-prints)\n  - [Pre-UART Console Output](#pre-uart-console-output)\n- [Test it](#test-it)\n- [Diff to previous](#diff-to-previous)\n\n## Introduction\n\nThe kernel is finally in a good place to add dynamic memory management. The entire kernel runs in\nthe higher half of the address space by now, and it has decent backtracing support, which can be\nleveraged to get rich tracing/debugging support for heap allocations.\n\nAlthough it is a vital part of implementing a heap, this tutorial will **not** cover\n`allocation/deallocation` of heap memory. Instead, we will re-use [@phil-opp]'s excellent\n[`linked_list_allocator`]. The reason is that while dynamic memory allocation algorithms are an\ninteresting topic, there would not be much added benefit in implementing a `linked list allocator`\nof our own, since it would turn out very similar to what Philipp and the other contributors have\nimplemented already. So we might just re-use that, even more so because it can be plugged seamlessly\ninto our kernel. [@phil-opp] has also written two great articles on [Heap Allocation] and [Allocator\nDesigns]. I really recommend to read those now before continuing with this tutorial.\n\n[@phil-opp]: https://github.com/phil-opp\n[`linked_list_allocator`]: https://crates.io/crates/linked_list_allocator\n[Heap Allocation]: https://os.phil-opp.com/heap-allocation/\n[Allocator Designs]: https://os.phil-opp.com/allocator-designs/\n\nThat being said, what this tutorial text will cover is supporting changes for _enabling_ the\nlinked_list_allocator, and changes to kernel code leveraging the heap.\n\n## Implementation\n\nFirst of all, we need to reserve some DRAM for the heap. Traditionally, this is done in the `linker\nscript`. We place it after the `.data` section and before the `MMIO remap` section.\n\n```ld.s\n    __data_end_exclusive = .;\n\n    /***********************************************************************************************\n    * Heap\n    ***********************************************************************************************/\n    __heap_start = .;\n    .heap (NOLOAD) :\n    {\n        . += 16 * 1024 * 1024;\n    } :segment_heap\n    __heap_end_exclusive = .;\n\n    ASSERT((. & PAGE_MASK) == 0, \"Heap is not page aligned\")\n\n    /***********************************************************************************************\n    * MMIO Remap Reserved\n    ***********************************************************************************************/\n    __mmio_remap_start = .;\n```\n\nIn the Rust code, the heap properties can now be queried using the added BSP-function\n`bsp::memory::mmu::virt_heap_region()`. The heap allocator itself is added in\n`src/memory/heap_alloc.rs`. There, we add the `linked_list_allocator`, wrap it into an\n`IRQSafeNullock`, and instantiate it the wrapper in a `static`. This way, global access to the\nallocator becomes concurrency-safe:\n\n```rust\nuse linked_list_allocator::Heap as LinkedListHeap;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// A heap allocator that can be lazyily initialized.\npub struct HeapAllocator {\n    inner: IRQSafeNullLock<LinkedListHeap>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n#[global_allocator]\nstatic KERNEL_HEAP_ALLOCATOR: HeapAllocator = HeapAllocator::new();\n```\n\nAll that is left to do now is to implement the [`GlobalAlloc`] trait for `HeapAllocator`:\n\n[`GlobalAlloc`]: https://doc.rust-lang.org/stable/core/alloc/trait.GlobalAlloc.html\n\n```rust\nunsafe impl GlobalAlloc for HeapAllocator {\n    unsafe fn alloc(&self, layout: Layout) -> *mut u8 {\n        let result = KERNEL_HEAP_ALLOCATOR\n            .inner\n            .lock(|inner| inner.allocate_first_fit(layout).ok());\n\n        match result {\n            None => core::ptr::null_mut(),\n            Some(allocation) => {\n                let ptr = allocation.as_ptr();\n\n                debug_print_alloc_dealloc(\"Allocation\", ptr, layout);\n\n                ptr\n            }\n        }\n    }\n\n    unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {\n        KERNEL_HEAP_ALLOCATOR\n            .inner\n            .lock(|inner| inner.deallocate(core::ptr::NonNull::new_unchecked(ptr), layout));\n\n        debug_print_alloc_dealloc(\"Free\", ptr, layout);\n    }\n}\n```\n\nDuring kernel init, `kernel_init_heap_allocator()` will be called, which basically points the\nwrapped allocator to the heap that we defined earlier:\n\n```rust\n/// Query the BSP for the heap region and initialize the kernel's heap allocator with it.\npub fn kernel_init_heap_allocator() {\n    static INIT_DONE: AtomicBool = AtomicBool::new(false);\n    if INIT_DONE.load(Ordering::Relaxed) {\n        warn!(\"Already initialized\");\n        return;\n    }\n\n    let region = bsp::memory::mmu::virt_heap_region();\n\n    KERNEL_HEAP_ALLOCATOR.inner.lock(|inner| unsafe {\n        inner.init(region.start_addr().as_usize() as *mut u8, region.size())\n    });\n\n    INIT_DONE.store(true, Ordering::Relaxed);\n}\n```\n\nThat's it already! We can now use `Box`, `Vec` and friends 🥳.\n\n### Debug Prints\n\nYou might have noticed the `debug_print_alloc_dealloc()` calls in above's snippet. Under the hood,\nthis function makes use of the `debug!` macro that has been added in this tutorial. This macro will\nonly print to the console when `make` is invoked with the `ENV` variable `DEBUG_PRINTS` set to\n\"**y**\". As you can see in the following snippet, this enables rich debug output for heap\nallocations and deallocations, containing information such as `size`, `start` and `end exclusive`\naddresses, as well as a backtrace that shows from where the (de)allocation originated.\n\n```console\n$ DEBUG_PRINTS=y make qemu\n\n[...]\n\n<D   0.040505> Kernel Heap: Allocation\n      Size:     0x10 (16 Byte)\n      Start:    0xffff_ffff_c00a_0010\n      End excl: 0xffff_ffff_c00a_0020\n\n      Backtrace:\n      ----------------------------------------------------------------------------------------------\n          Address            Function containing address\n      ----------------------------------------------------------------------------------------------\n       1. ffffffffc000cdf8 | <libkernel::bsp::device_driver::bcm::bcm2xxx_pl011_uart::PL011Uart as libkernel::console::interface::Write>::write_fmt\n       2. ffffffffc000b4f8 | <libkernel::memory::heap_alloc::HeapAllocator as core::alloc::global::GlobalAlloc>::alloc\n       3. ffffffffc000d940 | libkernel::memory::mmu::mapping_record::kernel_add\n       4. ffffffffc000adec | libkernel::bsp::raspberrypi::memory::mmu::kernel_add_mapping_records_for_precomputed\n       5. ffffffffc00016ac | kernel_init\n      ----------------------------------------------------------------------------------------------\n\n[    0.042872] mingo version 0.19.0\n[    0.043080] Booting on: Raspberry Pi 3\n```\n\n### Pre-UART Console Output\n\nHaving a heap allows us to simplify a few modules by switching static-length arrays to the dynamic\n`Vec` data structure. Examples are the `interrupt controller drivers` for their handler tables,\n`src/memory/mmu/mapping_record.rs` for bookkeeping virtual memory mappings and the `BSP driver\nmanager` for its instantiated device drivers.\n\nHowever, many of those allocations happen already **before** the UART driver comes online.\nTherefore, a lot of the (de)allocation debug prints would go into the void with the way pre-UART\nprints have been handled so far, which is undesirable. To solve this problem, the kernel's initial\n(aka pre-UART) console is now not a `NullConsole` anymore, but a `BufferConsole`. The latter owns a\nsmall static array of `chars`, that records any console prints before the actual UART driver comes\nonline. Once the UART driver is registered in the kernel to become the default console, the first\nthing that is done is to print any buffered records of the `BufferConsole`:\n\n```rust\npub fn register_console(new_console: &'static (dyn interface::All + Sync)) {\n    CUR_CONSOLE.write(|con| *con = new_console);\n\n    static FIRST_SWITCH: InitStateLock<bool> = InitStateLock::new(true);\n    FIRST_SWITCH.write(|first| {\n        if *first == true {\n            *first = false;\n\n            buffer_console::BUFFER_CONSOLE.dump();\n        }\n    });\n}\n```\n\n`BUFFER_CONSOLE.dump()` just drains its buffer to using the newly registered console.\n\n## Test it\n\nIf compiled without `DEBUG_PRINTS`, the heap can be observed in the mapping overview and through the\nnewly added usage statistics:\n\n```console\n$ make chainboot\n[...]\nMinipush 1.0\n\n[MP] ⏳ Waiting for /dev/ttyUSB0\n[MP] ✅ Serial connected\n[MP] 🔌 Please power the target now\n\n __  __ _      _ _                 _\n|  \\/  (_)_ _ (_) |   ___  __ _ __| |\n| |\\/| | | ' \\| | |__/ _ \\/ _` / _` |\n|_|  |_|_|_||_|_|____\\___/\\__,_\\__,_|\n\n           Raspberry Pi 3\n\n[ML] Requesting binary\n[MP] ⏩ Pushing 320 KiB ======================================🦀 100% 106 KiB/s Time: 00:00:03\n[ML] Loaded! Executing the payload now\n\n[    3.572716] mingo version 0.19.0\n[    3.572924] Booting on: Raspberry Pi 3\n[    3.573379] MMU online:\n[    3.573672]       -------------------------------------------------------------------------------------------------------------------------------------------\n[    3.575416]                         Virtual                                   Physical               Size       Attr                    Entity\n[    3.577160]       -------------------------------------------------------------------------------------------------------------------------------------------\n[    3.578905]       0xffff_ffff_c000_0000..0xffff_ffff_c001_ffff --> 0x00_0008_0000..0x00_0009_ffff | 128 KiB | C   RO X  | Kernel code and RO data\n[    3.580519]       0xffff_ffff_c002_0000..0xffff_ffff_c009_ffff --> 0x00_000a_0000..0x00_0011_ffff | 512 KiB | C   RW XN | Kernel data and bss\n[    3.582089]       0xffff_ffff_c00a_0000..0xffff_ffff_c109_ffff --> 0x00_0012_0000..0x00_0111_ffff |  16 MiB | C   RW XN | Kernel heap\n[    3.583573]       0xffff_ffff_c10a_0000..0xffff_ffff_c10a_ffff --> 0x00_3f20_0000..0x00_3f20_ffff |  64 KiB | Dev RW XN | BCM PL011 UART\n[    3.585090]                                                                                                             | BCM GPIO\n[    3.586542]       0xffff_ffff_c10b_0000..0xffff_ffff_c10b_ffff --> 0x00_3f00_0000..0x00_3f00_ffff |  64 KiB | Dev RW XN | BCM Interrupt Controller\n[    3.588167]       0xffff_ffff_c18b_0000..0xffff_ffff_c192_ffff --> 0x00_0000_0000..0x00_0007_ffff | 512 KiB | C   RW XN | Kernel boot-core stack\n[    3.589770]       -------------------------------------------------------------------------------------------------------------------------------------------\n[    3.591515] Current privilege level: EL1\n\n[...]\n\n[    3.597624] Kernel heap:\n[    3.597928]       Used: 2512 Byte (3 KiB)\n[    3.598415]       Free: 16774704 Byte (16 MiB)\n[    3.598957] Echoing input now\n```\n\n## Diff to previous\n```diff\n\ndiff -uNr 18_backtrace/kernel/Cargo.toml 19_kernel_heap/kernel/Cargo.toml\n--- 18_backtrace/kernel/Cargo.toml\n+++ 19_kernel_heap/kernel/Cargo.toml\n@@ -1,11 +1,12 @@\n [package]\n name = \"mingo\"\n-version = \"0.18.0\"\n+version = \"0.19.0\"\n authors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\n edition = \"2021\"\n\n [features]\n default = []\n+debug_prints = []\n bsp_rpi3 = [\"tock-registers\"]\n bsp_rpi4 = [\"tock-registers\"]\n test_build = [\"qemu-exit\"]\n@@ -17,6 +18,7 @@\n [dependencies]\n test-types = { path = \"../libraries/test-types\" }\n debug-symbol-types = { path = \"../libraries/debug-symbol-types\" }\n+linked_list_allocator = { version = \"0.10.x\", default-features = false, features = [\"const_mut_refs\"] }\n\n # Optional dependencies\n tock-registers = { version = \"0.8.x\", default-features = false, features = [\"register_types\"], optional = true }\n\ndiff -uNr 18_backtrace/kernel/src/bsp/device_driver/arm/gicv2.rs 19_kernel_heap/kernel/src/bsp/device_driver/arm/gicv2.rs\n--- 18_backtrace/kernel/src/bsp/device_driver/arm/gicv2.rs\n+++ 19_kernel_heap/kernel/src/bsp/device_driver/arm/gicv2.rs\n@@ -86,13 +86,13 @@\n     synchronization,\n     synchronization::InitStateLock,\n };\n+use alloc::vec::Vec;\n\n //--------------------------------------------------------------------------------------------------\n // Private Definitions\n //--------------------------------------------------------------------------------------------------\n\n-type HandlerTable = [Option<exception::asynchronous::IRQHandlerDescriptor<IRQNumber>>;\n-    IRQNumber::MAX_INCLUSIVE + 1];\n+type HandlerTable = Vec<Option<exception::asynchronous::IRQHandlerDescriptor<IRQNumber>>>;\n\n //--------------------------------------------------------------------------------------------------\n // Public Definitions\n@@ -118,7 +118,7 @@\n //--------------------------------------------------------------------------------------------------\n\n impl GICv2 {\n-    const MAX_IRQ_NUMBER: usize = 300; // Normally 1019, but keep it lower to save some space.\n+    const MAX_IRQ_NUMBER: usize = 1019;\n\n     pub const COMPATIBLE: &'static str = \"GICv2 (ARM Generic Interrupt Controller v2)\";\n\n@@ -134,7 +134,7 @@\n         Self {\n             gicd: gicd::GICD::new(gicd_mmio_start_addr),\n             gicc: gicc::GICC::new(gicc_mmio_start_addr),\n-            handler_table: InitStateLock::new([None; IRQNumber::MAX_INCLUSIVE + 1]),\n+            handler_table: InitStateLock::new(Vec::new()),\n         }\n     }\n }\n@@ -152,6 +152,9 @@\n     }\n\n     unsafe fn init(&self) -> Result<(), &'static str> {\n+        self.handler_table\n+            .write(|table| table.resize(IRQNumber::MAX_INCLUSIVE + 1, None));\n+\n         if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() {\n             self.gicd.boot_core_init();\n         }\n\ndiff -uNr 18_backtrace/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs 19_kernel_heap/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs\n--- 18_backtrace/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs\n+++ 19_kernel_heap/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs\n@@ -16,6 +16,7 @@\n     synchronization,\n     synchronization::{IRQSafeNullLock, InitStateLock},\n };\n+use alloc::vec::Vec;\n use tock_registers::{\n     interfaces::{Readable, Writeable},\n     register_structs,\n@@ -52,8 +53,7 @@\n /// Abstraction for the ReadOnly parts of the associated MMIO registers.\n type ReadOnlyRegisters = MMIODerefWrapper<RORegisterBlock>;\n\n-type HandlerTable = [Option<exception::asynchronous::IRQHandlerDescriptor<PeripheralIRQ>>;\n-    PeripheralIRQ::MAX_INCLUSIVE + 1];\n+type HandlerTable = Vec<Option<exception::asynchronous::IRQHandlerDescriptor<PeripheralIRQ>>>;\n\n //--------------------------------------------------------------------------------------------------\n // Public Definitions\n@@ -85,10 +85,16 @@\n         Self {\n             wo_registers: IRQSafeNullLock::new(WriteOnlyRegisters::new(mmio_start_addr)),\n             ro_registers: ReadOnlyRegisters::new(mmio_start_addr),\n-            handler_table: InitStateLock::new([None; PeripheralIRQ::MAX_INCLUSIVE + 1]),\n+            handler_table: InitStateLock::new(Vec::new()),\n         }\n     }\n\n+    /// Called by the kernel to bring up the device.\n+    pub fn init(&self) {\n+        self.handler_table\n+            .write(|table| table.resize(PeripheralIRQ::MAX_INCLUSIVE + 1, None));\n+    }\n+\n     /// Query the list of pending IRQs.\n     fn pending_irqs(&self) -> PendingIRQs {\n         let pending_mask: u64 = (u64::from(self.ro_registers.PENDING_2.get()) << 32)\n\ndiff -uNr 18_backtrace/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs 19_kernel_heap/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs\n--- 18_backtrace/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs\n+++ 19_kernel_heap/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs\n@@ -109,6 +109,12 @@\n     fn compatible(&self) -> &'static str {\n         Self::COMPATIBLE\n     }\n+\n+    unsafe fn init(&self) -> Result<(), &'static str> {\n+        self.periph.init();\n+\n+        Ok(())\n+    }\n }\n\n impl exception::asynchronous::interface::IRQManager for InterruptController {\n\ndiff -uNr 18_backtrace/kernel/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs 19_kernel_heap/kernel/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs\n--- 18_backtrace/kernel/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs\n+++ 19_kernel_heap/kernel/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs\n@@ -327,6 +327,13 @@\n         self.chars_written += 1;\n     }\n\n+    /// Send a slice of characters.\n+    fn write_array(&mut self, a: &[char]) {\n+        for c in a {\n+            self.write_char(*c);\n+        }\n+    }\n+\n     /// Block execution until the last buffered character has been physically put on the TX wire.\n     fn flush(&self) {\n         // Spin until the busy bit is cleared.\n@@ -443,6 +450,10 @@\n         self.inner.lock(|inner| inner.write_char(c));\n     }\n\n+    fn write_array(&self, a: &[char]) {\n+        self.inner.lock(|inner| inner.write_array(a));\n+    }\n+\n     fn write_fmt(&self, args: core::fmt::Arguments) -> fmt::Result {\n         // Fully qualified syntax for the call to `core::fmt::Write::write_fmt()` to increase\n         // readability.\n\ndiff -uNr 18_backtrace/kernel/src/bsp/raspberrypi/kernel.ld 19_kernel_heap/kernel/src/bsp/raspberrypi/kernel.ld\n--- 18_backtrace/kernel/src/bsp/raspberrypi/kernel.ld\n+++ 19_kernel_heap/kernel/src/bsp/raspberrypi/kernel.ld\n@@ -35,6 +35,7 @@\n {\n     segment_code            PT_LOAD FLAGS(5);\n     segment_data            PT_LOAD FLAGS(6);\n+    segment_heap            PT_LOAD FLAGS(6);\n     segment_boot_core_stack PT_LOAD FLAGS(6);\n }\n\n@@ -84,6 +85,18 @@\n     __data_end_exclusive = .;\n\n     /***********************************************************************************************\n+    * Heap\n+    ***********************************************************************************************/\n+    __heap_start = .;\n+    .heap (NOLOAD) :\n+    {\n+        . += 16 * 1024 * 1024;\n+    } :segment_heap\n+    __heap_end_exclusive = .;\n+\n+    ASSERT((. & PAGE_MASK) == 0, \"Heap is not page aligned\")\n+\n+    /***********************************************************************************************\n     * MMIO Remap Reserved\n     ***********************************************************************************************/\n     __mmio_remap_start = .;\n\ndiff -uNr 18_backtrace/kernel/src/bsp/raspberrypi/memory/mmu.rs 19_kernel_heap/kernel/src/bsp/raspberrypi/memory/mmu.rs\n--- 18_backtrace/kernel/src/bsp/raspberrypi/memory/mmu.rs\n+++ 19_kernel_heap/kernel/src/bsp/raspberrypi/memory/mmu.rs\n@@ -122,6 +122,16 @@\n     MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n }\n\n+/// The heap pages.\n+pub fn virt_heap_region() -> MemoryRegion<Virtual> {\n+    let num_pages = size_to_num_pages(super::heap_size());\n+\n+    let start_page_addr = super::virt_heap_start();\n+    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n+\n+    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n+}\n+\n /// The boot core stack pages.\n pub fn virt_boot_core_stack_region() -> MemoryRegion<Virtual> {\n     let num_pages = size_to_num_pages(super::boot_core_stack_size());\n@@ -169,6 +179,14 @@\n         &kernel_page_attributes(virt_data_region.start_page_addr()),\n     );\n\n+    let virt_heap_region = virt_heap_region();\n+    generic_mmu::kernel_add_mapping_record(\n+        \"Kernel heap\",\n+        &virt_heap_region,\n+        &kernel_virt_to_phys_region(virt_heap_region),\n+        &kernel_page_attributes(virt_heap_region.start_page_addr()),\n+    );\n+\n     let virt_boot_core_stack_region = virt_boot_core_stack_region();\n     generic_mmu::kernel_add_mapping_record(\n         \"Kernel boot-core stack\",\n\ndiff -uNr 18_backtrace/kernel/src/bsp/raspberrypi/memory.rs 19_kernel_heap/kernel/src/bsp/raspberrypi/memory.rs\n--- 18_backtrace/kernel/src/bsp/raspberrypi/memory.rs\n+++ 19_kernel_heap/kernel/src/bsp/raspberrypi/memory.rs\n@@ -28,7 +28,11 @@\n //! | .bss                                  |\n //! |                                       |\n //! +---------------------------------------+\n-//! |                                       | data_end_exclusive\n+//! |                                       | heap_start == data_end_exclusive\n+//! | .heap                                 |\n+//! |                                       |\n+//! +---------------------------------------+\n+//! |                                       | heap_end_exclusive\n //! |                                       |\n //!\n //!\n@@ -50,7 +54,11 @@\n //! | .bss                                  |\n //! |                                       |\n //! +---------------------------------------+\n-//! |                                       |  mmio_remap_start == data_end_exclusive\n+//! |                                       | heap_start == data_end_exclusive\n+//! | .heap                                 |\n+//! |                                       |\n+//! +---------------------------------------+\n+//! |                                       |  mmio_remap_start == heap_end_exclusive\n //! | VA region for MMIO remapping          |\n //! |                                       |\n //! +---------------------------------------+\n@@ -83,6 +91,9 @@\n     static __data_start: UnsafeCell<()>;\n     static __data_end_exclusive: UnsafeCell<()>;\n\n+    static __heap_start: UnsafeCell<()>;\n+    static __heap_end_exclusive: UnsafeCell<()>;\n+\n     static __mmio_remap_start: UnsafeCell<()>;\n     static __mmio_remap_end_exclusive: UnsafeCell<()>;\n\n@@ -179,6 +190,22 @@\n     unsafe { (__data_end_exclusive.get() as usize) - (__data_start.get() as usize) }\n }\n\n+/// Start page address of the heap segment.\n+#[inline(always)]\n+fn virt_heap_start() -> PageAddress<Virtual> {\n+    PageAddress::from(unsafe { __heap_start.get() as usize })\n+}\n+\n+/// Size of the heap segment.\n+///\n+/// # Safety\n+///\n+/// - Value is provided by the linker script and must be trusted as-is.\n+#[inline(always)]\n+fn heap_size() -> usize {\n+    unsafe { (__heap_end_exclusive.get() as usize) - (__heap_start.get() as usize) }\n+}\n+\n /// Start page address of the MMIO remap reservation.\n ///\n /// # Safety\n\ndiff -uNr 18_backtrace/kernel/src/console/buffer_console.rs 19_kernel_heap/kernel/src/console/buffer_console.rs\n--- 18_backtrace/kernel/src/console/buffer_console.rs\n+++ 19_kernel_heap/kernel/src/console/buffer_console.rs\n@@ -0,0 +1,108 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! A console that buffers input during the init phase.\n+\n+use super::interface;\n+use crate::{console, info, synchronization, synchronization::InitStateLock};\n+use core::fmt;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+const BUF_SIZE: usize = 1024 * 64;\n+\n+pub struct BufferConsoleInner {\n+    buf: [char; BUF_SIZE],\n+    write_ptr: usize,\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+pub struct BufferConsole {\n+    inner: InitStateLock<BufferConsoleInner>,\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Global instances\n+//--------------------------------------------------------------------------------------------------\n+\n+pub static BUFFER_CONSOLE: BufferConsole = BufferConsole {\n+    inner: InitStateLock::new(BufferConsoleInner {\n+        // Use the null character, so this lands in .bss and does not waste space in the binary.\n+        buf: ['\\0'; BUF_SIZE],\n+        write_ptr: 0,\n+    }),\n+};\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Code\n+//--------------------------------------------------------------------------------------------------\n+\n+impl BufferConsoleInner {\n+    fn write_char(&mut self, c: char) {\n+        if self.write_ptr < (BUF_SIZE - 1) {\n+            self.buf[self.write_ptr] = c;\n+            self.write_ptr += 1;\n+        }\n+    }\n+}\n+\n+impl fmt::Write for BufferConsoleInner {\n+    fn write_str(&mut self, s: &str) -> fmt::Result {\n+        for c in s.chars() {\n+            self.write_char(c);\n+        }\n+\n+        Ok(())\n+    }\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+use synchronization::interface::ReadWriteEx;\n+\n+impl BufferConsole {\n+    /// Dump the buffer.\n+    ///\n+    /// # Invariant\n+    ///\n+    /// It is expected that this is only called when self != crate::console::console().\n+    pub fn dump(&self) {\n+        self.inner.read(|inner| {\n+            console::console().write_array(&inner.buf[0..inner.write_ptr]);\n+\n+            if inner.write_ptr == (BUF_SIZE - 1) {\n+                info!(\"Pre-UART buffer overflowed\");\n+            } else if inner.write_ptr > 0 {\n+                info!(\"End of pre-UART buffer\")\n+            }\n+        });\n+    }\n+}\n+\n+impl interface::Write for BufferConsole {\n+    fn write_char(&self, c: char) {\n+        self.inner.write(|inner| inner.write_char(c));\n+    }\n+\n+    fn write_array(&self, _a: &[char]) {}\n+\n+    fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result {\n+        self.inner.write(|inner| fmt::Write::write_fmt(inner, args))\n+    }\n+\n+    fn flush(&self) {}\n+}\n+\n+impl interface::Read for BufferConsole {\n+    fn clear_rx(&self) {}\n+}\n+\n+impl interface::Statistics for BufferConsole {}\n+impl interface::All for BufferConsole {}\n\ndiff -uNr 18_backtrace/kernel/src/console/null_console.rs 19_kernel_heap/kernel/src/console/null_console.rs\n--- 18_backtrace/kernel/src/console/null_console.rs\n+++ 19_kernel_heap/kernel/src/console/null_console.rs\n@@ -1,41 +0,0 @@\n-// SPDX-License-Identifier: MIT OR Apache-2.0\n-//\n-// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n-\n-//! Null console.\n-\n-use super::interface;\n-use core::fmt;\n-\n-//--------------------------------------------------------------------------------------------------\n-// Public Definitions\n-//--------------------------------------------------------------------------------------------------\n-\n-pub struct NullConsole;\n-\n-//--------------------------------------------------------------------------------------------------\n-// Global instances\n-//--------------------------------------------------------------------------------------------------\n-\n-pub static NULL_CONSOLE: NullConsole = NullConsole {};\n-\n-//--------------------------------------------------------------------------------------------------\n-// Public Code\n-//--------------------------------------------------------------------------------------------------\n-\n-impl interface::Write for NullConsole {\n-    fn write_char(&self, _c: char) {}\n-\n-    fn write_fmt(&self, _args: fmt::Arguments) -> fmt::Result {\n-        fmt::Result::Ok(())\n-    }\n-\n-    fn flush(&self) {}\n-}\n-\n-impl interface::Read for NullConsole {\n-    fn clear_rx(&self) {}\n-}\n-\n-impl interface::Statistics for NullConsole {}\n-impl interface::All for NullConsole {}\n\ndiff -uNr 18_backtrace/kernel/src/console.rs 19_kernel_heap/kernel/src/console.rs\n--- 18_backtrace/kernel/src/console.rs\n+++ 19_kernel_heap/kernel/src/console.rs\n@@ -4,7 +4,7 @@\n\n //! System console.\n\n-mod null_console;\n+mod buffer_console;\n\n use crate::synchronization;\n\n@@ -21,6 +21,9 @@\n         /// Write a single character.\n         fn write_char(&self, c: char);\n\n+        /// Write a slice of characters.\n+        fn write_array(&self, a: &[char]);\n+\n         /// Write a Rust format string.\n         fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result;\n\n@@ -61,7 +64,7 @@\n //--------------------------------------------------------------------------------------------------\n\n static CUR_CONSOLE: InitStateLock<&'static (dyn interface::All + Sync)> =\n-    InitStateLock::new(&null_console::NULL_CONSOLE);\n+    InitStateLock::new(&buffer_console::BUFFER_CONSOLE);\n\n //--------------------------------------------------------------------------------------------------\n // Public Code\n@@ -71,6 +74,15 @@\n /// Register a new console.\n pub fn register_console(new_console: &'static (dyn interface::All + Sync)) {\n     CUR_CONSOLE.write(|con| *con = new_console);\n+\n+    static FIRST_SWITCH: InitStateLock<bool> = InitStateLock::new(true);\n+    FIRST_SWITCH.write(|first| {\n+        if *first {\n+            *first = false;\n+\n+            buffer_console::BUFFER_CONSOLE.dump();\n+        }\n+    });\n }\n\n /// Return a reference to the currently registered console.\n\ndiff -uNr 18_backtrace/kernel/src/driver.rs 19_kernel_heap/kernel/src/driver.rs\n--- 18_backtrace/kernel/src/driver.rs\n+++ 19_kernel_heap/kernel/src/driver.rs\n@@ -8,23 +8,10 @@\n     exception, info,\n     synchronization::{interface::ReadWriteEx, InitStateLock},\n };\n+use alloc::vec::Vec;\n use core::fmt;\n\n //--------------------------------------------------------------------------------------------------\n-// Private Definitions\n-//--------------------------------------------------------------------------------------------------\n-\n-const NUM_DRIVERS: usize = 5;\n-\n-struct DriverManagerInner<T>\n-where\n-    T: 'static,\n-{\n-    next_index: usize,\n-    descriptors: [Option<DeviceDriverDescriptor<T>>; NUM_DRIVERS],\n-}\n-\n-//--------------------------------------------------------------------------------------------------\n // Public Definitions\n //--------------------------------------------------------------------------------------------------\n\n@@ -68,7 +55,6 @@\n pub type DeviceDriverPostInitCallback = unsafe fn() -> Result<(), &'static str>;\n\n /// A descriptor for device drivers.\n-#[derive(Copy, Clone)]\n pub struct DeviceDriverDescriptor<T>\n where\n     T: 'static,\n@@ -83,7 +69,7 @@\n where\n     T: 'static,\n {\n-    inner: InitStateLock<DriverManagerInner<T>>,\n+    descriptors: InitStateLock<Vec<DeviceDriverDescriptor<T>>>,\n }\n\n //--------------------------------------------------------------------------------------------------\n@@ -93,23 +79,6 @@\n static DRIVER_MANAGER: DriverManager<exception::asynchronous::IRQNumber> = DriverManager::new();\n\n //--------------------------------------------------------------------------------------------------\n-// Private Code\n-//--------------------------------------------------------------------------------------------------\n-\n-impl<T> DriverManagerInner<T>\n-where\n-    T: 'static + Copy,\n-{\n-    /// Create an instance.\n-    pub const fn new() -> Self {\n-        Self {\n-            next_index: 0,\n-            descriptors: [None; NUM_DRIVERS],\n-        }\n-    }\n-}\n-\n-//--------------------------------------------------------------------------------------------------\n // Public Code\n //--------------------------------------------------------------------------------------------------\n\n@@ -135,32 +104,19 @@\n\n impl<T> DriverManager<T>\n where\n-    T: fmt::Display + Copy,\n+    T: fmt::Display,\n {\n     /// Create an instance.\n     pub const fn new() -> Self {\n         Self {\n-            inner: InitStateLock::new(DriverManagerInner::new()),\n+            descriptors: InitStateLock::new(Vec::new()),\n         }\n     }\n\n     /// Register a device driver with the kernel.\n     pub fn register_driver(&self, descriptor: DeviceDriverDescriptor<T>) {\n-        self.inner.write(|inner| {\n-            inner.descriptors[inner.next_index] = Some(descriptor);\n-            inner.next_index += 1;\n-        })\n-    }\n-\n-    /// Helper for iterating over registered drivers.\n-    fn for_each_descriptor<'a>(&'a self, f: impl FnMut(&'a DeviceDriverDescriptor<T>)) {\n-        self.inner.read(|inner| {\n-            inner\n-                .descriptors\n-                .iter()\n-                .filter_map(|x| x.as_ref())\n-                .for_each(f)\n-        })\n+        self.descriptors\n+            .write(|descriptors| descriptors.push(descriptor));\n     }\n\n     /// Fully initialize all drivers and their interrupts handlers.\n@@ -169,53 +125,54 @@\n     ///\n     /// - During init, drivers might do stuff with system-wide impact.\n     pub unsafe fn init_drivers_and_irqs(&self) {\n-        self.for_each_descriptor(|descriptor| {\n-            // 1. Initialize driver.\n-            if let Err(x) = descriptor.device_driver.init() {\n-                panic!(\n-                    \"Error initializing driver: {}: {}\",\n-                    descriptor.device_driver.compatible(),\n-                    x\n-                );\n-            }\n-\n-            // 2. Call corresponding post init callback.\n-            if let Some(callback) = &descriptor.post_init_callback {\n-                if let Err(x) = callback() {\n+        self.descriptors.read(|descriptors| {\n+            for descriptor in descriptors {\n+                // 1. Initialize driver.\n+                if let Err(x) = descriptor.device_driver.init() {\n                     panic!(\n-                        \"Error during driver post-init callback: {}: {}\",\n+                        \"Error initializing driver: {}: {}\",\n                         descriptor.device_driver.compatible(),\n                         x\n                     );\n                 }\n+\n+                // 2. Call corresponding post init callback.\n+                if let Some(callback) = &descriptor.post_init_callback {\n+                    if let Err(x) = callback() {\n+                        panic!(\n+                            \"Error during driver post-init callback: {}: {}\",\n+                            descriptor.device_driver.compatible(),\n+                            x\n+                        );\n+                    }\n+                }\n             }\n-        });\n\n-        // 3. After all post-init callbacks were done, the interrupt controller should be\n-        //    registered and functional. So let drivers register with it now.\n-        self.for_each_descriptor(|descriptor| {\n-            if let Some(irq_number) = &descriptor.irq_number {\n-                if let Err(x) = descriptor\n-                    .device_driver\n-                    .register_and_enable_irq_handler(irq_number)\n-                {\n-                    panic!(\n-                        \"Error during driver interrupt handler registration: {}: {}\",\n-                        descriptor.device_driver.compatible(),\n-                        x\n-                    );\n+            // 3. After all post-init callbacks were done, the interrupt controller should be\n+            //    registered and functional. So let drivers register with it now.\n+            for descriptor in descriptors {\n+                if let Some(irq_number) = &descriptor.irq_number {\n+                    if let Err(x) = descriptor\n+                        .device_driver\n+                        .register_and_enable_irq_handler(irq_number)\n+                    {\n+                        panic!(\n+                            \"Error during driver interrupt handler registration: {}: {}\",\n+                            descriptor.device_driver.compatible(),\n+                            x\n+                        );\n+                    }\n                 }\n             }\n-        });\n+        })\n     }\n\n     /// Enumerate all registered device drivers.\n     pub fn enumerate(&self) {\n-        let mut i: usize = 1;\n-        self.for_each_descriptor(|descriptor| {\n-            info!(\"      {}. {}\", i, descriptor.device_driver.compatible());\n-\n-            i += 1;\n+        self.descriptors.read(|descriptors| {\n+            for (i, desc) in descriptors.iter().enumerate() {\n+                info!(\"      {}. {}\", i + 1, desc.device_driver.compatible());\n+            }\n         });\n     }\n }\n\ndiff -uNr 18_backtrace/kernel/src/lib.rs 19_kernel_heap/kernel/src/lib.rs\n--- 18_backtrace/kernel/src/lib.rs\n+++ 19_kernel_heap/kernel/src/lib.rs\n@@ -110,6 +110,7 @@\n\n #![allow(clippy::upper_case_acronyms)]\n #![allow(incomplete_features)]\n+#![feature(alloc_error_handler)]\n #![feature(asm_const)]\n #![feature(const_option)]\n #![feature(core_intrinsics)]\n@@ -130,6 +131,8 @@\n #![reexport_test_harness_main = \"test_main\"]\n #![test_runner(crate::test_runner)]\n\n+extern crate alloc;\n+\n mod panic_wait;\n mod synchronization;\n\n\ndiff -uNr 18_backtrace/kernel/src/main.rs 19_kernel_heap/kernel/src/main.rs\n--- 18_backtrace/kernel/src/main.rs\n+++ 19_kernel_heap/kernel/src/main.rs\n@@ -13,6 +13,8 @@\n #![no_main]\n #![no_std]\n\n+extern crate alloc;\n+\n use libkernel::{bsp, cpu, driver, exception, info, memory, state, time};\n\n /// Early init code.\n@@ -73,6 +75,9 @@\n     info!(\"Registered IRQ handlers:\");\n     exception::asynchronous::irq_manager().print_handler();\n\n+    info!(\"Kernel heap:\");\n+    memory::heap_alloc::kernel_heap_allocator().print_usage();\n+\n     info!(\"Echoing input now\");\n     cpu::wait_forever();\n }\n\ndiff -uNr 18_backtrace/kernel/src/memory/heap_alloc.rs 19_kernel_heap/kernel/src/memory/heap_alloc.rs\n--- 18_backtrace/kernel/src/memory/heap_alloc.rs\n+++ 19_kernel_heap/kernel/src/memory/heap_alloc.rs\n@@ -0,0 +1,147 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Heap allocation.\n+\n+use crate::{\n+    backtrace, bsp, common, debug, info,\n+    memory::{Address, Virtual},\n+    synchronization,\n+    synchronization::IRQSafeNullLock,\n+    warn,\n+};\n+use alloc::alloc::{GlobalAlloc, Layout};\n+use core::sync::atomic::{AtomicBool, Ordering};\n+use linked_list_allocator::Heap as LinkedListHeap;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// A heap allocator that can be lazyily initialized.\n+pub struct HeapAllocator {\n+    inner: IRQSafeNullLock<LinkedListHeap>,\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Global instances\n+//--------------------------------------------------------------------------------------------------\n+\n+#[global_allocator]\n+static KERNEL_HEAP_ALLOCATOR: HeapAllocator = HeapAllocator::new();\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Code\n+//--------------------------------------------------------------------------------------------------\n+\n+#[inline(always)]\n+fn debug_print_alloc_dealloc(operation: &'static str, ptr: *mut u8, layout: Layout) {\n+    let size = layout.size();\n+    let (size_h, size_unit) = common::size_human_readable_ceil(size);\n+    let addr = Address::<Virtual>::new(ptr as usize);\n+\n+    debug!(\n+        \"Kernel Heap: {}\\n      \\\n+        Size:     {:#x} ({} {})\\n      \\\n+        Start:    {}\\n      \\\n+        End excl: {}\\n\\n      \\\n+        {}\",\n+        operation,\n+        size,\n+        size_h,\n+        size_unit,\n+        addr,\n+        addr + size,\n+        backtrace::Backtrace\n+    );\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+use synchronization::interface::Mutex;\n+\n+#[alloc_error_handler]\n+fn alloc_error_handler(layout: Layout) -> ! {\n+    panic!(\"Allocation error: {:?}\", layout)\n+}\n+\n+/// Return a reference to the kernel's heap allocator.\n+pub fn kernel_heap_allocator() -> &'static HeapAllocator {\n+    &KERNEL_HEAP_ALLOCATOR\n+}\n+\n+impl HeapAllocator {\n+    /// Create an instance.\n+    pub const fn new() -> Self {\n+        Self {\n+            inner: IRQSafeNullLock::new(LinkedListHeap::empty()),\n+        }\n+    }\n+\n+    /// Print the current heap usage.\n+    pub fn print_usage(&self) {\n+        let (used, free) = KERNEL_HEAP_ALLOCATOR\n+            .inner\n+            .lock(|inner| (inner.used(), inner.free()));\n+\n+        if used >= 1024 {\n+            let (used_h, used_unit) = common::size_human_readable_ceil(used);\n+            info!(\"      Used: {} Byte ({} {})\", used, used_h, used_unit);\n+        } else {\n+            info!(\"      Used: {} Byte\", used);\n+        }\n+\n+        if free >= 1024 {\n+            let (free_h, free_unit) = common::size_human_readable_ceil(free);\n+            info!(\"      Free: {} Byte ({} {})\", free, free_h, free_unit);\n+        } else {\n+            info!(\"      Free: {} Byte\", free);\n+        }\n+    }\n+}\n+\n+unsafe impl GlobalAlloc for HeapAllocator {\n+    unsafe fn alloc(&self, layout: Layout) -> *mut u8 {\n+        let result = KERNEL_HEAP_ALLOCATOR\n+            .inner\n+            .lock(|inner| inner.allocate_first_fit(layout).ok());\n+\n+        match result {\n+            None => core::ptr::null_mut(),\n+            Some(allocation) => {\n+                let ptr = allocation.as_ptr();\n+\n+                debug_print_alloc_dealloc(\"Allocation\", ptr, layout);\n+\n+                ptr\n+            }\n+        }\n+    }\n+\n+    unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {\n+        KERNEL_HEAP_ALLOCATOR\n+            .inner\n+            .lock(|inner| inner.deallocate(core::ptr::NonNull::new_unchecked(ptr), layout));\n+\n+        debug_print_alloc_dealloc(\"Free\", ptr, layout);\n+    }\n+}\n+\n+/// Query the BSP for the heap region and initialize the kernel's heap allocator with it.\n+pub fn kernel_init_heap_allocator() {\n+    static INIT_DONE: AtomicBool = AtomicBool::new(false);\n+    if INIT_DONE.load(Ordering::Relaxed) {\n+        warn!(\"Already initialized\");\n+        return;\n+    }\n+\n+    let region = bsp::memory::mmu::virt_heap_region();\n+\n+    KERNEL_HEAP_ALLOCATOR.inner.lock(|inner| unsafe {\n+        inner.init(region.start_addr().as_usize() as *mut u8, region.size())\n+    });\n+\n+    INIT_DONE.store(true, Ordering::Relaxed);\n+}\n\ndiff -uNr 18_backtrace/kernel/src/memory/mmu/mapping_record.rs 19_kernel_heap/kernel/src/memory/mmu/mapping_record.rs\n--- 18_backtrace/kernel/src/memory/mmu/mapping_record.rs\n+++ 19_kernel_heap/kernel/src/memory/mmu/mapping_record.rs\n@@ -8,7 +8,8 @@\n     AccessPermissions, Address, AttributeFields, MMIODescriptor, MemAttributes, MemoryRegion,\n     Physical, Virtual,\n };\n-use crate::{bsp, common, info, synchronization, synchronization::InitStateLock, warn};\n+use crate::{bsp, common, info, synchronization, synchronization::InitStateLock};\n+use alloc::{vec, vec::Vec};\n\n //--------------------------------------------------------------------------------------------------\n // Private Definitions\n@@ -16,9 +17,8 @@\n\n /// Type describing a virtual memory mapping.\n #[allow(missing_docs)]\n-#[derive(Copy, Clone)]\n struct MappingRecordEntry {\n-    pub users: [Option<&'static str>; 5],\n+    pub users: Vec<&'static str>,\n     pub phys_start_addr: Address<Physical>,\n     pub virt_start_addr: Address<Virtual>,\n     pub num_pages: usize,\n@@ -26,7 +26,7 @@\n }\n\n struct MappingRecord {\n-    inner: [Option<MappingRecordEntry>; 12],\n+    inner: Vec<MappingRecordEntry>,\n }\n\n //--------------------------------------------------------------------------------------------------\n@@ -48,7 +48,7 @@\n         attr: &AttributeFields,\n     ) -> Self {\n         Self {\n-            users: [Some(name), None, None, None, None],\n+            users: vec![name],\n             phys_start_addr: phys_region.start_addr(),\n             virt_start_addr: virt_region.start_addr(),\n             num_pages: phys_region.num_pages(),\n@@ -56,54 +56,28 @@\n         }\n     }\n\n-    fn find_next_free_user(&mut self) -> Result<&mut Option<&'static str>, &'static str> {\n-        if let Some(x) = self.users.iter_mut().find(|x| x.is_none()) {\n-            return Ok(x);\n-        };\n-\n-        Err(\"Storage for user info exhausted\")\n-    }\n-\n-    pub fn add_user(&mut self, user: &'static str) -> Result<(), &'static str> {\n-        let x = self.find_next_free_user()?;\n-        *x = Some(user);\n-        Ok(())\n+    pub fn add_user(&mut self, user: &'static str) {\n+        self.users.push(user);\n     }\n }\n\n impl MappingRecord {\n     pub const fn new() -> Self {\n-        Self { inner: [None; 12] }\n-    }\n-\n-    fn size(&self) -> usize {\n-        self.inner.iter().filter(|x| x.is_some()).count()\n+        Self { inner: Vec::new() }\n     }\n\n     fn sort(&mut self) {\n-        let upper_bound_exclusive = self.size();\n-        let entries = &mut self.inner[0..upper_bound_exclusive];\n-\n-        if !entries.is_sorted_by_key(|item| item.unwrap().virt_start_addr) {\n-            entries.sort_unstable_by_key(|item| item.unwrap().virt_start_addr)\n+        if !self.inner.is_sorted_by_key(|item| item.virt_start_addr) {\n+            self.inner.sort_unstable_by_key(|item| item.virt_start_addr)\n         }\n     }\n\n-    fn find_next_free(&mut self) -> Result<&mut Option<MappingRecordEntry>, &'static str> {\n-        if let Some(x) = self.inner.iter_mut().find(|x| x.is_none()) {\n-            return Ok(x);\n-        }\n-\n-        Err(\"Storage for mapping info exhausted\")\n-    }\n-\n     fn find_duplicate(\n         &mut self,\n         phys_region: &MemoryRegion<Physical>,\n     ) -> Option<&mut MappingRecordEntry> {\n         self.inner\n             .iter_mut()\n-            .filter_map(|x| x.as_mut())\n             .filter(|x| x.attribute_fields.mem_attributes == MemAttributes::Device)\n             .find(|x| {\n                 if x.phys_start_addr != phys_region.start_addr() {\n@@ -124,10 +98,8 @@\n         virt_region: &MemoryRegion<Virtual>,\n         phys_region: &MemoryRegion<Physical>,\n         attr: &AttributeFields,\n-    ) -> Result<(), &'static str> {\n-        let x = self.find_next_free()?;\n-\n-        *x = Some(MappingRecordEntry::new(\n+    ) {\n+        self.inner.push(MappingRecordEntry::new(\n             name,\n             virt_region,\n             phys_region,\n@@ -135,8 +107,6 @@\n         ));\n\n         self.sort();\n-\n-        Ok(())\n     }\n\n     pub fn print(&self) {\n@@ -147,7 +117,7 @@\n         );\n         info!(\"      -------------------------------------------------------------------------------------------------------------------------------------------\");\n\n-        for i in self.inner.iter().flatten() {\n+        for i in self.inner.iter() {\n             let size = i.num_pages * bsp::memory::mmu::KernelGranule::SIZE;\n             let virt_start = i.virt_start_addr;\n             let virt_end_inclusive = virt_start + (size - 1);\n@@ -183,16 +153,14 @@\n                 attr,\n                 acc_p,\n                 xn,\n-                i.users[0].unwrap()\n+                i.users[0]\n             );\n\n-            for k in i.users[1..].iter() {\n-                if let Some(additional_user) = *k {\n-                    info!(\n+            for k in &i.users[1..] {\n+                info!(\n                         \"                                                                                                            | {}\",\n-                        additional_user\n+                        k\n                     );\n-                }\n             }\n         }\n\n@@ -211,7 +179,7 @@\n     virt_region: &MemoryRegion<Virtual>,\n     phys_region: &MemoryRegion<Physical>,\n     attr: &AttributeFields,\n-) -> Result<(), &'static str> {\n+) {\n     KERNEL_MAPPING_RECORD.write(|mr| mr.add(name, virt_region, phys_region, attr))\n }\n\n@@ -224,9 +192,7 @@\n     KERNEL_MAPPING_RECORD.write(|mr| {\n         let dup = mr.find_duplicate(&phys_region)?;\n\n-        if let Err(x) = dup.add_user(new_user) {\n-            warn!(\"{}\", x);\n-        }\n+        dup.add_user(new_user);\n\n         Some(dup.virt_start_addr)\n     })\n\ndiff -uNr 18_backtrace/kernel/src/memory/mmu.rs 19_kernel_heap/kernel/src/memory/mmu.rs\n--- 18_backtrace/kernel/src/memory/mmu.rs\n+++ 19_kernel_heap/kernel/src/memory/mmu.rs\n@@ -17,7 +17,6 @@\n     bsp,\n     memory::{Address, Physical, Virtual},\n     synchronization::{self, interface::Mutex},\n-    warn,\n };\n use core::{fmt, num::NonZeroUsize};\n\n@@ -176,9 +175,7 @@\n     phys_region: &MemoryRegion<Physical>,\n     attr: &AttributeFields,\n ) {\n-    if let Err(x) = mapping_record::kernel_add(name, virt_region, phys_region, attr) {\n-        warn!(\"{}\", x);\n-    }\n+    mapping_record::kernel_add(name, virt_region, phys_region, attr);\n }\n\n /// MMIO remapping in the kernel translation tables.\n\ndiff -uNr 18_backtrace/kernel/src/memory.rs 19_kernel_heap/kernel/src/memory.rs\n--- 18_backtrace/kernel/src/memory.rs\n+++ 19_kernel_heap/kernel/src/memory.rs\n@@ -4,6 +4,7 @@\n\n //! Memory Management.\n\n+pub mod heap_alloc;\n pub mod mmu;\n\n use crate::{bsp, common};\n@@ -163,6 +164,7 @@\n /// Initialize the memory subsystem.\n pub fn init() {\n     mmu::kernel_init_mmio_va_allocator();\n+    heap_alloc::kernel_init_heap_allocator();\n }\n\n //--------------------------------------------------------------------------------------------------\n\ndiff -uNr 18_backtrace/kernel/src/print.rs 19_kernel_heap/kernel/src/print.rs\n--- 18_backtrace/kernel/src/print.rs\n+++ 19_kernel_heap/kernel/src/print.rs\n@@ -82,3 +82,31 @@\n         ));\n     })\n }\n+\n+/// Debug print, with a newline.\n+#[macro_export]\n+macro_rules! debug {\n+    ($string:expr) => ({\n+        if cfg!(feature = \"debug_prints\") {\n+            let timestamp = $crate::time::time_manager().uptime();\n+\n+            $crate::print::_print(format_args_nl!(\n+                concat!(\"<[>D {:>3}.{:06}> \", $string),\n+                timestamp.as_secs(),\n+                timestamp.subsec_micros(),\n+            ));\n+        }\n+    });\n+    ($format_string:expr, $($arg:tt)*) => ({\n+        if cfg!(feature = \"debug_prints\") {\n+            let timestamp = $crate::time::time_manager().uptime();\n+\n+            $crate::print::_print(format_args_nl!(\n+                concat!(\"<D {:>3}.{:06}> \", $format_string),\n+                timestamp.as_secs(),\n+                timestamp.subsec_micros(),\n+                $($arg)*\n+            ));\n+        }\n+    })\n+}\n\ndiff -uNr 18_backtrace/kernel/src/state.rs 19_kernel_heap/kernel/src/state.rs\n--- 18_backtrace/kernel/src/state.rs\n+++ 19_kernel_heap/kernel/src/state.rs\n@@ -52,7 +52,7 @@\n     const SINGLE_CORE_MAIN: u8 = 1;\n     const MULTI_CORE_MAIN: u8 = 2;\n\n-    /// Create an instance.\n+    /// Create a new instance.\n     pub const fn new() -> Self {\n         Self(AtomicU8::new(Self::INIT))\n     }\n\ndiff -uNr 18_backtrace/Makefile 19_kernel_heap/Makefile\n--- 18_backtrace/Makefile\n+++ 19_kernel_heap/Makefile\n@@ -16,6 +16,11 @@\n # Default to a serial device name that is common in Linux.\n DEV_SERIAL ?= /dev/ttyUSB0\n\n+# Optional debug prints.\n+ifdef DEBUG_PRINTS\n+    FEATURES = --features debug_prints\n+endif\n+\n # Optional integration test name.\n ifdef TEST\n     TEST_ARG = --test $(TEST)\n@@ -70,7 +75,7 @@\n ##--------------------------------------------------------------------------------------------------\n KERNEL_MANIFEST      = kernel/Cargo.toml\n KERNEL_LINKER_SCRIPT = kernel.ld\n-LAST_BUILD_CONFIG    = target/$(BSP).build_config\n+LAST_BUILD_CONFIG    = target/$(BSP)_$(DEBUG_PRINTS).build_config\n\n KERNEL_ELF_RAW      = target/$(TARGET)/release/kernel\n # This parses cargo's dep-info file.\n@@ -117,17 +122,17 @@\n     -D warnings                   \\\n     -D missing_docs\n\n-FEATURES      = --features bsp_$(BSP)\n+FEATURES     += --features bsp_$(BSP)\n COMPILER_ARGS = --target=$(TARGET) \\\n     $(FEATURES)                    \\\n     --release\n\n # build-std can be skipped for helper commands that do not rely on correct stack frames and other\n # custom compiler options. This results in a huge speedup.\n-RUSTC_CMD   = cargo rustc $(COMPILER_ARGS) -Z build-std=core --manifest-path $(KERNEL_MANIFEST)\n+RUSTC_CMD   = cargo rustc $(COMPILER_ARGS) -Z build-std=core,alloc --manifest-path $(KERNEL_MANIFEST)\n DOC_CMD     = cargo doc $(COMPILER_ARGS)\n CLIPPY_CMD  = cargo clippy $(COMPILER_ARGS)\n-TEST_CMD    = cargo test $(COMPILER_ARGS) -Z build-std=core --manifest-path $(KERNEL_MANIFEST)\n+TEST_CMD    = cargo test $(COMPILER_ARGS) -Z build-std=core,alloc --manifest-path $(KERNEL_MANIFEST)\n OBJCOPY_CMD = rust-objcopy \\\n     --strip-all            \\\n     -O binary\n\n```\n"
  },
  {
    "path": "19_kernel_heap/kernel/Cargo.toml",
    "content": "[package]\nname = \"mingo\"\nversion = \"0.19.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n\n[features]\ndefault = []\ndebug_prints = []\nbsp_rpi3 = [\"tock-registers\"]\nbsp_rpi4 = [\"tock-registers\"]\ntest_build = [\"qemu-exit\"]\n\n##--------------------------------------------------------------------------------------------------\n## Dependencies\n##--------------------------------------------------------------------------------------------------\n\n[dependencies]\ntest-types = { path = \"../libraries/test-types\" }\ndebug-symbol-types = { path = \"../libraries/debug-symbol-types\" }\nlinked_list_allocator = { version = \"0.10.x\", default-features = false, features = [\"const_mut_refs\"] }\n\n# Optional dependencies\ntock-registers = { version = \"0.8.x\", default-features = false, features = [\"register_types\"], optional = true }\nqemu-exit = { version = \"3.x.x\", optional = true }\n\n# Platform specific dependencies\n[target.'cfg(target_arch = \"aarch64\")'.dependencies]\naarch64-cpu = { version = \"9.x.x\" }\n\n##--------------------------------------------------------------------------------------------------\n## Testing\n##--------------------------------------------------------------------------------------------------\n\n[dev-dependencies]\ntest-macros = { path = \"../libraries/test-macros\" }\n\n# Unit tests are done in the library part of the kernel.\n[lib]\nname = \"libkernel\"\ntest = true\n\n# Disable unit tests for the kernel binary.\n[[bin]]\nname = \"kernel\"\npath = \"src/main.rs\"\ntest = false\n\n# List of tests without harness.\n[[test]]\nname = \"00_console_sanity\"\nharness = false\n\n[[test]]\nname = \"02_exception_sync_page_fault\"\nharness = false\n\n[[test]]\nname = \"03_exception_restore_sanity\"\nharness = false\n\n[[test]]\nname = \"05_backtrace_sanity\"\nharness = false\n\n[[test]]\nname = \"06_backtrace_invalid_frame\"\nharness = false\n\n[[test]]\nname = \"07_backtrace_invalid_link\"\nharness = false\n"
  },
  {
    "path": "19_kernel_heap/kernel/build.rs",
    "content": "use std::{env, fs, process};\n\nfn main() {\n    let ld_script_path = match env::var(\"LD_SCRIPT_PATH\") {\n        Ok(var) => var,\n        _ => process::exit(0),\n    };\n\n    let files = fs::read_dir(ld_script_path).unwrap();\n    files\n        .filter_map(Result::ok)\n        .filter(|d| {\n            if let Some(e) = d.path().extension() {\n                e == \"ld\"\n            } else {\n                false\n            }\n        })\n        .for_each(|f| println!(\"cargo:rerun-if-changed={}\", f.path().display()));\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/_arch/aarch64/backtrace.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural backtracing support.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::backtrace::arch_backtrace\n\nuse crate::{\n    backtrace::BacktraceItem,\n    memory::{Address, Virtual},\n};\nuse aarch64_cpu::registers::*;\nuse tock_registers::interfaces::Readable;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// A Stack frame record.\n///\n/// # Note\n///\n/// The convention is that `previous_record` is valid as long as it contains a non-null value.\n/// Therefore, it is possible to type the member as `Option<&StackFrameRecord>` because of Rust's\n/// `null-pointer optimization`.\n#[repr(C)]\nstruct StackFrameRecord<'a> {\n    previous_record: Option<&'a StackFrameRecord<'a>>,\n    link: Address<Virtual>,\n}\n\nstruct StackFrameRecordIterator<'a> {\n    cur: &'a StackFrameRecord<'a>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<'a> Iterator for StackFrameRecordIterator<'a> {\n    type Item = BacktraceItem;\n\n    fn next(&mut self) -> Option<Self::Item> {\n        static ABORT_FRAME: StackFrameRecord = StackFrameRecord {\n            previous_record: None,\n            link: Address::new(0),\n        };\n\n        // If previous is None, this is the root frame, so iteration will stop here.\n        let previous = self.cur.previous_record?;\n\n        // Need to abort if the pointer to the previous frame record is invalid.\n        let prev_addr = Address::<Virtual>::new(previous as *const _ as usize);\n        if !prev_addr.is_valid_stack_addr() {\n            // This allows to return the error and then stop on the next iteration.\n            self.cur = &ABORT_FRAME;\n            return Some(BacktraceItem::InvalidFramePointer(prev_addr));\n        }\n\n        let ret = if !self.cur.link.is_valid_code_addr() {\n            Some(BacktraceItem::InvalidLink(self.cur.link))\n        } else {\n            // The link points to the instruction to be executed _after_ returning from a branch.\n            // However, we want to show the instruction that caused the branch, so subtract by one\n            // instruction.\n            //\n            // This might be called from panic!, so it must not panic itself on the subtraction.\n            let link = if self.cur.link >= Address::new(4) {\n                self.cur.link - 4\n            } else {\n                self.cur.link\n            };\n\n            Some(BacktraceItem::Link(link))\n        };\n\n        // Advance the iterator.\n        self.cur = previous;\n\n        ret\n    }\n}\n\nfn stack_frame_record_iterator<'a>() -> Option<StackFrameRecordIterator<'a>> {\n    let fp = Address::<Virtual>::new(FP.get() as usize);\n    if !fp.is_valid_stack_addr() {\n        return None;\n    }\n\n    Some(StackFrameRecordIterator {\n        cur: unsafe { &*(fp.as_usize() as *const _) },\n    })\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Architectural implementation of the backtrace.\npub fn backtrace(f: impl FnOnce(Option<&mut dyn Iterator<Item = BacktraceItem>>)) {\n    f(stack_frame_record_iterator().as_mut().map(|s| s as _))\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(feature = \"test_build\")]\n#[inline(always)]\n/// Hack for corrupting the previous frame address in the current stack frame.\n///\n/// # Safety\n///\n/// - To be used only by testing code.\npub unsafe fn corrupt_previous_frame_addr() {\n    let sf = FP.get() as *mut usize;\n    *sf = 0x123;\n}\n\n#[cfg(feature = \"test_build\")]\n#[inline(always)]\n/// Hack for corrupting the link in the current stack frame.\n///\n/// # Safety\n///\n/// - To be used only by testing code.\npub unsafe fn corrupt_link() {\n    let sf = FP.get() as *mut StackFrameRecord;\n    (*sf).link = Address::new(0x456);\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/_arch/aarch64/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural boot code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::boot::arch_boot\n\nuse crate::{memory, memory::Address};\nuse aarch64_cpu::{asm, registers::*};\nuse core::{\n    arch::global_asm,\n    sync::atomic::{compiler_fence, Ordering},\n};\nuse tock_registers::interfaces::Writeable;\n\n// Assembly counterpart to this file.\nglobal_asm!(\n    include_str!(\"boot.s\"),\n    CONST_CURRENTEL_EL2 = const 0x8,\n    CONST_CORE_ID_MASK = const 0b11\n);\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Prepares the transition from EL2 to EL1.\n///\n/// # Safety\n///\n/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.\n/// - The HW state of EL1 must be prepared in a sound way.\n#[inline(always)]\nunsafe fn prepare_el2_to_el1_transition(\n    virt_boot_core_stack_end_exclusive_addr: u64,\n    virt_kernel_init_addr: u64,\n) {\n    // Enable timer counter registers for EL1.\n    CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET);\n\n    // No offset for reading the counters.\n    CNTVOFF_EL2.set(0);\n\n    // Set EL1 execution state to AArch64.\n    HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64);\n\n    // Set up a simulated exception return.\n    //\n    // First, fake a saved program status where all interrupts were masked and SP_EL1 was used as a\n    // stack pointer.\n    SPSR_EL2.write(\n        SPSR_EL2::D::Masked\n            + SPSR_EL2::A::Masked\n            + SPSR_EL2::I::Masked\n            + SPSR_EL2::F::Masked\n            + SPSR_EL2::M::EL1h,\n    );\n\n    // Second, let the link register point to kernel_init().\n    ELR_EL2.set(virt_kernel_init_addr);\n\n    // Set up SP_EL1 (stack pointer), which will be used by EL1 once we \"return\" to it. Since there\n    // are no plans to ever return to EL2, just re-use the same stack.\n    SP_EL1.set(virt_boot_core_stack_end_exclusive_addr);\n}\n\n/// Reset the backtrace by setting link register and frame pointer to zero.\n///\n/// # Safety\n///\n/// - This function must only be used immediately before entering EL1.\n#[inline(always)]\nunsafe fn prepare_backtrace_reset() {\n    compiler_fence(Ordering::SeqCst);\n    FP.set(0);\n    LR.set(0);\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The Rust entry of the `kernel` binary.\n///\n/// The function is called from the assembly `_start` function.\n///\n/// # Safety\n///\n/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`.\n#[no_mangle]\npub unsafe extern \"C\" fn _start_rust(\n    phys_kernel_tables_base_addr: u64,\n    virt_boot_core_stack_end_exclusive_addr: u64,\n    virt_kernel_init_addr: u64,\n) -> ! {\n    prepare_el2_to_el1_transition(\n        virt_boot_core_stack_end_exclusive_addr,\n        virt_kernel_init_addr,\n    );\n\n    // Turn on the MMU for EL1.\n    let addr = Address::new(phys_kernel_tables_base_addr as usize);\n    memory::mmu::enable_mmu_and_caching(addr).unwrap();\n\n    // Make the function we return to the root of a backtrace.\n    prepare_backtrace_reset();\n\n    // Use `eret` to \"return\" to EL1. Since virtual memory will already be enabled, this results in\n    // execution of kernel_init() in EL1 from its _virtual address_.\n    asm::eret()\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/_arch/aarch64/cpu/boot.s",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2022 Andre Richter <andre.o.richter@gmail.com>\n\n//--------------------------------------------------------------------------------------------------\n// Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Load the address of a symbol into a register, PC-relative.\n//\n// The symbol must lie within +/- 4 GiB of the Program Counter.\n//\n// # Resources\n//\n// - https://sourceware.org/binutils/docs-2.36/as/AArch64_002dRelocations.html\n.macro ADR_REL register, symbol\n\tadrp\t\\register, \\symbol\n\tadd\t\\register, \\register, #:lo12:\\symbol\n.endm\n\n// Load the address of a symbol into a register, absolute.\n//\n// # Resources\n//\n// - https://sourceware.org/binutils/docs-2.36/as/AArch64_002dRelocations.html\n.macro ADR_ABS register, symbol\n\tmovz\t\\register, #:abs_g3:\\symbol\n\tmovk\t\\register, #:abs_g2_nc:\\symbol\n\tmovk\t\\register, #:abs_g1_nc:\\symbol\n\tmovk\t\\register, #:abs_g0_nc:\\symbol\n.endm\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n.section .text._start\n\n//------------------------------------------------------------------------------\n// fn _start()\n//------------------------------------------------------------------------------\n_start:\n\t// Only proceed if the core executes in EL2. Park it otherwise.\n\tmrs\tx0, CurrentEL\n\tcmp\tx0, {CONST_CURRENTEL_EL2}\n\tb.ne\t.L_parking_loop\n\n\t// Only proceed on the boot core. Park it otherwise.\n\tmrs\tx1, MPIDR_EL1\n\tand\tx1, x1, {CONST_CORE_ID_MASK}\n\tldr\tx2, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs\n\tcmp\tx1, x2\n\tb.ne\t.L_parking_loop\n\n\t// If execution reaches here, it is the boot core.\n\n\t// Initialize DRAM.\n\tADR_REL\tx0, __bss_start\n\tADR_REL x1, __bss_end_exclusive\n\n.L_bss_init_loop:\n\tcmp\tx0, x1\n\tb.eq\t.L_prepare_rust\n\tstp\txzr, xzr, [x0], #16\n\tb\t.L_bss_init_loop\n\n\t// Prepare the jump to Rust code.\n.L_prepare_rust:\n\t// Load the base address of the kernel's translation tables.\n\tldr\tx0, PHYS_KERNEL_TABLES_BASE_ADDR // provided by bsp/__board_name__/memory/mmu.rs\n\n\t// Load the _absolute_ addresses of the following symbols. Since the kernel is linked at\n\t// the top of the 64 bit address space, these are effectively virtual addresses.\n\tADR_ABS\tx1, __boot_core_stack_end_exclusive\n\tADR_ABS\tx2, kernel_init\n\n\t// Load the PC-relative address of the stack and set the stack pointer.\n\t//\n\t// Since _start() is the first function that runs after the firmware has loaded the kernel\n\t// into memory, retrieving this symbol PC-relative returns the \"physical\" address.\n\t//\n\t// Setting the stack pointer to this value ensures that anything that still runs in EL2,\n\t// until the kernel returns to EL1 with the MMU enabled, works as well. After the return to\n\t// EL1, the virtual address of the stack retrieved above will be used.\n\tADR_REL\tx3, __boot_core_stack_end_exclusive\n\tmov\tsp, x3\n\n\t// Read the CPU's timer counter frequency and store it in ARCH_TIMER_COUNTER_FREQUENCY.\n\t// Abort if the frequency read back as 0.\n\tADR_REL\tx4, ARCH_TIMER_COUNTER_FREQUENCY // provided by aarch64/time.rs\n\tmrs\tx5, CNTFRQ_EL0\n\tcmp\tx5, xzr\n\tb.eq\t.L_parking_loop\n\tstr\tw5, [x4]\n\n\t// Jump to Rust code. x0, x1 and x2 hold the function arguments provided to _start_rust().\n\tb\t_start_rust\n\n\t// Infinitely wait for events (aka \"park the core\").\n.L_parking_loop:\n\twfe\n\tb\t.L_parking_loop\n\n.size\t_start, . - _start\n.type\t_start, function\n.global\t_start\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/_arch/aarch64/cpu/smp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural symmetric multiprocessing.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::smp::arch_smp\n\nuse aarch64_cpu::registers::*;\nuse tock_registers::interfaces::Readable;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return the executing core's id.\n#[inline(always)]\npub fn core_id<T>() -> T\nwhere\n    T: From<u8>,\n{\n    const CORE_MASK: u64 = 0b11;\n\n    T::from((MPIDR_EL1.get() & CORE_MASK) as u8)\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/_arch/aarch64/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural processor code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::arch_cpu\n\nuse aarch64_cpu::asm;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\npub use asm::nop;\n\n/// Pause execution on the core.\n#[inline(always)]\npub fn wait_forever() -> ! {\n    loop {\n        asm::wfe()\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n#[cfg(feature = \"test_build\")]\nuse qemu_exit::QEMUExit;\n\n#[cfg(feature = \"test_build\")]\nconst QEMU_EXIT_HANDLE: qemu_exit::AArch64 = qemu_exit::AArch64::new();\n\n/// Make the host QEMU binary execute `exit(1)`.\n#[cfg(feature = \"test_build\")]\npub fn qemu_exit_failure() -> ! {\n    QEMU_EXIT_HANDLE.exit_failure()\n}\n\n/// Make the host QEMU binary execute `exit(0)`.\n#[cfg(feature = \"test_build\")]\npub fn qemu_exit_success() -> ! {\n    QEMU_EXIT_HANDLE.exit_success()\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/_arch/aarch64/exception/asynchronous.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural asynchronous exception handling.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::exception::asynchronous::arch_asynchronous\n\nuse aarch64_cpu::registers::*;\nuse core::arch::asm;\nuse tock_registers::interfaces::{Readable, Writeable};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nmod daif_bits {\n    pub const IRQ: u8 = 0b0010;\n}\n\ntrait DaifField {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register>;\n}\n\nstruct Debug;\nstruct SError;\nstruct IRQ;\nstruct FIQ;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl DaifField for Debug {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::D\n    }\n}\n\nimpl DaifField for SError {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::A\n    }\n}\n\nimpl DaifField for IRQ {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::I\n    }\n}\n\nimpl DaifField for FIQ {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::F\n    }\n}\n\nfn is_masked<T>() -> bool\nwhere\n    T: DaifField,\n{\n    DAIF.is_set(T::daif_field())\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Returns whether IRQs are masked on the executing core.\npub fn is_local_irq_masked() -> bool {\n    !is_masked::<IRQ>()\n}\n\n/// Unmask IRQs on the executing core.\n///\n/// It is not needed to place an explicit instruction synchronization barrier after the `msr`.\n/// Quoting the Architecture Reference Manual for ARMv8-A, section C5.1.3:\n///\n/// \"Writes to PSTATE.{PAN, D, A, I, F} occur in program order without the need for additional\n/// synchronization.\"\n#[inline(always)]\npub fn local_irq_unmask() {\n    unsafe {\n        asm!(\n            \"msr DAIFClr, {arg}\",\n            arg = const daif_bits::IRQ,\n            options(nomem, nostack, preserves_flags)\n        );\n    }\n}\n\n/// Mask IRQs on the executing core.\n#[inline(always)]\npub fn local_irq_mask() {\n    unsafe {\n        asm!(\n            \"msr DAIFSet, {arg}\",\n            arg = const daif_bits::IRQ,\n            options(nomem, nostack, preserves_flags)\n        );\n    }\n}\n\n/// Mask IRQs on the executing core and return the previously saved interrupt mask bits (DAIF).\n#[inline(always)]\npub fn local_irq_mask_save() -> u64 {\n    let saved = DAIF.get();\n    local_irq_mask();\n\n    saved\n}\n\n/// Restore the interrupt mask bits (DAIF) using the callee's argument.\n///\n/// # Invariant\n///\n/// - No sanity checks on the input.\n#[inline(always)]\npub fn local_irq_restore(saved: u64) {\n    DAIF.set(saved);\n}\n\n/// Print the AArch64 exceptions status.\n#[rustfmt::skip]\npub fn print_state() {\n    use crate::info;\n\n    let to_mask_str = |x| -> _ {\n        if x { \"Masked\" } else { \"Unmasked\" }\n    };\n\n    info!(\"      Debug:  {}\", to_mask_str(is_masked::<Debug>()));\n    info!(\"      SError: {}\", to_mask_str(is_masked::<SError>()));\n    info!(\"      IRQ:    {}\", to_mask_str(is_masked::<IRQ>()));\n    info!(\"      FIQ:    {}\", to_mask_str(is_masked::<FIQ>()));\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/_arch/aarch64/exception.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural synchronous and asynchronous exception handling.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::exception::arch_exception\n\nuse crate::{exception, memory, symbols};\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::{arch::global_asm, cell::UnsafeCell, fmt};\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    registers::InMemoryRegister,\n};\n\n// Assembly counterpart to this file.\nglobal_asm!(\n    include_str!(\"exception.s\"),\n    CONST_ESR_EL1_EC_SHIFT = const 26,\n    CONST_ESR_EL1_EC_VALUE_SVC64 = const 0x15\n);\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Wrapper structs for memory copies of registers.\n#[repr(transparent)]\nstruct SpsrEL1(InMemoryRegister<u64, SPSR_EL1::Register>);\nstruct EsrEL1(InMemoryRegister<u64, ESR_EL1::Register>);\n\n/// The exception context as it is stored on the stack on exception entry.\n#[repr(C)]\nstruct ExceptionContext {\n    /// General Purpose Registers.\n    gpr: [u64; 30],\n\n    /// The link register, aka x30.\n    lr: u64,\n\n    /// Exception link register. The program counter at the time the exception happened.\n    elr_el1: u64,\n\n    /// Saved program status.\n    spsr_el1: SpsrEL1,\n\n    /// Exception syndrome register.\n    esr_el1: EsrEL1,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Prints verbose information about the exception and then panics.\nfn default_exception_handler(exc: &ExceptionContext) {\n    panic!(\n        \"CPU Exception!\\n\\n\\\n        {}\",\n        exc\n    );\n}\n\n//------------------------------------------------------------------------------\n// Current, EL0\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn current_el0_synchronous(_e: &mut ExceptionContext) {\n    panic!(\"Should not be here. Use of SP_EL0 in EL1 is not supported.\")\n}\n\n#[no_mangle]\nextern \"C\" fn current_el0_irq(_e: &mut ExceptionContext) {\n    panic!(\"Should not be here. Use of SP_EL0 in EL1 is not supported.\")\n}\n\n#[no_mangle]\nextern \"C\" fn current_el0_serror(_e: &mut ExceptionContext) {\n    panic!(\"Should not be here. Use of SP_EL0 in EL1 is not supported.\")\n}\n\n//------------------------------------------------------------------------------\n// Current, ELx\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn current_elx_synchronous(e: &mut ExceptionContext) {\n    #[cfg(feature = \"test_build\")]\n    {\n        const TEST_SVC_ID: u64 = 0x1337;\n\n        if let Some(ESR_EL1::EC::Value::SVC64) = e.esr_el1.exception_class() {\n            if e.esr_el1.iss() == TEST_SVC_ID {\n                return;\n            }\n        }\n    }\n\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn current_elx_irq(_e: &mut ExceptionContext) {\n    let token = unsafe { &exception::asynchronous::IRQContext::new() };\n    exception::asynchronous::irq_manager().handle_pending_irqs(token);\n}\n\n#[no_mangle]\nextern \"C\" fn current_elx_serror(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n//------------------------------------------------------------------------------\n// Lower, AArch64\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn lower_aarch64_synchronous(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch64_irq(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch64_serror(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n//------------------------------------------------------------------------------\n// Lower, AArch32\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn lower_aarch32_synchronous(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch32_irq(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch32_serror(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n//------------------------------------------------------------------------------\n// Misc\n//------------------------------------------------------------------------------\n\n/// Human readable SPSR_EL1.\n#[rustfmt::skip]\nimpl fmt::Display for SpsrEL1 {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        // Raw value.\n        writeln!(f, \"SPSR_EL1: {:#010x}\", self.0.get())?;\n\n        let to_flag_str = |x| -> _ {\n            if x { \"Set\" } else { \"Not set\" }\n         };\n\n        writeln!(f, \"      Flags:\")?;\n        writeln!(f, \"            Negative (N): {}\", to_flag_str(self.0.is_set(SPSR_EL1::N)))?;\n        writeln!(f, \"            Zero     (Z): {}\", to_flag_str(self.0.is_set(SPSR_EL1::Z)))?;\n        writeln!(f, \"            Carry    (C): {}\", to_flag_str(self.0.is_set(SPSR_EL1::C)))?;\n        writeln!(f, \"            Overflow (V): {}\", to_flag_str(self.0.is_set(SPSR_EL1::V)))?;\n\n        let to_mask_str = |x| -> _ {\n            if x { \"Masked\" } else { \"Unmasked\" }\n        };\n\n        writeln!(f, \"      Exception handling state:\")?;\n        writeln!(f, \"            Debug  (D): {}\", to_mask_str(self.0.is_set(SPSR_EL1::D)))?;\n        writeln!(f, \"            SError (A): {}\", to_mask_str(self.0.is_set(SPSR_EL1::A)))?;\n        writeln!(f, \"            IRQ    (I): {}\", to_mask_str(self.0.is_set(SPSR_EL1::I)))?;\n        writeln!(f, \"            FIQ    (F): {}\", to_mask_str(self.0.is_set(SPSR_EL1::F)))?;\n\n        write!(f, \"      Illegal Execution State (IL): {}\",\n            to_flag_str(self.0.is_set(SPSR_EL1::IL))\n        )\n    }\n}\n\nimpl EsrEL1 {\n    #[inline(always)]\n    fn exception_class(&self) -> Option<ESR_EL1::EC::Value> {\n        self.0.read_as_enum(ESR_EL1::EC)\n    }\n\n    #[cfg(feature = \"test_build\")]\n    #[inline(always)]\n    fn iss(&self) -> u64 {\n        self.0.read(ESR_EL1::ISS)\n    }\n}\n\n/// Human readable ESR_EL1.\n#[rustfmt::skip]\nimpl fmt::Display for EsrEL1 {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        // Raw print of whole register.\n        writeln!(f, \"ESR_EL1: {:#010x}\", self.0.get())?;\n\n        // Raw print of exception class.\n        write!(f, \"      Exception Class         (EC) : {:#x}\", self.0.read(ESR_EL1::EC))?;\n\n        // Exception class.\n        let ec_translation = match self.exception_class() {\n            Some(ESR_EL1::EC::Value::DataAbortCurrentEL) => \"Data Abort, current EL\",\n            _ => \"N/A\",\n        };\n        writeln!(f, \" - {}\", ec_translation)?;\n\n        // Raw print of instruction specific syndrome.\n        write!(f, \"      Instr Specific Syndrome (ISS): {:#x}\", self.0.read(ESR_EL1::ISS))\n    }\n}\n\nimpl ExceptionContext {\n    #[inline(always)]\n    fn exception_class(&self) -> Option<ESR_EL1::EC::Value> {\n        self.esr_el1.exception_class()\n    }\n\n    #[inline(always)]\n    fn fault_address_valid(&self) -> bool {\n        use ESR_EL1::EC::Value::*;\n\n        match self.exception_class() {\n            None => false,\n            Some(ec) => matches!(\n                ec,\n                InstrAbortLowerEL\n                    | InstrAbortCurrentEL\n                    | PCAlignmentFault\n                    | DataAbortLowerEL\n                    | DataAbortCurrentEL\n                    | WatchpointLowerEL\n                    | WatchpointCurrentEL\n            ),\n        }\n    }\n}\n\n/// Human readable print of the exception context.\nimpl fmt::Display for ExceptionContext {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        writeln!(f, \"{}\", self.esr_el1)?;\n\n        if self.fault_address_valid() {\n            writeln!(f, \"FAR_EL1: {:#018x}\", FAR_EL1.get() as usize)?;\n        }\n\n        writeln!(f, \"{}\", self.spsr_el1)?;\n        writeln!(f, \"ELR_EL1: {:#018x}\", self.elr_el1)?;\n        writeln!(\n            f,\n            \"      Symbol: {}\",\n            match symbols::lookup_symbol(memory::Address::new(self.elr_el1 as usize)) {\n                Some(sym) => sym.name(),\n                _ => \"Symbol not found\",\n            }\n        )?;\n        writeln!(f)?;\n        writeln!(f, \"General purpose register:\")?;\n\n        #[rustfmt::skip]\n        let alternating = |x| -> _ {\n            if x % 2 == 0 { \"   \" } else { \"\\n\" }\n        };\n\n        // Print two registers per line.\n        for (i, reg) in self.gpr.iter().enumerate() {\n            write!(f, \"      x{: <2}: {: >#018x}{}\", i, reg, alternating(i))?;\n        }\n        write!(f, \"      lr : {:#018x}\", self.lr)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse crate::exception::PrivilegeLevel;\n\n/// The processing element's current privilege level.\npub fn current_privilege_level() -> (PrivilegeLevel, &'static str) {\n    let el = CurrentEL.read_as_enum(CurrentEL::EL);\n    match el {\n        Some(CurrentEL::EL::Value::EL2) => (PrivilegeLevel::Hypervisor, \"EL2\"),\n        Some(CurrentEL::EL::Value::EL1) => (PrivilegeLevel::Kernel, \"EL1\"),\n        Some(CurrentEL::EL::Value::EL0) => (PrivilegeLevel::User, \"EL0\"),\n        _ => (PrivilegeLevel::Unknown, \"Unknown\"),\n    }\n}\n\n/// Init exception handling by setting the exception vector base address register.\n///\n/// # Safety\n///\n/// - Changes the HW state of the executing core.\n/// - The vector table and the symbol `__exception_vector_table_start` from the linker script must\n///   adhere to the alignment and size constraints demanded by the ARMv8-A Architecture Reference\n///   Manual.\npub unsafe fn handling_init() {\n    // Provided by exception.S.\n    extern \"Rust\" {\n        static __exception_vector_start: UnsafeCell<()>;\n    }\n\n    VBAR_EL1.set(__exception_vector_start.get() as u64);\n\n    // Force VBAR update to complete before next instruction.\n    barrier::isb(barrier::SY);\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/_arch/aarch64/exception.s",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2022 Andre Richter <andre.o.richter@gmail.com>\n\n//--------------------------------------------------------------------------------------------------\n// Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Call the function provided by parameter `\\handler` after saving the exception context. Provide\n/// the context as the first parameter to '\\handler'.\n.macro CALL_WITH_CONTEXT handler is_lower_el is_sync\n__vector_\\handler:\n\t// Make room on the stack for the exception context.\n\tsub\tsp,  sp,  #16 * 18\n\n\t// Store all general purpose registers on the stack.\n\tstp\tx0,  x1,  [sp, #16 * 0]\n\tstp\tx2,  x3,  [sp, #16 * 1]\n\tstp\tx4,  x5,  [sp, #16 * 2]\n\tstp\tx6,  x7,  [sp, #16 * 3]\n\tstp\tx8,  x9,  [sp, #16 * 4]\n\tstp\tx10, x11, [sp, #16 * 5]\n\tstp\tx12, x13, [sp, #16 * 6]\n\tstp\tx14, x15, [sp, #16 * 7]\n\tstp\tx16, x17, [sp, #16 * 8]\n\tstp\tx18, x19, [sp, #16 * 9]\n\tstp\tx20, x21, [sp, #16 * 10]\n\tstp\tx22, x23, [sp, #16 * 11]\n\tstp\tx24, x25, [sp, #16 * 12]\n\tstp\tx26, x27, [sp, #16 * 13]\n\tstp\tx28, x29, [sp, #16 * 14]\n\n\t// Add the exception link register (ELR_EL1), saved program status (SPSR_EL1) and exception\n\t// syndrome register (ESR_EL1).\n\tmrs\tx1,  ELR_EL1\n\tmrs\tx2,  SPSR_EL1\n\tmrs\tx3,  ESR_EL1\n\n\tstp\tlr,  x1,  [sp, #16 * 15]\n\tstp\tx2,  x3,  [sp, #16 * 16]\n\n\t// Build a stack frame for backtracing.\n.if \\is_lower_el == 1\n\t// If we came from a lower EL, make it a root frame (by storing zero) so that the kernel\n\t// does not attempt to trace into userspace.\n\tstp\txzr, xzr, [sp, #16 * 17]\n.else\n\t// For normal branches, the link address points to the instruction to be executed _after_\n\t// returning from a branch. In a backtrace, we want to show the instruction that caused the\n\t// branch, though. That is why code in backtrace.rs subtracts 4 (length of one instruction)\n\t// from the link address.\n\t//\n\t// Here we have a special case, though, because ELR_EL1 is used instead of LR to build the\n\t// stack frame, so that it becomes possible to trace beyond an exception. Hence, it must be\n\t// considered that semantics for ELR_EL1 differ from case to case.\n\t//\n\t// Unless an \"exception generating instruction\" was executed, ELR_EL1 already points to the\n\t// the correct instruction, and hence the subtraction by 4 in backtrace.rs would yield wrong\n\t// results. To cover for this, 4 is added to ELR_EL1 below unless the cause of exception was\n\t// an SVC instruction. BRK and HLT are \"exception generating instructions\" as well, but they\n\t// are not expected and therefore left out for now.\n\t//\n\t// For reference: Search for \"preferred exception return address\" in the Architecture\n\t// Reference Manual for ARMv8-A.\n.if \\is_sync == 1\n\tlsr\tw3,  w3, {CONST_ESR_EL1_EC_SHIFT}   // w3 = ESR_EL1.EC\n\tcmp\tw3,  {CONST_ESR_EL1_EC_VALUE_SVC64} // w3 == SVC64 ?\n\tb.eq\t1f\n.endif\n\tadd\tx1,  x1, #4\n1:\n\tstp\tx29, x1, [sp, #16 * 17]\n.endif\n\n\t// Set the frame pointer to the stack frame record.\n\tadd\tx29, sp, #16 * 17\n\n\t// x0 is the first argument for the function called through `\\handler`.\n\tmov\tx0,  sp\n\n\t// Call `\\handler`.\n\tbl\t\\handler\n\n\t// After returning from exception handling code, replay the saved context and return via\n\t// `eret`.\n\tb\t__exception_restore_context\n\n.size\t__vector_\\handler, . - __vector_\\handler\n.type\t__vector_\\handler, function\n.endm\n\n.macro FIQ_SUSPEND\n1:\twfe\n\tb\t1b\n.endm\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n.section .text\n\n//------------------------------------------------------------------------------\n// The exception vector table.\n//------------------------------------------------------------------------------\n\n// Align by 2^11 bytes, as demanded by ARMv8-A. Same as ALIGN(2048) in an ld script.\n.align 11\n\n// Export a symbol for the Rust code to use.\n__exception_vector_start:\n\n// Current exception level with SP_EL0.\n//\n// .org sets the offset relative to section start.\n//\n// # Safety\n//\n// - It must be ensured that `CALL_WITH_CONTEXT` <= 0x80 bytes.\n.org 0x000\n\tCALL_WITH_CONTEXT current_el0_synchronous, 0, 1\n.org 0x080\n\tCALL_WITH_CONTEXT current_el0_irq, 0, 0\n.org 0x100\n\tFIQ_SUSPEND\n.org 0x180\n\tCALL_WITH_CONTEXT current_el0_serror, 0, 0\n\n// Current exception level with SP_ELx, x > 0.\n.org 0x200\n\tCALL_WITH_CONTEXT current_elx_synchronous, 0, 1\n.org 0x280\n\tCALL_WITH_CONTEXT current_elx_irq, 0, 0\n.org 0x300\n\tFIQ_SUSPEND\n.org 0x380\n\tCALL_WITH_CONTEXT current_elx_serror, 0, 0\n\n// Lower exception level, AArch64\n.org 0x400\n\tCALL_WITH_CONTEXT lower_aarch64_synchronous, 1, 1\n.org 0x480\n\tCALL_WITH_CONTEXT lower_aarch64_irq, 1, 0\n.org 0x500\n\tFIQ_SUSPEND\n.org 0x580\n\tCALL_WITH_CONTEXT lower_aarch64_serror, 1, 0\n\n// Lower exception level, AArch32\n.org 0x600\n\tCALL_WITH_CONTEXT lower_aarch32_synchronous, 1, 0\n.org 0x680\n\tCALL_WITH_CONTEXT lower_aarch32_irq, 1, 0\n.org 0x700\n\tFIQ_SUSPEND\n.org 0x780\n\tCALL_WITH_CONTEXT lower_aarch32_serror, 1, 0\n.org 0x800\n\n//------------------------------------------------------------------------------\n// fn __exception_restore_context()\n//------------------------------------------------------------------------------\n__exception_restore_context:\n\tldr\tw19,      [sp, #16 * 16]\n\tldp\tlr,  x20, [sp, #16 * 15]\n\n\tmsr\tSPSR_EL1, x19\n\tmsr\tELR_EL1,  x20\n\n\tldp\tx0,  x1,  [sp, #16 * 0]\n\tldp\tx2,  x3,  [sp, #16 * 1]\n\tldp\tx4,  x5,  [sp, #16 * 2]\n\tldp\tx6,  x7,  [sp, #16 * 3]\n\tldp\tx8,  x9,  [sp, #16 * 4]\n\tldp\tx10, x11, [sp, #16 * 5]\n\tldp\tx12, x13, [sp, #16 * 6]\n\tldp\tx14, x15, [sp, #16 * 7]\n\tldp\tx16, x17, [sp, #16 * 8]\n\tldp\tx18, x19, [sp, #16 * 9]\n\tldp\tx20, x21, [sp, #16 * 10]\n\tldp\tx22, x23, [sp, #16 * 11]\n\tldp\tx24, x25, [sp, #16 * 12]\n\tldp\tx26, x27, [sp, #16 * 13]\n\tldp\tx28, x29, [sp, #16 * 14]\n\n\tadd\tsp,  sp,  #16 * 18\n\n\teret\n\n.size\t__exception_restore_context, . - __exception_restore_context\n.type\t__exception_restore_context, function\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/_arch/aarch64/memory/mmu/translation_table.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural translation table.\n//!\n//! Only 64 KiB granule is supported.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::memory::mmu::translation_table::arch_translation_table\n\nuse crate::{\n    bsp,\n    memory::{\n        self,\n        mmu::{\n            arch_mmu::{Granule512MiB, Granule64KiB},\n            AccessPermissions, AttributeFields, MemAttributes, MemoryRegion, PageAddress,\n        },\n        Address, Physical, Virtual,\n    },\n};\nuse core::convert;\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields,\n    registers::InMemoryRegister,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// A table descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-15.\nregister_bitfields! {u64,\n    STAGE1_TABLE_DESCRIPTOR [\n        /// Physical address of the next descriptor.\n        NEXT_LEVEL_TABLE_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16]\n\n        TYPE  OFFSET(1) NUMBITS(1) [\n            Block = 0,\n            Table = 1\n        ],\n\n        VALID OFFSET(0) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ]\n    ]\n}\n\n// A level 3 page descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-17.\nregister_bitfields! {u64,\n    STAGE1_PAGE_DESCRIPTOR [\n        /// Unprivileged execute-never.\n        UXN      OFFSET(54) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ],\n\n        /// Privileged execute-never.\n        PXN      OFFSET(53) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ],\n\n        /// Physical address of the next table descriptor (lvl2) or the page descriptor (lvl3).\n        OUTPUT_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16]\n\n        /// Access flag.\n        AF       OFFSET(10) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ],\n\n        /// Shareability field.\n        SH       OFFSET(8) NUMBITS(2) [\n            OuterShareable = 0b10,\n            InnerShareable = 0b11\n        ],\n\n        /// Access Permissions.\n        AP       OFFSET(6) NUMBITS(2) [\n            RW_EL1 = 0b00,\n            RW_EL1_EL0 = 0b01,\n            RO_EL1 = 0b10,\n            RO_EL1_EL0 = 0b11\n        ],\n\n        /// Memory attributes index into the MAIR_EL1 register.\n        AttrIndx OFFSET(2) NUMBITS(3) [],\n\n        TYPE     OFFSET(1) NUMBITS(1) [\n            Reserved_Invalid = 0,\n            Page = 1\n        ],\n\n        VALID    OFFSET(0) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ]\n    ]\n}\n\n/// A table descriptor for 64 KiB aperture.\n///\n/// The output points to the next table.\n#[derive(Copy, Clone)]\n#[repr(C)]\nstruct TableDescriptor {\n    value: u64,\n}\n\n/// A page descriptor with 64 KiB aperture.\n///\n/// The output points to physical memory.\n#[derive(Copy, Clone)]\n#[repr(C)]\nstruct PageDescriptor {\n    value: u64,\n}\n\ntrait StartAddr {\n    fn virt_start_addr(&self) -> Address<Virtual>;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB\n/// aligned, so the lvl3 is put first.\n#[repr(C)]\n#[repr(align(65536))]\npub struct FixedSizeTranslationTable<const NUM_TABLES: usize, const START_FROM_TOP: bool> {\n    /// Page descriptors, covering 64 KiB windows per entry.\n    lvl3: [[PageDescriptor; 8192]; NUM_TABLES],\n\n    /// Table descriptors, covering 512 MiB windows.\n    lvl2: [TableDescriptor; NUM_TABLES],\n\n    /// Have the tables been initialized?\n    initialized: bool,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T, const N: usize> StartAddr for [T; N] {\n    fn virt_start_addr(&self) -> Address<Virtual> {\n        Address::new(self as *const _ as usize)\n    }\n}\n\nimpl TableDescriptor {\n    /// Create an instance.\n    ///\n    /// Descriptor is invalid by default.\n    pub const fn new_zeroed() -> Self {\n        Self { value: 0 }\n    }\n\n    /// Create an instance pointing to the supplied address.\n    pub fn from_next_lvl_table_addr(phys_next_lvl_table_addr: Address<Physical>) -> Self {\n        let val = InMemoryRegister::<u64, STAGE1_TABLE_DESCRIPTOR::Register>::new(0);\n\n        let shifted = phys_next_lvl_table_addr.as_usize() >> Granule64KiB::SHIFT;\n        val.write(\n            STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64)\n                + STAGE1_TABLE_DESCRIPTOR::TYPE::Table\n                + STAGE1_TABLE_DESCRIPTOR::VALID::True,\n        );\n\n        TableDescriptor { value: val.get() }\n    }\n}\n\n/// Convert the kernel's generic memory attributes to HW-specific attributes of the MMU.\nimpl convert::From<AttributeFields>\n    for tock_registers::fields::FieldValue<u64, STAGE1_PAGE_DESCRIPTOR::Register>\n{\n    fn from(attribute_fields: AttributeFields) -> Self {\n        // Memory attributes.\n        let mut desc = match attribute_fields.mem_attributes {\n            MemAttributes::CacheableDRAM => {\n                STAGE1_PAGE_DESCRIPTOR::SH::InnerShareable\n                    + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::NORMAL)\n            }\n            MemAttributes::Device => {\n                STAGE1_PAGE_DESCRIPTOR::SH::OuterShareable\n                    + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::DEVICE)\n            }\n        };\n\n        // Access Permissions.\n        desc += match attribute_fields.acc_perms {\n            AccessPermissions::ReadOnly => STAGE1_PAGE_DESCRIPTOR::AP::RO_EL1,\n            AccessPermissions::ReadWrite => STAGE1_PAGE_DESCRIPTOR::AP::RW_EL1,\n        };\n\n        // The execute-never attribute is mapped to PXN in AArch64.\n        desc += if attribute_fields.execute_never {\n            STAGE1_PAGE_DESCRIPTOR::PXN::True\n        } else {\n            STAGE1_PAGE_DESCRIPTOR::PXN::False\n        };\n\n        // Always set unprivileged exectue-never as long as userspace is not implemented yet.\n        desc += STAGE1_PAGE_DESCRIPTOR::UXN::True;\n\n        desc\n    }\n}\n\n/// Convert the HW-specific attributes of the MMU to kernel's generic memory attributes.\nimpl convert::TryFrom<InMemoryRegister<u64, STAGE1_PAGE_DESCRIPTOR::Register>> for AttributeFields {\n    type Error = &'static str;\n\n    fn try_from(\n        desc: InMemoryRegister<u64, STAGE1_PAGE_DESCRIPTOR::Register>,\n    ) -> Result<AttributeFields, Self::Error> {\n        let mem_attributes = match desc.read(STAGE1_PAGE_DESCRIPTOR::AttrIndx) {\n            memory::mmu::arch_mmu::mair::NORMAL => MemAttributes::CacheableDRAM,\n            memory::mmu::arch_mmu::mair::DEVICE => MemAttributes::Device,\n            _ => return Err(\"Unexpected memory attribute\"),\n        };\n\n        let acc_perms = match desc.read_as_enum(STAGE1_PAGE_DESCRIPTOR::AP) {\n            Some(STAGE1_PAGE_DESCRIPTOR::AP::Value::RO_EL1) => AccessPermissions::ReadOnly,\n            Some(STAGE1_PAGE_DESCRIPTOR::AP::Value::RW_EL1) => AccessPermissions::ReadWrite,\n            _ => return Err(\"Unexpected access permission\"),\n        };\n\n        let execute_never = desc.read(STAGE1_PAGE_DESCRIPTOR::PXN) > 0;\n\n        Ok(AttributeFields {\n            mem_attributes,\n            acc_perms,\n            execute_never,\n        })\n    }\n}\n\nimpl PageDescriptor {\n    /// Create an instance.\n    ///\n    /// Descriptor is invalid by default.\n    pub const fn new_zeroed() -> Self {\n        Self { value: 0 }\n    }\n\n    /// Create an instance.\n    pub fn from_output_page_addr(\n        phys_output_page_addr: PageAddress<Physical>,\n        attribute_fields: &AttributeFields,\n    ) -> Self {\n        let val = InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(0);\n\n        let shifted = phys_output_page_addr.into_inner().as_usize() >> Granule64KiB::SHIFT;\n        val.write(\n            STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted as u64)\n                + STAGE1_PAGE_DESCRIPTOR::AF::True\n                + STAGE1_PAGE_DESCRIPTOR::TYPE::Page\n                + STAGE1_PAGE_DESCRIPTOR::VALID::True\n                + (*attribute_fields).into(),\n        );\n\n        Self { value: val.get() }\n    }\n\n    /// Returns the valid bit.\n    fn is_valid(&self) -> bool {\n        InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(self.value)\n            .is_set(STAGE1_PAGE_DESCRIPTOR::VALID)\n    }\n\n    /// Returns the output page.\n    fn output_page_addr(&self) -> PageAddress<Physical> {\n        let shifted = InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(self.value)\n            .read(STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB) as usize;\n\n        PageAddress::from(shifted << Granule64KiB::SHIFT)\n    }\n\n    /// Returns the attributes.\n    fn try_attributes(&self) -> Result<AttributeFields, &'static str> {\n        InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(self.value).try_into()\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<const AS_SIZE: usize> memory::mmu::AssociatedTranslationTable\n    for memory::mmu::AddressSpace<AS_SIZE>\nwhere\n    [u8; Self::SIZE >> Granule512MiB::SHIFT]: Sized,\n{\n    type TableStartFromTop =\n        FixedSizeTranslationTable<{ Self::SIZE >> Granule512MiB::SHIFT }, true>;\n\n    type TableStartFromBottom =\n        FixedSizeTranslationTable<{ Self::SIZE >> Granule512MiB::SHIFT }, false>;\n}\n\nimpl<const NUM_TABLES: usize, const START_FROM_TOP: bool>\n    FixedSizeTranslationTable<NUM_TABLES, START_FROM_TOP>\n{\n    const START_FROM_TOP_OFFSET: Address<Virtual> =\n        Address::new((usize::MAX - (Granule512MiB::SIZE * NUM_TABLES)) + 1);\n\n    /// Create an instance.\n    #[allow(clippy::assertions_on_constants)]\n    const fn _new(for_precompute: bool) -> Self {\n        assert!(bsp::memory::mmu::KernelGranule::SIZE == Granule64KiB::SIZE);\n\n        // Can't have a zero-sized address space.\n        assert!(NUM_TABLES > 0);\n\n        Self {\n            lvl3: [[PageDescriptor::new_zeroed(); 8192]; NUM_TABLES],\n            lvl2: [TableDescriptor::new_zeroed(); NUM_TABLES],\n            initialized: for_precompute,\n        }\n    }\n\n    pub const fn new_for_precompute() -> Self {\n        Self::_new(true)\n    }\n\n    #[cfg(test)]\n    pub fn new_for_runtime() -> Self {\n        Self::_new(false)\n    }\n\n    /// Helper to calculate the lvl2 and lvl3 indices from an address.\n    #[inline(always)]\n    fn lvl2_lvl3_index_from_page_addr(\n        &self,\n        virt_page_addr: PageAddress<Virtual>,\n    ) -> Result<(usize, usize), &'static str> {\n        let mut addr = virt_page_addr.into_inner();\n\n        if START_FROM_TOP {\n            addr = addr - Self::START_FROM_TOP_OFFSET;\n        }\n\n        let lvl2_index = addr.as_usize() >> Granule512MiB::SHIFT;\n        let lvl3_index = (addr.as_usize() & Granule512MiB::MASK) >> Granule64KiB::SHIFT;\n\n        if lvl2_index > (NUM_TABLES - 1) {\n            return Err(\"Virtual page is out of bounds of translation table\");\n        }\n\n        Ok((lvl2_index, lvl3_index))\n    }\n\n    /// Returns the PageDescriptor corresponding to the supplied page address.\n    #[inline(always)]\n    fn page_descriptor_from_page_addr(\n        &self,\n        virt_page_addr: PageAddress<Virtual>,\n    ) -> Result<&PageDescriptor, &'static str> {\n        let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from_page_addr(virt_page_addr)?;\n        let desc = &self.lvl3[lvl2_index][lvl3_index];\n\n        Ok(desc)\n    }\n\n    /// Sets the PageDescriptor corresponding to the supplied page address.\n    ///\n    /// Doesn't allow overriding an already valid page.\n    #[inline(always)]\n    fn set_page_descriptor_from_page_addr(\n        &mut self,\n        virt_page_addr: PageAddress<Virtual>,\n        new_desc: &PageDescriptor,\n    ) -> Result<(), &'static str> {\n        let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from_page_addr(virt_page_addr)?;\n        let desc = &mut self.lvl3[lvl2_index][lvl3_index];\n\n        if desc.is_valid() {\n            return Err(\"Virtual page is already mapped\");\n        }\n\n        *desc = *new_desc;\n        Ok(())\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\n\nimpl<const NUM_TABLES: usize, const START_FROM_TOP: bool>\n    memory::mmu::translation_table::interface::TranslationTable\n    for FixedSizeTranslationTable<NUM_TABLES, START_FROM_TOP>\n{\n    fn init(&mut self) -> Result<(), &'static str> {\n        if self.initialized {\n            return Ok(());\n        }\n\n        // Populate the l2 entries.\n        for (lvl2_nr, lvl2_entry) in self.lvl2.iter_mut().enumerate() {\n            let virt_table_addr = self.lvl3[lvl2_nr].virt_start_addr();\n            let phys_table_addr = memory::mmu::try_kernel_virt_addr_to_phys_addr(virt_table_addr)?;\n\n            let new_desc = TableDescriptor::from_next_lvl_table_addr(phys_table_addr);\n            *lvl2_entry = new_desc;\n        }\n\n        self.initialized = true;\n\n        Ok(())\n    }\n\n    unsafe fn map_at(\n        &mut self,\n        virt_region: &MemoryRegion<Virtual>,\n        phys_region: &MemoryRegion<Physical>,\n        attr: &AttributeFields,\n    ) -> Result<(), &'static str> {\n        assert!(self.initialized, \"Translation tables not initialized\");\n\n        if virt_region.size() != phys_region.size() {\n            return Err(\"Tried to map memory regions with unequal sizes\");\n        }\n\n        if phys_region.end_exclusive_page_addr() > bsp::memory::phys_addr_space_end_exclusive_addr()\n        {\n            return Err(\"Tried to map outside of physical address space\");\n        }\n\n        let iter = phys_region.into_iter().zip(virt_region.into_iter());\n        for (phys_page_addr, virt_page_addr) in iter {\n            let new_desc = PageDescriptor::from_output_page_addr(phys_page_addr, attr);\n            let virt_page = virt_page_addr;\n\n            self.set_page_descriptor_from_page_addr(virt_page, &new_desc)?;\n        }\n\n        Ok(())\n    }\n\n    fn try_virt_page_addr_to_phys_page_addr(\n        &self,\n        virt_page_addr: PageAddress<Virtual>,\n    ) -> Result<PageAddress<Physical>, &'static str> {\n        let page_desc = self.page_descriptor_from_page_addr(virt_page_addr)?;\n\n        if !page_desc.is_valid() {\n            return Err(\"Page marked invalid\");\n        }\n\n        Ok(page_desc.output_page_addr())\n    }\n\n    fn try_page_attributes(\n        &self,\n        virt_page_addr: PageAddress<Virtual>,\n    ) -> Result<AttributeFields, &'static str> {\n        let page_desc = self.page_descriptor_from_page_addr(virt_page_addr)?;\n\n        if !page_desc.is_valid() {\n            return Err(\"Page marked invalid\");\n        }\n\n        page_desc.try_attributes()\n    }\n\n    /// Try to translate a virtual address to a physical address.\n    ///\n    /// Will only succeed if there exists a valid mapping for the input address.\n    fn try_virt_addr_to_phys_addr(\n        &self,\n        virt_addr: Address<Virtual>,\n    ) -> Result<Address<Physical>, &'static str> {\n        let virt_page = PageAddress::from(virt_addr.align_down_page());\n        let phys_page = self.try_virt_page_addr_to_phys_page_addr(virt_page)?;\n\n        Ok(phys_page.into_inner() + virt_addr.offset_into_page())\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\npub type MinSizeTranslationTable = FixedSizeTranslationTable<1, true>;\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// Check if the size of `struct TableDescriptor` is as expected.\n    #[kernel_test]\n    fn size_of_tabledescriptor_equals_64_bit() {\n        assert_eq!(\n            core::mem::size_of::<TableDescriptor>(),\n            core::mem::size_of::<u64>()\n        );\n    }\n\n    /// Check if the size of `struct PageDescriptor` is as expected.\n    #[kernel_test]\n    fn size_of_pagedescriptor_equals_64_bit() {\n        assert_eq!(\n            core::mem::size_of::<PageDescriptor>(),\n            core::mem::size_of::<u64>()\n        );\n    }\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/_arch/aarch64/memory/mmu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management Unit Driver.\n//!\n//! Only 64 KiB granule is supported.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::memory::mmu::arch_mmu\n\nuse crate::{\n    bsp, memory,\n    memory::{mmu::TranslationGranule, Address, Physical},\n};\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::intrinsics::unlikely;\nuse tock_registers::interfaces::{ReadWriteable, Readable, Writeable};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Memory Management Unit type.\nstruct MemoryManagementUnit;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub type Granule512MiB = TranslationGranule<{ 512 * 1024 * 1024 }>;\npub type Granule64KiB = TranslationGranule<{ 64 * 1024 }>;\n\n/// Constants for indexing the MAIR_EL1.\n#[allow(dead_code)]\npub mod mair {\n    pub const DEVICE: u64 = 0;\n    pub const NORMAL: u64 = 1;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic MMU: MemoryManagementUnit = MemoryManagementUnit;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<const AS_SIZE: usize> memory::mmu::AddressSpace<AS_SIZE> {\n    /// Checks for architectural restrictions.\n    pub const fn arch_address_space_size_sanity_checks() {\n        // Size must be at least one full 512 MiB table.\n        assert!((AS_SIZE % Granule512MiB::SIZE) == 0);\n\n        // Check for 48 bit virtual address size as maximum, which is supported by any ARMv8\n        // version.\n        assert!(AS_SIZE <= (1 << 48));\n    }\n}\n\nimpl MemoryManagementUnit {\n    /// Setup function for the MAIR_EL1 register.\n    #[inline(always)]\n    fn set_up_mair(&self) {\n        // Define the memory types being mapped.\n        MAIR_EL1.write(\n            // Attribute 1 - Cacheable normal DRAM.\n            MAIR_EL1::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc +\n        MAIR_EL1::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc +\n\n        // Attribute 0 - Device.\n        MAIR_EL1::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck,\n        );\n    }\n\n    /// Configure various settings of stage 1 of the EL1 translation regime.\n    #[inline(always)]\n    fn configure_translation_control(&self) {\n        let t1sz = (64 - bsp::memory::mmu::KernelVirtAddrSpace::SIZE_SHIFT) as u64;\n\n        TCR_EL1.write(\n            TCR_EL1::TBI1::Used\n                + TCR_EL1::IPS::Bits_40\n                + TCR_EL1::TG1::KiB_64\n                + TCR_EL1::SH1::Inner\n                + TCR_EL1::ORGN1::WriteBack_ReadAlloc_WriteAlloc_Cacheable\n                + TCR_EL1::IRGN1::WriteBack_ReadAlloc_WriteAlloc_Cacheable\n                + TCR_EL1::EPD1::EnableTTBR1Walks\n                + TCR_EL1::A1::TTBR1\n                + TCR_EL1::T1SZ.val(t1sz)\n                + TCR_EL1::EPD0::DisableTTBR0Walks,\n        );\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the MMU instance.\npub fn mmu() -> &'static impl memory::mmu::interface::MMU {\n    &MMU\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse memory::mmu::MMUEnableError;\n\nimpl memory::mmu::interface::MMU for MemoryManagementUnit {\n    unsafe fn enable_mmu_and_caching(\n        &self,\n        phys_tables_base_addr: Address<Physical>,\n    ) -> Result<(), MMUEnableError> {\n        if unlikely(self.is_enabled()) {\n            return Err(MMUEnableError::AlreadyEnabled);\n        }\n\n        // Fail early if translation granule is not supported.\n        if unlikely(!ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran64::Supported)) {\n            return Err(MMUEnableError::Other(\n                \"Translation granule not supported in HW\",\n            ));\n        }\n\n        // Prepare the memory attribute indirection register.\n        self.set_up_mair();\n\n        // Set the \"Translation Table Base Register\".\n        TTBR1_EL1.set_baddr(phys_tables_base_addr.as_usize() as u64);\n\n        self.configure_translation_control();\n\n        // Switch the MMU on.\n        //\n        // First, force all previous changes to be seen before the MMU is enabled.\n        barrier::isb(barrier::SY);\n\n        // Enable the MMU and turn on data and instruction caching.\n        SCTLR_EL1.modify(SCTLR_EL1::M::Enable + SCTLR_EL1::C::Cacheable + SCTLR_EL1::I::Cacheable);\n\n        // Force MMU init to complete before next instruction.\n        barrier::isb(barrier::SY);\n\n        Ok(())\n    }\n\n    #[inline(always)]\n    fn is_enabled(&self) -> bool {\n        SCTLR_EL1.matches_all(SCTLR_EL1::M::Enable)\n    }\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/_arch/aarch64/time.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural timer primitives.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::time::arch_time\n\nuse crate::warn;\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::{\n    num::{NonZeroU128, NonZeroU32, NonZeroU64},\n    ops::{Add, Div},\n    time::Duration,\n};\nuse tock_registers::interfaces::Readable;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nconst NANOSEC_PER_SEC: NonZeroU64 = NonZeroU64::new(1_000_000_000).unwrap();\n\n#[derive(Copy, Clone, PartialOrd, PartialEq)]\nstruct GenericTimerCounterValue(u64);\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n/// Boot assembly code overwrites this value with the value of CNTFRQ_EL0 before any Rust code is\n/// executed. This given value here is just a (safe) dummy.\n#[no_mangle]\nstatic ARCH_TIMER_COUNTER_FREQUENCY: NonZeroU32 = NonZeroU32::MIN;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nfn arch_timer_counter_frequency() -> NonZeroU32 {\n    // Read volatile is needed here to prevent the compiler from optimizing\n    // ARCH_TIMER_COUNTER_FREQUENCY away.\n    //\n    // This is safe, because all the safety requirements as stated in read_volatile()'s\n    // documentation are fulfilled.\n    unsafe { core::ptr::read_volatile(&ARCH_TIMER_COUNTER_FREQUENCY) }\n}\n\nimpl GenericTimerCounterValue {\n    pub const MAX: Self = GenericTimerCounterValue(u64::MAX);\n}\n\nimpl Add for GenericTimerCounterValue {\n    type Output = Self;\n\n    fn add(self, other: Self) -> Self {\n        GenericTimerCounterValue(self.0.wrapping_add(other.0))\n    }\n}\n\nimpl From<GenericTimerCounterValue> for Duration {\n    fn from(counter_value: GenericTimerCounterValue) -> Self {\n        if counter_value.0 == 0 {\n            return Duration::ZERO;\n        }\n\n        let frequency: NonZeroU64 = arch_timer_counter_frequency().into();\n\n        // Div<NonZeroU64> implementation for u64 cannot panic.\n        let secs = counter_value.0.div(frequency);\n\n        // This is safe, because frequency can never be greater than u32::MAX, which means the\n        // largest theoretical value for sub_second_counter_value is (u32::MAX - 1). Therefore,\n        // (sub_second_counter_value * NANOSEC_PER_SEC) cannot overflow an u64.\n        //\n        // The subsequent division ensures the result fits into u32, since the max result is smaller\n        // than NANOSEC_PER_SEC. Therefore, just cast it to u32 using `as`.\n        let sub_second_counter_value = counter_value.0 % frequency;\n        let nanos = unsafe { sub_second_counter_value.unchecked_mul(u64::from(NANOSEC_PER_SEC)) }\n            .div(frequency) as u32;\n\n        Duration::new(secs, nanos)\n    }\n}\n\nfn max_duration() -> Duration {\n    Duration::from(GenericTimerCounterValue::MAX)\n}\n\nimpl TryFrom<Duration> for GenericTimerCounterValue {\n    type Error = &'static str;\n\n    fn try_from(duration: Duration) -> Result<Self, Self::Error> {\n        if duration < resolution() {\n            return Ok(GenericTimerCounterValue(0));\n        }\n\n        if duration > max_duration() {\n            return Err(\"Conversion error. Duration too big\");\n        }\n\n        let frequency: u128 = u32::from(arch_timer_counter_frequency()) as u128;\n        let duration: u128 = duration.as_nanos();\n\n        // This is safe, because frequency can never be greater than u32::MAX, and\n        // (Duration::MAX.as_nanos() * u32::MAX) < u128::MAX.\n        let counter_value =\n            unsafe { duration.unchecked_mul(frequency) }.div(NonZeroU128::from(NANOSEC_PER_SEC));\n\n        // Since we checked above that we are <= max_duration(), just cast to u64.\n        Ok(GenericTimerCounterValue(counter_value as u64))\n    }\n}\n\n#[inline(always)]\nfn read_cntpct() -> GenericTimerCounterValue {\n    // Prevent that the counter is read ahead of time due to out-of-order execution.\n    barrier::isb(barrier::SY);\n    let cnt = CNTPCT_EL0.get();\n\n    GenericTimerCounterValue(cnt)\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The timer's resolution.\npub fn resolution() -> Duration {\n    Duration::from(GenericTimerCounterValue(1))\n}\n\n/// The uptime since power-on of the device.\n///\n/// This includes time consumed by firmware and bootloaders.\npub fn uptime() -> Duration {\n    read_cntpct().into()\n}\n\n/// Spin for a given duration.\npub fn spin_for(duration: Duration) {\n    let curr_counter_value = read_cntpct();\n\n    let counter_value_delta: GenericTimerCounterValue = match duration.try_into() {\n        Err(msg) => {\n            warn!(\"spin_for: {}. Skipping\", msg);\n            return;\n        }\n        Ok(val) => val,\n    };\n    let counter_value_target = curr_counter_value + counter_value_delta;\n\n    // Busy wait.\n    //\n    // Read CNTPCT_EL0 directly to avoid the ISB that is part of [`read_cntpct`].\n    while GenericTimerCounterValue(CNTPCT_EL0.get()) < counter_value_target {}\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/backtrace.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Backtracing support.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/backtrace.rs\"]\nmod arch_backtrace;\n\nuse crate::{\n    memory::{Address, Virtual},\n    symbols,\n};\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\n#[cfg(feature = \"test_build\")]\npub use arch_backtrace::{corrupt_link, corrupt_previous_frame_addr};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// A backtrace item.\n#[allow(missing_docs)]\npub enum BacktraceItem {\n    InvalidFramePointer(Address<Virtual>),\n    InvalidLink(Address<Virtual>),\n    Link(Address<Virtual>),\n}\n\n/// Pseudo-struct for printing a backtrace using its fmt::Display implementation.\npub struct Backtrace;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl fmt::Display for Backtrace {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        writeln!(f, \"Backtrace:\")?;\n        writeln!(\n            f,\n            \"      ----------------------------------------------------------------------------------------------\"\n        )?;\n        writeln!(\n            f,\n            \"          Address            Function containing address\"\n        )?;\n        writeln!(\n            f,\n            \"      ----------------------------------------------------------------------------------------------\"\n        )?;\n\n        let mut fmt_res: fmt::Result = Ok(());\n        let trace_formatter =\n            |maybe_iter: Option<&mut dyn Iterator<Item = BacktraceItem>>| match maybe_iter {\n                None => fmt_res = writeln!(f, \"ERROR! No valid stack frame found\"),\n                Some(iter) => {\n                    // Since the backtrace is printed, the first function is always\n                    // core::fmt::write. Skip 1 so it is excluded and doesn't bloat the output.\n                    for (i, backtrace_res) in iter.skip(1).enumerate() {\n                        match backtrace_res {\n                            BacktraceItem::InvalidFramePointer(addr) => {\n                                fmt_res = writeln!(\n                                    f,\n                                    \"      {:>2}. ERROR! \\\n                                    Encountered invalid frame pointer ({}) during backtrace\",\n                                    i + 1,\n                                    addr\n                                );\n                            }\n                            BacktraceItem::InvalidLink(addr) => {\n                                fmt_res = writeln!(\n                                    f,\n                                    \"      {:>2}. ERROR! \\\n                                    Link address ({}) is not contained in kernel .text section\",\n                                    i + 1,\n                                    addr\n                                );\n                            }\n                            BacktraceItem::Link(addr) => {\n                                fmt_res = writeln!(\n                                    f,\n                                    \"      {:>2}. {:016x} | {:<50}\",\n                                    i + 1,\n                                    addr.as_usize(),\n                                    match symbols::lookup_symbol(addr) {\n                                        Some(sym) => sym.name(),\n                                        _ => \"Symbol not found\",\n                                    }\n                                )\n                            }\n                        };\n\n                        if fmt_res.is_err() {\n                            break;\n                        }\n                    }\n                }\n            };\n\n        arch_backtrace::backtrace(trace_formatter);\n        fmt_res?;\n\n        writeln!(\n            f,\n            \"      ----------------------------------------------------------------------------------------------\"\n        )\n    }\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/bsp/device_driver/arm/gicv2/gicc.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GICC Driver - GIC CPU interface.\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    exception,\n    memory::{Address, Virtual},\n};\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields, register_structs,\n    registers::ReadWrite,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nregister_bitfields! {\n    u32,\n\n    /// CPU Interface Control Register\n    CTLR [\n        Enable OFFSET(0) NUMBITS(1) []\n    ],\n\n    /// Interrupt Priority Mask Register\n    PMR [\n        Priority OFFSET(0) NUMBITS(8) []\n    ],\n\n    /// Interrupt Acknowledge Register\n    IAR [\n        InterruptID OFFSET(0) NUMBITS(10) []\n    ],\n\n    /// End of Interrupt Register\n    EOIR [\n        EOIINTID OFFSET(0) NUMBITS(10) []\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    pub RegisterBlock {\n        (0x000 => CTLR: ReadWrite<u32, CTLR::Register>),\n        (0x004 => PMR: ReadWrite<u32, PMR::Register>),\n        (0x008 => _reserved1),\n        (0x00C => IAR: ReadWrite<u32, IAR::Register>),\n        (0x010 => EOIR: ReadWrite<u32, EOIR::Register>),\n        (0x014  => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the GIC CPU interface.\npub struct GICC {\n    registers: Registers,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GICC {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n        }\n    }\n\n    /// Accept interrupts of any priority.\n    ///\n    /// Quoting the GICv2 Architecture Specification:\n    ///\n    ///   \"Writing 255 to the GICC_PMR always sets it to the largest supported priority field\n    ///    value.\"\n    ///\n    /// # Safety\n    ///\n    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n    ///   of `&mut self`.\n    pub fn priority_accept_all(&self) {\n        self.registers.PMR.write(PMR::Priority.val(255)); // Comment in arch spec.\n    }\n\n    /// Enable the interface - start accepting IRQs.\n    ///\n    /// # Safety\n    ///\n    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n    ///   of `&mut self`.\n    pub fn enable(&self) {\n        self.registers.CTLR.write(CTLR::Enable::SET);\n    }\n\n    /// Extract the number of the highest-priority pending IRQ.\n    ///\n    /// Can only be called from IRQ context, which is ensured by taking an `IRQContext` token.\n    ///\n    /// # Safety\n    ///\n    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n    ///   of `&mut self`.\n    #[allow(clippy::trivially_copy_pass_by_ref)]\n    pub fn pending_irq_number<'irq_context>(\n        &self,\n        _ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) -> usize {\n        self.registers.IAR.read(IAR::InterruptID) as usize\n    }\n\n    /// Complete handling of the currently active IRQ.\n    ///\n    /// Can only be called from IRQ context, which is ensured by taking an `IRQContext` token.\n    ///\n    /// To be called after `pending_irq_number()`.\n    ///\n    /// # Safety\n    ///\n    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n    ///   of `&mut self`.\n    #[allow(clippy::trivially_copy_pass_by_ref)]\n    pub fn mark_comleted<'irq_context>(\n        &self,\n        irq_number: u32,\n        _ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) {\n        self.registers.EOIR.write(EOIR::EOIINTID.val(irq_number));\n    }\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/bsp/device_driver/arm/gicv2/gicd.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GICD Driver - GIC Distributor.\n//!\n//! # Glossary\n//!   - SPI - Shared Peripheral Interrupt.\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    memory::{Address, Virtual},\n    state, synchronization,\n    synchronization::IRQSafeNullLock,\n};\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields, register_structs,\n    registers::{ReadOnly, ReadWrite},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nregister_bitfields! {\n    u32,\n\n    /// Distributor Control Register\n    CTLR [\n        Enable OFFSET(0) NUMBITS(1) []\n    ],\n\n    /// Interrupt Controller Type Register\n    TYPER [\n        ITLinesNumber OFFSET(0)  NUMBITS(5) []\n    ],\n\n    /// Interrupt Processor Targets Registers\n    ITARGETSR [\n        Offset3 OFFSET(24) NUMBITS(8) [],\n        Offset2 OFFSET(16) NUMBITS(8) [],\n        Offset1 OFFSET(8)  NUMBITS(8) [],\n        Offset0 OFFSET(0)  NUMBITS(8) []\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    SharedRegisterBlock {\n        (0x000 => CTLR: ReadWrite<u32, CTLR::Register>),\n        (0x004 => TYPER: ReadOnly<u32, TYPER::Register>),\n        (0x008 => _reserved1),\n        (0x104 => ISENABLER: [ReadWrite<u32>; 31]),\n        (0x180 => _reserved2),\n        (0x820 => ITARGETSR: [ReadWrite<u32, ITARGETSR::Register>; 248]),\n        (0xC00 => @END),\n    }\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    BankedRegisterBlock {\n        (0x000 => _reserved1),\n        (0x100 => ISENABLER: ReadWrite<u32>),\n        (0x104 => _reserved2),\n        (0x800 => ITARGETSR: [ReadOnly<u32, ITARGETSR::Register>; 8]),\n        (0x820 => @END),\n    }\n}\n\n/// Abstraction for the non-banked parts of the associated MMIO registers.\ntype SharedRegisters = MMIODerefWrapper<SharedRegisterBlock>;\n\n/// Abstraction for the banked parts of the associated MMIO registers.\ntype BankedRegisters = MMIODerefWrapper<BankedRegisterBlock>;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the GIC Distributor.\npub struct GICD {\n    /// Access to shared registers is guarded with a lock.\n    shared_registers: IRQSafeNullLock<SharedRegisters>,\n\n    /// Access to banked registers is unguarded.\n    banked_registers: BankedRegisters,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl SharedRegisters {\n    /// Return the number of IRQs that this HW implements.\n    #[inline(always)]\n    fn num_irqs(&mut self) -> usize {\n        // Query number of implemented IRQs.\n        //\n        // Refer to GICv2 Architecture Specification, Section 4.3.2.\n        ((self.TYPER.read(TYPER::ITLinesNumber) as usize) + 1) * 32\n    }\n\n    /// Return a slice of the implemented ITARGETSR.\n    #[inline(always)]\n    fn implemented_itargets_slice(&mut self) -> &[ReadWrite<u32, ITARGETSR::Register>] {\n        assert!(self.num_irqs() >= 36);\n\n        // Calculate the max index of the shared ITARGETSR array.\n        //\n        // The first 32 IRQs are private, so not included in `shared_registers`. Each ITARGETS\n        // register has four entries, so shift right by two. Subtract one because we start\n        // counting at zero.\n        let spi_itargetsr_max_index = ((self.num_irqs() - 32) >> 2) - 1;\n\n        // Rust automatically inserts slice range sanity check, i.e. max >= min.\n        &self.ITARGETSR[0..spi_itargetsr_max_index]\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl GICD {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            shared_registers: IRQSafeNullLock::new(SharedRegisters::new(mmio_start_addr)),\n            banked_registers: BankedRegisters::new(mmio_start_addr),\n        }\n    }\n\n    /// Use a banked ITARGETSR to retrieve the executing core's GIC target mask.\n    ///\n    /// Quoting the GICv2 Architecture Specification:\n    ///\n    ///   \"GICD_ITARGETSR0 to GICD_ITARGETSR7 are read-only, and each field returns a value that\n    ///    corresponds only to the processor reading the register.\"\n    fn local_gic_target_mask(&self) -> u32 {\n        self.banked_registers.ITARGETSR[0].read(ITARGETSR::Offset0)\n    }\n\n    /// Route all SPIs to the boot core and enable the distributor.\n    pub fn boot_core_init(&self) {\n        assert!(\n            state::state_manager().is_init(),\n            \"Only allowed during kernel init phase\"\n        );\n\n        // Target all SPIs to the boot core only.\n        let mask = self.local_gic_target_mask();\n\n        self.shared_registers.lock(|regs| {\n            for i in regs.implemented_itargets_slice().iter() {\n                i.write(\n                    ITARGETSR::Offset3.val(mask)\n                        + ITARGETSR::Offset2.val(mask)\n                        + ITARGETSR::Offset1.val(mask)\n                        + ITARGETSR::Offset0.val(mask),\n                );\n            }\n\n            regs.CTLR.write(CTLR::Enable::SET);\n        });\n    }\n\n    /// Enable an interrupt.\n    pub fn enable(&self, irq_num: &super::IRQNumber) {\n        let irq_num = irq_num.get();\n\n        // Each bit in the u32 enable register corresponds to one IRQ number. Shift right by 5\n        // (division by 32) and arrive at the index for the respective ISENABLER[i].\n        let enable_reg_index = irq_num >> 5;\n        let enable_bit: u32 = 1u32 << (irq_num % 32);\n\n        // Check if we are handling a private or shared IRQ.\n        match irq_num {\n            // Private.\n            0..=31 => {\n                let enable_reg = &self.banked_registers.ISENABLER;\n                enable_reg.set(enable_reg.get() | enable_bit);\n            }\n            // Shared.\n            _ => {\n                let enable_reg_index_shared = enable_reg_index - 1;\n\n                self.shared_registers.lock(|regs| {\n                    let enable_reg = &regs.ISENABLER[enable_reg_index_shared];\n                    enable_reg.set(enable_reg.get() | enable_bit);\n                });\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/bsp/device_driver/arm/gicv2.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GICv2 Driver - ARM Generic Interrupt Controller v2.\n//!\n//! The following is a collection of excerpts with useful information from\n//!   - `Programmer's Guide for ARMv8-A`\n//!   - `ARM Generic Interrupt Controller Architecture Specification`\n//!\n//! # Programmer's Guide - 10.6.1 Configuration\n//!\n//! The GIC is accessed as a memory-mapped peripheral.\n//!\n//! All cores can access the common Distributor, but the CPU interface is banked, that is, each core\n//! uses the same address to access its own private CPU interface.\n//!\n//! It is not possible for a core to access the CPU interface of another core.\n//!\n//! # Architecture Specification - 10.6.2 Initialization\n//!\n//! Both the Distributor and the CPU interfaces are disabled at reset. The GIC must be initialized\n//! after reset before it can deliver interrupts to the core.\n//!\n//! In the Distributor, software must configure the priority, target, security and enable individual\n//! interrupts. The Distributor must subsequently be enabled through its control register\n//! (GICD_CTLR). For each CPU interface, software must program the priority mask and preemption\n//! settings.\n//!\n//! Each CPU interface block itself must be enabled through its control register (GICD_CTLR). This\n//! prepares the GIC to deliver interrupts to the core.\n//!\n//! Before interrupts are expected in the core, software prepares the core to take interrupts by\n//! setting a valid interrupt vector in the vector table, and clearing interrupt mask bits in\n//! PSTATE, and setting the routing controls.\n//!\n//! The entire interrupt mechanism in the system can be disabled by disabling the Distributor.\n//! Interrupt delivery to an individual core can be disabled by disabling its CPU interface.\n//! Individual interrupts can also be disabled (or enabled) in the distributor.\n//!\n//! For an interrupt to reach the core, the individual interrupt, Distributor and CPU interface must\n//! all be enabled. The interrupt also needs to be of sufficient priority, that is, higher than the\n//! core's priority mask.\n//!\n//! # Architecture Specification - 1.4.2 Interrupt types\n//!\n//! - Peripheral interrupt\n//!     - Private Peripheral Interrupt (PPI)\n//!         - This is a peripheral interrupt that is specific to a single processor.\n//!     - Shared Peripheral Interrupt (SPI)\n//!         - This is a peripheral interrupt that the Distributor can route to any of a specified\n//!           combination of processors.\n//!\n//! - Software-generated interrupt (SGI)\n//!     - This is an interrupt generated by software writing to a GICD_SGIR register in the GIC. The\n//!       system uses SGIs for interprocessor communication.\n//!     - An SGI has edge-triggered properties. The software triggering of the interrupt is\n//!       equivalent to the edge transition of the interrupt request signal.\n//!     - When an SGI occurs in a multiprocessor implementation, the CPUID field in the Interrupt\n//!       Acknowledge Register, GICC_IAR, or the Aliased Interrupt Acknowledge Register, GICC_AIAR,\n//!       identifies the processor that requested the interrupt.\n//!\n//! # Architecture Specification - 2.2.1 Interrupt IDs\n//!\n//! Interrupts from sources are identified using ID numbers. Each CPU interface can see up to 1020\n//! interrupts. The banking of SPIs and PPIs increases the total number of interrupts supported by\n//! the Distributor.\n//!\n//! The GIC assigns interrupt ID numbers ID0-ID1019 as follows:\n//!   - Interrupt numbers 32..1019 are used for SPIs.\n//!   - Interrupt numbers 0..31 are used for interrupts that are private to a CPU interface. These\n//!     interrupts are banked in the Distributor.\n//!       - A banked interrupt is one where the Distributor can have multiple interrupts with the\n//!         same ID. A banked interrupt is identified uniquely by its ID number and its associated\n//!         CPU interface number. Of the banked interrupt IDs:\n//!           - 00..15 SGIs\n//!           - 16..31 PPIs\n\nmod gicc;\nmod gicd;\n\nuse crate::{\n    bsp::{self, device_driver::common::BoundedUsize},\n    cpu, driver, exception,\n    memory::{Address, Virtual},\n    synchronization,\n    synchronization::InitStateLock,\n};\nuse alloc::vec::Vec;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\ntype HandlerTable = Vec<Option<exception::asynchronous::IRQHandlerDescriptor<IRQNumber>>>;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Used for the associated type of trait [`exception::asynchronous::interface::IRQManager`].\npub type IRQNumber = BoundedUsize<{ GICv2::MAX_IRQ_NUMBER }>;\n\n/// Representation of the GIC.\npub struct GICv2 {\n    /// The Distributor.\n    gicd: gicd::GICD,\n\n    /// The CPU Interface.\n    gicc: gicc::GICC,\n\n    /// Stores registered IRQ handlers. Writable only during kernel init. RO afterwards.\n    handler_table: InitStateLock<HandlerTable>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GICv2 {\n    const MAX_IRQ_NUMBER: usize = 1019;\n\n    pub const COMPATIBLE: &'static str = \"GICv2 (ARM Generic Interrupt Controller v2)\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(\n        gicd_mmio_start_addr: Address<Virtual>,\n        gicc_mmio_start_addr: Address<Virtual>,\n    ) -> Self {\n        Self {\n            gicd: gicd::GICD::new(gicd_mmio_start_addr),\n            gicc: gicc::GICC::new(gicc_mmio_start_addr),\n            handler_table: InitStateLock::new(Vec::new()),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::ReadWriteEx;\n\nimpl driver::interface::DeviceDriver for GICv2 {\n    type IRQNumberType = IRQNumber;\n\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n\n    unsafe fn init(&self) -> Result<(), &'static str> {\n        self.handler_table\n            .write(|table| table.resize(IRQNumber::MAX_INCLUSIVE + 1, None));\n\n        if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() {\n            self.gicd.boot_core_init();\n        }\n\n        self.gicc.priority_accept_all();\n        self.gicc.enable();\n\n        Ok(())\n    }\n}\n\nimpl exception::asynchronous::interface::IRQManager for GICv2 {\n    type IRQNumberType = IRQNumber;\n\n    fn register_handler(\n        &self,\n        irq_handler_descriptor: exception::asynchronous::IRQHandlerDescriptor<Self::IRQNumberType>,\n    ) -> Result<(), &'static str> {\n        self.handler_table.write(|table| {\n            let irq_number = irq_handler_descriptor.number().get();\n\n            if table[irq_number].is_some() {\n                return Err(\"IRQ handler already registered\");\n            }\n\n            table[irq_number] = Some(irq_handler_descriptor);\n\n            Ok(())\n        })\n    }\n\n    fn enable(&self, irq_number: &Self::IRQNumberType) {\n        self.gicd.enable(irq_number);\n    }\n\n    fn handle_pending_irqs<'irq_context>(\n        &'irq_context self,\n        ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) {\n        // Extract the highest priority pending IRQ number from the Interrupt Acknowledge Register\n        // (IAR).\n        let irq_number = self.gicc.pending_irq_number(ic);\n\n        // Guard against spurious interrupts.\n        if irq_number > GICv2::MAX_IRQ_NUMBER {\n            return;\n        }\n\n        // Call the IRQ handler. Panic if there is none.\n        self.handler_table.read(|table| {\n            match table[irq_number] {\n                None => panic!(\"No handler registered for IRQ {}\", irq_number),\n                Some(descriptor) => {\n                    // Call the IRQ handler. Panics on failure.\n                    descriptor.handler().handle().expect(\"Error handling IRQ\");\n                }\n            }\n        });\n\n        // Signal completion of handling.\n        self.gicc.mark_comleted(irq_number as u32, ic);\n    }\n\n    fn print_handler(&self) {\n        use crate::info;\n\n        info!(\"      Peripheral handler:\");\n\n        self.handler_table.read(|table| {\n            for (i, opt) in table.iter().skip(32).enumerate() {\n                if let Some(handler) = opt {\n                    info!(\"            {: >3}. {}\", i + 32, handler.name());\n                }\n            }\n        });\n    }\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/bsp/device_driver/arm.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! ARM driver top level.\n\npub mod gicv2;\n\npub use gicv2::*;\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GPIO Driver.\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    driver,\n    exception::asynchronous::IRQNumber,\n    memory::{Address, Virtual},\n    synchronization,\n    synchronization::IRQSafeNullLock,\n};\nuse tock_registers::{\n    interfaces::{ReadWriteable, Writeable},\n    register_bitfields, register_structs,\n    registers::ReadWrite,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// GPIO registers.\n//\n// Descriptions taken from\n// - https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf\n// - https://datasheets.raspberrypi.org/bcm2711/bcm2711-peripherals.pdf\nregister_bitfields! {\n    u32,\n\n    /// GPIO Function Select 1\n    GPFSEL1 [\n        /// Pin 15\n        FSEL15 OFFSET(15) NUMBITS(3) [\n            Input = 0b000,\n            Output = 0b001,\n            AltFunc0 = 0b100  // PL011 UART RX\n\n        ],\n\n        /// Pin 14\n        FSEL14 OFFSET(12) NUMBITS(3) [\n            Input = 0b000,\n            Output = 0b001,\n            AltFunc0 = 0b100  // PL011 UART TX\n        ]\n    ],\n\n    /// GPIO Pull-up/down Register\n    ///\n    /// BCM2837 only.\n    GPPUD [\n        /// Controls the actuation of the internal pull-up/down control line to ALL the GPIO pins.\n        PUD OFFSET(0) NUMBITS(2) [\n            Off = 0b00,\n            PullDown = 0b01,\n            PullUp = 0b10\n        ]\n    ],\n\n    /// GPIO Pull-up/down Clock Register 0\n    ///\n    /// BCM2837 only.\n    GPPUDCLK0 [\n        /// Pin 15\n        PUDCLK15 OFFSET(15) NUMBITS(1) [\n            NoEffect = 0,\n            AssertClock = 1\n        ],\n\n        /// Pin 14\n        PUDCLK14 OFFSET(14) NUMBITS(1) [\n            NoEffect = 0,\n            AssertClock = 1\n        ]\n    ],\n\n    /// GPIO Pull-up / Pull-down Register 0\n    ///\n    /// BCM2711 only.\n    GPIO_PUP_PDN_CNTRL_REG0 [\n        /// Pin 15\n        GPIO_PUP_PDN_CNTRL15 OFFSET(30) NUMBITS(2) [\n            NoResistor = 0b00,\n            PullUp = 0b01\n        ],\n\n        /// Pin 14\n        GPIO_PUP_PDN_CNTRL14 OFFSET(28) NUMBITS(2) [\n            NoResistor = 0b00,\n            PullUp = 0b01\n        ]\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    RegisterBlock {\n        (0x00 => _reserved1),\n        (0x04 => GPFSEL1: ReadWrite<u32, GPFSEL1::Register>),\n        (0x08 => _reserved2),\n        (0x94 => GPPUD: ReadWrite<u32, GPPUD::Register>),\n        (0x98 => GPPUDCLK0: ReadWrite<u32, GPPUDCLK0::Register>),\n        (0x9C => _reserved3),\n        (0xE4 => GPIO_PUP_PDN_CNTRL_REG0: ReadWrite<u32, GPIO_PUP_PDN_CNTRL_REG0::Register>),\n        (0xE8 => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\nstruct GPIOInner {\n    registers: Registers,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the GPIO HW.\npub struct GPIO {\n    inner: IRQSafeNullLock<GPIOInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GPIOInner {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n        }\n    }\n\n    /// Disable pull-up/down on pins 14 and 15.\n    #[cfg(feature = \"bsp_rpi3\")]\n    fn disable_pud_14_15_bcm2837(&mut self) {\n        use crate::time;\n        use core::time::Duration;\n\n        // The Linux 2837 GPIO driver waits 1 µs between the steps.\n        const DELAY: Duration = Duration::from_micros(1);\n\n        self.registers.GPPUD.write(GPPUD::PUD::Off);\n        time::time_manager().spin_for(DELAY);\n\n        self.registers\n            .GPPUDCLK0\n            .write(GPPUDCLK0::PUDCLK15::AssertClock + GPPUDCLK0::PUDCLK14::AssertClock);\n        time::time_manager().spin_for(DELAY);\n\n        self.registers.GPPUD.write(GPPUD::PUD::Off);\n        self.registers.GPPUDCLK0.set(0);\n    }\n\n    /// Disable pull-up/down on pins 14 and 15.\n    #[cfg(feature = \"bsp_rpi4\")]\n    fn disable_pud_14_15_bcm2711(&mut self) {\n        self.registers.GPIO_PUP_PDN_CNTRL_REG0.write(\n            GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL15::PullUp\n                + GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL14::PullUp,\n        );\n    }\n\n    /// Map PL011 UART as standard output.\n    ///\n    /// TX to pin 14\n    /// RX to pin 15\n    pub fn map_pl011_uart(&mut self) {\n        // Select the UART on pins 14 and 15.\n        self.registers\n            .GPFSEL1\n            .modify(GPFSEL1::FSEL15::AltFunc0 + GPFSEL1::FSEL14::AltFunc0);\n\n        // Disable pull-up/down on pins 14 and 15.\n        #[cfg(feature = \"bsp_rpi3\")]\n        self.disable_pud_14_15_bcm2837();\n\n        #[cfg(feature = \"bsp_rpi4\")]\n        self.disable_pud_14_15_bcm2711();\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GPIO {\n    pub const COMPATIBLE: &'static str = \"BCM GPIO\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            inner: IRQSafeNullLock::new(GPIOInner::new(mmio_start_addr)),\n        }\n    }\n\n    /// Concurrency safe version of `GPIOInner.map_pl011_uart()`\n    pub fn map_pl011_uart(&self) {\n        self.inner.lock(|inner| inner.map_pl011_uart())\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl driver::interface::DeviceDriver for GPIO {\n    type IRQNumberType = IRQNumber;\n\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Peripheral Interrupt Controller Driver.\n//!\n//! # Resources\n//!\n//! - <https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf>\n\nuse super::{PendingIRQs, PeripheralIRQ};\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    exception,\n    memory::{Address, Virtual},\n    synchronization,\n    synchronization::{IRQSafeNullLock, InitStateLock},\n};\nuse alloc::vec::Vec;\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_structs,\n    registers::{ReadOnly, WriteOnly},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    WORegisterBlock {\n        (0x00 => _reserved1),\n        (0x10 => ENABLE_1: WriteOnly<u32>),\n        (0x14 => ENABLE_2: WriteOnly<u32>),\n        (0x18 => @END),\n    }\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    RORegisterBlock {\n        (0x00 => _reserved1),\n        (0x04 => PENDING_1: ReadOnly<u32>),\n        (0x08 => PENDING_2: ReadOnly<u32>),\n        (0x0c => @END),\n    }\n}\n\n/// Abstraction for the WriteOnly parts of the associated MMIO registers.\ntype WriteOnlyRegisters = MMIODerefWrapper<WORegisterBlock>;\n\n/// Abstraction for the ReadOnly parts of the associated MMIO registers.\ntype ReadOnlyRegisters = MMIODerefWrapper<RORegisterBlock>;\n\ntype HandlerTable = Vec<Option<exception::asynchronous::IRQHandlerDescriptor<PeripheralIRQ>>>;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the peripheral interrupt controller.\npub struct PeripheralIC {\n    /// Access to write registers is guarded with a lock.\n    wo_registers: IRQSafeNullLock<WriteOnlyRegisters>,\n\n    /// Register read access is unguarded.\n    ro_registers: ReadOnlyRegisters,\n\n    /// Stores registered IRQ handlers. Writable only during kernel init. RO afterwards.\n    handler_table: InitStateLock<HandlerTable>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PeripheralIC {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            wo_registers: IRQSafeNullLock::new(WriteOnlyRegisters::new(mmio_start_addr)),\n            ro_registers: ReadOnlyRegisters::new(mmio_start_addr),\n            handler_table: InitStateLock::new(Vec::new()),\n        }\n    }\n\n    /// Called by the kernel to bring up the device.\n    pub fn init(&self) {\n        self.handler_table\n            .write(|table| table.resize(PeripheralIRQ::MAX_INCLUSIVE + 1, None));\n    }\n\n    /// Query the list of pending IRQs.\n    fn pending_irqs(&self) -> PendingIRQs {\n        let pending_mask: u64 = (u64::from(self.ro_registers.PENDING_2.get()) << 32)\n            | u64::from(self.ro_registers.PENDING_1.get());\n\n        PendingIRQs::new(pending_mask)\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::{Mutex, ReadWriteEx};\n\nimpl exception::asynchronous::interface::IRQManager for PeripheralIC {\n    type IRQNumberType = PeripheralIRQ;\n\n    fn register_handler(\n        &self,\n        irq_handler_descriptor: exception::asynchronous::IRQHandlerDescriptor<Self::IRQNumberType>,\n    ) -> Result<(), &'static str> {\n        self.handler_table.write(|table| {\n            let irq_number = irq_handler_descriptor.number().get();\n\n            if table[irq_number].is_some() {\n                return Err(\"IRQ handler already registered\");\n            }\n\n            table[irq_number] = Some(irq_handler_descriptor);\n\n            Ok(())\n        })\n    }\n\n    fn enable(&self, irq: &Self::IRQNumberType) {\n        self.wo_registers.lock(|regs| {\n            let enable_reg = if irq.get() <= 31 {\n                &regs.ENABLE_1\n            } else {\n                &regs.ENABLE_2\n            };\n\n            let enable_bit: u32 = 1 << (irq.get() % 32);\n\n            // Writing a 1 to a bit will set the corresponding IRQ enable bit. All other IRQ enable\n            // bits are unaffected. So we don't need read and OR'ing here.\n            enable_reg.set(enable_bit);\n        });\n    }\n\n    fn handle_pending_irqs<'irq_context>(\n        &'irq_context self,\n        _ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) {\n        self.handler_table.read(|table| {\n            for irq_number in self.pending_irqs() {\n                match table[irq_number] {\n                    None => panic!(\"No handler registered for IRQ {}\", irq_number),\n                    Some(descriptor) => {\n                        // Call the IRQ handler. Panics on failure.\n                        descriptor.handler().handle().expect(\"Error handling IRQ\");\n                    }\n                }\n            }\n        })\n    }\n\n    fn print_handler(&self) {\n        use crate::info;\n\n        info!(\"      Peripheral handler:\");\n\n        self.handler_table.read(|table| {\n            for (i, opt) in table.iter().enumerate() {\n                if let Some(handler) = opt {\n                    info!(\"            {: >3}. {}\", i, handler.name());\n                }\n            }\n        });\n    }\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Interrupt Controller Driver.\n\nmod peripheral_ic;\n\nuse crate::{\n    bsp::device_driver::common::BoundedUsize,\n    driver,\n    exception::{self, asynchronous::IRQHandlerDescriptor},\n    memory::{Address, Virtual},\n};\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Wrapper struct for a bitmask indicating pending IRQ numbers.\nstruct PendingIRQs {\n    bitmask: u64,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub type LocalIRQ = BoundedUsize<{ InterruptController::MAX_LOCAL_IRQ_NUMBER }>;\npub type PeripheralIRQ = BoundedUsize<{ InterruptController::MAX_PERIPHERAL_IRQ_NUMBER }>;\n\n/// Used for the associated type of trait [`exception::asynchronous::interface::IRQManager`].\n#[derive(Copy, Clone)]\n#[allow(missing_docs)]\npub enum IRQNumber {\n    Local(LocalIRQ),\n    Peripheral(PeripheralIRQ),\n}\n\n/// Representation of the Interrupt Controller.\npub struct InterruptController {\n    periph: peripheral_ic::PeripheralIC,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PendingIRQs {\n    pub fn new(bitmask: u64) -> Self {\n        Self { bitmask }\n    }\n}\n\nimpl Iterator for PendingIRQs {\n    type Item = usize;\n\n    fn next(&mut self) -> Option<Self::Item> {\n        if self.bitmask == 0 {\n            return None;\n        }\n\n        let next = self.bitmask.trailing_zeros() as usize;\n        self.bitmask &= self.bitmask.wrapping_sub(1);\n        Some(next)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl fmt::Display for IRQNumber {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        match self {\n            Self::Local(number) => write!(f, \"Local({})\", number),\n            Self::Peripheral(number) => write!(f, \"Peripheral({})\", number),\n        }\n    }\n}\n\nimpl InterruptController {\n    // Restrict to 3 for now. This makes future code for local_ic.rs more straight forward.\n    const MAX_LOCAL_IRQ_NUMBER: usize = 3;\n    const MAX_PERIPHERAL_IRQ_NUMBER: usize = 63;\n\n    pub const COMPATIBLE: &'static str = \"BCM Interrupt Controller\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(periph_mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            periph: peripheral_ic::PeripheralIC::new(periph_mmio_start_addr),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\n\nimpl driver::interface::DeviceDriver for InterruptController {\n    type IRQNumberType = IRQNumber;\n\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n\n    unsafe fn init(&self) -> Result<(), &'static str> {\n        self.periph.init();\n\n        Ok(())\n    }\n}\n\nimpl exception::asynchronous::interface::IRQManager for InterruptController {\n    type IRQNumberType = IRQNumber;\n\n    fn register_handler(\n        &self,\n        irq_handler_descriptor: exception::asynchronous::IRQHandlerDescriptor<Self::IRQNumberType>,\n    ) -> Result<(), &'static str> {\n        match irq_handler_descriptor.number() {\n            IRQNumber::Local(_) => unimplemented!(\"Local IRQ controller not implemented.\"),\n            IRQNumber::Peripheral(pirq) => {\n                let periph_descriptor = IRQHandlerDescriptor::new(\n                    pirq,\n                    irq_handler_descriptor.name(),\n                    irq_handler_descriptor.handler(),\n                );\n\n                self.periph.register_handler(periph_descriptor)\n            }\n        }\n    }\n\n    fn enable(&self, irq: &Self::IRQNumberType) {\n        match irq {\n            IRQNumber::Local(_) => unimplemented!(\"Local IRQ controller not implemented.\"),\n            IRQNumber::Peripheral(pirq) => self.periph.enable(pirq),\n        }\n    }\n\n    fn handle_pending_irqs<'irq_context>(\n        &'irq_context self,\n        ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) {\n        // It can only be a peripheral IRQ pending because enable() does not support local IRQs yet.\n        self.periph.handle_pending_irqs(ic)\n    }\n\n    fn print_handler(&self) {\n        self.periph.print_handler();\n    }\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! PL011 UART driver.\n//!\n//! # Resources\n//!\n//! - <https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf>\n//! - <https://developer.arm.com/documentation/ddi0183/latest>\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    console, cpu, driver,\n    exception::{self, asynchronous::IRQNumber},\n    memory::{Address, Virtual},\n    synchronization,\n    synchronization::IRQSafeNullLock,\n};\nuse core::fmt;\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields, register_structs,\n    registers::{ReadOnly, ReadWrite, WriteOnly},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// PL011 UART registers.\n//\n// Descriptions taken from \"PrimeCell UART (PL011) Technical Reference Manual\" r1p5.\nregister_bitfields! {\n    u32,\n\n    /// Flag Register.\n    FR [\n        /// Transmit FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n        /// Line Control Register, LCR_H.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the transmit holding register is empty.\n        /// - If the FIFO is enabled, the TXFE bit is set when the transmit FIFO is empty.\n        /// - This bit does not indicate if there is data in the transmit shift register.\n        TXFE OFFSET(7) NUMBITS(1) [],\n\n        /// Transmit FIFO full. The meaning of this bit depends on the state of the FEN bit in the\n        /// LCR_H Register.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the transmit holding register is full.\n        /// - If the FIFO is enabled, the TXFF bit is set when the transmit FIFO is full.\n        TXFF OFFSET(5) NUMBITS(1) [],\n\n        /// Receive FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n        /// LCR_H Register.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the receive holding register is empty.\n        /// - If the FIFO is enabled, the RXFE bit is set when the receive FIFO is empty.\n        RXFE OFFSET(4) NUMBITS(1) [],\n\n        /// UART busy. If this bit is set to 1, the UART is busy transmitting data. This bit remains\n        /// set until the complete byte, including all the stop bits, has been sent from the shift\n        /// register.\n        ///\n        /// This bit is set as soon as the transmit FIFO becomes non-empty, regardless of whether\n        /// the UART is enabled or not.\n        BUSY OFFSET(3) NUMBITS(1) []\n    ],\n\n    /// Integer Baud Rate Divisor.\n    IBRD [\n        /// The integer baud rate divisor.\n        BAUD_DIVINT OFFSET(0) NUMBITS(16) []\n    ],\n\n    /// Fractional Baud Rate Divisor.\n    FBRD [\n        ///  The fractional baud rate divisor.\n        BAUD_DIVFRAC OFFSET(0) NUMBITS(6) []\n    ],\n\n    /// Line Control Register.\n    LCR_H [\n        /// Word length. These bits indicate the number of data bits transmitted or received in a\n        /// frame.\n        #[allow(clippy::enum_variant_names)]\n        WLEN OFFSET(5) NUMBITS(2) [\n            FiveBit = 0b00,\n            SixBit = 0b01,\n            SevenBit = 0b10,\n            EightBit = 0b11\n        ],\n\n        /// Enable FIFOs:\n        ///\n        /// 0 = FIFOs are disabled (character mode) that is, the FIFOs become 1-byte-deep holding\n        /// registers.\n        ///\n        /// 1 = Transmit and receive FIFO buffers are enabled (FIFO mode).\n        FEN  OFFSET(4) NUMBITS(1) [\n            FifosDisabled = 0,\n            FifosEnabled = 1\n        ]\n    ],\n\n    /// Control Register.\n    CR [\n        /// Receive enable. If this bit is set to 1, the receive section of the UART is enabled.\n        /// Data reception occurs for either UART signals or SIR signals depending on the setting of\n        /// the SIREN bit. When the UART is disabled in the middle of reception, it completes the\n        /// current character before stopping.\n        RXE OFFSET(9) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// Transmit enable. If this bit is set to 1, the transmit section of the UART is enabled.\n        /// Data transmission occurs for either UART signals, or SIR signals depending on the\n        /// setting of the SIREN bit. When the UART is disabled in the middle of transmission, it\n        /// completes the current character before stopping.\n        TXE OFFSET(8) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// UART enable:\n        ///\n        /// 0 = UART is disabled. If the UART is disabled in the middle of transmission or\n        /// reception, it completes the current character before stopping.\n        ///\n        /// 1 = The UART is enabled. Data transmission and reception occurs for either UART signals\n        /// or SIR signals depending on the setting of the SIREN bit\n        UARTEN OFFSET(0) NUMBITS(1) [\n            /// If the UART is disabled in the middle of transmission or reception, it completes the\n            /// current character before stopping.\n            Disabled = 0,\n            Enabled = 1\n        ]\n    ],\n\n    /// Interrupt FIFO Level Select Register.\n    IFLS [\n        /// Receive interrupt FIFO level select. The trigger points for the receive interrupt are as\n        /// follows.\n        RXIFLSEL OFFSET(3) NUMBITS(5) [\n            OneEigth = 0b000,\n            OneQuarter = 0b001,\n            OneHalf = 0b010,\n            ThreeQuarters = 0b011,\n            SevenEights = 0b100\n        ]\n    ],\n\n    /// Interrupt Mask Set/Clear Register.\n    IMSC [\n        /// Receive timeout interrupt mask. A read returns the current mask for the UARTRTINTR\n        /// interrupt.\n        ///\n        /// - On a write of 1, the mask of the UARTRTINTR interrupt is set.\n        /// - A write of 0 clears the mask.\n        RTIM OFFSET(6) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// Receive interrupt mask. A read returns the current mask for the UARTRXINTR interrupt.\n        ///\n        /// - On a write of 1, the mask of the UARTRXINTR interrupt is set.\n        /// - A write of 0 clears the mask.\n        RXIM OFFSET(4) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ]\n    ],\n\n    /// Masked Interrupt Status Register.\n    MIS [\n        /// Receive timeout masked interrupt status. Returns the masked interrupt state of the\n        /// UARTRTINTR interrupt.\n        RTMIS OFFSET(6) NUMBITS(1) [],\n\n        /// Receive masked interrupt status. Returns the masked interrupt state of the UARTRXINTR\n        /// interrupt.\n        RXMIS OFFSET(4) NUMBITS(1) []\n    ],\n\n    /// Interrupt Clear Register.\n    ICR [\n        /// Meta field for all pending interrupts.\n        ALL OFFSET(0) NUMBITS(11) []\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    pub RegisterBlock {\n        (0x00 => DR: ReadWrite<u32>),\n        (0x04 => _reserved1),\n        (0x18 => FR: ReadOnly<u32, FR::Register>),\n        (0x1c => _reserved2),\n        (0x24 => IBRD: WriteOnly<u32, IBRD::Register>),\n        (0x28 => FBRD: WriteOnly<u32, FBRD::Register>),\n        (0x2c => LCR_H: WriteOnly<u32, LCR_H::Register>),\n        (0x30 => CR: WriteOnly<u32, CR::Register>),\n        (0x34 => IFLS: ReadWrite<u32, IFLS::Register>),\n        (0x38 => IMSC: ReadWrite<u32, IMSC::Register>),\n        (0x3C => _reserved3),\n        (0x40 => MIS: ReadOnly<u32, MIS::Register>),\n        (0x44 => ICR: WriteOnly<u32, ICR::Register>),\n        (0x48 => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\n#[derive(PartialEq)]\nenum BlockingMode {\n    Blocking,\n    NonBlocking,\n}\n\nstruct PL011UartInner {\n    registers: Registers,\n    chars_written: usize,\n    chars_read: usize,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the UART.\npub struct PL011Uart {\n    inner: IRQSafeNullLock<PL011UartInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PL011UartInner {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n            chars_written: 0,\n            chars_read: 0,\n        }\n    }\n\n    /// Set up baud rate and characteristics.\n    ///\n    /// This results in 8N1 and 921_600 baud.\n    ///\n    /// The calculation for the BRD is (we set the clock to 48 MHz in config.txt):\n    /// `(48_000_000 / 16) / 921_600 = 3.2552083`.\n    ///\n    /// This means the integer part is `3` and goes into the `IBRD`.\n    /// The fractional part is `0.2552083`.\n    ///\n    /// `FBRD` calculation according to the PL011 Technical Reference Manual:\n    /// `INTEGER((0.2552083 * 64) + 0.5) = 16`.\n    ///\n    /// Therefore, the generated baud rate divider is: `3 + 16/64 = 3.25`. Which results in a\n    /// genrated baud rate of `48_000_000 / (16 * 3.25) = 923_077`.\n    ///\n    /// Error = `((923_077 - 921_600) / 921_600) * 100 = 0.16%`.\n    pub fn init(&mut self) {\n        // Execution can arrive here while there are still characters queued in the TX FIFO and\n        // actively being sent out by the UART hardware. If the UART is turned off in this case,\n        // those queued characters would be lost.\n        //\n        // For example, this can happen during runtime on a call to panic!(), because panic!()\n        // initializes its own UART instance and calls init().\n        //\n        // Hence, flush first to ensure all pending characters are transmitted.\n        self.flush();\n\n        // Turn the UART off temporarily.\n        self.registers.CR.set(0);\n\n        // Clear all pending interrupts.\n        self.registers.ICR.write(ICR::ALL::CLEAR);\n\n        // From the PL011 Technical Reference Manual:\n        //\n        // The LCR_H, IBRD, and FBRD registers form the single 30-bit wide LCR Register that is\n        // updated on a single write strobe generated by a LCR_H write. So, to internally update the\n        // contents of IBRD or FBRD, a LCR_H write must always be performed at the end.\n        //\n        // Set the baud rate, 8N1 and FIFO enabled.\n        self.registers.IBRD.write(IBRD::BAUD_DIVINT.val(3));\n        self.registers.FBRD.write(FBRD::BAUD_DIVFRAC.val(16));\n        self.registers\n            .LCR_H\n            .write(LCR_H::WLEN::EightBit + LCR_H::FEN::FifosEnabled);\n\n        // Set RX FIFO fill level at 1/8.\n        self.registers.IFLS.write(IFLS::RXIFLSEL::OneEigth);\n\n        // Enable RX IRQ + RX timeout IRQ.\n        self.registers\n            .IMSC\n            .write(IMSC::RXIM::Enabled + IMSC::RTIM::Enabled);\n\n        // Turn the UART on.\n        self.registers\n            .CR\n            .write(CR::UARTEN::Enabled + CR::TXE::Enabled + CR::RXE::Enabled);\n    }\n\n    /// Send a character.\n    fn write_char(&mut self, c: char) {\n        // Spin while TX FIFO full is set, waiting for an empty slot.\n        while self.registers.FR.matches_all(FR::TXFF::SET) {\n            cpu::nop();\n        }\n\n        // Write the character to the buffer.\n        self.registers.DR.set(c as u32);\n\n        self.chars_written += 1;\n    }\n\n    /// Send a slice of characters.\n    fn write_array(&mut self, a: &[char]) {\n        for c in a {\n            self.write_char(*c);\n        }\n    }\n\n    /// Block execution until the last buffered character has been physically put on the TX wire.\n    fn flush(&self) {\n        // Spin until the busy bit is cleared.\n        while self.registers.FR.matches_all(FR::BUSY::SET) {\n            cpu::nop();\n        }\n    }\n\n    /// Retrieve a character.\n    fn read_char_converting(&mut self, blocking_mode: BlockingMode) -> Option<char> {\n        // If RX FIFO is empty,\n        if self.registers.FR.matches_all(FR::RXFE::SET) {\n            // immediately return in non-blocking mode.\n            if blocking_mode == BlockingMode::NonBlocking {\n                return None;\n            }\n\n            // Otherwise, wait until a char was received.\n            while self.registers.FR.matches_all(FR::RXFE::SET) {\n                cpu::nop();\n            }\n        }\n\n        // Read one character.\n        let mut ret = self.registers.DR.get() as u8 as char;\n\n        // Convert carrige return to newline.\n        if ret == '\\r' {\n            ret = '\\n'\n        }\n\n        // Update statistics.\n        self.chars_read += 1;\n\n        Some(ret)\n    }\n}\n\n/// Implementing `core::fmt::Write` enables usage of the `format_args!` macros, which in turn are\n/// used to implement the `kernel`'s `print!` and `println!` macros. By implementing `write_str()`,\n/// we get `write_fmt()` automatically.\n///\n/// The function takes an `&mut self`, so it must be implemented for the inner struct.\n///\n/// See [`src/print.rs`].\n///\n/// [`src/print.rs`]: ../../print/index.html\nimpl fmt::Write for PL011UartInner {\n    fn write_str(&mut self, s: &str) -> fmt::Result {\n        for c in s.chars() {\n            self.write_char(c);\n        }\n\n        Ok(())\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PL011Uart {\n    pub const COMPATIBLE: &'static str = \"BCM PL011 UART\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            inner: IRQSafeNullLock::new(PL011UartInner::new(mmio_start_addr)),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl driver::interface::DeviceDriver for PL011Uart {\n    type IRQNumberType = IRQNumber;\n\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n\n    unsafe fn init(&self) -> Result<(), &'static str> {\n        self.inner.lock(|inner| inner.init());\n\n        Ok(())\n    }\n\n    fn register_and_enable_irq_handler(\n        &'static self,\n        irq_number: &Self::IRQNumberType,\n    ) -> Result<(), &'static str> {\n        use exception::asynchronous::{irq_manager, IRQHandlerDescriptor};\n\n        let descriptor = IRQHandlerDescriptor::new(*irq_number, Self::COMPATIBLE, self);\n\n        irq_manager().register_handler(descriptor)?;\n        irq_manager().enable(irq_number);\n\n        Ok(())\n    }\n}\n\nimpl console::interface::Write for PL011Uart {\n    /// Passthrough of `args` to the `core::fmt::Write` implementation, but guarded by a Mutex to\n    /// serialize access.\n    fn write_char(&self, c: char) {\n        self.inner.lock(|inner| inner.write_char(c));\n    }\n\n    fn write_array(&self, a: &[char]) {\n        self.inner.lock(|inner| inner.write_array(a));\n    }\n\n    fn write_fmt(&self, args: core::fmt::Arguments) -> fmt::Result {\n        // Fully qualified syntax for the call to `core::fmt::Write::write_fmt()` to increase\n        // readability.\n        self.inner.lock(|inner| fmt::Write::write_fmt(inner, args))\n    }\n\n    fn flush(&self) {\n        // Spin until TX FIFO empty is set.\n        self.inner.lock(|inner| inner.flush());\n    }\n}\n\nimpl console::interface::Read for PL011Uart {\n    fn read_char(&self) -> char {\n        self.inner\n            .lock(|inner| inner.read_char_converting(BlockingMode::Blocking).unwrap())\n    }\n\n    fn clear_rx(&self) {\n        // Read from the RX FIFO until it is indicating empty.\n        while self\n            .inner\n            .lock(|inner| inner.read_char_converting(BlockingMode::NonBlocking))\n            .is_some()\n        {}\n    }\n}\n\nimpl console::interface::Statistics for PL011Uart {\n    fn chars_written(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_written)\n    }\n\n    fn chars_read(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_read)\n    }\n}\n\nimpl console::interface::All for PL011Uart {}\n\nimpl exception::asynchronous::interface::IRQHandler for PL011Uart {\n    fn handle(&self) -> Result<(), &'static str> {\n        self.inner.lock(|inner| {\n            let pending = inner.registers.MIS.extract();\n\n            // Clear all pending IRQs.\n            inner.registers.ICR.write(ICR::ALL::CLEAR);\n\n            // Check for any kind of RX interrupt.\n            if pending.matches_any(MIS::RXMIS::SET + MIS::RTMIS::SET) {\n                // Echo any received characters.\n                while let Some(c) = inner.read_char_converting(BlockingMode::NonBlocking) {\n                    inner.write_char(c)\n                }\n            }\n        });\n\n        Ok(())\n    }\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/bsp/device_driver/bcm.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BCM driver top level.\n\nmod bcm2xxx_gpio;\n#[cfg(feature = \"bsp_rpi3\")]\nmod bcm2xxx_interrupt_controller;\nmod bcm2xxx_pl011_uart;\n\npub use bcm2xxx_gpio::*;\n#[cfg(feature = \"bsp_rpi3\")]\npub use bcm2xxx_interrupt_controller::*;\npub use bcm2xxx_pl011_uart::*;\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/bsp/device_driver/common.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Common device driver code.\n\nuse crate::memory::{Address, Virtual};\nuse core::{fmt, marker::PhantomData, ops};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct MMIODerefWrapper<T> {\n    start_addr: Address<Virtual>,\n    phantom: PhantomData<fn() -> T>,\n}\n\n/// A wrapper type for usize with integrated range bound check.\n#[derive(Copy, Clone)]\npub struct BoundedUsize<const MAX_INCLUSIVE: usize>(usize);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T> MMIODerefWrapper<T> {\n    /// Create an instance.\n    pub const unsafe fn new(start_addr: Address<Virtual>) -> Self {\n        Self {\n            start_addr,\n            phantom: PhantomData,\n        }\n    }\n}\n\nimpl<T> ops::Deref for MMIODerefWrapper<T> {\n    type Target = T;\n\n    fn deref(&self) -> &Self::Target {\n        unsafe { &*(self.start_addr.as_usize() as *const _) }\n    }\n}\n\nimpl<const MAX_INCLUSIVE: usize> BoundedUsize<{ MAX_INCLUSIVE }> {\n    pub const MAX_INCLUSIVE: usize = MAX_INCLUSIVE;\n\n    /// Creates a new instance if number <= MAX_INCLUSIVE.\n    pub const fn new(number: usize) -> Self {\n        assert!(number <= MAX_INCLUSIVE);\n\n        Self(number)\n    }\n\n    /// Return the wrapped number.\n    pub const fn get(self) -> usize {\n        self.0\n    }\n}\n\nimpl<const MAX_INCLUSIVE: usize> fmt::Display for BoundedUsize<{ MAX_INCLUSIVE }> {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        write!(f, \"{}\", self.0)\n    }\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/bsp/device_driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Device driver.\n\n#[cfg(feature = \"bsp_rpi4\")]\nmod arm;\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod bcm;\nmod common;\n\n#[cfg(feature = \"bsp_rpi4\")]\npub use arm::*;\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use bcm::*;\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/bsp/raspberrypi/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Processor code.\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Used by `arch` code to find the early boot core.\n#[no_mangle]\n#[link_section = \".text._start_arguments\"]\npub static BOOT_CORE_ID: u64 = 0;\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/bsp/raspberrypi/driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP driver support.\n\nuse super::{exception, memory::map::mmio};\nuse crate::{\n    bsp::device_driver,\n    console, driver as generic_driver,\n    exception::{self as generic_exception},\n    memory,\n    memory::mmu::MMIODescriptor,\n};\nuse core::{\n    mem::MaybeUninit,\n    sync::atomic::{AtomicBool, Ordering},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic mut PL011_UART: MaybeUninit<device_driver::PL011Uart> = MaybeUninit::uninit();\nstatic mut GPIO: MaybeUninit<device_driver::GPIO> = MaybeUninit::uninit();\n\n#[cfg(feature = \"bsp_rpi3\")]\nstatic mut INTERRUPT_CONTROLLER: MaybeUninit<device_driver::InterruptController> =\n    MaybeUninit::uninit();\n\n#[cfg(feature = \"bsp_rpi4\")]\nstatic mut INTERRUPT_CONTROLLER: MaybeUninit<device_driver::GICv2> = MaybeUninit::uninit();\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// This must be called only after successful init of the memory subsystem.\nunsafe fn instantiate_uart() -> Result<(), &'static str> {\n    let mmio_descriptor = MMIODescriptor::new(mmio::PL011_UART_START, mmio::PL011_UART_SIZE);\n    let virt_addr =\n        memory::mmu::kernel_map_mmio(device_driver::PL011Uart::COMPATIBLE, &mmio_descriptor)?;\n\n    PL011_UART.write(device_driver::PL011Uart::new(virt_addr));\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the UART driver.\nunsafe fn post_init_uart() -> Result<(), &'static str> {\n    console::register_console(PL011_UART.assume_init_ref());\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the memory subsystem.\nunsafe fn instantiate_gpio() -> Result<(), &'static str> {\n    let mmio_descriptor = MMIODescriptor::new(mmio::GPIO_START, mmio::GPIO_SIZE);\n    let virt_addr =\n        memory::mmu::kernel_map_mmio(device_driver::GPIO::COMPATIBLE, &mmio_descriptor)?;\n\n    GPIO.write(device_driver::GPIO::new(virt_addr));\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the GPIO driver.\nunsafe fn post_init_gpio() -> Result<(), &'static str> {\n    GPIO.assume_init_ref().map_pl011_uart();\n    Ok(())\n}\n\n/// This must be called only after successful init of the memory subsystem.\n#[cfg(feature = \"bsp_rpi3\")]\nunsafe fn instantiate_interrupt_controller() -> Result<(), &'static str> {\n    let periph_mmio_descriptor =\n        MMIODescriptor::new(mmio::PERIPHERAL_IC_START, mmio::PERIPHERAL_IC_SIZE);\n    let periph_virt_addr = memory::mmu::kernel_map_mmio(\n        device_driver::InterruptController::COMPATIBLE,\n        &periph_mmio_descriptor,\n    )?;\n\n    INTERRUPT_CONTROLLER.write(device_driver::InterruptController::new(periph_virt_addr));\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the memory subsystem.\n#[cfg(feature = \"bsp_rpi4\")]\nunsafe fn instantiate_interrupt_controller() -> Result<(), &'static str> {\n    let gicd_mmio_descriptor = MMIODescriptor::new(mmio::GICD_START, mmio::GICD_SIZE);\n    let gicd_virt_addr = memory::mmu::kernel_map_mmio(\"GICv2 GICD\", &gicd_mmio_descriptor)?;\n\n    let gicc_mmio_descriptor = MMIODescriptor::new(mmio::GICC_START, mmio::GICC_SIZE);\n    let gicc_virt_addr = memory::mmu::kernel_map_mmio(\"GICV2 GICC\", &gicc_mmio_descriptor)?;\n\n    INTERRUPT_CONTROLLER.write(device_driver::GICv2::new(gicd_virt_addr, gicc_virt_addr));\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the interrupt controller driver.\nunsafe fn post_init_interrupt_controller() -> Result<(), &'static str> {\n    generic_exception::asynchronous::register_irq_manager(INTERRUPT_CONTROLLER.assume_init_ref());\n\n    Ok(())\n}\n\n/// Function needs to ensure that driver registration happens only after correct instantiation.\nunsafe fn driver_uart() -> Result<(), &'static str> {\n    instantiate_uart()?;\n\n    let uart_descriptor = generic_driver::DeviceDriverDescriptor::new(\n        PL011_UART.assume_init_ref(),\n        Some(post_init_uart),\n        Some(exception::asynchronous::irq_map::PL011_UART),\n    );\n    generic_driver::driver_manager().register_driver(uart_descriptor);\n\n    Ok(())\n}\n\n/// Function needs to ensure that driver registration happens only after correct instantiation.\nunsafe fn driver_gpio() -> Result<(), &'static str> {\n    instantiate_gpio()?;\n\n    let gpio_descriptor = generic_driver::DeviceDriverDescriptor::new(\n        GPIO.assume_init_ref(),\n        Some(post_init_gpio),\n        None,\n    );\n    generic_driver::driver_manager().register_driver(gpio_descriptor);\n\n    Ok(())\n}\n\n/// Function needs to ensure that driver registration happens only after correct instantiation.\nunsafe fn driver_interrupt_controller() -> Result<(), &'static str> {\n    instantiate_interrupt_controller()?;\n\n    let interrupt_controller_descriptor = generic_driver::DeviceDriverDescriptor::new(\n        INTERRUPT_CONTROLLER.assume_init_ref(),\n        Some(post_init_interrupt_controller),\n        None,\n    );\n    generic_driver::driver_manager().register_driver(interrupt_controller_descriptor);\n\n    Ok(())\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Initialize the driver subsystem.\n///\n/// # Safety\n///\n/// See child function calls.\npub unsafe fn init() -> Result<(), &'static str> {\n    static INIT_DONE: AtomicBool = AtomicBool::new(false);\n    if INIT_DONE.load(Ordering::Relaxed) {\n        return Err(\"Init already done\");\n    }\n\n    driver_uart()?;\n    driver_gpio()?;\n    driver_interrupt_controller()?;\n\n    INIT_DONE.store(true, Ordering::Relaxed);\n    Ok(())\n}\n\n/// Minimal code needed to bring up the console in QEMU (for testing only). This is often less steps\n/// than on real hardware due to QEMU's abstractions.\n#[cfg(feature = \"test_build\")]\npub fn qemu_bring_up_console() {\n    use crate::cpu;\n\n    unsafe {\n        instantiate_uart().unwrap_or_else(|_| cpu::qemu_exit_failure());\n        console::register_console(PL011_UART.assume_init_ref());\n    };\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/bsp/raspberrypi/exception/asynchronous.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP asynchronous exception handling.\n\nuse crate::bsp;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Export for reuse in generic asynchronous.rs.\npub use bsp::device_driver::IRQNumber;\n\n#[cfg(feature = \"bsp_rpi3\")]\npub(in crate::bsp) mod irq_map {\n    use super::bsp::device_driver::{IRQNumber, PeripheralIRQ};\n\n    pub const PL011_UART: IRQNumber = IRQNumber::Peripheral(PeripheralIRQ::new(57));\n}\n\n#[cfg(feature = \"bsp_rpi4\")]\npub(in crate::bsp) mod irq_map {\n    use super::bsp::device_driver::IRQNumber;\n\n    pub const PL011_UART: IRQNumber = IRQNumber::new(153);\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/bsp/raspberrypi/exception.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP synchronous and asynchronous exception handling.\n\npub mod asynchronous;\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/bsp/raspberrypi/kernel.ld",
    "content": "/* SPDX-License-Identifier: MIT OR Apache-2.0\n *\n * Copyright (c) 2018-2022 Andre Richter <andre.o.richter@gmail.com>\n */\n\nINCLUDE kernel_virt_addr_space_size.ld;\n\nPAGE_SIZE = 64K;\nPAGE_MASK = PAGE_SIZE - 1;\n\n/* The kernel's virtual address range will be:\n *\n * [END_ADDRESS_INCLUSIVE, START_ADDRESS]\n * [u64::MAX             , (u64::MAX - __kernel_virt_addr_space_size) + 1]\n */\n__kernel_virt_start_addr = ((0xffffffffffffffff - __kernel_virt_addr_space_size) + 1);\n\n__rpi_phys_dram_start_addr = 0;\n\n/* The physical address at which the the kernel binary will be loaded by the Raspberry's firmware */\n__rpi_phys_binary_load_addr = 0x80000;\n\n\nENTRY(__rpi_phys_binary_load_addr)\n\n/* Flags:\n *     4 == R\n *     5 == RX\n *     6 == RW\n *\n * Segments are marked PT_LOAD below so that the ELF file provides virtual and physical addresses.\n * It doesn't mean all of them need actually be loaded.\n */\nPHDRS\n{\n    segment_code            PT_LOAD FLAGS(5);\n    segment_data            PT_LOAD FLAGS(6);\n    segment_heap            PT_LOAD FLAGS(6);\n    segment_boot_core_stack PT_LOAD FLAGS(6);\n}\n\nSECTIONS\n{\n    . =  __kernel_virt_start_addr;\n\n    ASSERT((. & PAGE_MASK) == 0, \"Start of address space is not page aligned\")\n\n    /***********************************************************************************************\n    * Code + RO Data + Global Offset Table\n    ***********************************************************************************************/\n    __code_start = .;\n    .text : AT(__rpi_phys_binary_load_addr)\n    {\n        KEEP(*(.text._start))\n        *(.text._start_arguments) /* Constants (or statics in Rust speak) read by _start(). */\n        *(.text._start_rust)      /* The Rust entry point */\n        *(.text*)                 /* Everything else */\n    } :segment_code\n\n    .rodata         : ALIGN(8) { *(.rodata*) } :segment_code\n    .kernel_symbols : ALIGN(8) {\n        __kernel_symbols_start = .;\n        . += 32 * 1024;\n    } :segment_code\n\n    . = ALIGN(PAGE_SIZE);\n    __code_end_exclusive = .;\n\n    /***********************************************************************************************\n    * Data + BSS\n    ***********************************************************************************************/\n    __data_start = .;\n    .data : { *(.data*) } :segment_data\n\n    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */\n    .bss (NOLOAD) : ALIGN(16)\n    {\n        __bss_start = .;\n        *(.bss*);\n        . = ALIGN(16);\n        __bss_end_exclusive = .;\n    } :segment_data\n\n    . = ALIGN(PAGE_SIZE);\n    __data_end_exclusive = .;\n\n    /***********************************************************************************************\n    * Heap\n    ***********************************************************************************************/\n    __heap_start = .;\n    .heap (NOLOAD) :\n    {\n        . += 16 * 1024 * 1024;\n    } :segment_heap\n    __heap_end_exclusive = .;\n\n    ASSERT((. & PAGE_MASK) == 0, \"Heap is not page aligned\")\n\n    /***********************************************************************************************\n    * MMIO Remap Reserved\n    ***********************************************************************************************/\n    __mmio_remap_start = .;\n    . += 8 * 1024 * 1024;\n    __mmio_remap_end_exclusive = .;\n\n    ASSERT((. & PAGE_MASK) == 0, \"MMIO remap reservation is not page aligned\")\n\n    /***********************************************************************************************\n    * Guard Page\n    ***********************************************************************************************/\n    . += PAGE_SIZE;\n\n    /***********************************************************************************************\n    * Boot Core Stack\n    ***********************************************************************************************/\n    .boot_core_stack (NOLOAD) : AT(__rpi_phys_dram_start_addr)\n    {\n        __boot_core_stack_start = .;         /*   ^             */\n                                             /*   | stack       */\n        . += __rpi_phys_binary_load_addr;    /*   | growth      */\n                                             /*   | direction   */\n        __boot_core_stack_end_exclusive = .; /*   |             */\n    } :segment_boot_core_stack\n\n    ASSERT((. & PAGE_MASK) == 0, \"End of boot core stack is not page aligned\")\n\n    /***********************************************************************************************\n    * Misc\n    ***********************************************************************************************/\n    .got : { *(.got*) }\n    ASSERT(SIZEOF(.got) == 0, \"Relocation support not expected\")\n\n    /DISCARD/ : { *(.comment*) }\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/bsp/raspberrypi/kernel_virt_addr_space_size.ld",
    "content": "__kernel_virt_addr_space_size = 1024 * 1024 * 1024\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/bsp/raspberrypi/memory/mmu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Memory Management Unit.\n\nuse crate::{\n    memory::{\n        mmu::{\n            self as generic_mmu, AddressSpace, AssociatedTranslationTable, AttributeFields,\n            MemoryRegion, PageAddress, TranslationGranule,\n        },\n        Physical, Virtual,\n    },\n    synchronization::InitStateLock,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\ntype KernelTranslationTable =\n    <KernelVirtAddrSpace as AssociatedTranslationTable>::TableStartFromTop;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// The translation granule chosen by this BSP. This will be used everywhere else in the kernel to\n/// derive respective data structures and their sizes. For example, the `crate::memory::mmu::Page`.\npub type KernelGranule = TranslationGranule<{ 64 * 1024 }>;\n\n/// The kernel's virtual address space defined by this BSP.\npub type KernelVirtAddrSpace = AddressSpace<{ kernel_virt_addr_space_size() }>;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n/// The kernel translation tables.\n///\n/// It is mandatory that InitStateLock is transparent.\n///\n/// That is, `size_of(InitStateLock<KernelTranslationTable>) == size_of(KernelTranslationTable)`.\n/// There is a unit tests that checks this porperty.\n#[link_section = \".data\"]\n#[no_mangle]\nstatic KERNEL_TABLES: InitStateLock<KernelTranslationTable> =\n    InitStateLock::new(KernelTranslationTable::new_for_precompute());\n\n/// This value is needed during early boot for MMU setup.\n///\n/// This will be patched to the correct value by the \"translation table tool\" after linking. This\n/// given value here is just a dummy.\n#[link_section = \".text._start_arguments\"]\n#[no_mangle]\nstatic PHYS_KERNEL_TABLES_BASE_ADDR: u64 = 0xCCCCAAAAFFFFEEEE;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// This is a hack for retrieving the value for the kernel's virtual address space size as a\n/// constant from a common place, since it is needed as a compile-time/link-time constant in both,\n/// the linker script and the Rust sources.\n#[allow(clippy::needless_late_init)]\nconst fn kernel_virt_addr_space_size() -> usize {\n    let __kernel_virt_addr_space_size;\n\n    include!(\"../kernel_virt_addr_space_size.ld\");\n\n    __kernel_virt_addr_space_size\n}\n\n/// Helper function for calculating the number of pages the given parameter spans.\nconst fn size_to_num_pages(size: usize) -> usize {\n    assert!(size > 0);\n    assert!(size % KernelGranule::SIZE == 0);\n\n    size >> KernelGranule::SHIFT\n}\n\n/// The data pages of the kernel binary.\nfn virt_data_region() -> MemoryRegion<Virtual> {\n    let num_pages = size_to_num_pages(super::data_size());\n\n    let start_page_addr = super::virt_data_start();\n    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n\n    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n}\n\n// There is no reason to expect the following conversions to fail, since they were generated offline\n// by the `translation table tool`. If it doesn't work, a panic due to the unwraps is justified.\nfn kernel_virt_to_phys_region(virt_region: MemoryRegion<Virtual>) -> MemoryRegion<Physical> {\n    let phys_start_page_addr =\n        generic_mmu::try_kernel_virt_page_addr_to_phys_page_addr(virt_region.start_page_addr())\n            .unwrap();\n\n    let phys_end_exclusive_page_addr = phys_start_page_addr\n        .checked_offset(virt_region.num_pages() as isize)\n        .unwrap();\n\n    MemoryRegion::new(phys_start_page_addr, phys_end_exclusive_page_addr)\n}\n\nfn kernel_page_attributes(virt_page_addr: PageAddress<Virtual>) -> AttributeFields {\n    generic_mmu::try_kernel_page_attributes(virt_page_addr).unwrap()\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The code pages of the kernel binary.\npub fn virt_code_region() -> MemoryRegion<Virtual> {\n    let num_pages = size_to_num_pages(super::code_size());\n\n    let start_page_addr = super::virt_code_start();\n    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n\n    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n}\n\n/// The heap pages.\npub fn virt_heap_region() -> MemoryRegion<Virtual> {\n    let num_pages = size_to_num_pages(super::heap_size());\n\n    let start_page_addr = super::virt_heap_start();\n    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n\n    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n}\n\n/// The boot core stack pages.\npub fn virt_boot_core_stack_region() -> MemoryRegion<Virtual> {\n    let num_pages = size_to_num_pages(super::boot_core_stack_size());\n\n    let start_page_addr = super::virt_boot_core_stack_start();\n    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n\n    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n}\n\n/// Return a reference to the kernel's translation tables.\npub fn kernel_translation_tables() -> &'static InitStateLock<KernelTranslationTable> {\n    &KERNEL_TABLES\n}\n\n/// The MMIO remap pages.\npub fn virt_mmio_remap_region() -> MemoryRegion<Virtual> {\n    let num_pages = size_to_num_pages(super::mmio_remap_size());\n\n    let start_page_addr = super::virt_mmio_remap_start();\n    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n\n    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n}\n\n/// Add mapping records for the kernel binary.\n///\n/// The actual translation table entries for the kernel binary are generated using the offline\n/// `translation table tool` and patched into the kernel binary. This function just adds the mapping\n/// record entries.\npub fn kernel_add_mapping_records_for_precomputed() {\n    let virt_code_region = virt_code_region();\n    generic_mmu::kernel_add_mapping_record(\n        \"Kernel code and RO data\",\n        &virt_code_region,\n        &kernel_virt_to_phys_region(virt_code_region),\n        &kernel_page_attributes(virt_code_region.start_page_addr()),\n    );\n\n    let virt_data_region = virt_data_region();\n    generic_mmu::kernel_add_mapping_record(\n        \"Kernel data and bss\",\n        &virt_data_region,\n        &kernel_virt_to_phys_region(virt_data_region),\n        &kernel_page_attributes(virt_data_region.start_page_addr()),\n    );\n\n    let virt_heap_region = virt_heap_region();\n    generic_mmu::kernel_add_mapping_record(\n        \"Kernel heap\",\n        &virt_heap_region,\n        &kernel_virt_to_phys_region(virt_heap_region),\n        &kernel_page_attributes(virt_heap_region.start_page_addr()),\n    );\n\n    let virt_boot_core_stack_region = virt_boot_core_stack_region();\n    generic_mmu::kernel_add_mapping_record(\n        \"Kernel boot-core stack\",\n        &virt_boot_core_stack_region,\n        &kernel_virt_to_phys_region(virt_boot_core_stack_region),\n        &kernel_page_attributes(virt_boot_core_stack_region.start_page_addr()),\n    );\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/bsp/raspberrypi/memory.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Memory Management.\n//!\n//! The physical memory layout.\n//!\n//! The Raspberry's firmware copies the kernel binary to 0x8_0000. The preceding region will be used\n//! as the boot core's stack.\n//!\n//! +---------------------------------------+\n//! |                                       | boot_core_stack_start @ 0x0\n//! |                                       |                                ^\n//! | Boot-core Stack                       |                                | stack\n//! |                                       |                                | growth\n//! |                                       |                                | direction\n//! +---------------------------------------+\n//! |                                       | code_start @ 0x8_0000 == boot_core_stack_end_exclusive\n//! | .text                                 |\n//! | .rodata                               |\n//! | .got                                  |\n//! | .kernel_symbols                       |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       | data_start == code_end_exclusive\n//! | .data                                 |\n//! | .bss                                  |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       | heap_start == data_end_exclusive\n//! | .heap                                 |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       | heap_end_exclusive\n//! |                                       |\n//!\n//!\n//!\n//!\n//!\n//! The virtual memory layout is as follows:\n//!\n//! +---------------------------------------+\n//! |                                       | code_start @ __kernel_virt_start_addr\n//! | .text                                 |\n//! | .rodata                               |\n//! | .got                                  |\n//! | .kernel_symbols                       |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       | data_start == code_end_exclusive\n//! | .data                                 |\n//! | .bss                                  |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       | heap_start == data_end_exclusive\n//! | .heap                                 |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       |  mmio_remap_start == heap_end_exclusive\n//! | VA region for MMIO remapping          |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       |  mmio_remap_end_exclusive\n//! | Unmapped guard page                   |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       | boot_core_stack_start\n//! |                                       |                                ^\n//! | Boot-core Stack                       |                                | stack\n//! |                                       |                                | growth\n//! |                                       |                                | direction\n//! +---------------------------------------+\n//! |                                       | boot_core_stack_end_exclusive\n//! |                                       |\npub mod mmu;\n\nuse crate::memory::{mmu::PageAddress, Address, Physical, Virtual};\nuse core::cell::UnsafeCell;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Symbols from the linker script.\nextern \"Rust\" {\n    static __code_start: UnsafeCell<()>;\n    static __code_end_exclusive: UnsafeCell<()>;\n\n    static __data_start: UnsafeCell<()>;\n    static __data_end_exclusive: UnsafeCell<()>;\n\n    static __heap_start: UnsafeCell<()>;\n    static __heap_end_exclusive: UnsafeCell<()>;\n\n    static __mmio_remap_start: UnsafeCell<()>;\n    static __mmio_remap_end_exclusive: UnsafeCell<()>;\n\n    static __boot_core_stack_start: UnsafeCell<()>;\n    static __boot_core_stack_end_exclusive: UnsafeCell<()>;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// The board's physical memory map.\n#[rustfmt::skip]\npub(super) mod map {\n    use super::*;\n\n    /// Physical devices.\n    #[cfg(feature = \"bsp_rpi3\")]\n    pub mod mmio {\n        use super::*;\n\n        pub const PERIPHERAL_IC_START: Address<Physical> = Address::new(0x3F00_B200);\n        pub const PERIPHERAL_IC_SIZE:  usize             =              0x24;\n\n        pub const GPIO_START:          Address<Physical> = Address::new(0x3F20_0000);\n        pub const GPIO_SIZE:           usize             =              0xA0;\n\n        pub const PL011_UART_START:    Address<Physical> = Address::new(0x3F20_1000);\n        pub const PL011_UART_SIZE:     usize             =              0x48;\n\n        pub const END:                 Address<Physical> = Address::new(0x4001_0000);\n    }\n\n    /// Physical devices.\n    #[cfg(feature = \"bsp_rpi4\")]\n    pub mod mmio {\n        use super::*;\n\n        pub const GPIO_START:       Address<Physical> = Address::new(0xFE20_0000);\n        pub const GPIO_SIZE:        usize             =              0xA0;\n\n        pub const PL011_UART_START: Address<Physical> = Address::new(0xFE20_1000);\n        pub const PL011_UART_SIZE:  usize             =              0x48;\n\n        pub const GICD_START:       Address<Physical> = Address::new(0xFF84_1000);\n        pub const GICD_SIZE:        usize             =              0x824;\n\n        pub const GICC_START:       Address<Physical> = Address::new(0xFF84_2000);\n        pub const GICC_SIZE:        usize             =              0x14;\n\n        pub const END:              Address<Physical> = Address::new(0xFF85_0000);\n    }\n\n    pub const END: Address<Physical> = mmio::END;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Start page address of the code segment.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn virt_code_start() -> PageAddress<Virtual> {\n    PageAddress::from(unsafe { __code_start.get() as usize })\n}\n\n/// Size of the code segment.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn code_size() -> usize {\n    unsafe { (__code_end_exclusive.get() as usize) - (__code_start.get() as usize) }\n}\n\n/// Start page address of the data segment.\n#[inline(always)]\nfn virt_data_start() -> PageAddress<Virtual> {\n    PageAddress::from(unsafe { __data_start.get() as usize })\n}\n\n/// Size of the data segment.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn data_size() -> usize {\n    unsafe { (__data_end_exclusive.get() as usize) - (__data_start.get() as usize) }\n}\n\n/// Start page address of the heap segment.\n#[inline(always)]\nfn virt_heap_start() -> PageAddress<Virtual> {\n    PageAddress::from(unsafe { __heap_start.get() as usize })\n}\n\n/// Size of the heap segment.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn heap_size() -> usize {\n    unsafe { (__heap_end_exclusive.get() as usize) - (__heap_start.get() as usize) }\n}\n\n/// Start page address of the MMIO remap reservation.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn virt_mmio_remap_start() -> PageAddress<Virtual> {\n    PageAddress::from(unsafe { __mmio_remap_start.get() as usize })\n}\n\n/// Size of the MMIO remap reservation.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn mmio_remap_size() -> usize {\n    unsafe { (__mmio_remap_end_exclusive.get() as usize) - (__mmio_remap_start.get() as usize) }\n}\n\n/// Start page address of the boot core's stack.\n#[inline(always)]\nfn virt_boot_core_stack_start() -> PageAddress<Virtual> {\n    PageAddress::from(unsafe { __boot_core_stack_start.get() as usize })\n}\n\n/// Size of the boot core's stack.\n#[inline(always)]\nfn boot_core_stack_size() -> usize {\n    unsafe {\n        (__boot_core_stack_end_exclusive.get() as usize) - (__boot_core_stack_start.get() as usize)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Exclusive end address of the physical address space.\n#[inline(always)]\npub fn phys_addr_space_end_exclusive_addr() -> PageAddress<Physical> {\n    PageAddress::from(map::END)\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/bsp/raspberrypi.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Top-level BSP file for the Raspberry Pi 3 and 4.\n\npub mod cpu;\npub mod driver;\npub mod exception;\npub mod memory;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Board identification.\npub fn board_name() -> &'static str {\n    #[cfg(feature = \"bsp_rpi3\")]\n    {\n        \"Raspberry Pi 3\"\n    }\n\n    #[cfg(feature = \"bsp_rpi4\")]\n    {\n        \"Raspberry Pi 4\"\n    }\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/bsp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Conditional reexporting of Board Support Packages.\n\nmod device_driver;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod raspberrypi;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use raspberrypi::*;\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/common.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! General purpose code.\n\n/// Check if a value is aligned to a given size.\n#[inline(always)]\npub const fn is_aligned(value: usize, alignment: usize) -> bool {\n    assert!(alignment.is_power_of_two());\n\n    (value & (alignment - 1)) == 0\n}\n\n/// Align down.\n#[inline(always)]\npub const fn align_down(value: usize, alignment: usize) -> usize {\n    assert!(alignment.is_power_of_two());\n\n    value & !(alignment - 1)\n}\n\n/// Align up.\n#[inline(always)]\npub const fn align_up(value: usize, alignment: usize) -> usize {\n    assert!(alignment.is_power_of_two());\n\n    (value + alignment - 1) & !(alignment - 1)\n}\n\n/// Convert a size into human readable format.\npub const fn size_human_readable_ceil(size: usize) -> (usize, &'static str) {\n    const KIB: usize = 1024;\n    const MIB: usize = 1024 * 1024;\n    const GIB: usize = 1024 * 1024 * 1024;\n\n    if (size / GIB) > 0 {\n        (size.div_ceil(GIB), \"GiB\")\n    } else if (size / MIB) > 0 {\n        (size.div_ceil(MIB), \"MiB\")\n    } else if (size / KIB) > 0 {\n        (size.div_ceil(KIB), \"KiB\")\n    } else {\n        (size, \"Byte\")\n    }\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/console/buffer_console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A console that buffers input during the init phase.\n\nuse super::interface;\nuse crate::{console, info, synchronization, synchronization::InitStateLock};\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nconst BUF_SIZE: usize = 1024 * 64;\n\npub struct BufferConsoleInner {\n    buf: [char; BUF_SIZE],\n    write_ptr: usize,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct BufferConsole {\n    inner: InitStateLock<BufferConsoleInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\npub static BUFFER_CONSOLE: BufferConsole = BufferConsole {\n    inner: InitStateLock::new(BufferConsoleInner {\n        // Use the null character, so this lands in .bss and does not waste space in the binary.\n        buf: ['\\0'; BUF_SIZE],\n        write_ptr: 0,\n    }),\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl BufferConsoleInner {\n    fn write_char(&mut self, c: char) {\n        if self.write_ptr < (BUF_SIZE - 1) {\n            self.buf[self.write_ptr] = c;\n            self.write_ptr += 1;\n        }\n    }\n}\n\nimpl fmt::Write for BufferConsoleInner {\n    fn write_str(&mut self, s: &str) -> fmt::Result {\n        for c in s.chars() {\n            self.write_char(c);\n        }\n\n        Ok(())\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::interface::ReadWriteEx;\n\nimpl BufferConsole {\n    /// Dump the buffer.\n    ///\n    /// # Invariant\n    ///\n    /// It is expected that this is only called when self != crate::console::console().\n    pub fn dump(&self) {\n        self.inner.read(|inner| {\n            console::console().write_array(&inner.buf[0..inner.write_ptr]);\n\n            if inner.write_ptr == (BUF_SIZE - 1) {\n                info!(\"Pre-UART buffer overflowed\");\n            } else if inner.write_ptr > 0 {\n                info!(\"End of pre-UART buffer\")\n            }\n        });\n    }\n}\n\nimpl interface::Write for BufferConsole {\n    fn write_char(&self, c: char) {\n        self.inner.write(|inner| inner.write_char(c));\n    }\n\n    fn write_array(&self, _a: &[char]) {}\n\n    fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result {\n        self.inner.write(|inner| fmt::Write::write_fmt(inner, args))\n    }\n\n    fn flush(&self) {}\n}\n\nimpl interface::Read for BufferConsole {\n    fn clear_rx(&self) {}\n}\n\nimpl interface::Statistics for BufferConsole {}\nimpl interface::All for BufferConsole {}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! System console.\n\nmod buffer_console;\n\nuse crate::synchronization;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Console interfaces.\npub mod interface {\n    use core::fmt;\n\n    /// Console write functions.\n    pub trait Write {\n        /// Write a single character.\n        fn write_char(&self, c: char);\n\n        /// Write a slice of characters.\n        fn write_array(&self, a: &[char]);\n\n        /// Write a Rust format string.\n        fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result;\n\n        /// Block until the last buffered character has been physically put on the TX wire.\n        fn flush(&self);\n    }\n\n    /// Console read functions.\n    pub trait Read {\n        /// Read a single character.\n        fn read_char(&self) -> char {\n            ' '\n        }\n\n        /// Clear RX buffers, if any.\n        fn clear_rx(&self);\n    }\n\n    /// Console statistics.\n    pub trait Statistics {\n        /// Return the number of characters written.\n        fn chars_written(&self) -> usize {\n            0\n        }\n\n        /// Return the number of characters read.\n        fn chars_read(&self) -> usize {\n            0\n        }\n    }\n\n    /// Trait alias for a full-fledged console.\n    pub trait All: Write + Read + Statistics {}\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic CUR_CONSOLE: InitStateLock<&'static (dyn interface::All + Sync)> =\n    InitStateLock::new(&buffer_console::BUFFER_CONSOLE);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::{interface::ReadWriteEx, InitStateLock};\n\n/// Register a new console.\npub fn register_console(new_console: &'static (dyn interface::All + Sync)) {\n    CUR_CONSOLE.write(|con| *con = new_console);\n\n    static FIRST_SWITCH: InitStateLock<bool> = InitStateLock::new(true);\n    FIRST_SWITCH.write(|first| {\n        if *first {\n            *first = false;\n\n            buffer_console::BUFFER_CONSOLE.dump();\n        }\n    });\n}\n\n/// Return a reference to the currently registered console.\n///\n/// This is the global console used by all printing macros.\npub fn console() -> &'static dyn interface::All {\n    CUR_CONSOLE.read(|con| *con)\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Boot code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/cpu/boot.rs\"]\nmod arch_boot;\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/cpu/smp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Symmetric multiprocessing.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/cpu/smp.rs\"]\nmod arch_smp;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_smp::core_id;\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Processor code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/cpu.rs\"]\nmod arch_cpu;\n\nmod boot;\n\npub mod smp;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_cpu::{nop, wait_forever};\n\n#[cfg(feature = \"test_build\")]\npub use arch_cpu::{qemu_exit_failure, qemu_exit_success};\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Driver support.\n\nuse crate::{\n    exception, info,\n    synchronization::{interface::ReadWriteEx, InitStateLock},\n};\nuse alloc::vec::Vec;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Driver interfaces.\npub mod interface {\n    /// Device Driver functions.\n    pub trait DeviceDriver {\n        /// Different interrupt controllers might use different types for IRQ number.\n        type IRQNumberType: super::fmt::Display;\n\n        /// Return a compatibility string for identifying the driver.\n        fn compatible(&self) -> &'static str;\n\n        /// Called by the kernel to bring up the device.\n        ///\n        /// # Safety\n        ///\n        /// - During init, drivers might do stuff with system-wide impact.\n        unsafe fn init(&self) -> Result<(), &'static str> {\n            Ok(())\n        }\n\n        /// Called by the kernel to register and enable the device's IRQ handler.\n        ///\n        /// Rust's type system will prevent a call to this function unless the calling instance\n        /// itself has static lifetime.\n        fn register_and_enable_irq_handler(\n            &'static self,\n            irq_number: &Self::IRQNumberType,\n        ) -> Result<(), &'static str> {\n            panic!(\n                \"Attempt to enable IRQ {} for device {}, but driver does not support this\",\n                irq_number,\n                self.compatible()\n            )\n        }\n    }\n}\n\n/// Tpye to be used as an optional callback after a driver's init() has run.\npub type DeviceDriverPostInitCallback = unsafe fn() -> Result<(), &'static str>;\n\n/// A descriptor for device drivers.\npub struct DeviceDriverDescriptor<T>\nwhere\n    T: 'static,\n{\n    device_driver: &'static (dyn interface::DeviceDriver<IRQNumberType = T> + Sync),\n    post_init_callback: Option<DeviceDriverPostInitCallback>,\n    irq_number: Option<T>,\n}\n\n/// Provides device driver management functions.\npub struct DriverManager<T>\nwhere\n    T: 'static,\n{\n    descriptors: InitStateLock<Vec<DeviceDriverDescriptor<T>>>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic DRIVER_MANAGER: DriverManager<exception::asynchronous::IRQNumber> = DriverManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T> DeviceDriverDescriptor<T> {\n    /// Create an instance.\n    pub fn new(\n        device_driver: &'static (dyn interface::DeviceDriver<IRQNumberType = T> + Sync),\n        post_init_callback: Option<DeviceDriverPostInitCallback>,\n        irq_number: Option<T>,\n    ) -> Self {\n        Self {\n            device_driver,\n            post_init_callback,\n            irq_number,\n        }\n    }\n}\n\n/// Return a reference to the global DriverManager.\npub fn driver_manager() -> &'static DriverManager<exception::asynchronous::IRQNumber> {\n    &DRIVER_MANAGER\n}\n\nimpl<T> DriverManager<T>\nwhere\n    T: fmt::Display,\n{\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            descriptors: InitStateLock::new(Vec::new()),\n        }\n    }\n\n    /// Register a device driver with the kernel.\n    pub fn register_driver(&self, descriptor: DeviceDriverDescriptor<T>) {\n        self.descriptors\n            .write(|descriptors| descriptors.push(descriptor));\n    }\n\n    /// Fully initialize all drivers and their interrupts handlers.\n    ///\n    /// # Safety\n    ///\n    /// - During init, drivers might do stuff with system-wide impact.\n    pub unsafe fn init_drivers_and_irqs(&self) {\n        self.descriptors.read(|descriptors| {\n            for descriptor in descriptors {\n                // 1. Initialize driver.\n                if let Err(x) = descriptor.device_driver.init() {\n                    panic!(\n                        \"Error initializing driver: {}: {}\",\n                        descriptor.device_driver.compatible(),\n                        x\n                    );\n                }\n\n                // 2. Call corresponding post init callback.\n                if let Some(callback) = &descriptor.post_init_callback {\n                    if let Err(x) = callback() {\n                        panic!(\n                            \"Error during driver post-init callback: {}: {}\",\n                            descriptor.device_driver.compatible(),\n                            x\n                        );\n                    }\n                }\n            }\n\n            // 3. After all post-init callbacks were done, the interrupt controller should be\n            //    registered and functional. So let drivers register with it now.\n            for descriptor in descriptors {\n                if let Some(irq_number) = &descriptor.irq_number {\n                    if let Err(x) = descriptor\n                        .device_driver\n                        .register_and_enable_irq_handler(irq_number)\n                    {\n                        panic!(\n                            \"Error during driver interrupt handler registration: {}: {}\",\n                            descriptor.device_driver.compatible(),\n                            x\n                        );\n                    }\n                }\n            }\n        })\n    }\n\n    /// Enumerate all registered device drivers.\n    pub fn enumerate(&self) {\n        self.descriptors.read(|descriptors| {\n            for (i, desc) in descriptors.iter().enumerate() {\n                info!(\"      {}. {}\", i + 1, desc.device_driver.compatible());\n            }\n        });\n    }\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/exception/asynchronous/null_irq_manager.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Null IRQ Manager.\n\nuse super::{interface, IRQContext, IRQHandlerDescriptor};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct NullIRQManager;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\npub static NULL_IRQ_MANAGER: NullIRQManager = NullIRQManager {};\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl interface::IRQManager for NullIRQManager {\n    type IRQNumberType = super::IRQNumber;\n\n    fn register_handler(\n        &self,\n        _descriptor: IRQHandlerDescriptor<Self::IRQNumberType>,\n    ) -> Result<(), &'static str> {\n        panic!(\"No IRQ Manager registered yet\");\n    }\n\n    fn enable(&self, _irq_number: &Self::IRQNumberType) {\n        panic!(\"No IRQ Manager registered yet\");\n    }\n\n    fn handle_pending_irqs<'irq_context>(&'irq_context self, _ic: &IRQContext<'irq_context>) {\n        panic!(\"No IRQ Manager registered yet\");\n    }\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/exception/asynchronous.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Asynchronous exception handling.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/exception/asynchronous.rs\"]\nmod arch_asynchronous;\nmod null_irq_manager;\n\nuse crate::{bsp, synchronization};\nuse core::marker::PhantomData;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_asynchronous::{\n    is_local_irq_masked, local_irq_mask, local_irq_mask_save, local_irq_restore, local_irq_unmask,\n    print_state,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Interrupt number as defined by the BSP.\npub type IRQNumber = bsp::exception::asynchronous::IRQNumber;\n\n/// Interrupt descriptor.\n#[derive(Copy, Clone)]\npub struct IRQHandlerDescriptor<T>\nwhere\n    T: Copy,\n{\n    /// The IRQ number.\n    number: T,\n\n    /// Descriptive name.\n    name: &'static str,\n\n    /// Reference to handler trait object.\n    handler: &'static (dyn interface::IRQHandler + Sync),\n}\n\n/// IRQContext token.\n///\n/// An instance of this type indicates that the local core is currently executing in IRQ\n/// context, aka executing an interrupt vector or subcalls of it.\n///\n/// Concept and implementation derived from the `CriticalSection` introduced in\n/// <https://github.com/rust-embedded/bare-metal>\n#[derive(Clone, Copy)]\npub struct IRQContext<'irq_context> {\n    _0: PhantomData<&'irq_context ()>,\n}\n\n/// Asynchronous exception handling interfaces.\npub mod interface {\n\n    /// Implemented by types that handle IRQs.\n    pub trait IRQHandler {\n        /// Called when the corresponding interrupt is asserted.\n        fn handle(&self) -> Result<(), &'static str>;\n    }\n\n    /// IRQ management functions.\n    ///\n    /// The `BSP` is supposed to supply one global instance. Typically implemented by the\n    /// platform's interrupt controller.\n    pub trait IRQManager {\n        /// The IRQ number type depends on the implementation.\n        type IRQNumberType: Copy;\n\n        /// Register a handler.\n        fn register_handler(\n            &self,\n            irq_handler_descriptor: super::IRQHandlerDescriptor<Self::IRQNumberType>,\n        ) -> Result<(), &'static str>;\n\n        /// Enable an interrupt in the controller.\n        fn enable(&self, irq_number: &Self::IRQNumberType);\n\n        /// Handle pending interrupts.\n        ///\n        /// This function is called directly from the CPU's IRQ exception vector. On AArch64,\n        /// this means that the respective CPU core has disabled exception handling.\n        /// This function can therefore not be preempted and runs start to finish.\n        ///\n        /// Takes an IRQContext token to ensure it can only be called from IRQ context.\n        #[allow(clippy::trivially_copy_pass_by_ref)]\n        fn handle_pending_irqs<'irq_context>(\n            &'irq_context self,\n            ic: &super::IRQContext<'irq_context>,\n        );\n\n        /// Print list of registered handlers.\n        fn print_handler(&self) {}\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic CUR_IRQ_MANAGER: InitStateLock<\n    &'static (dyn interface::IRQManager<IRQNumberType = IRQNumber> + Sync),\n> = InitStateLock::new(&null_irq_manager::NULL_IRQ_MANAGER);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::{interface::ReadWriteEx, InitStateLock};\n\nimpl<T> IRQHandlerDescriptor<T>\nwhere\n    T: Copy,\n{\n    /// Create an instance.\n    pub const fn new(\n        number: T,\n        name: &'static str,\n        handler: &'static (dyn interface::IRQHandler + Sync),\n    ) -> Self {\n        Self {\n            number,\n            name,\n            handler,\n        }\n    }\n\n    /// Return the number.\n    pub const fn number(&self) -> T {\n        self.number\n    }\n\n    /// Return the name.\n    pub const fn name(&self) -> &'static str {\n        self.name\n    }\n\n    /// Return the handler.\n    pub const fn handler(&self) -> &'static (dyn interface::IRQHandler + Sync) {\n        self.handler\n    }\n}\n\nimpl<'irq_context> IRQContext<'irq_context> {\n    /// Creates an IRQContext token.\n    ///\n    /// # Safety\n    ///\n    /// - This must only be called when the current core is in an interrupt context and will not\n    ///   live beyond the end of it. That is, creation is allowed in interrupt vector functions. For\n    ///   example, in the ARMv8-A case, in `extern \"C\" fn current_elx_irq()`.\n    /// - Note that the lifetime `'irq_context` of the returned instance is unconstrained. User code\n    ///   must not be able to influence the lifetime picked for this type, since that might cause it\n    ///   to be inferred to `'static`.\n    #[inline(always)]\n    pub unsafe fn new() -> Self {\n        IRQContext { _0: PhantomData }\n    }\n}\n\n/// Executes the provided closure while IRQs are masked on the executing core.\n///\n/// While the function temporarily changes the HW state of the executing core, it restores it to the\n/// previous state before returning, so this is deemed safe.\n#[inline(always)]\npub fn exec_with_irq_masked<T>(f: impl FnOnce() -> T) -> T {\n    let saved = local_irq_mask_save();\n    let ret = f();\n    local_irq_restore(saved);\n\n    ret\n}\n\n/// Register a new IRQ manager.\npub fn register_irq_manager(\n    new_manager: &'static (dyn interface::IRQManager<IRQNumberType = IRQNumber> + Sync),\n) {\n    CUR_IRQ_MANAGER.write(|manager| *manager = new_manager);\n}\n\n/// Return a reference to the currently registered IRQ manager.\n///\n/// This is the IRQ manager used by the architectural interrupt handling code.\npub fn irq_manager() -> &'static dyn interface::IRQManager<IRQNumberType = IRQNumber> {\n    CUR_IRQ_MANAGER.read(|manager| *manager)\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/exception.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Synchronous and asynchronous exception handling.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/exception.rs\"]\nmod arch_exception;\n\npub mod asynchronous;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_exception::{current_privilege_level, handling_init};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Kernel privilege levels.\n#[allow(missing_docs)]\n#[derive(Eq, PartialEq)]\npub enum PrivilegeLevel {\n    User,\n    Kernel,\n    Hypervisor,\n    Unknown,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// Libkernel unit tests must execute in kernel mode.\n    #[kernel_test]\n    fn test_runner_executes_in_kernel_mode() {\n        let (level, _) = current_privilege_level();\n\n        assert!(level == PrivilegeLevel::Kernel)\n    }\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/lib.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n// Rust embedded logo for `make doc`.\n#![doc(\n    html_logo_url = \"https://raw.githubusercontent.com/rust-embedded/wg/master/assets/logo/ewg-logo-blue-white-on-transparent.png\"\n)]\n\n//! The `kernel` library.\n//!\n//! Used to compose the final kernel binary.\n//!\n//! # Code organization and architecture\n//!\n//! The code is divided into different *modules*, each representing a typical **subsystem** of the\n//! `kernel`. Top-level module files of subsystems reside directly in the `src` folder. For example,\n//! `src/memory.rs` contains code that is concerned with all things memory management.\n//!\n//! ## Visibility of processor architecture code\n//!\n//! Some of the `kernel`'s subsystems depend on low-level code that is specific to the target\n//! processor architecture. For each supported processor architecture, there exists a subfolder in\n//! `src/_arch`, for example, `src/_arch/aarch64`.\n//!\n//! The architecture folders mirror the subsystem modules laid out in `src`. For example,\n//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go\n//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in\n//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic\n//! module's name prefixed with `arch_`.\n//!\n//! For example, this is the top of `src/memory/mmu.rs`:\n//!\n//! ```\n//! #[cfg(target_arch = \"aarch64\")]\n//! #[path = \"../_arch/aarch64/memory/mmu.rs\"]\n//! mod arch_mmu;\n//! ```\n//!\n//! Often times, items from the `arch_ module` will be publicly reexported by the parent module.\n//! This way, each architecture specific module can provide its implementation of an item, while the\n//! caller must not be concerned which architecture has been conditionally compiled.\n//!\n//! ## BSP code\n//!\n//! `BSP` stands for Board Support Package. `BSP` code is organized under `src/bsp.rs` and contains\n//! target board specific definitions and functions. These are things such as the board's memory map\n//! or instances of drivers for devices that are featured on the respective board.\n//!\n//! Just like processor architecture code, the `BSP` code's module structure tries to mirror the\n//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is\n//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`.\n//!\n//! ## Kernel interfaces\n//!\n//! Both `arch` and `bsp` contain code that is conditionally compiled depending on the actual target\n//! and board for which the kernel is compiled. For example, the `interrupt controller` hardware of\n//! the `Raspberry Pi 3` and the `Raspberry Pi 4` is different, but we want the rest of the `kernel`\n//! code to play nicely with any of the two without much hassle.\n//!\n//! In order to provide a clean abstraction between `arch`, `bsp` and `generic kernel code`,\n//! `interface` traits are provided *whenever possible* and *where it makes sense*. They are defined\n//! in the respective subsystem module and help to enforce the idiom of *program to an interface,\n//! not an implementation*. For example, there will be a common IRQ handling interface which the two\n//! different interrupt controller `drivers` of both Raspberrys will implement, and only export the\n//! interface to the rest of the `kernel`.\n//!\n//! ```\n//!         +-------------------+\n//!         | Interface (Trait) |\n//!         |                   |\n//!         +--+-------------+--+\n//!            ^             ^\n//!            |             |\n//!            |             |\n//! +----------+--+       +--+----------+\n//! | kernel code |       |  bsp code   |\n//! |             |       |  arch code  |\n//! +-------------+       +-------------+\n//! ```\n//!\n//! # Summary\n//!\n//! For a logical `kernel` subsystem, corresponding code can be distributed over several physical\n//! locations. Here is an example for the **memory** subsystem:\n//!\n//! - `src/memory.rs` and `src/memory/**/*`\n//!   - Common code that is agnostic of target processor architecture and `BSP` characteristics.\n//!     - Example: A function to zero a chunk of memory.\n//!   - Interfaces for the memory subsystem that are implemented by `arch` or `BSP` code.\n//!     - Example: An `MMU` interface that defines `MMU` function prototypes.\n//! - `src/bsp/__board_name__/memory.rs` and `src/bsp/__board_name__/memory/**/*`\n//!   - `BSP` specific code.\n//!   - Example: The board's memory map (physical addresses of DRAM and MMIO devices).\n//! - `src/_arch/__arch_name__/memory.rs` and `src/_arch/__arch_name__/memory/**/*`\n//!   - Processor architecture specific code.\n//!   - Example: Implementation of the `MMU` interface for the `__arch_name__` processor\n//!     architecture.\n//!\n//! From a namespace perspective, **memory** subsystem code lives in:\n//!\n//! - `crate::memory::*`\n//! - `crate::bsp::memory::*`\n//!\n//! # Boot flow\n//!\n//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.\n//!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.\n//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.\n\n#![allow(clippy::upper_case_acronyms)]\n#![allow(incomplete_features)]\n#![feature(alloc_error_handler)]\n#![feature(asm_const)]\n#![feature(const_option)]\n#![feature(core_intrinsics)]\n#![feature(format_args_nl)]\n#![feature(generic_const_exprs)]\n#![feature(int_roundings)]\n#![feature(is_sorted)]\n#![feature(linkage)]\n#![feature(nonzero_min_max)]\n#![feature(panic_info_message)]\n#![feature(step_trait)]\n#![feature(trait_alias)]\n#![feature(unchecked_math)]\n#![no_std]\n// Testing\n#![cfg_attr(test, no_main)]\n#![feature(custom_test_frameworks)]\n#![reexport_test_harness_main = \"test_main\"]\n#![test_runner(crate::test_runner)]\n\nextern crate alloc;\n\nmod panic_wait;\nmod synchronization;\n\npub mod backtrace;\npub mod bsp;\npub mod common;\npub mod console;\npub mod cpu;\npub mod driver;\npub mod exception;\npub mod memory;\npub mod print;\npub mod state;\npub mod symbols;\npub mod time;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Version string.\npub fn version() -> &'static str {\n    concat!(\n        env!(\"CARGO_PKG_NAME\"),\n        \" version \",\n        env!(\"CARGO_PKG_VERSION\")\n    )\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n/// The default runner for unit tests.\npub fn test_runner(tests: &[&test_types::UnitTest]) {\n    // This line will be printed as the test header.\n    println!(\"Running {} tests\", tests.len());\n\n    for (i, test) in tests.iter().enumerate() {\n        print!(\"{:>3}. {:.<58}\", i + 1, test.name);\n\n        // Run the actual test.\n        (test.test_func)();\n\n        // Failed tests call panic!(). Execution reaches here only if the test has passed.\n        println!(\"[ok]\")\n    }\n}\n\n/// The `kernel_init()` for unit tests.\n#[cfg(test)]\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    test_main();\n\n    cpu::qemu_exit_success()\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/main.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n// Rust embedded logo for `make doc`.\n#![doc(\n    html_logo_url = \"https://raw.githubusercontent.com/rust-embedded/wg/master/assets/logo/ewg-logo-blue-white-on-transparent.png\"\n)]\n\n//! The `kernel` binary.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\nextern crate alloc;\n\nuse libkernel::{bsp, cpu, driver, exception, info, memory, state, time};\n\n/// Early init code.\n///\n/// When this code runs, virtual memory is already enabled.\n///\n/// # Safety\n///\n/// - Only a single core must be active and running this function.\n/// - Printing will not work until the respective driver's MMIO is remapped.\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n\n    // Initialize the BSP driver subsystem.\n    if let Err(x) = bsp::driver::init() {\n        panic!(\"Error initializing BSP driver subsystem: {}\", x);\n    }\n\n    // Initialize all device drivers.\n    driver::driver_manager().init_drivers_and_irqs();\n\n    bsp::memory::mmu::kernel_add_mapping_records_for_precomputed();\n\n    // Unmask interrupts on the boot CPU core.\n    exception::asynchronous::local_irq_unmask();\n\n    // Announce conclusion of the kernel_init() phase.\n    state::state_manager().transition_to_single_core_main();\n\n    // Transition from unsafe to safe.\n    kernel_main()\n}\n\n/// The main function running after the early init.\nfn kernel_main() -> ! {\n    info!(\"{}\", libkernel::version());\n    info!(\"Booting on: {}\", bsp::board_name());\n\n    info!(\"MMU online:\");\n    memory::mmu::kernel_print_mappings();\n\n    let (_, privilege_level) = exception::current_privilege_level();\n    info!(\"Current privilege level: {}\", privilege_level);\n\n    info!(\"Exception handling state:\");\n    exception::asynchronous::print_state();\n\n    info!(\n        \"Architectural timer resolution: {} ns\",\n        time::time_manager().resolution().as_nanos()\n    );\n\n    info!(\"Drivers loaded:\");\n    driver::driver_manager().enumerate();\n\n    info!(\"Registered IRQ handlers:\");\n    exception::asynchronous::irq_manager().print_handler();\n\n    info!(\"Kernel heap:\");\n    memory::heap_alloc::kernel_heap_allocator().print_usage();\n\n    info!(\"Echoing input now\");\n    cpu::wait_forever();\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/memory/heap_alloc.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Heap allocation.\n\nuse crate::{\n    backtrace, bsp, common, debug, info,\n    memory::{Address, Virtual},\n    synchronization,\n    synchronization::IRQSafeNullLock,\n    warn,\n};\nuse alloc::alloc::{GlobalAlloc, Layout};\nuse core::sync::atomic::{AtomicBool, Ordering};\nuse linked_list_allocator::Heap as LinkedListHeap;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// A heap allocator that can be lazyily initialized.\npub struct HeapAllocator {\n    inner: IRQSafeNullLock<LinkedListHeap>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n#[global_allocator]\nstatic KERNEL_HEAP_ALLOCATOR: HeapAllocator = HeapAllocator::new();\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n#[inline(always)]\nfn debug_print_alloc_dealloc(operation: &'static str, ptr: *mut u8, layout: Layout) {\n    let size = layout.size();\n    let (size_h, size_unit) = common::size_human_readable_ceil(size);\n    let addr = Address::<Virtual>::new(ptr as usize);\n\n    debug!(\n        \"Kernel Heap: {}\\n      \\\n        Size:     {:#x} ({} {})\\n      \\\n        Start:    {}\\n      \\\n        End excl: {}\\n\\n      \\\n        {}\",\n        operation,\n        size,\n        size_h,\n        size_unit,\n        addr,\n        addr + size,\n        backtrace::Backtrace\n    );\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\n#[alloc_error_handler]\nfn alloc_error_handler(layout: Layout) -> ! {\n    panic!(\"Allocation error: {:?}\", layout)\n}\n\n/// Return a reference to the kernel's heap allocator.\npub fn kernel_heap_allocator() -> &'static HeapAllocator {\n    &KERNEL_HEAP_ALLOCATOR\n}\n\nimpl HeapAllocator {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            inner: IRQSafeNullLock::new(LinkedListHeap::empty()),\n        }\n    }\n\n    /// Print the current heap usage.\n    pub fn print_usage(&self) {\n        let (used, free) = KERNEL_HEAP_ALLOCATOR\n            .inner\n            .lock(|inner| (inner.used(), inner.free()));\n\n        if used >= 1024 {\n            let (used_h, used_unit) = common::size_human_readable_ceil(used);\n            info!(\"      Used: {} Byte ({} {})\", used, used_h, used_unit);\n        } else {\n            info!(\"      Used: {} Byte\", used);\n        }\n\n        if free >= 1024 {\n            let (free_h, free_unit) = common::size_human_readable_ceil(free);\n            info!(\"      Free: {} Byte ({} {})\", free, free_h, free_unit);\n        } else {\n            info!(\"      Free: {} Byte\", free);\n        }\n    }\n}\n\nunsafe impl GlobalAlloc for HeapAllocator {\n    unsafe fn alloc(&self, layout: Layout) -> *mut u8 {\n        let result = KERNEL_HEAP_ALLOCATOR\n            .inner\n            .lock(|inner| inner.allocate_first_fit(layout).ok());\n\n        match result {\n            None => core::ptr::null_mut(),\n            Some(allocation) => {\n                let ptr = allocation.as_ptr();\n\n                debug_print_alloc_dealloc(\"Allocation\", ptr, layout);\n\n                ptr\n            }\n        }\n    }\n\n    unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {\n        KERNEL_HEAP_ALLOCATOR\n            .inner\n            .lock(|inner| inner.deallocate(core::ptr::NonNull::new_unchecked(ptr), layout));\n\n        debug_print_alloc_dealloc(\"Free\", ptr, layout);\n    }\n}\n\n/// Query the BSP for the heap region and initialize the kernel's heap allocator with it.\npub fn kernel_init_heap_allocator() {\n    static INIT_DONE: AtomicBool = AtomicBool::new(false);\n    if INIT_DONE.load(Ordering::Relaxed) {\n        warn!(\"Already initialized\");\n        return;\n    }\n\n    let region = bsp::memory::mmu::virt_heap_region();\n\n    KERNEL_HEAP_ALLOCATOR.inner.lock(|inner| unsafe {\n        inner.init(region.start_addr().as_usize() as *mut u8, region.size())\n    });\n\n    INIT_DONE.store(true, Ordering::Relaxed);\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/memory/mmu/mapping_record.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A record of mapped pages.\n\nuse super::{\n    AccessPermissions, Address, AttributeFields, MMIODescriptor, MemAttributes, MemoryRegion,\n    Physical, Virtual,\n};\nuse crate::{bsp, common, info, synchronization, synchronization::InitStateLock};\nuse alloc::{vec, vec::Vec};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Type describing a virtual memory mapping.\n#[allow(missing_docs)]\nstruct MappingRecordEntry {\n    pub users: Vec<&'static str>,\n    pub phys_start_addr: Address<Physical>,\n    pub virt_start_addr: Address<Virtual>,\n    pub num_pages: usize,\n    pub attribute_fields: AttributeFields,\n}\n\nstruct MappingRecord {\n    inner: Vec<MappingRecordEntry>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic KERNEL_MAPPING_RECORD: InitStateLock<MappingRecord> =\n    InitStateLock::new(MappingRecord::new());\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl MappingRecordEntry {\n    pub fn new(\n        name: &'static str,\n        virt_region: &MemoryRegion<Virtual>,\n        phys_region: &MemoryRegion<Physical>,\n        attr: &AttributeFields,\n    ) -> Self {\n        Self {\n            users: vec![name],\n            phys_start_addr: phys_region.start_addr(),\n            virt_start_addr: virt_region.start_addr(),\n            num_pages: phys_region.num_pages(),\n            attribute_fields: *attr,\n        }\n    }\n\n    pub fn add_user(&mut self, user: &'static str) {\n        self.users.push(user);\n    }\n}\n\nimpl MappingRecord {\n    pub const fn new() -> Self {\n        Self { inner: Vec::new() }\n    }\n\n    fn sort(&mut self) {\n        if !self.inner.is_sorted_by_key(|item| item.virt_start_addr) {\n            self.inner.sort_unstable_by_key(|item| item.virt_start_addr)\n        }\n    }\n\n    fn find_duplicate(\n        &mut self,\n        phys_region: &MemoryRegion<Physical>,\n    ) -> Option<&mut MappingRecordEntry> {\n        self.inner\n            .iter_mut()\n            .filter(|x| x.attribute_fields.mem_attributes == MemAttributes::Device)\n            .find(|x| {\n                if x.phys_start_addr != phys_region.start_addr() {\n                    return false;\n                }\n\n                if x.num_pages != phys_region.num_pages() {\n                    return false;\n                }\n\n                true\n            })\n    }\n\n    pub fn add(\n        &mut self,\n        name: &'static str,\n        virt_region: &MemoryRegion<Virtual>,\n        phys_region: &MemoryRegion<Physical>,\n        attr: &AttributeFields,\n    ) {\n        self.inner.push(MappingRecordEntry::new(\n            name,\n            virt_region,\n            phys_region,\n            attr,\n        ));\n\n        self.sort();\n    }\n\n    pub fn print(&self) {\n        info!(\"      -------------------------------------------------------------------------------------------------------------------------------------------\");\n        info!(\n            \"      {:^44}     {:^30}   {:^7}   {:^9}   {:^35}\",\n            \"Virtual\", \"Physical\", \"Size\", \"Attr\", \"Entity\"\n        );\n        info!(\"      -------------------------------------------------------------------------------------------------------------------------------------------\");\n\n        for i in self.inner.iter() {\n            let size = i.num_pages * bsp::memory::mmu::KernelGranule::SIZE;\n            let virt_start = i.virt_start_addr;\n            let virt_end_inclusive = virt_start + (size - 1);\n            let phys_start = i.phys_start_addr;\n            let phys_end_inclusive = phys_start + (size - 1);\n\n            let (size, unit) = common::size_human_readable_ceil(size);\n\n            let attr = match i.attribute_fields.mem_attributes {\n                MemAttributes::CacheableDRAM => \"C\",\n                MemAttributes::Device => \"Dev\",\n            };\n\n            let acc_p = match i.attribute_fields.acc_perms {\n                AccessPermissions::ReadOnly => \"RO\",\n                AccessPermissions::ReadWrite => \"RW\",\n            };\n\n            let xn = if i.attribute_fields.execute_never {\n                \"XN\"\n            } else {\n                \"X\"\n            };\n\n            info!(\n                \"      {}..{} --> {}..{} | {:>3} {} | {:<3} {} {:<2} | {}\",\n                virt_start,\n                virt_end_inclusive,\n                phys_start,\n                phys_end_inclusive,\n                size,\n                unit,\n                attr,\n                acc_p,\n                xn,\n                i.users[0]\n            );\n\n            for k in &i.users[1..] {\n                info!(\n                        \"                                                                                                            | {}\",\n                        k\n                    );\n            }\n        }\n\n        info!(\"      -------------------------------------------------------------------------------------------------------------------------------------------\");\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::interface::ReadWriteEx;\n\n/// Add an entry to the mapping info record.\npub fn kernel_add(\n    name: &'static str,\n    virt_region: &MemoryRegion<Virtual>,\n    phys_region: &MemoryRegion<Physical>,\n    attr: &AttributeFields,\n) {\n    KERNEL_MAPPING_RECORD.write(|mr| mr.add(name, virt_region, phys_region, attr))\n}\n\npub fn kernel_find_and_insert_mmio_duplicate(\n    mmio_descriptor: &MMIODescriptor,\n    new_user: &'static str,\n) -> Option<Address<Virtual>> {\n    let phys_region: MemoryRegion<Physical> = (*mmio_descriptor).into();\n\n    KERNEL_MAPPING_RECORD.write(|mr| {\n        let dup = mr.find_duplicate(&phys_region)?;\n\n        dup.add_user(new_user);\n\n        Some(dup.virt_start_addr)\n    })\n}\n\n/// Human-readable print of all recorded kernel mappings.\npub fn kernel_print() {\n    KERNEL_MAPPING_RECORD.read(|mr| mr.print());\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/memory/mmu/page_alloc.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Page allocation.\n\nuse super::MemoryRegion;\nuse crate::{\n    memory::{AddressType, Virtual},\n    synchronization::IRQSafeNullLock,\n    warn,\n};\nuse core::num::NonZeroUsize;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// A page allocator that can be lazyily initialized.\npub struct PageAllocator<ATYPE: AddressType> {\n    pool: Option<MemoryRegion<ATYPE>>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic KERNEL_MMIO_VA_ALLOCATOR: IRQSafeNullLock<PageAllocator<Virtual>> =\n    IRQSafeNullLock::new(PageAllocator::new());\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the kernel's MMIO virtual address allocator.\npub fn kernel_mmio_va_allocator() -> &'static IRQSafeNullLock<PageAllocator<Virtual>> {\n    &KERNEL_MMIO_VA_ALLOCATOR\n}\n\nimpl<ATYPE: AddressType> PageAllocator<ATYPE> {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self { pool: None }\n    }\n\n    /// Initialize the allocator.\n    pub fn init(&mut self, pool: MemoryRegion<ATYPE>) {\n        if self.pool.is_some() {\n            warn!(\"Already initialized\");\n            return;\n        }\n\n        self.pool = Some(pool);\n    }\n\n    /// Allocate a number of pages.\n    pub fn alloc(\n        &mut self,\n        num_requested_pages: NonZeroUsize,\n    ) -> Result<MemoryRegion<ATYPE>, &'static str> {\n        if self.pool.is_none() {\n            return Err(\"Allocator not initialized\");\n        }\n\n        self.pool\n            .as_mut()\n            .unwrap()\n            .take_first_n_pages(num_requested_pages)\n    }\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/memory/mmu/translation_table.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Translation table.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../../_arch/aarch64/memory/mmu/translation_table.rs\"]\nmod arch_translation_table;\n\nuse super::{AttributeFields, MemoryRegion};\nuse crate::memory::{Address, Physical, Virtual};\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\n#[cfg(target_arch = \"aarch64\")]\npub use arch_translation_table::FixedSizeTranslationTable;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Translation table interfaces.\npub mod interface {\n    use crate::memory::mmu::PageAddress;\n\n    use super::*;\n\n    /// Translation table operations.\n    pub trait TranslationTable {\n        /// Anything that needs to run before any of the other provided functions can be used.\n        ///\n        /// # Safety\n        ///\n        /// - Implementor must ensure that this function can run only once or is harmless if invoked\n        ///   multiple times.\n        fn init(&mut self) -> Result<(), &'static str>;\n\n        /// Map the given virtual memory region to the given physical memory region.\n        ///\n        /// # Safety\n        ///\n        /// - Using wrong attributes can cause multiple issues of different nature in the system.\n        /// - It is not required that the architectural implementation prevents aliasing. That is,\n        ///   mapping to the same physical memory using multiple virtual addresses, which would\n        ///   break Rust's ownership assumptions. This should be protected against in the kernel's\n        ///   generic MMU code.\n        unsafe fn map_at(\n            &mut self,\n            virt_region: &MemoryRegion<Virtual>,\n            phys_region: &MemoryRegion<Physical>,\n            attr: &AttributeFields,\n        ) -> Result<(), &'static str>;\n\n        /// Try to translate a virtual page address to a physical page address.\n        ///\n        /// Will only succeed if there exists a valid mapping for the input page.\n        fn try_virt_page_addr_to_phys_page_addr(\n            &self,\n            virt_page_addr: PageAddress<Virtual>,\n        ) -> Result<PageAddress<Physical>, &'static str>;\n\n        /// Try to get the attributes of a page.\n        ///\n        /// Will only succeed if there exists a valid mapping for the input page.\n        fn try_page_attributes(\n            &self,\n            virt_page_addr: PageAddress<Virtual>,\n        ) -> Result<AttributeFields, &'static str>;\n\n        /// Try to translate a virtual address to a physical address.\n        ///\n        /// Will only succeed if there exists a valid mapping for the input address.\n        fn try_virt_addr_to_phys_addr(\n            &self,\n            virt_addr: Address<Virtual>,\n        ) -> Result<Address<Physical>, &'static str>;\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use crate::memory::mmu::{AccessPermissions, MemAttributes, PageAddress};\n    use arch_translation_table::MinSizeTranslationTable;\n    use interface::TranslationTable;\n    use test_macros::kernel_test;\n\n    /// Sanity checks for the TranslationTable implementation.\n    #[kernel_test]\n    fn translationtable_implementation_sanity() {\n        // This will occupy a lot of space on the stack.\n        let mut tables = MinSizeTranslationTable::new_for_runtime();\n\n        assert_eq!(tables.init(), Ok(()));\n\n        let virt_end_exclusive_page_addr: PageAddress<Virtual> = PageAddress::MAX;\n        let virt_start_page_addr: PageAddress<Virtual> =\n            virt_end_exclusive_page_addr.checked_offset(-5).unwrap();\n\n        let phys_start_page_addr: PageAddress<Physical> = PageAddress::from(0);\n        let phys_end_exclusive_page_addr: PageAddress<Physical> =\n            phys_start_page_addr.checked_offset(5).unwrap();\n\n        let virt_region = MemoryRegion::new(virt_start_page_addr, virt_end_exclusive_page_addr);\n        let phys_region = MemoryRegion::new(phys_start_page_addr, phys_end_exclusive_page_addr);\n\n        let attr = AttributeFields {\n            mem_attributes: MemAttributes::CacheableDRAM,\n            acc_perms: AccessPermissions::ReadWrite,\n            execute_never: true,\n        };\n\n        unsafe { assert_eq!(tables.map_at(&virt_region, &phys_region, &attr), Ok(())) };\n\n        assert_eq!(\n            tables.try_virt_page_addr_to_phys_page_addr(virt_start_page_addr),\n            Ok(phys_start_page_addr)\n        );\n\n        assert_eq!(\n            tables.try_page_attributes(virt_start_page_addr.checked_offset(-1).unwrap()),\n            Err(\"Page marked invalid\")\n        );\n\n        assert_eq!(tables.try_page_attributes(virt_start_page_addr), Ok(attr));\n\n        let virt_addr = virt_start_page_addr.into_inner() + 0x100;\n        let phys_addr = phys_start_page_addr.into_inner() + 0x100;\n        assert_eq!(tables.try_virt_addr_to_phys_addr(virt_addr), Ok(phys_addr));\n    }\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/memory/mmu/types.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management Unit types.\n\nuse crate::{\n    bsp, common,\n    memory::{Address, AddressType, Physical},\n};\nuse core::{convert::From, iter::Step, num::NonZeroUsize, ops::Range};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// A wrapper type around [Address] that ensures page alignment.\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub struct PageAddress<ATYPE: AddressType> {\n    inner: Address<ATYPE>,\n}\n\n/// A type that describes a region of memory in quantities of pages.\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub struct MemoryRegion<ATYPE: AddressType> {\n    start: PageAddress<ATYPE>,\n    end_exclusive: PageAddress<ATYPE>,\n}\n\n/// Architecture agnostic memory attributes.\n#[allow(missing_docs)]\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub enum MemAttributes {\n    CacheableDRAM,\n    Device,\n}\n\n/// Architecture agnostic access permissions.\n#[allow(missing_docs)]\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub enum AccessPermissions {\n    ReadOnly,\n    ReadWrite,\n}\n\n/// Collection of memory attributes.\n#[allow(missing_docs)]\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub struct AttributeFields {\n    pub mem_attributes: MemAttributes,\n    pub acc_perms: AccessPermissions,\n    pub execute_never: bool,\n}\n\n/// An MMIO descriptor for use in device drivers.\n#[derive(Copy, Clone)]\npub struct MMIODescriptor {\n    start_addr: Address<Physical>,\n    end_addr_exclusive: Address<Physical>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n//------------------------------------------------------------------------------\n// PageAddress\n//------------------------------------------------------------------------------\nimpl<ATYPE: AddressType> PageAddress<ATYPE> {\n    /// The largest value that can be represented by this type.\n    pub const MAX: Self = PageAddress {\n        inner: Address::new(usize::MAX).align_down_page(),\n    };\n\n    /// Unwraps the value.\n    pub fn into_inner(self) -> Address<ATYPE> {\n        self.inner\n    }\n\n    /// Calculates the offset from the page address.\n    ///\n    /// `count` is in units of [PageAddress]. For example, a count of 2 means `result = self + 2 *\n    /// page_size`.\n    pub fn checked_offset(self, count: isize) -> Option<Self> {\n        if count == 0 {\n            return Some(self);\n        }\n\n        let delta = count\n            .unsigned_abs()\n            .checked_mul(bsp::memory::mmu::KernelGranule::SIZE)?;\n        let result = if count.is_positive() {\n            self.inner.as_usize().checked_add(delta)?\n        } else {\n            self.inner.as_usize().checked_sub(delta)?\n        };\n\n        Some(Self {\n            inner: Address::new(result),\n        })\n    }\n}\n\nimpl<ATYPE: AddressType> From<usize> for PageAddress<ATYPE> {\n    fn from(addr: usize) -> Self {\n        assert!(\n            common::is_aligned(addr, bsp::memory::mmu::KernelGranule::SIZE),\n            \"Input usize not page aligned\"\n        );\n\n        Self {\n            inner: Address::new(addr),\n        }\n    }\n}\n\nimpl<ATYPE: AddressType> From<Address<ATYPE>> for PageAddress<ATYPE> {\n    fn from(addr: Address<ATYPE>) -> Self {\n        assert!(addr.is_page_aligned(), \"Input Address not page aligned\");\n\n        Self { inner: addr }\n    }\n}\n\nimpl<ATYPE: AddressType> Step for PageAddress<ATYPE> {\n    fn steps_between(start: &Self, end: &Self) -> Option<usize> {\n        if start > end {\n            return None;\n        }\n\n        // Since start <= end, do unchecked arithmetic.\n        Some(\n            (end.inner.as_usize() - start.inner.as_usize())\n                >> bsp::memory::mmu::KernelGranule::SHIFT,\n        )\n    }\n\n    fn forward_checked(start: Self, count: usize) -> Option<Self> {\n        start.checked_offset(count as isize)\n    }\n\n    fn backward_checked(start: Self, count: usize) -> Option<Self> {\n        start.checked_offset(-(count as isize))\n    }\n}\n\n//------------------------------------------------------------------------------\n// MemoryRegion\n//------------------------------------------------------------------------------\nimpl<ATYPE: AddressType> MemoryRegion<ATYPE> {\n    /// Create an instance.\n    pub fn new(start: PageAddress<ATYPE>, end_exclusive: PageAddress<ATYPE>) -> Self {\n        assert!(start <= end_exclusive);\n\n        Self {\n            start,\n            end_exclusive,\n        }\n    }\n\n    fn as_range(&self) -> Range<PageAddress<ATYPE>> {\n        self.into_iter()\n    }\n\n    /// Returns the start page address.\n    pub fn start_page_addr(&self) -> PageAddress<ATYPE> {\n        self.start\n    }\n\n    /// Returns the start address.\n    pub fn start_addr(&self) -> Address<ATYPE> {\n        self.start.into_inner()\n    }\n\n    /// Returns the exclusive end page address.\n    pub fn end_exclusive_page_addr(&self) -> PageAddress<ATYPE> {\n        self.end_exclusive\n    }\n\n    /// Returns the exclusive end page address.\n    pub fn end_inclusive_page_addr(&self) -> PageAddress<ATYPE> {\n        self.end_exclusive.checked_offset(-1).unwrap()\n    }\n\n    /// Checks if self contains an address.\n    pub fn contains(&self, addr: Address<ATYPE>) -> bool {\n        let page_addr = PageAddress::from(addr.align_down_page());\n        self.as_range().contains(&page_addr)\n    }\n\n    /// Checks if there is an overlap with another memory region.\n    pub fn overlaps(&self, other_region: &Self) -> bool {\n        let self_range = self.as_range();\n\n        self_range.contains(&other_region.start_page_addr())\n            || self_range.contains(&other_region.end_inclusive_page_addr())\n    }\n\n    /// Returns the number of pages contained in this region.\n    pub fn num_pages(&self) -> usize {\n        PageAddress::steps_between(&self.start, &self.end_exclusive).unwrap()\n    }\n\n    /// Returns the size in bytes of this region.\n    pub fn size(&self) -> usize {\n        // Invariant: start <= end_exclusive, so do unchecked arithmetic.\n        let end_exclusive = self.end_exclusive.into_inner().as_usize();\n        let start = self.start.into_inner().as_usize();\n\n        end_exclusive - start\n    }\n\n    /// Splits the MemoryRegion like:\n    ///\n    /// --------------------------------------------------------------------------------\n    /// |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |\n    /// --------------------------------------------------------------------------------\n    ///   ^                               ^                                       ^\n    ///   |                               |                                       |\n    ///   left_start     left_end_exclusive                                       |\n    ///                                                                           |\n    ///                                   ^                                       |\n    ///                                   |                                       |\n    ///                                   right_start           right_end_exclusive\n    ///\n    /// Left region is returned to the caller. Right region is the new region for this struct.\n    pub fn take_first_n_pages(&mut self, num_pages: NonZeroUsize) -> Result<Self, &'static str> {\n        let count: usize = num_pages.into();\n\n        let left_end_exclusive = self.start.checked_offset(count as isize);\n        let left_end_exclusive = match left_end_exclusive {\n            None => return Err(\"Overflow while calculating left_end_exclusive\"),\n            Some(x) => x,\n        };\n\n        if left_end_exclusive > self.end_exclusive {\n            return Err(\"Not enough free pages\");\n        }\n\n        let allocation = Self {\n            start: self.start,\n            end_exclusive: left_end_exclusive,\n        };\n        self.start = left_end_exclusive;\n\n        Ok(allocation)\n    }\n}\n\nimpl<ATYPE: AddressType> IntoIterator for MemoryRegion<ATYPE> {\n    type Item = PageAddress<ATYPE>;\n    type IntoIter = Range<Self::Item>;\n\n    fn into_iter(self) -> Self::IntoIter {\n        Range {\n            start: self.start,\n            end: self.end_exclusive,\n        }\n    }\n}\n\nimpl From<MMIODescriptor> for MemoryRegion<Physical> {\n    fn from(desc: MMIODescriptor) -> Self {\n        let start = PageAddress::from(desc.start_addr.align_down_page());\n        let end_exclusive = PageAddress::from(desc.end_addr_exclusive().align_up_page());\n\n        Self {\n            start,\n            end_exclusive,\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// MMIODescriptor\n//------------------------------------------------------------------------------\n\nimpl MMIODescriptor {\n    /// Create an instance.\n    pub const fn new(start_addr: Address<Physical>, size: usize) -> Self {\n        assert!(size > 0);\n        let end_addr_exclusive = Address::new(start_addr.as_usize() + size);\n\n        Self {\n            start_addr,\n            end_addr_exclusive,\n        }\n    }\n\n    /// Return the start address.\n    pub const fn start_addr(&self) -> Address<Physical> {\n        self.start_addr\n    }\n\n    /// Return the exclusive end address.\n    pub fn end_addr_exclusive(&self) -> Address<Physical> {\n        self.end_addr_exclusive\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use crate::memory::Virtual;\n    use test_macros::kernel_test;\n\n    /// Sanity of [PageAddress] methods.\n    #[kernel_test]\n    fn pageaddress_type_method_sanity() {\n        let page_addr: PageAddress<Virtual> =\n            PageAddress::from(bsp::memory::mmu::KernelGranule::SIZE * 2);\n\n        assert_eq!(\n            page_addr.checked_offset(-2),\n            Some(PageAddress::<Virtual>::from(0))\n        );\n\n        assert_eq!(\n            page_addr.checked_offset(2),\n            Some(PageAddress::<Virtual>::from(\n                bsp::memory::mmu::KernelGranule::SIZE * 4\n            ))\n        );\n\n        assert_eq!(\n            PageAddress::<Virtual>::from(0).checked_offset(0),\n            Some(PageAddress::<Virtual>::from(0))\n        );\n        assert_eq!(PageAddress::<Virtual>::from(0).checked_offset(-1), None);\n\n        let max_page_addr = Address::<Virtual>::new(usize::MAX).align_down_page();\n        assert_eq!(\n            PageAddress::<Virtual>::from(max_page_addr).checked_offset(1),\n            None\n        );\n\n        let zero = PageAddress::<Virtual>::from(0);\n        let three = PageAddress::<Virtual>::from(bsp::memory::mmu::KernelGranule::SIZE * 3);\n        assert_eq!(PageAddress::steps_between(&zero, &three), Some(3));\n    }\n\n    /// Sanity of [MemoryRegion] methods.\n    #[kernel_test]\n    fn memoryregion_type_method_sanity() {\n        let zero = PageAddress::<Virtual>::from(0);\n        let zero_region = MemoryRegion::new(zero, zero);\n        assert_eq!(zero_region.num_pages(), 0);\n        assert_eq!(zero_region.size(), 0);\n\n        let one = PageAddress::<Virtual>::from(bsp::memory::mmu::KernelGranule::SIZE);\n        let one_region = MemoryRegion::new(zero, one);\n        assert_eq!(one_region.num_pages(), 1);\n        assert_eq!(one_region.size(), bsp::memory::mmu::KernelGranule::SIZE);\n\n        let three = PageAddress::<Virtual>::from(bsp::memory::mmu::KernelGranule::SIZE * 3);\n        let mut three_region = MemoryRegion::new(zero, three);\n        assert!(three_region.contains(zero.into_inner()));\n        assert!(!three_region.contains(three.into_inner()));\n        assert!(three_region.overlaps(&one_region));\n\n        let allocation = three_region\n            .take_first_n_pages(NonZeroUsize::new(2).unwrap())\n            .unwrap();\n        assert_eq!(allocation.num_pages(), 2);\n        assert_eq!(three_region.num_pages(), 1);\n\n        for (i, alloc) in allocation.into_iter().enumerate() {\n            assert_eq!(\n                alloc.into_inner().as_usize(),\n                i * bsp::memory::mmu::KernelGranule::SIZE\n            );\n        }\n    }\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/memory/mmu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management Unit.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/memory/mmu.rs\"]\nmod arch_mmu;\n\nmod mapping_record;\nmod page_alloc;\nmod translation_table;\nmod types;\n\nuse crate::{\n    bsp,\n    memory::{Address, Physical, Virtual},\n    synchronization::{self, interface::Mutex},\n};\nuse core::{fmt, num::NonZeroUsize};\n\npub use types::*;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// MMU enable errors variants.\n#[allow(missing_docs)]\n#[derive(Debug)]\npub enum MMUEnableError {\n    AlreadyEnabled,\n    Other(&'static str),\n}\n\n/// Memory Management interfaces.\npub mod interface {\n    use super::*;\n\n    /// MMU functions.\n    pub trait MMU {\n        /// Turns on the MMU for the first time and enables data and instruction caching.\n        ///\n        /// # Safety\n        ///\n        /// - Changes the HW's global state.\n        unsafe fn enable_mmu_and_caching(\n            &self,\n            phys_tables_base_addr: Address<Physical>,\n        ) -> Result<(), MMUEnableError>;\n\n        /// Returns true if the MMU is enabled, false otherwise.\n        fn is_enabled(&self) -> bool;\n    }\n}\n\n/// Describes the characteristics of a translation granule.\npub struct TranslationGranule<const GRANULE_SIZE: usize>;\n\n/// Describes properties of an address space.\npub struct AddressSpace<const AS_SIZE: usize>;\n\n/// Intended to be implemented for [`AddressSpace`].\npub trait AssociatedTranslationTable {\n    /// A translation table whose address range is:\n    ///\n    /// [u64::MAX, (u64::MAX - AS_SIZE) + 1]\n    type TableStartFromTop;\n\n    /// A translation table whose address range is:\n    ///\n    /// [AS_SIZE - 1, 0]\n    type TableStartFromBottom;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\nuse interface::MMU;\nuse synchronization::interface::ReadWriteEx;\nuse translation_table::interface::TranslationTable;\n\n/// Map a region in the kernel's translation tables.\n///\n/// No input checks done, input is passed through to the architectural implementation.\n///\n/// # Safety\n///\n/// - See `map_at()`.\n/// - Does not prevent aliasing.\nunsafe fn kernel_map_at_unchecked(\n    name: &'static str,\n    virt_region: &MemoryRegion<Virtual>,\n    phys_region: &MemoryRegion<Physical>,\n    attr: &AttributeFields,\n) -> Result<(), &'static str> {\n    bsp::memory::mmu::kernel_translation_tables()\n        .write(|tables| tables.map_at(virt_region, phys_region, attr))?;\n\n    kernel_add_mapping_record(name, virt_region, phys_region, attr);\n\n    Ok(())\n}\n\n/// Try to translate a kernel virtual address to a physical address.\n///\n/// Will only succeed if there exists a valid mapping for the input address.\nfn try_kernel_virt_addr_to_phys_addr(\n    virt_addr: Address<Virtual>,\n) -> Result<Address<Physical>, &'static str> {\n    bsp::memory::mmu::kernel_translation_tables()\n        .read(|tables| tables.try_virt_addr_to_phys_addr(virt_addr))\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl fmt::Display for MMUEnableError {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        match self {\n            MMUEnableError::AlreadyEnabled => write!(f, \"MMU is already enabled\"),\n            MMUEnableError::Other(x) => write!(f, \"{}\", x),\n        }\n    }\n}\n\nimpl<const GRANULE_SIZE: usize> TranslationGranule<GRANULE_SIZE> {\n    /// The granule's size.\n    pub const SIZE: usize = Self::size_checked();\n\n    /// The granule's mask.\n    pub const MASK: usize = Self::SIZE - 1;\n\n    /// The granule's shift, aka log2(size).\n    pub const SHIFT: usize = Self::SIZE.trailing_zeros() as usize;\n\n    const fn size_checked() -> usize {\n        assert!(GRANULE_SIZE.is_power_of_two());\n\n        GRANULE_SIZE\n    }\n}\n\nimpl<const AS_SIZE: usize> AddressSpace<AS_SIZE> {\n    /// The address space size.\n    pub const SIZE: usize = Self::size_checked();\n\n    /// The address space shift, aka log2(size).\n    pub const SIZE_SHIFT: usize = Self::SIZE.trailing_zeros() as usize;\n\n    const fn size_checked() -> usize {\n        assert!(AS_SIZE.is_power_of_two());\n\n        // Check for architectural restrictions as well.\n        Self::arch_address_space_size_sanity_checks();\n\n        AS_SIZE\n    }\n}\n\n/// Query the BSP for the reserved virtual addresses for MMIO remapping and initialize the kernel's\n/// MMIO VA allocator with it.\npub fn kernel_init_mmio_va_allocator() {\n    let region = bsp::memory::mmu::virt_mmio_remap_region();\n\n    page_alloc::kernel_mmio_va_allocator().lock(|allocator| allocator.init(region));\n}\n\n/// Add an entry to the mapping info record.\npub fn kernel_add_mapping_record(\n    name: &'static str,\n    virt_region: &MemoryRegion<Virtual>,\n    phys_region: &MemoryRegion<Physical>,\n    attr: &AttributeFields,\n) {\n    mapping_record::kernel_add(name, virt_region, phys_region, attr);\n}\n\n/// MMIO remapping in the kernel translation tables.\n///\n/// Typically used by device drivers.\n///\n/// # Safety\n///\n/// - Same as `kernel_map_at_unchecked()`, minus the aliasing part.\npub unsafe fn kernel_map_mmio(\n    name: &'static str,\n    mmio_descriptor: &MMIODescriptor,\n) -> Result<Address<Virtual>, &'static str> {\n    let phys_region = MemoryRegion::from(*mmio_descriptor);\n    let offset_into_start_page = mmio_descriptor.start_addr().offset_into_page();\n\n    // Check if an identical region has been mapped for another driver. If so, reuse it.\n    let virt_addr = if let Some(addr) =\n        mapping_record::kernel_find_and_insert_mmio_duplicate(mmio_descriptor, name)\n    {\n        addr\n    // Otherwise, allocate a new region and map it.\n    } else {\n        let num_pages = match NonZeroUsize::new(phys_region.num_pages()) {\n            None => return Err(\"Requested 0 pages\"),\n            Some(x) => x,\n        };\n\n        let virt_region =\n            page_alloc::kernel_mmio_va_allocator().lock(|allocator| allocator.alloc(num_pages))?;\n\n        kernel_map_at_unchecked(\n            name,\n            &virt_region,\n            &phys_region,\n            &AttributeFields {\n                mem_attributes: MemAttributes::Device,\n                acc_perms: AccessPermissions::ReadWrite,\n                execute_never: true,\n            },\n        )?;\n\n        virt_region.start_addr()\n    };\n\n    Ok(virt_addr + offset_into_start_page)\n}\n\n/// Try to translate a kernel virtual page address to a physical page address.\n///\n/// Will only succeed if there exists a valid mapping for the input page.\npub fn try_kernel_virt_page_addr_to_phys_page_addr(\n    virt_page_addr: PageAddress<Virtual>,\n) -> Result<PageAddress<Physical>, &'static str> {\n    bsp::memory::mmu::kernel_translation_tables()\n        .read(|tables| tables.try_virt_page_addr_to_phys_page_addr(virt_page_addr))\n}\n\n/// Try to get the attributes of a kernel page.\n///\n/// Will only succeed if there exists a valid mapping for the input page.\npub fn try_kernel_page_attributes(\n    virt_page_addr: PageAddress<Virtual>,\n) -> Result<AttributeFields, &'static str> {\n    bsp::memory::mmu::kernel_translation_tables()\n        .read(|tables| tables.try_page_attributes(virt_page_addr))\n}\n\n/// Human-readable print of all recorded kernel mappings.\npub fn kernel_print_mappings() {\n    mapping_record::kernel_print()\n}\n\n/// Enable the MMU and data + instruction caching.\n///\n/// # Safety\n///\n/// - Crucial function during kernel init. Changes the the complete memory view of the processor.\n#[inline(always)]\npub unsafe fn enable_mmu_and_caching(\n    phys_tables_base_addr: Address<Physical>,\n) -> Result<(), MMUEnableError> {\n    arch_mmu::mmu().enable_mmu_and_caching(phys_tables_base_addr)\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/memory.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management.\n\npub mod heap_alloc;\npub mod mmu;\n\nuse crate::{bsp, common};\nuse core::{\n    fmt,\n    marker::PhantomData,\n    ops::{Add, Sub},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Metadata trait for marking the type of an address.\npub trait AddressType: Copy + Clone + PartialOrd + PartialEq + Ord + Eq {}\n\n/// Zero-sized type to mark a physical address.\n#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq)]\npub enum Physical {}\n\n/// Zero-sized type to mark a virtual address.\n#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq)]\npub enum Virtual {}\n\n/// Generic address type.\n#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq)]\npub struct Address<ATYPE: AddressType> {\n    value: usize,\n    _address_type: PhantomData<fn() -> ATYPE>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl AddressType for Physical {}\nimpl AddressType for Virtual {}\n\nimpl<ATYPE: AddressType> Address<ATYPE> {\n    /// Create an instance.\n    pub const fn new(value: usize) -> Self {\n        Self {\n            value,\n            _address_type: PhantomData,\n        }\n    }\n\n    /// Convert to usize.\n    pub const fn as_usize(self) -> usize {\n        self.value\n    }\n\n    /// Align down to page size.\n    #[must_use]\n    pub const fn align_down_page(self) -> Self {\n        let aligned = common::align_down(self.value, bsp::memory::mmu::KernelGranule::SIZE);\n\n        Self::new(aligned)\n    }\n\n    /// Align up to page size.\n    #[must_use]\n    pub const fn align_up_page(self) -> Self {\n        let aligned = common::align_up(self.value, bsp::memory::mmu::KernelGranule::SIZE);\n\n        Self::new(aligned)\n    }\n\n    /// Checks if the address is page aligned.\n    pub const fn is_page_aligned(&self) -> bool {\n        common::is_aligned(self.value, bsp::memory::mmu::KernelGranule::SIZE)\n    }\n\n    /// Return the address' offset into the corresponding page.\n    pub const fn offset_into_page(&self) -> usize {\n        self.value & bsp::memory::mmu::KernelGranule::MASK\n    }\n}\n\nimpl<ATYPE: AddressType> Add<usize> for Address<ATYPE> {\n    type Output = Self;\n\n    #[inline(always)]\n    fn add(self, rhs: usize) -> Self::Output {\n        match self.value.checked_add(rhs) {\n            None => panic!(\"Overflow on Address::add\"),\n            Some(x) => Self::new(x),\n        }\n    }\n}\n\nimpl<ATYPE: AddressType> Sub<usize> for Address<ATYPE> {\n    type Output = Self;\n\n    #[inline(always)]\n    fn sub(self, rhs: usize) -> Self::Output {\n        match self.value.checked_sub(rhs) {\n            None => panic!(\"Overflow on Address::sub\"),\n            Some(x) => Self::new(x),\n        }\n    }\n}\n\nimpl<ATYPE: AddressType> Sub<Address<ATYPE>> for Address<ATYPE> {\n    type Output = Self;\n\n    #[inline(always)]\n    fn sub(self, rhs: Address<ATYPE>) -> Self::Output {\n        match self.value.checked_sub(rhs.value) {\n            None => panic!(\"Overflow on Address::sub\"),\n            Some(x) => Self::new(x),\n        }\n    }\n}\n\nimpl Address<Virtual> {\n    /// Checks if the address is part of the boot core stack region.\n    pub fn is_valid_stack_addr(&self) -> bool {\n        bsp::memory::mmu::virt_boot_core_stack_region().contains(*self)\n    }\n\n    /// Checks if the address is part of the kernel code region.\n    pub fn is_valid_code_addr(&self) -> bool {\n        bsp::memory::mmu::virt_code_region().contains(*self)\n    }\n}\n\nimpl fmt::Display for Address<Physical> {\n    // Don't expect to see physical addresses greater than 40 bit.\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        let q3: u8 = ((self.value >> 32) & 0xff) as u8;\n        let q2: u16 = ((self.value >> 16) & 0xffff) as u16;\n        let q1: u16 = (self.value & 0xffff) as u16;\n\n        write!(f, \"0x\")?;\n        write!(f, \"{:02x}_\", q3)?;\n        write!(f, \"{:04x}_\", q2)?;\n        write!(f, \"{:04x}\", q1)\n    }\n}\n\nimpl fmt::Display for Address<Virtual> {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        let q4: u16 = ((self.value >> 48) & 0xffff) as u16;\n        let q3: u16 = ((self.value >> 32) & 0xffff) as u16;\n        let q2: u16 = ((self.value >> 16) & 0xffff) as u16;\n        let q1: u16 = (self.value & 0xffff) as u16;\n\n        write!(f, \"0x\")?;\n        write!(f, \"{:04x}_\", q4)?;\n        write!(f, \"{:04x}_\", q3)?;\n        write!(f, \"{:04x}_\", q2)?;\n        write!(f, \"{:04x}\", q1)\n    }\n}\n\n/// Initialize the memory subsystem.\npub fn init() {\n    mmu::kernel_init_mmio_va_allocator();\n    heap_alloc::kernel_init_heap_allocator();\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// Sanity of [Address] methods.\n    #[kernel_test]\n    fn address_type_method_sanity() {\n        let addr = Address::<Virtual>::new(bsp::memory::mmu::KernelGranule::SIZE + 100);\n\n        assert_eq!(\n            addr.align_down_page().as_usize(),\n            bsp::memory::mmu::KernelGranule::SIZE\n        );\n\n        assert_eq!(\n            addr.align_up_page().as_usize(),\n            bsp::memory::mmu::KernelGranule::SIZE * 2\n        );\n\n        assert!(!addr.is_page_aligned());\n\n        assert_eq!(addr.offset_into_page(), 100);\n    }\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/panic_wait.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A panic handler that infinitely waits.\n\nuse crate::{backtrace, cpu, exception, println};\nuse core::panic::PanicInfo;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// The point of exit for `libkernel`.\n///\n/// It is linked weakly, so that the integration tests can overload its standard behavior.\n#[linkage = \"weak\"]\n#[no_mangle]\nfn _panic_exit() -> ! {\n    #[cfg(not(feature = \"test_build\"))]\n    {\n        cpu::wait_forever()\n    }\n\n    #[cfg(feature = \"test_build\")]\n    {\n        cpu::qemu_exit_failure()\n    }\n}\n\n/// Stop immediately if called a second time.\n///\n/// # Note\n///\n/// Using atomics here relieves us from needing to use `unsafe` for the static variable.\n///\n/// On `AArch64`, which is the only implemented architecture at the time of writing this,\n/// [`AtomicBool::load`] and [`AtomicBool::store`] are lowered to ordinary load and store\n/// instructions. They are therefore safe to use even with MMU + caching deactivated.\n///\n/// [`AtomicBool::load`]: core::sync::atomic::AtomicBool::load\n/// [`AtomicBool::store`]: core::sync::atomic::AtomicBool::store\nfn panic_prevent_reenter() {\n    use core::sync::atomic::{AtomicBool, Ordering};\n\n    #[cfg(not(target_arch = \"aarch64\"))]\n    compile_error!(\"Add the target_arch to above's check if the following code is safe to use\");\n\n    static PANIC_IN_PROGRESS: AtomicBool = AtomicBool::new(false);\n\n    if !PANIC_IN_PROGRESS.load(Ordering::Relaxed) {\n        PANIC_IN_PROGRESS.store(true, Ordering::Relaxed);\n\n        return;\n    }\n\n    _panic_exit()\n}\n\n#[panic_handler]\nfn panic(info: &PanicInfo) -> ! {\n    exception::asynchronous::local_irq_mask();\n\n    // Protect against panic infinite loops if any of the following code panics itself.\n    panic_prevent_reenter();\n\n    let timestamp = crate::time::time_manager().uptime();\n    let (location, line, column) = match info.location() {\n        Some(loc) => (loc.file(), loc.line(), loc.column()),\n        _ => (\"???\", 0, 0),\n    };\n\n    println!(\n        \"[  {:>3}.{:06}] Kernel panic!\\n\\n\\\n        Panic location:\\n      File '{}', line {}, column {}\\n\\n\\\n        {}\\n\\n\\\n        {}\",\n        timestamp.as_secs(),\n        timestamp.subsec_micros(),\n        location,\n        line,\n        column,\n        info.message().unwrap_or(&format_args!(\"\")),\n        backtrace::Backtrace\n    );\n\n    _panic_exit()\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/print.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Printing.\n\nuse crate::console;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n#[doc(hidden)]\npub fn _print(args: fmt::Arguments) {\n    console::console().write_fmt(args).unwrap();\n}\n\n/// Prints without a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! print {\n    ($($arg:tt)*) => ($crate::print::_print(format_args!($($arg)*)));\n}\n\n/// Prints with a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! println {\n    () => ($crate::print!(\"\\n\"));\n    ($($arg:tt)*) => ({\n        $crate::print::_print(format_args_nl!($($arg)*));\n    })\n}\n\n/// Prints an info, with a newline.\n#[macro_export]\nmacro_rules! info {\n    ($string:expr) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[  {:>3}.{:06}] \", $string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n        ));\n    });\n    ($format_string:expr, $($arg:tt)*) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[  {:>3}.{:06}] \", $format_string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n            $($arg)*\n        ));\n    })\n}\n\n/// Prints a warning, with a newline.\n#[macro_export]\nmacro_rules! warn {\n    ($string:expr) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[W {:>3}.{:06}] \", $string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n        ));\n    });\n    ($format_string:expr, $($arg:tt)*) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[W {:>3}.{:06}] \", $format_string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n            $($arg)*\n        ));\n    })\n}\n\n/// Debug print, with a newline.\n#[macro_export]\nmacro_rules! debug {\n    ($string:expr) => ({\n        if cfg!(feature = \"debug_prints\") {\n            let timestamp = $crate::time::time_manager().uptime();\n\n            $crate::print::_print(format_args_nl!(\n                concat!(\"<[>D {:>3}.{:06}> \", $string),\n                timestamp.as_secs(),\n                timestamp.subsec_micros(),\n            ));\n        }\n    });\n    ($format_string:expr, $($arg:tt)*) => ({\n        if cfg!(feature = \"debug_prints\") {\n            let timestamp = $crate::time::time_manager().uptime();\n\n            $crate::print::_print(format_args_nl!(\n                concat!(\"<D {:>3}.{:06}> \", $format_string),\n                timestamp.as_secs(),\n                timestamp.subsec_micros(),\n                $($arg)*\n            ));\n        }\n    })\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/state.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! State information about the kernel itself.\n\nuse core::sync::atomic::{AtomicU8, Ordering};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Different stages in the kernel execution.\n#[derive(Copy, Clone, Eq, PartialEq)]\nenum State {\n    /// The kernel starts booting in this state.\n    Init,\n\n    /// The kernel transitions to this state when jumping to `kernel_main()` (at the end of\n    /// `kernel_init()`, after all init calls are done).\n    SingleCoreMain,\n\n    /// The kernel transitions to this state when it boots the secondary cores, aka switches\n    /// exectution mode to symmetric multiprocessing (SMP).\n    MultiCoreMain,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Maintains the kernel state and state transitions.\npub struct StateManager(AtomicU8);\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic STATE_MANAGER: StateManager = StateManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the global StateManager.\npub fn state_manager() -> &'static StateManager {\n    &STATE_MANAGER\n}\n\nimpl StateManager {\n    const INIT: u8 = 0;\n    const SINGLE_CORE_MAIN: u8 = 1;\n    const MULTI_CORE_MAIN: u8 = 2;\n\n    /// Create a new instance.\n    pub const fn new() -> Self {\n        Self(AtomicU8::new(Self::INIT))\n    }\n\n    /// Return the current state.\n    fn state(&self) -> State {\n        let state = self.0.load(Ordering::Acquire);\n\n        match state {\n            Self::INIT => State::Init,\n            Self::SINGLE_CORE_MAIN => State::SingleCoreMain,\n            Self::MULTI_CORE_MAIN => State::MultiCoreMain,\n            _ => panic!(\"Invalid KERNEL_STATE\"),\n        }\n    }\n\n    /// Return if the kernel is init state.\n    pub fn is_init(&self) -> bool {\n        self.state() == State::Init\n    }\n\n    /// Transition from Init to SingleCoreMain.\n    pub fn transition_to_single_core_main(&self) {\n        if self\n            .0\n            .compare_exchange(\n                Self::INIT,\n                Self::SINGLE_CORE_MAIN,\n                Ordering::Acquire,\n                Ordering::Relaxed,\n            )\n            .is_err()\n        {\n            panic!(\"transition_to_single_core_main() called while state != Init\");\n        }\n    }\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/symbols.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Debug symbol support.\n\nuse crate::memory::{Address, Virtual};\nuse core::{cell::UnsafeCell, slice};\nuse debug_symbol_types::Symbol;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Symbol from the linker script.\nextern \"Rust\" {\n    static __kernel_symbols_start: UnsafeCell<()>;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n/// This will be patched to the correct value by the \"kernel symbols tool\" after linking. This given\n/// value here is just a (safe) dummy.\n#[no_mangle]\nstatic NUM_KERNEL_SYMBOLS: u64 = 0;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nfn kernel_symbol_section_virt_start_addr() -> Address<Virtual> {\n    Address::new(unsafe { __kernel_symbols_start.get() as usize })\n}\n\nfn num_kernel_symbols() -> usize {\n    unsafe {\n        // Read volatile is needed here to prevent the compiler from optimizing NUM_KERNEL_SYMBOLS\n        // away.\n        core::ptr::read_volatile(&NUM_KERNEL_SYMBOLS as *const u64) as usize\n    }\n}\n\nfn kernel_symbols_slice() -> &'static [Symbol] {\n    let ptr = kernel_symbol_section_virt_start_addr().as_usize() as *const Symbol;\n\n    unsafe { slice::from_raw_parts(ptr, num_kernel_symbols()) }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Retrieve the symbol corresponding to a virtual address, if any.\npub fn lookup_symbol(addr: Address<Virtual>) -> Option<&'static Symbol> {\n    kernel_symbols_slice()\n        .iter()\n        .find(|&i| i.contains(addr.as_usize()))\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// Sanity of symbols module.\n    #[kernel_test]\n    fn symbols_sanity() {\n        let first_sym = lookup_symbol(Address::new(\n            crate::common::is_aligned as *const usize as usize,\n        ))\n        .unwrap()\n        .name();\n\n        assert_eq!(first_sym, \"libkernel::common::is_aligned\");\n\n        let second_sym = lookup_symbol(Address::new(crate::version as *const usize as usize))\n            .unwrap()\n            .name();\n\n        assert_eq!(second_sym, \"libkernel::version\");\n    }\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/synchronization.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Synchronization primitives.\n//!\n//! # Resources\n//!\n//!   - <https://doc.rust-lang.org/book/ch16-04-extensible-concurrency-sync-and-send.html>\n//!   - <https://stackoverflow.com/questions/59428096/understanding-the-send-trait>\n//!   - <https://doc.rust-lang.org/std/cell/index.html>\n\nuse core::cell::UnsafeCell;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Synchronization interfaces.\npub mod interface {\n\n    /// Any object implementing this trait guarantees exclusive access to the data wrapped within\n    /// the Mutex for the duration of the provided closure.\n    pub trait Mutex {\n        /// The type of the data that is wrapped by this mutex.\n        type Data;\n\n        /// Locks the mutex and grants the closure temporary mutable access to the wrapped data.\n        fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R;\n    }\n\n    /// A reader-writer exclusion type.\n    ///\n    /// The implementing object allows either a number of readers or at most one writer at any point\n    /// in time.\n    pub trait ReadWriteEx {\n        /// The type of encapsulated data.\n        type Data;\n\n        /// Grants temporary mutable access to the encapsulated data.\n        fn write<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R;\n\n        /// Grants temporary immutable access to the encapsulated data.\n        fn read<'a, R>(&'a self, f: impl FnOnce(&'a Self::Data) -> R) -> R;\n    }\n}\n\n/// A pseudo-lock for teaching purposes.\n///\n/// In contrast to a real Mutex implementation, does not protect against concurrent access from\n/// other cores to the contained data. This part is preserved for later lessons.\n///\n/// The lock will only be used as long as it is safe to do so, i.e. as long as the kernel is\n/// executing on a single core.\npub struct IRQSafeNullLock<T>\nwhere\n    T: ?Sized,\n{\n    data: UnsafeCell<T>,\n}\n\n/// A pseudo-lock that is RW during the single-core kernel init phase and RO afterwards.\n///\n/// Intended to encapsulate data that is populated during kernel init when no concurrency exists.\npub struct InitStateLock<T>\nwhere\n    T: ?Sized,\n{\n    data: UnsafeCell<T>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nunsafe impl<T> Send for IRQSafeNullLock<T> where T: ?Sized + Send {}\nunsafe impl<T> Sync for IRQSafeNullLock<T> where T: ?Sized + Send {}\n\nimpl<T> IRQSafeNullLock<T> {\n    /// Create an instance.\n    pub const fn new(data: T) -> Self {\n        Self {\n            data: UnsafeCell::new(data),\n        }\n    }\n}\n\nunsafe impl<T> Send for InitStateLock<T> where T: ?Sized + Send {}\nunsafe impl<T> Sync for InitStateLock<T> where T: ?Sized + Send {}\n\nimpl<T> InitStateLock<T> {\n    /// Create an instance.\n    pub const fn new(data: T) -> Self {\n        Self {\n            data: UnsafeCell::new(data),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse crate::{exception, state};\n\nimpl<T> interface::Mutex for IRQSafeNullLock<T> {\n    type Data = T;\n\n    fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R {\n        // In a real lock, there would be code encapsulating this line that ensures that this\n        // mutable reference will ever only be given out once at a time.\n        let data = unsafe { &mut *self.data.get() };\n\n        // Execute the closure while IRQs are masked.\n        exception::asynchronous::exec_with_irq_masked(|| f(data))\n    }\n}\n\nimpl<T> interface::ReadWriteEx for InitStateLock<T> {\n    type Data = T;\n\n    fn write<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R {\n        assert!(\n            state::state_manager().is_init(),\n            \"InitStateLock::write called after kernel init phase\"\n        );\n        assert!(\n            !exception::asynchronous::is_local_irq_masked(),\n            \"InitStateLock::write called with IRQs unmasked\"\n        );\n\n        let data = unsafe { &mut *self.data.get() };\n\n        f(data)\n    }\n\n    fn read<'a, R>(&'a self, f: impl FnOnce(&'a Self::Data) -> R) -> R {\n        let data = unsafe { &*self.data.get() };\n\n        f(data)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// InitStateLock must be transparent.\n    #[kernel_test]\n    fn init_state_lock_is_transparent() {\n        use core::mem::size_of;\n\n        assert_eq!(size_of::<InitStateLock<u64>>(), size_of::<u64>());\n    }\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/src/time.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Timer primitives.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/time.rs\"]\nmod arch_time;\n\nuse core::time::Duration;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Provides time management functions.\npub struct TimeManager;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic TIME_MANAGER: TimeManager = TimeManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the global TimeManager.\npub fn time_manager() -> &'static TimeManager {\n    &TIME_MANAGER\n}\n\nimpl TimeManager {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self\n    }\n\n    /// The timer's resolution.\n    pub fn resolution(&self) -> Duration {\n        arch_time::resolution()\n    }\n\n    /// The uptime since power-on of the device.\n    ///\n    /// This includes time consumed by firmware and bootloaders.\n    pub fn uptime(&self) -> Duration {\n        arch_time::uptime()\n    }\n\n    /// Spin for a given duration.\n    pub fn spin_for(&self, duration: Duration) {\n        arch_time::spin_for(duration)\n    }\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/tests/00_console_sanity.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'console_io_test'\n\n# Verify sending and receiving works as expected.\nclass TxRxHandshakeTest < SubtestBase\n    def name\n        'Transmit and Receive handshake'\n    end\n\n    def run(qemu_out, qemu_in)\n        qemu_in.write_nonblock('ABC')\n        expect_or_raise(qemu_out, 'OK1234')\n    end\nend\n\n# Check for correct TX statistics implementation. Depends on test 1 being run first.\nclass TxStatisticsTest < SubtestBase\n    def name\n        'Transmit statistics'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, '6')\n    end\nend\n\n# Check for correct RX statistics implementation. Depends on test 1 being run first.\nclass RxStatisticsTest < SubtestBase\n    def name\n        'Receive statistics'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, '3')\n    end\nend\n\n## -------------------------------------------------------------------------------------------------\n## Test registration\n## -------------------------------------------------------------------------------------------------\ndef subtest_collection\n    [TxRxHandshakeTest.new, TxStatisticsTest.new, RxStatisticsTest.new]\nend\n"
  },
  {
    "path": "19_kernel_heap/kernel/tests/00_console_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Console sanity tests - RX, TX and statistics.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Console tests should time out on the I/O harness in case of panic.\nmod panic_wait_forever;\n\nuse libkernel::{bsp, console, cpu, exception, memory, print};\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    use console::console;\n\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    // Handshake\n    assert_eq!(console().read_char(), 'A');\n    assert_eq!(console().read_char(), 'B');\n    assert_eq!(console().read_char(), 'C');\n    print!(\"OK1234\");\n\n    // 6\n    print!(\"{}\", console().chars_written());\n\n    // 3\n    print!(\"{}\", console().chars_read());\n\n    // The QEMU process running this test will be closed by the I/O test harness.\n    cpu::wait_forever();\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/tests/01_timer_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Timer sanity tests.\n\n#![feature(custom_test_frameworks)]\n#![no_main]\n#![no_std]\n#![reexport_test_harness_main = \"test_main\"]\n#![test_runner(libkernel::test_runner)]\n\nuse core::time::Duration;\nuse libkernel::{bsp, cpu, exception, memory, time};\nuse test_macros::kernel_test;\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    // Depending on CPU arch, some timer bring-up code could go here. Not needed for the RPi.\n\n    test_main();\n\n    cpu::qemu_exit_success()\n}\n\n/// Simple check that the timer is running.\n#[kernel_test]\nfn timer_is_counting() {\n    assert!(time::time_manager().uptime().as_nanos() > 0)\n}\n\n/// Timer resolution must be sufficient.\n#[kernel_test]\nfn timer_resolution_is_sufficient() {\n    assert!(time::time_manager().resolution().as_nanos() > 0);\n    assert!(time::time_manager().resolution().as_nanos() < 100)\n}\n\n/// Sanity check spin_for() implementation.\n#[kernel_test]\nfn spin_accuracy_check_1_second() {\n    let t1 = time::time_manager().uptime();\n    time::time_manager().spin_for(Duration::from_secs(1));\n    let t2 = time::time_manager().uptime();\n\n    assert_eq!((t2 - t1).as_secs(), 1)\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/tests/02_exception_sync_page_fault.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Page faults must result in synchronous exceptions.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Overwrites libkernel's `panic_wait::_panic_exit()` so that it returns a \"success\" code.\n///\n/// In this test, reaching the panic is a success, because it is called from the synchronous\n/// exception handler, which is what this test wants to achieve.\n///\n/// It also means that this integration test can not use any other code that calls panic!() directly\n/// or indirectly.\nmod panic_exit_success;\n\nuse libkernel::{bsp, cpu, exception, info, memory, println};\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    // This line will be printed as the test header.\n    println!(\"Testing synchronous exception handling by causing a page fault\");\n\n    info!(\"Writing to bottom of address space to address 1 GiB...\");\n    let big_addr: u64 = 1024 * 1024 * 1024;\n    core::ptr::read_volatile(big_addr as *mut u64);\n\n    // If execution reaches here, the memory access above did not cause a page fault exception.\n    cpu::qemu_exit_failure()\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/tests/03_exception_restore_sanity.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'console_io_test'\n\n# Verify that exception restore works.\nclass ExceptionRestoreTest < SubtestBase\n    def name\n        'Exception restore'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, 'Back from system call!')\n    end\nend\n\n## -------------------------------------------------------------------------------------------------\n## Test registration\n## -------------------------------------------------------------------------------------------------\ndef subtest_collection\n    [ExceptionRestoreTest.new]\nend\n"
  },
  {
    "path": "19_kernel_heap/kernel/tests/03_exception_restore_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A simple sanity test to see if exception restore code works.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Console tests should time out on the I/O harness in case of panic.\nmod panic_wait_forever;\n\nuse core::arch::asm;\nuse libkernel::{bsp, cpu, exception, info, memory, println};\n\n#[inline(never)]\nfn nested_system_call() {\n    #[cfg(target_arch = \"aarch64\")]\n    unsafe {\n        asm!(\"svc #0x1337\", options(nomem, nostack, preserves_flags));\n    }\n\n    #[cfg(not(target_arch = \"aarch64\"))]\n    {\n        info!(\"Not supported yet\");\n        cpu::wait_forever();\n    }\n}\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    // This line will be printed as the test header.\n    println!(\"Testing exception restore\");\n\n    info!(\"Making a dummy system call\");\n\n    // Calling this inside a function indirectly tests if the link register is restored properly.\n    nested_system_call();\n\n    info!(\"Back from system call!\");\n\n    // The QEMU process running this test will be closed by the I/O test harness.\n    cpu::wait_forever();\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/tests/04_exception_irq_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! IRQ handling sanity tests.\n\n#![feature(custom_test_frameworks)]\n#![no_main]\n#![no_std]\n#![reexport_test_harness_main = \"test_main\"]\n#![test_runner(libkernel::test_runner)]\n\nuse libkernel::{bsp, cpu, exception, memory};\nuse test_macros::kernel_test;\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    exception::handling_init();\n    exception::asynchronous::local_irq_unmask();\n\n    test_main();\n\n    cpu::qemu_exit_success()\n}\n\n/// Check that IRQ masking works.\n#[kernel_test]\nfn local_irq_mask_works() {\n    // Precondition: IRQs are unmasked.\n    assert!(exception::asynchronous::is_local_irq_masked());\n\n    exception::asynchronous::local_irq_mask();\n    assert!(!exception::asynchronous::is_local_irq_masked());\n\n    // Restore earlier state.\n    exception::asynchronous::local_irq_unmask();\n}\n\n/// Check that IRQ unmasking works.\n#[kernel_test]\nfn local_irq_unmask_works() {\n    // Precondition: IRQs are masked.\n    exception::asynchronous::local_irq_mask();\n    assert!(!exception::asynchronous::is_local_irq_masked());\n\n    exception::asynchronous::local_irq_unmask();\n    assert!(exception::asynchronous::is_local_irq_masked());\n}\n\n/// Check that IRQ mask save is saving \"something\".\n#[kernel_test]\nfn local_irq_mask_save_works() {\n    // Precondition: IRQs are unmasked.\n    assert!(exception::asynchronous::is_local_irq_masked());\n\n    let first = exception::asynchronous::local_irq_mask_save();\n    assert!(!exception::asynchronous::is_local_irq_masked());\n\n    let second = exception::asynchronous::local_irq_mask_save();\n    assert_ne!(first, second);\n\n    exception::asynchronous::local_irq_restore(first);\n    assert!(exception::asynchronous::is_local_irq_masked());\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/tests/05_backtrace_sanity.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'console_io_test'\n\n# Verify that panic produces a backtrace.\nclass PanicBacktraceTest < SubtestBase\n    def name\n        'Panic produces backtrace'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, 'Kernel panic!')\n        expect_or_raise(qemu_out, 'Backtrace:')\n    end\nend\n\n# Verify backtrace correctness.\nclass BacktraceCorrectnessTest < SubtestBase\n    def name\n        'Backtrace is correct'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, '| core::panicking::panic')\n        expect_or_raise(qemu_out, '| _05_backtrace_sanity::nested')\n        expect_or_raise(qemu_out, '| kernel_init')\n    end\nend\n\n## -------------------------------------------------------------------------------------------------\n## Test registration\n## -------------------------------------------------------------------------------------------------\ndef subtest_collection\n    [PanicBacktraceTest.new, BacktraceCorrectnessTest.new]\nend\n"
  },
  {
    "path": "19_kernel_heap/kernel/tests/05_backtrace_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Test if backtracing code detects an invalid frame pointer.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Console tests should time out on the I/O harness in case of panic.\nmod panic_wait_forever;\n\nuse libkernel::{bsp, cpu, exception, memory};\n\n#[inline(never)]\nfn nested() {\n    panic!()\n}\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    nested();\n\n    // The QEMU process running this test will be closed by the I/O test harness.\n    cpu::wait_forever()\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/tests/06_backtrace_invalid_frame.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'console_io_test'\n\n# Test detection of invalid frame pointers.\nclass InvalidFramePointerTest < SubtestBase\n    def name\n        'Detect invalid frame pointer'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out,\n                        /Encountered invalid frame pointer \\(.*\\) during backtrace/)\n    end\nend\n\n## -------------------------------------------------------------------------------------------------\n## Test registration\n## -------------------------------------------------------------------------------------------------\ndef subtest_collection\n    [InvalidFramePointerTest.new]\nend\n"
  },
  {
    "path": "19_kernel_heap/kernel/tests/06_backtrace_invalid_frame.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Test if backtracing code detects an invalid frame pointer.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Console tests should time out on the I/O harness in case of panic.\nmod panic_wait_forever;\n\nuse libkernel::{backtrace, bsp, cpu, exception, memory};\n\n#[inline(never)]\nfn nested() {\n    unsafe { backtrace::corrupt_previous_frame_addr() };\n\n    panic!()\n}\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    nested();\n\n    // The QEMU process running this test will be closed by the I/O test harness.\n    cpu::wait_forever()\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/tests/07_backtrace_invalid_link.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'console_io_test'\n\n# Test detection of invalid link.\nclass InvalidLinkTest < SubtestBase\n    def name\n        'Detect invalid link'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, /Link address \\(.*\\) is not contained in kernel .text section/)\n    end\nend\n\n## -------------------------------------------------------------------------------------------------\n## Test registration\n## -------------------------------------------------------------------------------------------------\ndef subtest_collection\n    [InvalidLinkTest.new]\nend\n"
  },
  {
    "path": "19_kernel_heap/kernel/tests/07_backtrace_invalid_link.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Test if backtracing code detects an invalid link.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Console tests should time out on the I/O harness in case of panic.\nmod panic_wait_forever;\n\nuse libkernel::{backtrace, bsp, cpu, exception, memory};\n\n#[inline(never)]\nfn nested_2() -> &'static str {\n    unsafe { backtrace::corrupt_link() };\n    libkernel::println!(\"{}\", libkernel::backtrace::Backtrace);\n    \"foo\"\n}\n\n#[inline(never)]\nfn nested_1() {\n    libkernel::println!(\"{}\", nested_2())\n}\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    nested_1();\n\n    // The QEMU process running this test will be closed by the I/O test harness.\n    cpu::wait_forever()\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/tests/boot_test_string.rb",
    "content": "# frozen_string_literal: true\n\nEXPECTED_PRINT = 'Echoing input now'\n"
  },
  {
    "path": "19_kernel_heap/kernel/tests/panic_exit_success/mod.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n/// Overwrites libkernel's `panic_wait::_panic_exit()` with the QEMU-exit version.\n#[no_mangle]\nfn _panic_exit() -> ! {\n    libkernel::cpu::qemu_exit_success()\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel/tests/panic_wait_forever/mod.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n/// Overwrites libkernel's `panic_wait::_panic_exit()` with wait_forever.\n#[no_mangle]\nfn _panic_exit() -> ! {\n    libkernel::cpu::wait_forever()\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel_symbols/Cargo.toml",
    "content": "[package]\nname = \"kernel_symbols\"\nversion = \"0.1.0\"\nedition = \"2021\"\n\n[features]\ndefault = []\ngenerated_symbols_available = []\n\n##--------------------------------------------------------------------------------------------------\n## Dependencies\n##--------------------------------------------------------------------------------------------------\n\n[dependencies]\ndebug-symbol-types = { path = \"../libraries/debug-symbol-types\" }\n"
  },
  {
    "path": "19_kernel_heap/kernel_symbols/build.rs",
    "content": "use std::{env, path::Path};\n\nfn main() {\n    if let Ok(path) = env::var(\"KERNEL_SYMBOLS_DEMANGLED_RS\") {\n        if Path::new(&path).exists() {\n            println!(\"cargo:rustc-cfg=feature=\\\"generated_symbols_available\\\"\")\n        }\n    }\n\n    println!(\n        \"cargo:rerun-if-changed={}\",\n        Path::new(\"kernel_symbols.ld\").display()\n    );\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel_symbols/kernel_symbols.ld",
    "content": "/* SPDX-License-Identifier: MIT OR Apache-2.0\n *\n * Copyright (c) 2022 Andre Richter <andre.o.richter@gmail.com>\n */\n\nSECTIONS\n{\n    .rodata : {\n        ASSERT(. > 0xffffffff00000000, \"Expected higher half address\")\n\n        KEEP(*(.rodata.symbol_desc*))\n        . = ALIGN(8);\n        *(.rodata*)\n    }\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel_symbols/src/main.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Generation of kernel symbols.\n\n#![no_std]\n#![no_main]\n\n#[cfg(feature = \"generated_symbols_available\")]\ninclude!(env!(\"KERNEL_SYMBOLS_DEMANGLED_RS\"));\n\n#[panic_handler]\nfn panic(_info: &core::panic::PanicInfo) -> ! {\n    unimplemented!()\n}\n"
  },
  {
    "path": "19_kernel_heap/kernel_symbols.mk",
    "content": "## SPDX-License-Identifier: MIT OR Apache-2.0\n##\n## Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\ninclude ../common/format.mk\ninclude ../common/docker.mk\n\n##--------------------------------------------------------------------------------------------------\n## Check for input variables that need be exported by the calling Makefile\n##--------------------------------------------------------------------------------------------------\nifndef KERNEL_SYMBOLS_TOOL_PATH\n$(error KERNEL_SYMBOLS_TOOL_PATH is not set)\nendif\n\nifndef TARGET\n$(error TARGET is not set)\nendif\n\nifndef KERNEL_SYMBOLS_INPUT_ELF\n$(error KERNEL_SYMBOLS_INPUT_ELF is not set)\nendif\n\nifndef KERNEL_SYMBOLS_OUTPUT_ELF\n$(error KERNEL_SYMBOLS_OUTPUT_ELF is not set)\nendif\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets and Prerequisites\n##--------------------------------------------------------------------------------------------------\nKERNEL_SYMBOLS_MANIFEST      = kernel_symbols/Cargo.toml\nKERNEL_SYMBOLS_LINKER_SCRIPT = kernel_symbols/kernel_symbols.ld\n\nKERNEL_SYMBOLS_RS           = $(KERNEL_SYMBOLS_INPUT_ELF)_symbols.rs\nKERNEL_SYMBOLS_DEMANGLED_RS = $(shell pwd)/$(KERNEL_SYMBOLS_INPUT_ELF)_symbols_demangled.rs\n\nKERNEL_SYMBOLS_ELF      = target/$(TARGET)/release/kernel_symbols\nKERNEL_SYMBOLS_STRIPPED = target/$(TARGET)/release/kernel_symbols_stripped\n\n# Export for build.rs of kernel_symbols crate.\nexport KERNEL_SYMBOLS_DEMANGLED_RS\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Command building blocks\n##--------------------------------------------------------------------------------------------------\nGET_SYMBOLS_SECTION_VIRT_ADDR = $(DOCKER_TOOLS) $(EXEC_SYMBOLS_TOOL) \\\n    --get_symbols_section_virt_addr $(KERNEL_SYMBOLS_OUTPUT_ELF)\n\nRUSTFLAGS = -C link-arg=--script=$(KERNEL_SYMBOLS_LINKER_SCRIPT) \\\n    -C link-arg=--section-start=.rodata=$$($(GET_SYMBOLS_SECTION_VIRT_ADDR))\n\nRUSTFLAGS_PEDANTIC = $(RUSTFLAGS) \\\n    -D warnings                   \\\n    -D missing_docs\n\nCOMPILER_ARGS = --target=$(TARGET) \\\n    --release\n\nRUSTC_CMD   = cargo rustc $(COMPILER_ARGS) --manifest-path $(KERNEL_SYMBOLS_MANIFEST)\nOBJCOPY_CMD = rust-objcopy \\\n    --strip-all            \\\n    -O binary\n\nEXEC_SYMBOLS_TOOL  = ruby $(KERNEL_SYMBOLS_TOOL_PATH)/main.rb\n\n##------------------------------------------------------------------------------\n## Dockerization\n##------------------------------------------------------------------------------\nDOCKER_CMD = docker run -t --rm -v $(shell pwd):/work/tutorial -w /work/tutorial\n\n# DOCKER_IMAGE defined in include file (see top of this file).\nDOCKER_TOOLS = $(DOCKER_CMD) $(DOCKER_IMAGE)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: all symbols measure_time_start measure_time_finish\n\nall: measure_time_start symbols measure_time_finish\n\nsymbols:\n\t@cp $(KERNEL_SYMBOLS_INPUT_ELF) $(KERNEL_SYMBOLS_OUTPUT_ELF)\n\n\t@$(DOCKER_TOOLS) $(EXEC_SYMBOLS_TOOL) --gen_symbols $(KERNEL_SYMBOLS_OUTPUT_ELF) \\\n                $(KERNEL_SYMBOLS_RS)\n\n\t$(call color_progress_prefix, \"Demangling\")\n\t@echo Symbol names\n\t@cat $(KERNEL_SYMBOLS_RS) | rustfilt > $(KERNEL_SYMBOLS_DEMANGLED_RS)\n\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(RUSTC_CMD)\n\n\t$(call color_progress_prefix, \"Stripping\")\n\t@echo Symbols ELF file\n\t@$(OBJCOPY_CMD) $(KERNEL_SYMBOLS_ELF) $(KERNEL_SYMBOLS_STRIPPED)\n\n\t@$(DOCKER_TOOLS) $(EXEC_SYMBOLS_TOOL) --patch_data $(KERNEL_SYMBOLS_OUTPUT_ELF) \\\n                $(KERNEL_SYMBOLS_STRIPPED)\n\n# Note: The following is the only _trivial_ way I could think of that works out of the box on both\n# Linux and macOS. Since macOS does not have the %N nanosecond format string option, the\n# resolution is restricted to whole seconds.\nmeasure_time_start:\n\t@date +%s > /tmp/kernel_symbols_start.date\n\nmeasure_time_finish:\n\t@date +%s > /tmp/kernel_symbols_end.date\n\n\t$(call color_progress_prefix, \"Finished\")\n\t@echo \"in $$((`cat /tmp/kernel_symbols_end.date` - `cat /tmp/kernel_symbols_start.date`)).0s\"\n\n\t@rm /tmp/kernel_symbols_end.date /tmp/kernel_symbols_start.date\n"
  },
  {
    "path": "19_kernel_heap/libraries/debug-symbol-types/Cargo.toml",
    "content": "[package]\nname = \"debug-symbol-types\"\nversion = \"0.1.0\"\nedition = \"2021\"\n"
  },
  {
    "path": "19_kernel_heap/libraries/debug-symbol-types/src/lib.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Types for implementing debug symbol support.\n\n#![no_std]\n\nuse core::ops::Range;\n\n/// A symbol containing a size.\n#[repr(C)]\n#[derive(Clone)]\npub struct Symbol {\n    addr_range: Range<usize>,\n    name: &'static str,\n}\n\nimpl Symbol {\n    /// Create an instance.\n    pub const fn new(start: usize, size: usize, name: &'static str) -> Symbol {\n        Symbol {\n            addr_range: Range {\n                start,\n                end: start + size,\n            },\n            name,\n        }\n    }\n\n    /// Returns true if addr is contained in the range.\n    pub fn contains(&self, addr: usize) -> bool {\n        self.addr_range.contains(&addr)\n    }\n\n    /// Returns the symbol's name.\n    pub fn name(&self) -> &'static str {\n        self.name\n    }\n\n    /// Returns the symbol's size.\n    pub fn size(&self) -> usize {\n        self.addr_range.end - self.addr_range.start\n    }\n}\n"
  },
  {
    "path": "19_kernel_heap/libraries/test-macros/Cargo.toml",
    "content": "[package]\nname = \"test-macros\"\nversion = \"0.1.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n\n[lib]\nproc-macro = true\n\n[dependencies]\nproc-macro2 = \"1.x\"\nquote = \"1.x\"\nsyn = { version = \"1.x\", features = [\"full\"] }\ntest-types = { path = \"../test-types\" }\n"
  },
  {
    "path": "19_kernel_heap/libraries/test-macros/src/lib.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\nuse proc_macro::TokenStream;\nuse proc_macro2::Span;\nuse quote::quote;\nuse syn::{parse_macro_input, Ident, ItemFn};\n\n#[proc_macro_attribute]\npub fn kernel_test(_attr: TokenStream, input: TokenStream) -> TokenStream {\n    let f = parse_macro_input!(input as ItemFn);\n\n    let test_name = &format!(\"{}\", f.sig.ident);\n    let test_ident = Ident::new(\n        &format!(\"{}_TEST_CONTAINER\", f.sig.ident.to_string().to_uppercase()),\n        Span::call_site(),\n    );\n    let test_code_block = f.block;\n\n    quote!(\n        #[test_case]\n        const #test_ident: test_types::UnitTest = test_types::UnitTest {\n            name: #test_name,\n            test_func: || #test_code_block,\n        };\n    )\n    .into()\n}\n"
  },
  {
    "path": "19_kernel_heap/libraries/test-types/Cargo.toml",
    "content": "[package]\nname = \"test-types\"\nversion = \"0.1.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n"
  },
  {
    "path": "19_kernel_heap/libraries/test-types/src/lib.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Types for the `custom_test_frameworks` implementation.\n\n#![no_std]\n\n/// Unit test container.\npub struct UnitTest {\n    /// Name of the test.\n    pub name: &'static str,\n\n    /// Function pointer to the test.\n    pub test_func: fn(),\n}\n"
  },
  {
    "path": "19_kernel_heap/tools/kernel_symbols_tool/cmds.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\ndef generate_symbols(kernel_elf, output_file)\n    File.open(output_file, 'w') do |file|\n        header = <<~HEREDOC\n            use debug_symbol_types::Symbol;\n\n            # [no_mangle]\n            # [link_section = \".rodata.symbol_desc\"]\n            static KERNEL_SYMBOLS: [Symbol; #{kernel_elf.num_symbols}] = [\n        HEREDOC\n\n        file.write(header)\n        kernel_elf.symbols.each do |sym|\n            value = sym.header.st_value\n            size = sym.header.st_size\n            name = sym.name\n\n            file.write(\"    Symbol::new(#{value}, #{size}, \\\"#{name}\\\"),\\n\")\n        end\n        file.write(\"];\\n\")\n    end\nend\n\ndef get_symbols_section_virt_addr(kernel_elf)\n    kernel_elf.kernel_symbols_section_virt_addr\nend\n\ndef patch_symbol_data(kernel_elf, symbols_blob_path)\n    symbols_blob = File.binread(symbols_blob_path)\n\n    raise if symbols_blob.size > kernel_elf.kernel_symbols_section_size\n\n    File.binwrite(kernel_elf.path, File.binread(symbols_blob_path),\n                  kernel_elf.kernel_symbols_section_offset_in_file)\nend\n\ndef patch_num_symbols(kernel_elf)\n    num_packed = [kernel_elf.num_symbols].pack('Q<*') # \"Q\" == uint64_t, \"<\" == little endian\n    File.binwrite(kernel_elf.path, num_packed, kernel_elf.num_kernel_symbols_offset_in_file)\nend\n"
  },
  {
    "path": "19_kernel_heap/tools/kernel_symbols_tool/kernel_elf.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n# KernelELF\nclass KernelELF\n    attr_reader :path\n\n    def initialize(kernel_elf_path, kernel_symbols_section, num_kernel_symbols)\n        @elf = ELFTools::ELFFile.new(File.open(kernel_elf_path))\n        @symtab_section = @elf.section_by_name('.symtab')\n\n        @path = kernel_elf_path\n        fetch_values(kernel_symbols_section, num_kernel_symbols)\n    end\n\n    private\n\n    def fetch_values(kernel_symbols_section, num_kernel_symbols)\n        sym = @symtab_section.symbol_by_name(num_kernel_symbols)\n        raise \"Symbol \\\"#{num_kernel_symbols}\\\" not found\" if sym.nil?\n\n        @num_kernel_symbols = sym\n\n        section = @elf.section_by_name(kernel_symbols_section)\n        raise \"Section \\\"#{kernel_symbols_section}\\\" not found\" if section.nil?\n\n        @kernel_symbols_section = section\n    end\n\n    def num_kernel_symbols_virt_addr\n        @num_kernel_symbols.header.st_value\n    end\n\n    def segment_containing_virt_addr(virt_addr)\n        @elf.each_segments do |segment|\n            return segment if segment.vma_in?(virt_addr)\n        end\n    end\n\n    def virt_addr_to_file_offset(virt_addr)\n        segment = segment_containing_virt_addr(virt_addr)\n        segment.vma_to_offset(virt_addr)\n    end\n\n    public\n\n    def symbols\n        non_zero_symbols = @symtab_section.symbols.reject { |sym| sym.header.st_size.zero? }\n        non_zero_symbols.sort_by { |sym| sym.header.st_value }\n    end\n\n    def num_symbols\n        symbols.size\n    end\n\n    def kernel_symbols_section_virt_addr\n        @kernel_symbols_section.header.sh_addr.to_i\n    end\n\n    def kernel_symbols_section_size\n        @kernel_symbols_section.header.sh_size.to_i\n    end\n\n    def kernel_symbols_section_offset_in_file\n        virt_addr_to_file_offset(kernel_symbols_section_virt_addr)\n    end\n\n    def num_kernel_symbols_offset_in_file\n        virt_addr_to_file_offset(num_kernel_symbols_virt_addr)\n    end\nend\n"
  },
  {
    "path": "19_kernel_heap/tools/kernel_symbols_tool/main.rb",
    "content": "#!/usr/bin/env ruby\n# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'rubygems'\nrequire 'bundler/setup'\nrequire 'colorize'\nrequire 'elftools'\n\nrequire_relative 'kernel_elf'\nrequire_relative 'cmds'\n\nKERNEL_SYMBOLS_SECTION = '.kernel_symbols'\nNUM_KERNEL_SYMBOLS = 'NUM_KERNEL_SYMBOLS'\n\ncmd = ARGV[0]\n\nkernel_elf_path = ARGV[1]\nkernel_elf = KernelELF.new(kernel_elf_path, KERNEL_SYMBOLS_SECTION, NUM_KERNEL_SYMBOLS)\n\ncase cmd\nwhen '--gen_symbols'\n    output_file = ARGV[2]\n\n    print 'Generating'.rjust(12).green.bold\n    puts ' Symbols source file'\n\n    generate_symbols(kernel_elf, output_file)\nwhen '--get_symbols_section_virt_addr'\n    addr = get_symbols_section_virt_addr(kernel_elf)\n\n    puts \"0x#{addr.to_s(16)}\"\nwhen '--patch_data'\n    symbols_blob_path = ARGV[2]\n    num_symbols = kernel_elf.num_symbols\n\n    print 'Patching'.rjust(12).green.bold\n    puts \" Symbols blob and number of symbols (#{num_symbols}) into ELF\"\n\n    patch_symbol_data(kernel_elf, symbols_blob_path)\n    patch_num_symbols(kernel_elf)\nelse\n    raise\nend\n"
  },
  {
    "path": "19_kernel_heap/tools/translation_table_tool/arch.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n# Bitfield manipulation.\nclass BitField\n    def initialize\n        @value = 0\n    end\n\n    def self.attr_bitfield(name, offset, num_bits)\n        define_method(\"#{name}=\") do |bits|\n            mask = (2**num_bits) - 1\n\n            raise \"Input out of range: #{name} = 0x#{bits.to_s(16)}\" if (bits & ~mask).positive?\n\n            # Clear bitfield\n            @value &= ~(mask << offset)\n\n            # Set it\n            @value |= (bits << offset)\n        end\n    end\n\n    def to_i\n        @value\n    end\n\n    def size_in_byte\n        8\n    end\nend\n\n# An array class that knows its memory location.\nclass CArray < Array\n    attr_reader :phys_start_addr\n\n    def initialize(phys_start_addr, size, &block)\n        @phys_start_addr = phys_start_addr\n\n        super(size, &block)\n    end\n\n    def size_in_byte\n        inject(0) { |sum, n| sum + n.size_in_byte }\n    end\nend\n\n#---------------------------------------------------------------------------------------------------\n# Arch::\n#---------------------------------------------------------------------------------------------------\nmodule Arch\n#---------------------------------------------------------------------------------------------------\n# Arch::ARMv8\n#---------------------------------------------------------------------------------------------------\nmodule ARMv8\n# ARMv8 Table Descriptor.\nclass Stage1TableDescriptor < BitField\n    module NextLevelTableAddr\n        OFFSET = 16\n        NUMBITS = 32\n    end\n\n    module Type\n        OFFSET = 1\n        NUMBITS = 1\n\n        BLOCK = 0\n        TABLE = 1\n    end\n\n    module Valid\n        OFFSET = 0\n        NUMBITS = 1\n\n        FALSE = 0\n        TRUE = 1\n    end\n\n    attr_bitfield(:__next_level_table_addr, NextLevelTableAddr::OFFSET, NextLevelTableAddr::NUMBITS)\n    attr_bitfield(:type, Type::OFFSET, Type::NUMBITS)\n    attr_bitfield(:valid, Valid::OFFSET, Valid::NUMBITS)\n\n    def next_level_table_addr=(addr)\n        addr >>= Granule64KiB::SHIFT\n\n        self.__next_level_table_addr = addr\n    end\n\n    private :__next_level_table_addr=\nend\n\n# ARMv8 level 3 page descriptor.\nclass Stage1PageDescriptor < BitField\n    module UXN\n        OFFSET = 54\n        NUMBITS = 1\n\n        FALSE = 0\n        TRUE = 1\n    end\n\n    module PXN\n        OFFSET = 53\n        NUMBITS = 1\n\n        FALSE = 0\n        TRUE = 1\n    end\n\n    module OutputAddr\n        OFFSET = 16\n        NUMBITS = 32\n    end\n\n    module AF\n        OFFSET = 10\n        NUMBITS = 1\n\n        FALSE = 0\n        TRUE = 1\n    end\n\n    module SH\n        OFFSET = 8\n        NUMBITS = 2\n\n        INNER_SHAREABLE = 0b11\n    end\n\n    module AP\n        OFFSET = 6\n        NUMBITS = 2\n\n        RW_EL1 = 0b00\n        RO_EL1 = 0b10\n    end\n\n    module AttrIndx\n        OFFSET = 2\n        NUMBITS = 3\n    end\n\n    module Type\n        OFFSET = 1\n        NUMBITS = 1\n\n        RESERVED_INVALID = 0\n        PAGE = 1\n    end\n\n    module Valid\n        OFFSET = 0\n        NUMBITS = 1\n\n        FALSE = 0\n        TRUE = 1\n    end\n\n    attr_bitfield(:uxn, UXN::OFFSET, UXN::NUMBITS)\n    attr_bitfield(:pxn, PXN::OFFSET, PXN::NUMBITS)\n    attr_bitfield(:__output_addr, OutputAddr::OFFSET, OutputAddr::NUMBITS)\n    attr_bitfield(:af, AF::OFFSET, AF::NUMBITS)\n    attr_bitfield(:sh, SH::OFFSET, SH::NUMBITS)\n    attr_bitfield(:ap, AP::OFFSET, AP::NUMBITS)\n    attr_bitfield(:attr_indx, AttrIndx::OFFSET, AttrIndx::NUMBITS)\n    attr_bitfield(:type, Type::OFFSET, Type::NUMBITS)\n    attr_bitfield(:valid, Valid::OFFSET, Valid::NUMBITS)\n\n    def output_addr=(addr)\n        addr >>= Granule64KiB::SHIFT\n\n        self.__output_addr = addr\n    end\n\n    private :__output_addr=\nend\n\n# Translation table representing the structure defined in translation_table.rs.\nclass TranslationTable\n    module MAIR\n        NORMAL = 1\n    end\n\n    def initialize\n        do_sanity_checks\n\n        num_lvl2_tables = BSP.kernel_virt_addr_space_size >> Granule512MiB::SHIFT\n\n        @lvl3 = new_lvl3(num_lvl2_tables, BSP.phys_addr_of_kernel_tables)\n\n        @lvl2_phys_start_addr = @lvl3.phys_start_addr + @lvl3.size_in_byte\n        @lvl2 = new_lvl2(num_lvl2_tables, @lvl2_phys_start_addr)\n\n        populate_lvl2_entries\n    end\n\n    def map_at(virt_region, phys_region, attributes)\n        return if virt_region.empty?\n\n        raise if virt_region.size != phys_region.size\n        raise if phys_region.last > BSP.phys_addr_space_end_page\n\n        virt_region.zip(phys_region).each do |virt_page, phys_page|\n            desc = page_descriptor_from(virt_page)\n            set_lvl3_entry(desc, phys_page, attributes)\n        end\n    end\n\n    def to_binary\n        data = @lvl3.flatten.map(&:to_i) + @lvl2.map(&:to_i)\n        data.pack('Q<*') # \"Q\" == uint64_t, \"<\" == little endian\n    end\n\n    def phys_tables_base_addr_binary\n        [@lvl2_phys_start_addr].pack('Q<*') # \"Q\" == uint64_t, \"<\" == little endian\n    end\n\n    def phys_tables_base_addr\n        @lvl2_phys_start_addr\n    end\n\n    private\n\n    def do_sanity_checks\n        raise unless BSP.kernel_granule::SIZE == Granule64KiB::SIZE\n        raise unless (BSP.kernel_virt_addr_space_size % Granule512MiB::SIZE).zero?\n    end\n\n    def new_lvl3(num_lvl2_tables, start_addr)\n        CArray.new(start_addr, num_lvl2_tables) do\n            temp = CArray.new(start_addr, 8192) do\n                Stage1PageDescriptor.new\n            end\n            start_addr += temp.size_in_byte\n\n            temp\n        end\n    end\n\n    def new_lvl2(num_lvl2_tables, start_addr)\n        CArray.new(start_addr, num_lvl2_tables) do\n            Stage1TableDescriptor.new\n        end\n    end\n\n    def populate_lvl2_entries\n        @lvl2.each_with_index do |descriptor, i|\n            descriptor.next_level_table_addr = @lvl3[i].phys_start_addr\n            descriptor.type = Stage1TableDescriptor::Type::TABLE\n            descriptor.valid = Stage1TableDescriptor::Valid::TRUE\n        end\n    end\n\n    def lvl2_lvl3_index_from(addr)\n        addr -= BSP.kernel_virt_start_addr\n\n        lvl2_index = addr >> Granule512MiB::SHIFT\n        lvl3_index = (addr & Granule512MiB::MASK) >> Granule64KiB::SHIFT\n\n        raise unless lvl2_index < @lvl2.size\n\n        [lvl2_index, lvl3_index]\n    end\n\n    def page_descriptor_from(virt_addr)\n        lvl2_index, lvl3_index = lvl2_lvl3_index_from(virt_addr)\n\n        @lvl3[lvl2_index][lvl3_index]\n    end\n\n    # rubocop:disable Metrics/MethodLength\n    def set_attributes(desc, attributes)\n        case attributes.mem_attributes\n        when :CacheableDRAM\n            desc.sh = Stage1PageDescriptor::SH::INNER_SHAREABLE\n            desc.attr_indx = MAIR::NORMAL\n        else\n            raise 'Invalid input'\n        end\n\n        desc.ap = case attributes.acc_perms\n                  when :ReadOnly\n                      Stage1PageDescriptor::AP::RO_EL1\n                  when :ReadWrite\n                      Stage1PageDescriptor::AP::RW_EL1\n                  else\n                      raise 'Invalid input'\n\n                  end\n\n        desc.pxn = if attributes.execute_never\n                       Stage1PageDescriptor::PXN::TRUE\n                   else\n                       Stage1PageDescriptor::PXN::FALSE\n                   end\n\n        desc.uxn = Stage1PageDescriptor::UXN::TRUE\n    end\n    # rubocop:enable Metrics/MethodLength\n\n    def set_lvl3_entry(desc, output_addr, attributes)\n        desc.output_addr = output_addr\n        desc.af = Stage1PageDescriptor::AF::TRUE\n        desc.type = Stage1PageDescriptor::Type::PAGE\n        desc.valid = Stage1PageDescriptor::Valid::TRUE\n\n        set_attributes(desc, attributes)\n    end\nend\nend\nend\n"
  },
  {
    "path": "19_kernel_heap/tools/translation_table_tool/bsp.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n# Raspberry Pi 3 + 4\nclass RaspberryPi\n    attr_reader :kernel_granule, :kernel_virt_addr_space_size, :kernel_virt_start_addr\n\n    MEMORY_SRC = File.read('kernel/src/bsp/raspberrypi/memory.rs').split(\"\\n\")\n\n    def initialize\n        @kernel_granule = Granule64KiB\n\n        @kernel_virt_addr_space_size = KERNEL_ELF.symbol_value('__kernel_virt_addr_space_size')\n        @kernel_virt_start_addr = KERNEL_ELF.symbol_value('__kernel_virt_start_addr')\n\n        @virt_addr_of_kernel_tables = KERNEL_ELF.symbol_value('KERNEL_TABLES')\n        @virt_addr_of_phys_kernel_tables_base_addr = KERNEL_ELF.symbol_value(\n            'PHYS_KERNEL_TABLES_BASE_ADDR'\n        )\n    end\n\n    def phys_addr_of_kernel_tables\n        KERNEL_ELF.virt_to_phys(@virt_addr_of_kernel_tables)\n    end\n\n    def kernel_tables_offset_in_file\n        KERNEL_ELF.virt_addr_to_file_offset(@virt_addr_of_kernel_tables)\n    end\n\n    def phys_kernel_tables_base_addr_offset_in_file\n        KERNEL_ELF.virt_addr_to_file_offset(@virt_addr_of_phys_kernel_tables_base_addr)\n    end\n\n    def phys_addr_space_end_page\n        x = MEMORY_SRC.grep(/pub const END/)\n        x = case BSP_TYPE\n            when :rpi3\n                x[0]\n            when :rpi4\n                x[1]\n            else\n                raise\n            end\n\n        # Extract the hex literal with underscores like 0x0123_abcd.\n        x = x.scan(/0x[\\h_]*/)[0]\n\n        # Further remove x and _ and convert to int.\n        x.scan(/\\h+/).join.to_i(16)\n    end\nend\n"
  },
  {
    "path": "19_kernel_heap/tools/translation_table_tool/generic.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\nmodule Granule64KiB\n    SIZE = 64 * 1024\n    SHIFT = Math.log2(SIZE).to_i\nend\n\nmodule Granule512MiB\n    SIZE = 512 * 1024 * 1024\n    SHIFT = Math.log2(SIZE).to_i\n    MASK = SIZE - 1\nend\n\n# Monkey-patch Integer with some helper functions.\nclass Integer\n    def power_of_two?\n        self[0].zero?\n    end\n\n    def aligned?(alignment)\n        raise unless alignment.power_of_two?\n\n        (self & (alignment - 1)).zero?\n    end\n\n    def align_up(alignment)\n        raise unless alignment.power_of_two?\n\n        (self + alignment - 1) & ~(alignment - 1)\n    end\n\n    def to_hex_underscore(with_leading_zeros: false)\n        fmt = with_leading_zeros ? '%016x' : '%x'\n        value = format(fmt, self).to_s.reverse.scan(/.{4}|.+/).join('_').reverse\n\n        format('0x%s', value)\n    end\nend\n\n# An array where each value is the start address of a Page.\nclass MemoryRegion < Array\n    def initialize(start_addr, size, granule_size)\n        raise unless start_addr.aligned?(granule_size)\n        raise unless size.positive?\n        raise unless (size % granule_size).zero?\n\n        num_pages = size / granule_size\n        super(num_pages) do |i|\n            (i * granule_size) + start_addr\n        end\n    end\nend\n\n# Collection of memory attributes.\nclass AttributeFields\n    attr_reader :mem_attributes, :acc_perms, :execute_never\n\n    def initialize(mem_attributes, acc_perms, execute_never)\n        @mem_attributes = mem_attributes\n        @acc_perms = acc_perms\n        @execute_never = execute_never\n    end\n\n    def to_s\n        x = case @mem_attributes\n            when :CacheableDRAM\n                'C'\n            else\n                '?'\n            end\n\n        y = case @acc_perms\n            when :ReadWrite\n                'RW'\n            when :ReadOnly\n                'RO'\n            else\n                '??'\n            end\n\n        z = @execute_never ? 'XN' : 'X '\n\n        \"#{x} #{y} #{z}\"\n    end\nend\n\n# A container that describes a virt-to-phys region mapping.\nclass MappingDescriptor\n    @max_section_name_length = 'Sections'.length\n\n    class << self\n        attr_accessor :max_section_name_length\n\n        def update_max_section_name_length(length)\n            @max_section_name_length = [@max_section_name_length, length].max\n        end\n    end\n\n    attr_reader :name, :virt_region, :phys_region, :attributes\n\n    def initialize(name, virt_region, phys_region, attributes)\n        @name = name\n        @virt_region = virt_region\n        @phys_region = phys_region\n        @attributes = attributes\n    end\n\n    def size_human_readable(size)\n        if size >= (1024 * 1024)\n            \"#{(size / (1024 * 1024)).to_s.rjust(3)} MiB\"\n        elsif size >= 1024\n            \"#{(size / 1024).to_s.rjust(3)} KiB\"\n        else\n            raise\n        end\n    end\n\n    def to_s\n        name = @name.ljust(self.class.max_section_name_length)\n        virt_start = @virt_region.first.to_hex_underscore(with_leading_zeros: true)\n        phys_start = @phys_region.first.to_hex_underscore(with_leading_zeros: true)\n        size = size_human_readable(@virt_region.size * 65_536)\n\n        \"#{name} | #{virt_start} | #{phys_start} | #{size} | #{@attributes}\"\n    end\n\n    def self.print_divider\n        print '             '\n        print '-' * max_section_name_length\n        puts '--------------------------------------------------------------------'\n    end\n\n    def self.print_header\n        print_divider\n        print '             '\n        print 'Sections'.center(max_section_name_length)\n        print '   '\n        print 'Virt Start Addr'.center(21)\n        print '   '\n        print 'Phys Start Addr'.center(21)\n        print '   '\n        print 'Size'.center(7)\n        print '   '\n        print 'Attr'.center(7)\n        puts\n        print_divider\n    end\nend\n\ndef kernel_map_binary\n    mapping_descriptors = KERNEL_ELF.generate_mapping_descriptors\n\n    # Generate_mapping_descriptors updates the header being printed with this call. So it must come\n    # afterwards.\n    MappingDescriptor.print_header\n\n    mapping_descriptors.each do |i|\n        print 'Generating'.rjust(12).green.bold\n        print ' '\n        puts i\n\n        TRANSLATION_TABLES.map_at(i.virt_region, i.phys_region, i.attributes)\n    end\n\n    MappingDescriptor.print_divider\nend\n\ndef kernel_patch_tables(kernel_elf_path)\n    print 'Patching'.rjust(12).green.bold\n    print ' Kernel table struct at ELF file offset '\n    puts BSP.kernel_tables_offset_in_file.to_hex_underscore\n\n    File.binwrite(kernel_elf_path, TRANSLATION_TABLES.to_binary, BSP.kernel_tables_offset_in_file)\nend\n\ndef kernel_patch_base_addr(kernel_elf_path)\n    print 'Patching'.rjust(12).green.bold\n    print ' Kernel tables physical base address start argument to value '\n    print TRANSLATION_TABLES.phys_tables_base_addr.to_hex_underscore\n    print ' at ELF file offset '\n    puts BSP.phys_kernel_tables_base_addr_offset_in_file.to_hex_underscore\n\n    File.binwrite(kernel_elf_path, TRANSLATION_TABLES.phys_tables_base_addr_binary,\n                  BSP.phys_kernel_tables_base_addr_offset_in_file)\nend\n"
  },
  {
    "path": "19_kernel_heap/tools/translation_table_tool/kernel_elf.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n# KernelELF\nclass KernelELF\n    SECTION_FLAG_ALLOC = 2\n\n    def initialize(kernel_elf_path)\n        @elf = ELFTools::ELFFile.new(File.open(kernel_elf_path))\n        @symtab_section = @elf.section_by_name('.symtab')\n    end\n\n    def machine\n        @elf.machine.to_sym\n    end\n\n    def symbol_value(symbol_name)\n        @symtab_section.symbol_by_name(symbol_name).header.st_value\n    end\n\n    def segment_containing_virt_addr(virt_addr)\n        @elf.each_segments do |segment|\n            return segment if segment.vma_in?(virt_addr)\n        end\n    end\n\n    def virt_to_phys(virt_addr)\n        segment = segment_containing_virt_addr(virt_addr)\n        translation_offset = segment.header.p_vaddr - segment.header.p_paddr\n\n        virt_addr - translation_offset\n    end\n\n    def virt_addr_to_file_offset(virt_addr)\n        segment = segment_containing_virt_addr(virt_addr)\n        segment.vma_to_offset(virt_addr)\n    end\n\n    def sections_in_segment(segment)\n        head = segment.mem_head\n        tail = segment.mem_tail\n\n        sections = @elf.each_sections.select do |section|\n            file_offset = section.header.sh_addr\n            flags = section.header.sh_flags\n\n            file_offset >= head && file_offset < tail && (flags & SECTION_FLAG_ALLOC != 0)\n        end\n\n        sections.map(&:name).join(' ')\n    end\n\n    def select_load_segments\n        @elf.each_segments.select do |segment|\n            segment.instance_of?(ELFTools::Segments::LoadSegment)\n        end\n    end\n\n    def segment_get_acc_perms(segment)\n        if segment.readable? && segment.writable?\n            :ReadWrite\n        elsif segment.readable?\n            :ReadOnly\n        else\n            :Invalid\n        end\n    end\n\n    def update_max_section_name_length(descriptors)\n        MappingDescriptor.update_max_section_name_length(descriptors.map { |i| i.name.size }.max)\n    end\n\n    def generate_mapping_descriptors\n        descriptors = select_load_segments.map do |segment|\n            # Assume each segment is page aligned.\n            size = segment.mem_size.align_up(BSP.kernel_granule::SIZE)\n            virt_start_addr = segment.header.p_vaddr\n            phys_start_addr = segment.header.p_paddr\n            acc_perms = segment_get_acc_perms(segment)\n            execute_never = !segment.executable?\n            section_names = sections_in_segment(segment)\n\n            virt_region = MemoryRegion.new(virt_start_addr, size, BSP.kernel_granule::SIZE)\n            phys_region = MemoryRegion.new(phys_start_addr, size, BSP.kernel_granule::SIZE)\n            attributes = AttributeFields.new(:CacheableDRAM, acc_perms, execute_never)\n\n            MappingDescriptor.new(section_names, virt_region, phys_region, attributes)\n        end\n\n        update_max_section_name_length(descriptors)\n        descriptors\n    end\nend\n"
  },
  {
    "path": "19_kernel_heap/tools/translation_table_tool/main.rb",
    "content": "#!/usr/bin/env ruby\n# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'rubygems'\nrequire 'bundler/setup'\nrequire 'colorize'\nrequire 'elftools'\n\nrequire_relative 'generic'\nrequire_relative 'kernel_elf'\nrequire_relative 'bsp'\nrequire_relative 'arch'\n\nBSP_TYPE = ARGV[0].to_sym\nkernel_elf_path = ARGV[1]\n\nstart = Time.now\n\nKERNEL_ELF = KernelELF.new(kernel_elf_path)\n\nBSP = case BSP_TYPE\n      when :rpi3, :rpi4\n          RaspberryPi.new\n      else\n          raise\n      end\n\nTRANSLATION_TABLES = case KERNEL_ELF.machine\n                     when :AArch64\n                         Arch::ARMv8::TranslationTable.new\n                     else\n                         raise\n                     end\n\nkernel_map_binary\nkernel_patch_tables(kernel_elf_path)\nkernel_patch_base_addr(kernel_elf_path)\n\nelapsed = Time.now - start\n\nprint 'Finished'.rjust(12).green.bold\nputs \" in #{elapsed.round(2)}s\"\n"
  },
  {
    "path": "20_timer_callbacks/.cargo/config.toml",
    "content": "[target.'cfg(target_os = \"none\")']\nrunner = \"target/kernel_test_runner.sh\"\n"
  },
  {
    "path": "20_timer_callbacks/.vscode/settings.json",
    "content": "{\n    \"editor.formatOnSave\": true,\n    \"editor.rulers\": [100],\n    \"rust-analyzer.cargo.target\": \"aarch64-unknown-none-softfloat\",\n    \"rust-analyzer.cargo.features\": [\"bsp_rpi3\"],\n    \"rust-analyzer.checkOnSave.allTargets\": false,\n    \"rust-analyzer.checkOnSave.extraArgs\": [\"--lib\", \"--bins\"],\n    \"rust-analyzer.lens.debug\": false,\n    \"rust-analyzer.lens.run\": false\n}\n"
  },
  {
    "path": "20_timer_callbacks/Cargo.toml",
    "content": "[workspace]\n\nmembers = [\n        \"libraries/*\",\n        \"kernel\",\n        \"kernel_symbols\"\n]\n\n[profile.release]\nlto = true\ndebug = true\n"
  },
  {
    "path": "20_timer_callbacks/Makefile",
    "content": "## SPDX-License-Identifier: MIT OR Apache-2.0\n##\n## Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\ninclude ../common/docker.mk\ninclude ../common/format.mk\ninclude ../common/operating_system.mk\n\n##--------------------------------------------------------------------------------------------------\n## Optional, user-provided configuration values\n##--------------------------------------------------------------------------------------------------\n\n# Default to the RPi3.\nBSP ?= rpi3\n\n# Default to a serial device name that is common in Linux.\nDEV_SERIAL ?= /dev/ttyUSB0\n\n# Optional debug prints.\nifdef DEBUG_PRINTS\n    FEATURES = --features debug_prints\nendif\n\n# Optional integration test name.\nifdef TEST\n    TEST_ARG = --test $(TEST)\nelse\n    TEST_ARG = --test '*'\nendif\n\n\n\n##--------------------------------------------------------------------------------------------------\n## BSP-specific configuration values\n##--------------------------------------------------------------------------------------------------\nQEMU_MISSING_STRING = \"This board is not yet supported for QEMU.\"\n\nifeq ($(BSP),rpi3)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE = raspi3\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    QEMU_TEST_ARGS    = $(QEMU_RELEASE_ARGS) -semihosting\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    OPENOCD_ARG       = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi3.cfg\n    JTAG_BOOT_IMAGE   = ../X1_JTAG_boot/jtag_boot_rpi3.img\n    LD_SCRIPT_PATH    = $(shell pwd)/kernel/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a53 -C force-frame-pointers\nelse ifeq ($(BSP),rpi4)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE =\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    QEMU_TEST_ARGS    = $(QEMU_RELEASE_ARGS) -semihosting\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    OPENOCD_ARG       = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi4.cfg\n    JTAG_BOOT_IMAGE   = ../X1_JTAG_boot/jtag_boot_rpi4.img\n    LD_SCRIPT_PATH    = $(shell pwd)/kernel/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a72 -C force-frame-pointers\nendif\n\n# Export for build.rs.\nexport LD_SCRIPT_PATH\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets and Prerequisites\n##--------------------------------------------------------------------------------------------------\nKERNEL_MANIFEST      = kernel/Cargo.toml\nKERNEL_LINKER_SCRIPT = kernel.ld\nLAST_BUILD_CONFIG    = target/$(BSP)_$(DEBUG_PRINTS).build_config\n\nKERNEL_ELF_RAW      = target/$(TARGET)/release/kernel\n# This parses cargo's dep-info file.\n# https://doc.rust-lang.org/cargo/guide/build-cache.html#dep-info-files\nKERNEL_ELF_RAW_DEPS = $(filter-out %: ,$(file < $(KERNEL_ELF_RAW).d)) $(KERNEL_MANIFEST) $(LAST_BUILD_CONFIG)\n\n##------------------------------------------------------------------------------\n## Translation tables\n##------------------------------------------------------------------------------\nTT_TOOL_PATH = tools/translation_table_tool\n\nKERNEL_ELF_TTABLES      = target/$(TARGET)/release/kernel+ttables\nKERNEL_ELF_TTABLES_DEPS = $(KERNEL_ELF_RAW) $(wildcard $(TT_TOOL_PATH)/*)\n\n##------------------------------------------------------------------------------\n## Kernel symbols\n##------------------------------------------------------------------------------\nexport KERNEL_SYMBOLS_TOOL_PATH = tools/kernel_symbols_tool\n\nKERNEL_ELF_TTABLES_SYMS = target/$(TARGET)/release/kernel+ttables+symbols\n\n# Unlike with KERNEL_ELF_RAW, we are not relying on dep-info here. One of the reasons being that the\n# name of the generated symbols file varies between runs, which can cause confusion.\nKERNEL_ELF_TTABLES_SYMS_DEPS = $(KERNEL_ELF_TTABLES) \\\n    $(wildcard kernel_symbols/*)                     \\\n    $(wildcard $(KERNEL_SYMBOLS_TOOL_PATH)/*)\n\nexport TARGET\nexport KERNEL_SYMBOLS_INPUT_ELF  = $(KERNEL_ELF_TTABLES)\nexport KERNEL_SYMBOLS_OUTPUT_ELF = $(KERNEL_ELF_TTABLES_SYMS)\n\nKERNEL_ELF = $(KERNEL_ELF_TTABLES_SYMS)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Command building blocks\n##--------------------------------------------------------------------------------------------------\nRUSTFLAGS = $(RUSTC_MISC_ARGS)                   \\\n    -C link-arg=--library-path=$(LD_SCRIPT_PATH) \\\n    -C link-arg=--script=$(KERNEL_LINKER_SCRIPT)\n\nRUSTFLAGS_PEDANTIC = $(RUSTFLAGS) \\\n    -D warnings                   \\\n    -D missing_docs\n\nFEATURES     += --features bsp_$(BSP)\nCOMPILER_ARGS = --target=$(TARGET) \\\n    $(FEATURES)                    \\\n    --release\n\n# build-std can be skipped for helper commands that do not rely on correct stack frames and other\n# custom compiler options. This results in a huge speedup.\nRUSTC_CMD   = cargo rustc $(COMPILER_ARGS) -Z build-std=core,alloc --manifest-path $(KERNEL_MANIFEST)\nDOC_CMD     = cargo doc $(COMPILER_ARGS)\nCLIPPY_CMD  = cargo clippy $(COMPILER_ARGS)\nTEST_CMD    = cargo test $(COMPILER_ARGS) -Z build-std=core,alloc --manifest-path $(KERNEL_MANIFEST)\nOBJCOPY_CMD = rust-objcopy \\\n    --strip-all            \\\n    -O binary\n\nEXEC_QEMU          = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE)\nEXEC_TT_TOOL       = ruby $(TT_TOOL_PATH)/main.rb\nEXEC_TEST_DISPATCH = ruby ../common/tests/dispatch.rb\nEXEC_MINIPUSH      = ruby ../common/serial/minipush.rb\n\n##------------------------------------------------------------------------------\n## Dockerization\n##------------------------------------------------------------------------------\nDOCKER_CMD            = docker run -t --rm -v $(shell pwd):/work/tutorial -w /work/tutorial\nDOCKER_CMD_INTERACT   = $(DOCKER_CMD) -i\nDOCKER_ARG_DIR_COMMON = -v $(shell pwd)/../common:/work/common\nDOCKER_ARG_DIR_JTAG   = -v $(shell pwd)/../X1_JTAG_boot:/work/X1_JTAG_boot\nDOCKER_ARG_DEV        = --privileged -v /dev:/dev\nDOCKER_ARG_NET        = --network host\n\n# DOCKER_IMAGE defined in include file (see top of this file).\nDOCKER_QEMU  = $(DOCKER_CMD_INTERACT) $(DOCKER_IMAGE)\nDOCKER_TOOLS = $(DOCKER_CMD) $(DOCKER_IMAGE)\nDOCKER_TEST  = $(DOCKER_CMD) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\nDOCKER_GDB   = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_NET) $(DOCKER_IMAGE)\n\n# Dockerize commands, which require USB device passthrough, only on Linux.\nifeq ($(shell uname -s),Linux)\n    DOCKER_CMD_DEV = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_DEV)\n\n    DOCKER_CHAINBOOT = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\n    DOCKER_JTAGBOOT  = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_ARG_DIR_JTAG) $(DOCKER_IMAGE)\n    DOCKER_OPENOCD   = $(DOCKER_CMD_DEV) $(DOCKER_ARG_NET) $(DOCKER_IMAGE)\nelse\n    DOCKER_OPENOCD   = echo \"Not yet supported on non-Linux systems.\"; \\#\nendif\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: all doc qemu chainboot clippy clean readelf objdump nm check\n\nall: $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Save the configuration as a file, so make understands if it changed.\n##------------------------------------------------------------------------------\n$(LAST_BUILD_CONFIG):\n\t@rm -f target/*.build_config\n\t@mkdir -p target\n\t@touch $(LAST_BUILD_CONFIG)\n\n##------------------------------------------------------------------------------\n## Compile the kernel ELF\n##------------------------------------------------------------------------------\n$(KERNEL_ELF_RAW): $(KERNEL_ELF_RAW_DEPS)\n\t$(call color_header, \"Compiling kernel ELF - $(BSP)\")\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(RUSTC_CMD)\n\n##------------------------------------------------------------------------------\n## Precompute the kernel translation tables and patch them into the kernel ELF\n##------------------------------------------------------------------------------\n$(KERNEL_ELF_TTABLES): $(KERNEL_ELF_TTABLES_DEPS)\n\t$(call color_header, \"Precomputing kernel translation tables and patching kernel ELF\")\n\t@cp $(KERNEL_ELF_RAW) $(KERNEL_ELF_TTABLES)\n\t@$(DOCKER_TOOLS) $(EXEC_TT_TOOL) $(BSP) $(KERNEL_ELF_TTABLES)\n\n##------------------------------------------------------------------------------\n## Generate kernel symbols and patch them into the kernel ELF\n##------------------------------------------------------------------------------\n$(KERNEL_ELF_TTABLES_SYMS): $(KERNEL_ELF_TTABLES_SYMS_DEPS)\n\t$(call color_header, \"Generating kernel symbols and patching kernel ELF\")\n\t@$(MAKE) --no-print-directory -f kernel_symbols.mk\n\n##------------------------------------------------------------------------------\n## Generate the stripped kernel binary\n##------------------------------------------------------------------------------\n$(KERNEL_BIN): $(KERNEL_ELF_TTABLES_SYMS)\n\t$(call color_header, \"Generating stripped binary\")\n\t@$(OBJCOPY_CMD) $(KERNEL_ELF_TTABLES_SYMS) $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Name\")\n\t@echo $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Size\")\n\t$(call disk_usage_KiB, $(KERNEL_BIN))\n\n##------------------------------------------------------------------------------\n## Generate the documentation\n##------------------------------------------------------------------------------\ndoc: clean\n\t$(call color_header, \"Generating docs\")\n\t@$(DOC_CMD) --document-private-items --open\n\n##------------------------------------------------------------------------------\n## Run the kernel in QEMU\n##------------------------------------------------------------------------------\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\nqemu:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\nqemu: $(KERNEL_BIN)\n\t$(call color_header, \"Launching QEMU\")\n\t@$(DOCKER_QEMU) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\nendif\n\n##------------------------------------------------------------------------------\n## Push the kernel to the real HW target\n##------------------------------------------------------------------------------\nchainboot: $(KERNEL_BIN)\n\t@$(DOCKER_CHAINBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run clippy\n##------------------------------------------------------------------------------\nclippy:\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(CLIPPY_CMD)\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(CLIPPY_CMD) --features test_build --tests \\\n                --manifest-path $(KERNEL_MANIFEST)\n\n##------------------------------------------------------------------------------\n## Clean\n##------------------------------------------------------------------------------\nclean:\n\trm -rf target $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run readelf\n##------------------------------------------------------------------------------\nreadelf: $(KERNEL_ELF)\n\t$(call color_header, \"Launching readelf\")\n\t@$(DOCKER_TOOLS) $(READELF_BINARY) --headers $(KERNEL_ELF)\n\n##------------------------------------------------------------------------------\n## Run objdump\n##------------------------------------------------------------------------------\nobjdump: $(KERNEL_ELF)\n\t$(call color_header, \"Launching objdump\")\n\t@$(DOCKER_TOOLS) $(OBJDUMP_BINARY) --disassemble --demangle \\\n                --section .text   \\\n                --section .rodata \\\n                $(KERNEL_ELF) | rustfilt\n\n##------------------------------------------------------------------------------\n## Run nm\n##------------------------------------------------------------------------------\nnm: $(KERNEL_ELF)\n\t$(call color_header, \"Launching nm\")\n\t@$(DOCKER_TOOLS) $(NM_BINARY) --demangle --print-size $(KERNEL_ELF) | sort | rustfilt\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Debugging targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: jtagboot openocd gdb gdb-opt0\n\n##------------------------------------------------------------------------------\n## Push the JTAG boot image to the real HW target\n##------------------------------------------------------------------------------\njtagboot:\n\t@$(DOCKER_JTAGBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(JTAG_BOOT_IMAGE)\n\n##------------------------------------------------------------------------------\n## Start OpenOCD session\n##------------------------------------------------------------------------------\nopenocd:\n\t$(call color_header, \"Launching OpenOCD\")\n\t@$(DOCKER_OPENOCD) openocd $(OPENOCD_ARG)\n\n##------------------------------------------------------------------------------\n## Start GDB session\n##------------------------------------------------------------------------------\ngdb-opt0: RUSTC_MISC_ARGS += -C opt-level=0\ngdb gdb-opt0: $(KERNEL_ELF)\n\t$(call color_header, \"Launching GDB\")\n\t@$(DOCKER_GDB) gdb-multiarch -q $(KERNEL_ELF)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Testing targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: test test_boot test_unit test_integration\n\ntest_unit test_integration: FEATURES += --features test_build\n\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\ntest_boot test_unit test_integration test:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\n##------------------------------------------------------------------------------\n## Run boot test\n##------------------------------------------------------------------------------\ntest_boot: $(KERNEL_BIN)\n\t$(call color_header, \"Boot test - $(BSP)\")\n\t@$(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Helpers for unit and integration test targets\n##------------------------------------------------------------------------------\ndefine KERNEL_TEST_RUNNER\n#!/usr/bin/env bash\n\n    # The cargo test runner seems to change into the crate under test's directory. Therefore, ensure\n    # this script executes from the root.\n    cd $(shell pwd)\n\n    TEST_ELF=$$(echo $$1 | sed -e 's/.*target/target/g')\n    TEST_ELF_SYMS=\"$${TEST_ELF}_syms\"\n    TEST_BINARY=$$(echo $$1.img | sed -e 's/.*target/target/g')\n\n    $(DOCKER_TOOLS) $(EXEC_TT_TOOL) $(BSP) $$TEST_ELF > /dev/null\n\n    # This overrides the two ENV variables. The other ENV variables that are required as input for\n    # the .mk file are set already because they are exported by this Makefile and this script is\n    # started by the same.\n    KERNEL_SYMBOLS_INPUT_ELF=$$TEST_ELF           \\\n        KERNEL_SYMBOLS_OUTPUT_ELF=$$TEST_ELF_SYMS \\\n        $(MAKE) --no-print-directory -f kernel_symbols.mk > /dev/null 2>&1\n\n    $(OBJCOPY_CMD) $$TEST_ELF_SYMS $$TEST_BINARY\n    $(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_TEST_ARGS) -kernel $$TEST_BINARY\nendef\n\nexport KERNEL_TEST_RUNNER\n\ndefine test_prepare\n    @mkdir -p target\n    @echo \"$$KERNEL_TEST_RUNNER\" > target/kernel_test_runner.sh\n    @chmod +x target/kernel_test_runner.sh\nendef\n\n##------------------------------------------------------------------------------\n## Run unit test(s)\n##------------------------------------------------------------------------------\ntest_unit:\n\t$(call color_header, \"Compiling unit test(s) - $(BSP)\")\n\t$(call test_prepare)\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(TEST_CMD) --lib\n\n##------------------------------------------------------------------------------\n## Run integration test(s)\n##------------------------------------------------------------------------------\ntest_integration:\n\t$(call color_header, \"Compiling integration test(s) - $(BSP)\")\n\t$(call test_prepare)\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(TEST_CMD) $(TEST_ARG)\n\ntest: test_boot test_unit test_integration\n\nendif\n"
  },
  {
    "path": "20_timer_callbacks/README.md",
    "content": "# Tutorial 20 - Timer Callbacks\n\n## tl;dr\n\n- The timer subsystem is extended so that it can be used to execute timeout callbacks in IRQ\n  context.\n\n## Note\n\nThis chapter's code will be tightly coupled to follow-up tutorials which are yet to be developed. It\nis therefore expected that this chapter's code is subject to change depending upon findings that are\nyet to be made.\n\nTherefore, content for this README will be provided sometime later when all the pieces fit together.\n\n## Diff to previous\n```diff\n\ndiff -uNr 19_kernel_heap/kernel/Cargo.toml 20_timer_callbacks/kernel/Cargo.toml\n--- 19_kernel_heap/kernel/Cargo.toml\n+++ 20_timer_callbacks/kernel/Cargo.toml\n@@ -1,6 +1,6 @@\n [package]\n name = \"mingo\"\n-version = \"0.19.0\"\n+version = \"0.20.0\"\n authors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\n edition = \"2021\"\n\n\ndiff -uNr 19_kernel_heap/kernel/src/_arch/aarch64/time.rs 20_timer_callbacks/kernel/src/_arch/aarch64/time.rs\n--- 19_kernel_heap/kernel/src/_arch/aarch64/time.rs\n+++ 20_timer_callbacks/kernel/src/_arch/aarch64/time.rs\n@@ -11,14 +11,17 @@\n //!\n //! crate::time::arch_time\n\n-use crate::warn;\n+use crate::{\n+    bsp::{self, exception},\n+    warn,\n+};\n use aarch64_cpu::{asm::barrier, registers::*};\n use core::{\n     num::{NonZeroU128, NonZeroU32, NonZeroU64},\n     ops::{Add, Div},\n     time::Duration,\n };\n-use tock_registers::interfaces::Readable;\n+use tock_registers::interfaces::{ReadWriteable, Readable, Writeable};\n\n //--------------------------------------------------------------------------------------------------\n // Private Definitions\n@@ -160,3 +163,31 @@\n     // Read CNTPCT_EL0 directly to avoid the ISB that is part of [`read_cntpct`].\n     while GenericTimerCounterValue(CNTPCT_EL0.get()) < counter_value_target {}\n }\n+\n+/// The associated IRQ number.\n+pub const fn timeout_irq() -> exception::asynchronous::IRQNumber {\n+    bsp::exception::asynchronous::irq_map::ARM_NS_PHYSICAL_TIMER\n+}\n+\n+/// Program a timer IRQ to be fired after `delay` has passed.\n+pub fn set_timeout_irq(due_time: Duration) {\n+    let counter_value_target: GenericTimerCounterValue = match due_time.try_into() {\n+        Err(msg) => {\n+            warn!(\"set_timeout: {}. Skipping\", msg);\n+            return;\n+        }\n+        Ok(val) => val,\n+    };\n+\n+    // Set the compare value register.\n+    CNTP_CVAL_EL0.set(counter_value_target.0);\n+\n+    // Kick off the timer.\n+    CNTP_CTL_EL0.modify(CNTP_CTL_EL0::ENABLE::SET + CNTP_CTL_EL0::IMASK::CLEAR);\n+}\n+\n+/// Conclude a pending timeout IRQ.\n+pub fn conclude_timeout_irq() {\n+    // Disable counting. De-asserts the IRQ.\n+    CNTP_CTL_EL0.modify(CNTP_CTL_EL0::ENABLE::CLEAR);\n+}\n\ndiff -uNr 19_kernel_heap/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/local_ic.rs 20_timer_callbacks/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/local_ic.rs\n--- 19_kernel_heap/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/local_ic.rs\n+++ 20_timer_callbacks/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/local_ic.rs\n@@ -0,0 +1,173 @@\n+// SPDX-License-Identifier: MIT OR Apache-2.0\n+//\n+// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n+\n+//! Local Interrupt Controller Driver.\n+//!\n+//! # Resources\n+//!\n+//! - <https://datasheets.raspberrypi.com/bcm2836/bcm2836-peripherals.pdf>\n+\n+use super::{LocalIRQ, PendingIRQs};\n+use crate::{\n+    bsp::device_driver::common::MMIODerefWrapper,\n+    exception,\n+    memory::{Address, Virtual},\n+    synchronization,\n+    synchronization::{IRQSafeNullLock, InitStateLock},\n+};\n+use alloc::vec::Vec;\n+use tock_registers::{\n+    interfaces::{Readable, Writeable},\n+    register_structs,\n+    registers::{ReadOnly, WriteOnly},\n+};\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+register_structs! {\n+    #[allow(non_snake_case)]\n+    WORegisterBlock {\n+        (0x00 => _reserved1),\n+        (0x40 => CORE0_TIMER_INTERRUPT_CONTROL: WriteOnly<u32>),\n+        (0x44 => @END),\n+    }\n+}\n+\n+register_structs! {\n+    #[allow(non_snake_case)]\n+    RORegisterBlock {\n+        (0x00 => _reserved1),\n+        (0x60 => CORE0_INTERRUPT_SOURCE: ReadOnly<u32>),\n+        (0x64 => @END),\n+    }\n+}\n+\n+/// Abstraction for the WriteOnly parts of the associated MMIO registers.\n+type WriteOnlyRegisters = MMIODerefWrapper<WORegisterBlock>;\n+\n+/// Abstraction for the ReadOnly parts of the associated MMIO registers.\n+type ReadOnlyRegisters = MMIODerefWrapper<RORegisterBlock>;\n+\n+type HandlerTable = Vec<Option<exception::asynchronous::IRQHandlerDescriptor<LocalIRQ>>>;\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+/// Representation of the peripheral interrupt controller.\n+pub struct LocalIC {\n+    /// Access to write registers is guarded with a lock.\n+    wo_registers: IRQSafeNullLock<WriteOnlyRegisters>,\n+\n+    /// Register read access is unguarded.\n+    ro_registers: ReadOnlyRegisters,\n+\n+    /// Stores registered IRQ handlers. Writable only during kernel init. RO afterwards.\n+    handler_table: InitStateLock<HandlerTable>,\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n+// Public Code\n+//--------------------------------------------------------------------------------------------------\n+\n+impl LocalIC {\n+    // See datasheet.\n+    const PERIPH_IRQ_MASK: u32 = (1 << 8);\n+\n+    /// Create an instance.\n+    ///\n+    /// # Safety\n+    ///\n+    /// - The user must ensure to provide a correct MMIO start address.\n+    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n+        Self {\n+            wo_registers: IRQSafeNullLock::new(WriteOnlyRegisters::new(mmio_start_addr)),\n+            ro_registers: ReadOnlyRegisters::new(mmio_start_addr),\n+            handler_table: InitStateLock::new(Vec::new()),\n+        }\n+    }\n+\n+    /// Called by the kernel to bring up the device.\n+    pub fn init(&self) {\n+        self.handler_table\n+            .write(|table| table.resize(LocalIRQ::MAX_INCLUSIVE + 1, None));\n+    }\n+\n+    /// Query the list of pending IRQs.\n+    fn pending_irqs(&self) -> PendingIRQs {\n+        // Ignore the indicator bit for a peripheral IRQ.\n+        PendingIRQs::new(\n+            (self.ro_registers.CORE0_INTERRUPT_SOURCE.get() & !Self::PERIPH_IRQ_MASK).into(),\n+        )\n+    }\n+}\n+\n+//------------------------------------------------------------------------------\n+// OS Interface Code\n+//------------------------------------------------------------------------------\n+use synchronization::interface::{Mutex, ReadWriteEx};\n+\n+impl exception::asynchronous::interface::IRQManager for LocalIC {\n+    type IRQNumberType = LocalIRQ;\n+\n+    fn register_handler(\n+        &self,\n+        irq_handler_descriptor: exception::asynchronous::IRQHandlerDescriptor<Self::IRQNumberType>,\n+    ) -> Result<(), &'static str> {\n+        self.handler_table.write(|table| {\n+            let irq_number = irq_handler_descriptor.number().get();\n+\n+            if table[irq_number].is_some() {\n+                return Err(\"IRQ handler already registered\");\n+            }\n+\n+            table[irq_number] = Some(irq_handler_descriptor);\n+\n+            Ok(())\n+        })\n+    }\n+\n+    fn enable(&self, irq: &Self::IRQNumberType) {\n+        self.wo_registers.lock(|regs| {\n+            let enable_bit: u32 = 1 << (irq.get());\n+\n+            // Writing a 1 to a bit will set the corresponding IRQ enable bit. All other IRQ enable\n+            // bits are unaffected. So we don't need read and OR'ing here.\n+            regs.CORE0_TIMER_INTERRUPT_CONTROL.set(enable_bit);\n+        });\n+    }\n+\n+    fn handle_pending_irqs<'irq_context>(\n+        &'irq_context self,\n+        _ic: &exception::asynchronous::IRQContext<'irq_context>,\n+    ) {\n+        self.handler_table.read(|table| {\n+            for irq_number in self.pending_irqs() {\n+                match table[irq_number] {\n+                    None => panic!(\"No handler registered for IRQ {}\", irq_number),\n+                    Some(descriptor) => {\n+                        // Call the IRQ handler. Panics on failure.\n+                        descriptor.handler().handle().expect(\"Error handling IRQ\");\n+                    }\n+                }\n+            }\n+        })\n+    }\n+\n+    fn print_handler(&self) {\n+        use crate::info;\n+\n+        info!(\"      Local handler:\");\n+\n+        self.handler_table.read(|table| {\n+            for (i, opt) in table.iter().enumerate() {\n+                if let Some(handler) = opt {\n+                    info!(\"            {: >3}. {}\", i, handler.name());\n+                }\n+            }\n+        });\n+    }\n+}\n\ndiff -uNr 19_kernel_heap/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs 20_timer_callbacks/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs\n--- 19_kernel_heap/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs\n+++ 20_timer_callbacks/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs\n@@ -4,6 +4,7 @@\n\n //! Interrupt Controller Driver.\n\n+mod local_ic;\n mod peripheral_ic;\n\n use crate::{\n@@ -40,6 +41,7 @@\n\n /// Representation of the Interrupt Controller.\n pub struct InterruptController {\n+    local: local_ic::LocalIC,\n     periph: peripheral_ic::PeripheralIC,\n }\n\n@@ -81,7 +83,7 @@\n }\n\n impl InterruptController {\n-    // Restrict to 3 for now. This makes future code for local_ic.rs more straight forward.\n+    // Restrict to 3 for now. This makes the code for local_ic.rs more straight forward.\n     const MAX_LOCAL_IRQ_NUMBER: usize = 3;\n     const MAX_PERIPHERAL_IRQ_NUMBER: usize = 63;\n\n@@ -92,8 +94,12 @@\n     /// # Safety\n     ///\n     /// - The user must ensure to provide a correct MMIO start address.\n-    pub const unsafe fn new(periph_mmio_start_addr: Address<Virtual>) -> Self {\n+    pub const unsafe fn new(\n+        local_mmio_start_addr: Address<Virtual>,\n+        periph_mmio_start_addr: Address<Virtual>,\n+    ) -> Self {\n         Self {\n+            local: local_ic::LocalIC::new(local_mmio_start_addr),\n             periph: peripheral_ic::PeripheralIC::new(periph_mmio_start_addr),\n         }\n     }\n@@ -111,6 +117,7 @@\n     }\n\n     unsafe fn init(&self) -> Result<(), &'static str> {\n+        self.local.init();\n         self.periph.init();\n\n         Ok(())\n@@ -125,7 +132,15 @@\n         irq_handler_descriptor: exception::asynchronous::IRQHandlerDescriptor<Self::IRQNumberType>,\n     ) -> Result<(), &'static str> {\n         match irq_handler_descriptor.number() {\n-            IRQNumber::Local(_) => unimplemented!(\"Local IRQ controller not implemented.\"),\n+            IRQNumber::Local(lirq) => {\n+                let local_descriptor = IRQHandlerDescriptor::new(\n+                    lirq,\n+                    irq_handler_descriptor.name(),\n+                    irq_handler_descriptor.handler(),\n+                );\n+\n+                self.local.register_handler(local_descriptor)\n+            }\n             IRQNumber::Peripheral(pirq) => {\n                 let periph_descriptor = IRQHandlerDescriptor::new(\n                     pirq,\n@@ -140,7 +155,7 @@\n\n     fn enable(&self, irq: &Self::IRQNumberType) {\n         match irq {\n-            IRQNumber::Local(_) => unimplemented!(\"Local IRQ controller not implemented.\"),\n+            IRQNumber::Local(lirq) => self.local.enable(lirq),\n             IRQNumber::Peripheral(pirq) => self.periph.enable(pirq),\n         }\n     }\n@@ -149,11 +164,12 @@\n         &'irq_context self,\n         ic: &exception::asynchronous::IRQContext<'irq_context>,\n     ) {\n-        // It can only be a peripheral IRQ pending because enable() does not support local IRQs yet.\n+        self.local.handle_pending_irqs(ic);\n         self.periph.handle_pending_irqs(ic)\n     }\n\n     fn print_handler(&self) {\n+        self.local.print_handler();\n         self.periph.print_handler();\n     }\n }\n\ndiff -uNr 19_kernel_heap/kernel/src/bsp/raspberrypi/driver.rs 20_timer_callbacks/kernel/src/bsp/raspberrypi/driver.rs\n--- 19_kernel_heap/kernel/src/bsp/raspberrypi/driver.rs\n+++ 20_timer_callbacks/kernel/src/bsp/raspberrypi/driver.rs\n@@ -73,6 +73,12 @@\n /// This must be called only after successful init of the memory subsystem.\n #[cfg(feature = \"bsp_rpi3\")]\n unsafe fn instantiate_interrupt_controller() -> Result<(), &'static str> {\n+    let local_mmio_descriptor = MMIODescriptor::new(mmio::LOCAL_IC_START, mmio::LOCAL_IC_SIZE);\n+    let local_virt_addr = memory::mmu::kernel_map_mmio(\n+        device_driver::InterruptController::COMPATIBLE,\n+        &local_mmio_descriptor,\n+    )?;\n+\n     let periph_mmio_descriptor =\n         MMIODescriptor::new(mmio::PERIPHERAL_IC_START, mmio::PERIPHERAL_IC_SIZE);\n     let periph_virt_addr = memory::mmu::kernel_map_mmio(\n@@ -80,7 +86,10 @@\n         &periph_mmio_descriptor,\n     )?;\n\n-    INTERRUPT_CONTROLLER.write(device_driver::InterruptController::new(periph_virt_addr));\n+    INTERRUPT_CONTROLLER.write(device_driver::InterruptController::new(\n+        local_virt_addr,\n+        periph_virt_addr,\n+    ));\n\n     Ok(())\n }\n\ndiff -uNr 19_kernel_heap/kernel/src/bsp/raspberrypi/exception/asynchronous.rs 20_timer_callbacks/kernel/src/bsp/raspberrypi/exception/asynchronous.rs\n--- 19_kernel_heap/kernel/src/bsp/raspberrypi/exception/asynchronous.rs\n+++ 20_timer_callbacks/kernel/src/bsp/raspberrypi/exception/asynchronous.rs\n@@ -13,16 +13,24 @@\n /// Export for reuse in generic asynchronous.rs.\n pub use bsp::device_driver::IRQNumber;\n\n+/// The IRQ map.\n #[cfg(feature = \"bsp_rpi3\")]\n-pub(in crate::bsp) mod irq_map {\n-    use super::bsp::device_driver::{IRQNumber, PeripheralIRQ};\n+pub mod irq_map {\n+    use super::bsp::device_driver::{IRQNumber, LocalIRQ, PeripheralIRQ};\n\n-    pub const PL011_UART: IRQNumber = IRQNumber::Peripheral(PeripheralIRQ::new(57));\n+    /// The non-secure physical timer IRQ number.\n+    pub const ARM_NS_PHYSICAL_TIMER: IRQNumber = IRQNumber::Local(LocalIRQ::new(1));\n+\n+    pub(in crate::bsp) const PL011_UART: IRQNumber = IRQNumber::Peripheral(PeripheralIRQ::new(57));\n }\n\n+/// The IRQ map.\n #[cfg(feature = \"bsp_rpi4\")]\n-pub(in crate::bsp) mod irq_map {\n+pub mod irq_map {\n     use super::bsp::device_driver::IRQNumber;\n\n-    pub const PL011_UART: IRQNumber = IRQNumber::new(153);\n+    /// The non-secure physical timer IRQ number.\n+    pub const ARM_NS_PHYSICAL_TIMER: IRQNumber = IRQNumber::new(30);\n+\n+    pub(in crate::bsp) const PL011_UART: IRQNumber = IRQNumber::new(153);\n }\n\ndiff -uNr 19_kernel_heap/kernel/src/bsp/raspberrypi/memory.rs 20_timer_callbacks/kernel/src/bsp/raspberrypi/memory.rs\n--- 19_kernel_heap/kernel/src/bsp/raspberrypi/memory.rs\n+++ 20_timer_callbacks/kernel/src/bsp/raspberrypi/memory.rs\n@@ -124,6 +124,9 @@\n         pub const PL011_UART_START:    Address<Physical> = Address::new(0x3F20_1000);\n         pub const PL011_UART_SIZE:     usize             =              0x48;\n\n+        pub const LOCAL_IC_START:      Address<Physical> = Address::new(0x4000_0000);\n+        pub const LOCAL_IC_SIZE:       usize             =              0x100;\n+\n         pub const END:                 Address<Physical> = Address::new(0x4001_0000);\n     }\n\n\ndiff -uNr 19_kernel_heap/kernel/src/main.rs 20_timer_callbacks/kernel/src/main.rs\n--- 19_kernel_heap/kernel/src/main.rs\n+++ 20_timer_callbacks/kernel/src/main.rs\n@@ -30,6 +30,11 @@\n     exception::handling_init();\n     memory::init();\n\n+    // Initialize the timer subsystem.\n+    if let Err(x) = time::init() {\n+        panic!(\"Error initializing timer subsystem: {}\", x);\n+    }\n+\n     // Initialize the BSP driver subsystem.\n     if let Err(x) = bsp::driver::init() {\n         panic!(\"Error initializing BSP driver subsystem: {}\", x);\n@@ -52,6 +57,9 @@\n\n /// The main function running after the early init.\n fn kernel_main() -> ! {\n+    use alloc::boxed::Box;\n+    use core::time::Duration;\n+\n     info!(\"{}\", libkernel::version());\n     info!(\"Booting on: {}\", bsp::board_name());\n\n@@ -78,6 +86,11 @@\n     info!(\"Kernel heap:\");\n     memory::heap_alloc::kernel_heap_allocator().print_usage();\n\n+    time::time_manager().set_timeout_once(Duration::from_secs(5), Box::new(|| info!(\"Once 5\")));\n+    time::time_manager().set_timeout_once(Duration::from_secs(3), Box::new(|| info!(\"Once 2\")));\n+    time::time_manager()\n+        .set_timeout_periodic(Duration::from_secs(1), Box::new(|| info!(\"Periodic 1 sec\")));\n+\n     info!(\"Echoing input now\");\n     cpu::wait_forever();\n }\n\ndiff -uNr 19_kernel_heap/kernel/src/time.rs 20_timer_callbacks/kernel/src/time.rs\n--- 19_kernel_heap/kernel/src/time.rs\n+++ 20_timer_callbacks/kernel/src/time.rs\n@@ -3,19 +3,54 @@\n // Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n //! Timer primitives.\n+//!\n+//! # Resources\n+//!\n+//! - <https://stackoverflow.com/questions/41081240/idiomatic-callbacks-in-rust>\n+//! - <https://doc.rust-lang.org/stable/std/panic/fn.set_hook.html>\n\n #[cfg(target_arch = \"aarch64\")]\n #[path = \"_arch/aarch64/time.rs\"]\n mod arch_time;\n\n-use core::time::Duration;\n+use crate::{\n+    driver, exception,\n+    exception::asynchronous::IRQNumber,\n+    synchronization::{interface::Mutex, IRQSafeNullLock},\n+    warn,\n+};\n+use alloc::{boxed::Box, vec::Vec};\n+use core::{\n+    sync::atomic::{AtomicBool, Ordering},\n+    time::Duration,\n+};\n+\n+//--------------------------------------------------------------------------------------------------\n+// Private Definitions\n+//--------------------------------------------------------------------------------------------------\n+\n+struct Timeout {\n+    due_time: Duration,\n+    period: Option<Duration>,\n+    callback: TimeoutCallback,\n+}\n+\n+struct OrderedTimeoutQueue {\n+    // Can be replaced with a BinaryHeap once it's new() becomes const.\n+    inner: Vec<Timeout>,\n+}\n\n //--------------------------------------------------------------------------------------------------\n // Public Definitions\n //--------------------------------------------------------------------------------------------------\n\n+/// The callback type used by timer IRQs.\n+pub type TimeoutCallback = Box<dyn Fn() + Send>;\n+\n /// Provides time management functions.\n-pub struct TimeManager;\n+pub struct TimeManager {\n+    queue: IRQSafeNullLock<OrderedTimeoutQueue>,\n+}\n\n //--------------------------------------------------------------------------------------------------\n // Global instances\n@@ -24,6 +59,46 @@\n static TIME_MANAGER: TimeManager = TimeManager::new();\n\n //--------------------------------------------------------------------------------------------------\n+// Private Code\n+//--------------------------------------------------------------------------------------------------\n+\n+impl Timeout {\n+    pub fn is_periodic(&self) -> bool {\n+        self.period.is_some()\n+    }\n+\n+    pub fn refresh(&mut self) {\n+        if let Some(delay) = self.period {\n+            self.due_time += delay;\n+        }\n+    }\n+}\n+\n+impl OrderedTimeoutQueue {\n+    pub const fn new() -> Self {\n+        Self { inner: Vec::new() }\n+    }\n+\n+    pub fn push(&mut self, timeout: Timeout) {\n+        self.inner.push(timeout);\n+\n+        // Note reverse compare order so that earliest expiring item is at end of vec. We do this so\n+        // that we can use Vec::pop below to retrieve the item that is next due.\n+        self.inner.sort_by(|a, b| b.due_time.cmp(&a.due_time));\n+    }\n+\n+    pub fn peek_next_due_time(&self) -> Option<Duration> {\n+        let timeout = self.inner.last()?;\n+\n+        Some(timeout.due_time)\n+    }\n+\n+    pub fn pop(&mut self) -> Option<Timeout> {\n+        self.inner.pop()\n+    }\n+}\n+\n+//--------------------------------------------------------------------------------------------------\n // Public Code\n //--------------------------------------------------------------------------------------------------\n\n@@ -33,9 +108,14 @@\n }\n\n impl TimeManager {\n+    /// Compatibility string.\n+    pub const COMPATIBLE: &'static str = \"ARM Architectural Timer\";\n+\n     /// Create an instance.\n     pub const fn new() -> Self {\n-        Self\n+        Self {\n+            queue: IRQSafeNullLock::new(OrderedTimeoutQueue::new()),\n+        }\n     }\n\n     /// The timer's resolution.\n@@ -54,4 +134,130 @@\n     pub fn spin_for(&self, duration: Duration) {\n         arch_time::spin_for(duration)\n     }\n+\n+    /// Set a timeout.\n+    fn set_timeout(&self, timeout: Timeout) {\n+        self.queue.lock(|queue| {\n+            queue.push(timeout);\n+\n+            arch_time::set_timeout_irq(queue.peek_next_due_time().unwrap());\n+        });\n+    }\n+\n+    /// Set a one-shot timeout.\n+    pub fn set_timeout_once(&self, delay: Duration, callback: TimeoutCallback) {\n+        let timeout = Timeout {\n+            due_time: self.uptime() + delay,\n+            period: None,\n+            callback,\n+        };\n+\n+        self.set_timeout(timeout);\n+    }\n+\n+    /// Set a periodic timeout.\n+    pub fn set_timeout_periodic(&self, delay: Duration, callback: TimeoutCallback) {\n+        let timeout = Timeout {\n+            due_time: self.uptime() + delay,\n+            period: Some(delay),\n+            callback,\n+        };\n+\n+        self.set_timeout(timeout);\n+    }\n+}\n+\n+/// Initialize the timer subsystem.\n+pub fn init() -> Result<(), &'static str> {\n+    static INIT_DONE: AtomicBool = AtomicBool::new(false);\n+    if INIT_DONE.load(Ordering::Relaxed) {\n+        return Err(\"Init already done\");\n+    }\n+\n+    let timer_descriptor =\n+        driver::DeviceDriverDescriptor::new(time_manager(), None, Some(arch_time::timeout_irq()));\n+    driver::driver_manager().register_driver(timer_descriptor);\n+\n+    INIT_DONE.store(true, Ordering::Relaxed);\n+    Ok(())\n+}\n+\n+//------------------------------------------------------------------------------\n+// OS Interface Code\n+//------------------------------------------------------------------------------\n+\n+impl driver::interface::DeviceDriver for TimeManager {\n+    type IRQNumberType = IRQNumber;\n+\n+    fn compatible(&self) -> &'static str {\n+        Self::COMPATIBLE\n+    }\n+\n+    fn register_and_enable_irq_handler(\n+        &'static self,\n+        irq_number: &Self::IRQNumberType,\n+    ) -> Result<(), &'static str> {\n+        use exception::asynchronous::{irq_manager, IRQHandlerDescriptor};\n+\n+        let descriptor = IRQHandlerDescriptor::new(*irq_number, Self::COMPATIBLE, self);\n+\n+        irq_manager().register_handler(descriptor)?;\n+        irq_manager().enable(irq_number);\n+\n+        Ok(())\n+    }\n+}\n+\n+impl exception::asynchronous::interface::IRQHandler for TimeManager {\n+    fn handle(&self) -> Result<(), &'static str> {\n+        arch_time::conclude_timeout_irq();\n+\n+        let maybe_timeout: Option<Timeout> = self.queue.lock(|queue| {\n+            let next_due_time = queue.peek_next_due_time()?;\n+            if next_due_time > self.uptime() {\n+                return None;\n+            }\n+\n+            let mut timeout = queue.pop().unwrap();\n+\n+            // Refresh as early as possible to prevent drift.\n+            if timeout.is_periodic() {\n+                timeout.refresh();\n+            }\n+\n+            Some(timeout)\n+        });\n+\n+        let timeout = match maybe_timeout {\n+            None => {\n+                warn!(\"Spurious timeout IRQ\");\n+                return Ok(());\n+            }\n+            Some(t) => t,\n+        };\n+\n+        // Important: Call the callback while not holding any lock, because the callback might\n+        // attempt to modify data that is protected by a lock (in particular, the timeout queue\n+        // itself).\n+        (timeout.callback)();\n+\n+        self.queue.lock(|queue| {\n+            if timeout.is_periodic() {\n+                // There might be some overhead involved in the periodic path, because the timeout\n+                // item is first popped from the underlying Vec and then pushed back again. It could\n+                // be faster to keep the item in the queue and find a way to work with a reference\n+                // to it.\n+                //\n+                // We are not going this route on purpose, though. It allows to keep the code simple\n+                // and the focus on the high-level concepts.\n+                queue.push(timeout);\n+            };\n+\n+            if let Some(due_time) = queue.peek_next_due_time() {\n+                arch_time::set_timeout_irq(due_time);\n+            }\n+        });\n+\n+        Ok(())\n+    }\n }\n\ndiff -uNr 19_kernel_heap/kernel/tests/boot_test_string.rb 20_timer_callbacks/kernel/tests/boot_test_string.rb\n--- 19_kernel_heap/kernel/tests/boot_test_string.rb\n+++ 20_timer_callbacks/kernel/tests/boot_test_string.rb\n@@ -1,3 +1,3 @@\n # frozen_string_literal: true\n\n-EXPECTED_PRINT = 'Echoing input now'\n+EXPECTED_PRINT = 'Once 5'\n\n```\n"
  },
  {
    "path": "20_timer_callbacks/kernel/Cargo.toml",
    "content": "[package]\nname = \"mingo\"\nversion = \"0.20.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n\n[features]\ndefault = []\ndebug_prints = []\nbsp_rpi3 = [\"tock-registers\"]\nbsp_rpi4 = [\"tock-registers\"]\ntest_build = [\"qemu-exit\"]\n\n##-------------------------------------------------------------------------------------------------\n## Dependencies\n##-------------------------------------------------------------------------------------------------\n\n[dependencies]\ntest-types = { path = \"../libraries/test-types\" }\ndebug-symbol-types = { path = \"../libraries/debug-symbol-types\" }\nlinked_list_allocator = { version = \"0.10.x\", default-features = false, features = [\"const_mut_refs\"] }\n\n# Optional dependencies\ntock-registers = { version = \"0.8.x\", default-features = false, features = [\"register_types\"], optional = true }\nqemu-exit = { version = \"3.x.x\", optional = true }\n\n# Platform specific dependencies\n[target.'cfg(target_arch = \"aarch64\")'.dependencies]\naarch64-cpu = { version = \"9.x.x\" }\n\n##-------------------------------------------------------------------------------------------------\n## Testing\n##-------------------------------------------------------------------------------------------------\n\n[dev-dependencies]\ntest-macros = { path = \"../libraries/test-macros\" }\n\n# Unit tests are done in the library part of the kernel.\n[lib]\nname = \"libkernel\"\ntest = true\n\n# Disable unit tests for the kernel binary.\n[[bin]]\nname = \"kernel\"\npath = \"src/main.rs\"\ntest = false\n\n# List of tests without harness.\n[[test]]\nname = \"00_console_sanity\"\nharness = false\n\n[[test]]\nname = \"02_exception_sync_page_fault\"\nharness = false\n\n[[test]]\nname = \"03_exception_restore_sanity\"\nharness = false\n\n[[test]]\nname = \"05_backtrace_sanity\"\nharness = false\n\n[[test]]\nname = \"06_backtrace_invalid_frame\"\nharness = false\n\n[[test]]\nname = \"07_backtrace_invalid_link\"\nharness = false\n"
  },
  {
    "path": "20_timer_callbacks/kernel/build.rs",
    "content": "use std::{env, fs, process};\n\nfn main() {\n    let ld_script_path = match env::var(\"LD_SCRIPT_PATH\") {\n        Ok(var) => var,\n        _ => process::exit(0),\n    };\n\n    let files = fs::read_dir(ld_script_path).unwrap();\n    files\n        .filter_map(Result::ok)\n        .filter(|d| {\n            if let Some(e) = d.path().extension() {\n                e == \"ld\"\n            } else {\n                false\n            }\n        })\n        .for_each(|f| println!(\"cargo:rerun-if-changed={}\", f.path().display()));\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/_arch/aarch64/backtrace.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural backtracing support.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::backtrace::arch_backtrace\n\nuse crate::{\n    backtrace::BacktraceItem,\n    memory::{Address, Virtual},\n};\nuse aarch64_cpu::registers::*;\nuse tock_registers::interfaces::Readable;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// A Stack frame record.\n///\n/// # Note\n///\n/// The convention is that `previous_record` is valid as long as it contains a non-null value.\n/// Therefore, it is possible to type the member as `Option<&StackFrameRecord>` because of Rust's\n/// `null-pointer optimization`.\n#[repr(C)]\nstruct StackFrameRecord<'a> {\n    previous_record: Option<&'a StackFrameRecord<'a>>,\n    link: Address<Virtual>,\n}\n\nstruct StackFrameRecordIterator<'a> {\n    cur: &'a StackFrameRecord<'a>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<'a> Iterator for StackFrameRecordIterator<'a> {\n    type Item = BacktraceItem;\n\n    fn next(&mut self) -> Option<Self::Item> {\n        static ABORT_FRAME: StackFrameRecord = StackFrameRecord {\n            previous_record: None,\n            link: Address::new(0),\n        };\n\n        // If previous is None, this is the root frame, so iteration will stop here.\n        let previous = self.cur.previous_record?;\n\n        // Need to abort if the pointer to the previous frame record is invalid.\n        let prev_addr = Address::<Virtual>::new(previous as *const _ as usize);\n        if !prev_addr.is_valid_stack_addr() {\n            // This allows to return the error and then stop on the next iteration.\n            self.cur = &ABORT_FRAME;\n            return Some(BacktraceItem::InvalidFramePointer(prev_addr));\n        }\n\n        let ret = if !self.cur.link.is_valid_code_addr() {\n            Some(BacktraceItem::InvalidLink(self.cur.link))\n        } else {\n            // The link points to the instruction to be executed _after_ returning from a branch.\n            // However, we want to show the instruction that caused the branch, so subtract by one\n            // instruction.\n            //\n            // This might be called from panic!, so it must not panic itself on the subtraction.\n            let link = if self.cur.link >= Address::new(4) {\n                self.cur.link - 4\n            } else {\n                self.cur.link\n            };\n\n            Some(BacktraceItem::Link(link))\n        };\n\n        // Advance the iterator.\n        self.cur = previous;\n\n        ret\n    }\n}\n\nfn stack_frame_record_iterator<'a>() -> Option<StackFrameRecordIterator<'a>> {\n    let fp = Address::<Virtual>::new(FP.get() as usize);\n    if !fp.is_valid_stack_addr() {\n        return None;\n    }\n\n    Some(StackFrameRecordIterator {\n        cur: unsafe { &*(fp.as_usize() as *const _) },\n    })\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Architectural implementation of the backtrace.\npub fn backtrace(f: impl FnOnce(Option<&mut dyn Iterator<Item = BacktraceItem>>)) {\n    f(stack_frame_record_iterator().as_mut().map(|s| s as _))\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(feature = \"test_build\")]\n#[inline(always)]\n/// Hack for corrupting the previous frame address in the current stack frame.\n///\n/// # Safety\n///\n/// - To be used only by testing code.\npub unsafe fn corrupt_previous_frame_addr() {\n    let sf = FP.get() as *mut usize;\n    *sf = 0x123;\n}\n\n#[cfg(feature = \"test_build\")]\n#[inline(always)]\n/// Hack for corrupting the link in the current stack frame.\n///\n/// # Safety\n///\n/// - To be used only by testing code.\npub unsafe fn corrupt_link() {\n    let sf = FP.get() as *mut StackFrameRecord;\n    (*sf).link = Address::new(0x456);\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/_arch/aarch64/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural boot code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::boot::arch_boot\n\nuse crate::{memory, memory::Address};\nuse aarch64_cpu::{asm, registers::*};\nuse core::{\n    arch::global_asm,\n    sync::atomic::{compiler_fence, Ordering},\n};\nuse tock_registers::interfaces::Writeable;\n\n// Assembly counterpart to this file.\nglobal_asm!(\n    include_str!(\"boot.s\"),\n    CONST_CURRENTEL_EL2 = const 0x8,\n    CONST_CORE_ID_MASK = const 0b11\n);\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Prepares the transition from EL2 to EL1.\n///\n/// # Safety\n///\n/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.\n/// - The HW state of EL1 must be prepared in a sound way.\n#[inline(always)]\nunsafe fn prepare_el2_to_el1_transition(\n    virt_boot_core_stack_end_exclusive_addr: u64,\n    virt_kernel_init_addr: u64,\n) {\n    // Enable timer counter registers for EL1.\n    CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET);\n\n    // No offset for reading the counters.\n    CNTVOFF_EL2.set(0);\n\n    // Set EL1 execution state to AArch64.\n    HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64);\n\n    // Set up a simulated exception return.\n    //\n    // First, fake a saved program status where all interrupts were masked and SP_EL1 was used as a\n    // stack pointer.\n    SPSR_EL2.write(\n        SPSR_EL2::D::Masked\n            + SPSR_EL2::A::Masked\n            + SPSR_EL2::I::Masked\n            + SPSR_EL2::F::Masked\n            + SPSR_EL2::M::EL1h,\n    );\n\n    // Second, let the link register point to kernel_init().\n    ELR_EL2.set(virt_kernel_init_addr);\n\n    // Set up SP_EL1 (stack pointer), which will be used by EL1 once we \"return\" to it. Since there\n    // are no plans to ever return to EL2, just re-use the same stack.\n    SP_EL1.set(virt_boot_core_stack_end_exclusive_addr);\n}\n\n/// Reset the backtrace by setting link register and frame pointer to zero.\n///\n/// # Safety\n///\n/// - This function must only be used immediately before entering EL1.\n#[inline(always)]\nunsafe fn prepare_backtrace_reset() {\n    compiler_fence(Ordering::SeqCst);\n    FP.set(0);\n    LR.set(0);\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The Rust entry of the `kernel` binary.\n///\n/// The function is called from the assembly `_start` function.\n///\n/// # Safety\n///\n/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`.\n#[no_mangle]\npub unsafe extern \"C\" fn _start_rust(\n    phys_kernel_tables_base_addr: u64,\n    virt_boot_core_stack_end_exclusive_addr: u64,\n    virt_kernel_init_addr: u64,\n) -> ! {\n    prepare_el2_to_el1_transition(\n        virt_boot_core_stack_end_exclusive_addr,\n        virt_kernel_init_addr,\n    );\n\n    // Turn on the MMU for EL1.\n    let addr = Address::new(phys_kernel_tables_base_addr as usize);\n    memory::mmu::enable_mmu_and_caching(addr).unwrap();\n\n    // Make the function we return to the root of a backtrace.\n    prepare_backtrace_reset();\n\n    // Use `eret` to \"return\" to EL1. Since virtual memory will already be enabled, this results in\n    // execution of kernel_init() in EL1 from its _virtual address_.\n    asm::eret()\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/_arch/aarch64/cpu/boot.s",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//--------------------------------------------------------------------------------------------------\n// Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Load the address of a symbol into a register, PC-relative.\n//\n// The symbol must lie within +/- 4 GiB of the Program Counter.\n//\n// # Resources\n//\n// - https://sourceware.org/binutils/docs-2.36/as/AArch64_002dRelocations.html\n.macro ADR_REL register, symbol\n\tadrp\t\\register, \\symbol\n\tadd\t\\register, \\register, #:lo12:\\symbol\n.endm\n\n// Load the address of a symbol into a register, absolute.\n//\n// # Resources\n//\n// - https://sourceware.org/binutils/docs-2.36/as/AArch64_002dRelocations.html\n.macro ADR_ABS register, symbol\n\tmovz\t\\register, #:abs_g3:\\symbol\n\tmovk\t\\register, #:abs_g2_nc:\\symbol\n\tmovk\t\\register, #:abs_g1_nc:\\symbol\n\tmovk\t\\register, #:abs_g0_nc:\\symbol\n.endm\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n.section .text._start\n\n//------------------------------------------------------------------------------\n// fn _start()\n//------------------------------------------------------------------------------\n_start:\n\t// Only proceed if the core executes in EL2. Park it otherwise.\n\tmrs\tx0, CurrentEL\n\tcmp\tx0, {CONST_CURRENTEL_EL2}\n\tb.ne\t.L_parking_loop\n\n\t// Only proceed on the boot core. Park it otherwise.\n\tmrs\tx1, MPIDR_EL1\n\tand\tx1, x1, {CONST_CORE_ID_MASK}\n\tldr\tx2, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs\n\tcmp\tx1, x2\n\tb.ne\t.L_parking_loop\n\n\t// If execution reaches here, it is the boot core.\n\n\t// Initialize DRAM.\n\tADR_REL\tx0, __bss_start\n\tADR_REL x1, __bss_end_exclusive\n\n.L_bss_init_loop:\n\tcmp\tx0, x1\n\tb.eq\t.L_prepare_rust\n\tstp\txzr, xzr, [x0], #16\n\tb\t.L_bss_init_loop\n\n\t// Prepare the jump to Rust code.\n.L_prepare_rust:\n\t// Load the base address of the kernel's translation tables.\n\tldr\tx0, PHYS_KERNEL_TABLES_BASE_ADDR // provided by bsp/__board_name__/memory/mmu.rs\n\n\t// Load the _absolute_ addresses of the following symbols. Since the kernel is linked at\n\t// the top of the 64 bit address space, these are effectively virtual addresses.\n\tADR_ABS\tx1, __boot_core_stack_end_exclusive\n\tADR_ABS\tx2, kernel_init\n\n\t// Load the PC-relative address of the stack and set the stack pointer.\n\t//\n\t// Since _start() is the first function that runs after the firmware has loaded the kernel\n\t// into memory, retrieving this symbol PC-relative returns the \"physical\" address.\n\t//\n\t// Setting the stack pointer to this value ensures that anything that still runs in EL2,\n\t// until the kernel returns to EL1 with the MMU enabled, works as well. After the return to\n\t// EL1, the virtual address of the stack retrieved above will be used.\n\tADR_REL\tx3, __boot_core_stack_end_exclusive\n\tmov\tsp, x3\n\n\t// Read the CPU's timer counter frequency and store it in ARCH_TIMER_COUNTER_FREQUENCY.\n\t// Abort if the frequency read back as 0.\n\tADR_REL\tx4, ARCH_TIMER_COUNTER_FREQUENCY // provided by aarch64/time.rs\n\tmrs\tx5, CNTFRQ_EL0\n\tcmp\tx5, xzr\n\tb.eq\t.L_parking_loop\n\tstr\tw5, [x4]\n\n\t// Jump to Rust code. x0, x1 and x2 hold the function arguments provided to _start_rust().\n\tb\t_start_rust\n\n\t// Infinitely wait for events (aka \"park the core\").\n.L_parking_loop:\n\twfe\n\tb\t.L_parking_loop\n\n.size\t_start, . - _start\n.type\t_start, function\n.global\t_start\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/_arch/aarch64/cpu/smp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural symmetric multiprocessing.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::smp::arch_smp\n\nuse aarch64_cpu::registers::*;\nuse tock_registers::interfaces::Readable;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return the executing core's id.\n#[inline(always)]\npub fn core_id<T>() -> T\nwhere\n    T: From<u8>,\n{\n    const CORE_MASK: u64 = 0b11;\n\n    T::from((MPIDR_EL1.get() & CORE_MASK) as u8)\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/_arch/aarch64/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural processor code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::arch_cpu\n\nuse aarch64_cpu::asm;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\npub use asm::nop;\n\n/// Pause execution on the core.\n#[inline(always)]\npub fn wait_forever() -> ! {\n    loop {\n        asm::wfe()\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n#[cfg(feature = \"test_build\")]\nuse qemu_exit::QEMUExit;\n\n#[cfg(feature = \"test_build\")]\nconst QEMU_EXIT_HANDLE: qemu_exit::AArch64 = qemu_exit::AArch64::new();\n\n/// Make the host QEMU binary execute `exit(1)`.\n#[cfg(feature = \"test_build\")]\npub fn qemu_exit_failure() -> ! {\n    QEMU_EXIT_HANDLE.exit_failure()\n}\n\n/// Make the host QEMU binary execute `exit(0)`.\n#[cfg(feature = \"test_build\")]\npub fn qemu_exit_success() -> ! {\n    QEMU_EXIT_HANDLE.exit_success()\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/_arch/aarch64/exception/asynchronous.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural asynchronous exception handling.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::exception::asynchronous::arch_asynchronous\n\nuse aarch64_cpu::registers::*;\nuse core::arch::asm;\nuse tock_registers::interfaces::{Readable, Writeable};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nmod daif_bits {\n    pub const IRQ: u8 = 0b0010;\n}\n\ntrait DaifField {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register>;\n}\n\nstruct Debug;\nstruct SError;\nstruct IRQ;\nstruct FIQ;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl DaifField for Debug {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::D\n    }\n}\n\nimpl DaifField for SError {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::A\n    }\n}\n\nimpl DaifField for IRQ {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::I\n    }\n}\n\nimpl DaifField for FIQ {\n    fn daif_field() -> tock_registers::fields::Field<u64, DAIF::Register> {\n        DAIF::F\n    }\n}\n\nfn is_masked<T>() -> bool\nwhere\n    T: DaifField,\n{\n    DAIF.is_set(T::daif_field())\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Returns whether IRQs are masked on the executing core.\npub fn is_local_irq_masked() -> bool {\n    !is_masked::<IRQ>()\n}\n\n/// Unmask IRQs on the executing core.\n///\n/// It is not needed to place an explicit instruction synchronization barrier after the `msr`.\n/// Quoting the Architecture Reference Manual for ARMv8-A, section C5.1.3:\n///\n/// \"Writes to PSTATE.{PAN, D, A, I, F} occur in program order without the need for additional\n/// synchronization.\"\n#[inline(always)]\npub fn local_irq_unmask() {\n    unsafe {\n        asm!(\n            \"msr DAIFClr, {arg}\",\n            arg = const daif_bits::IRQ,\n            options(nomem, nostack, preserves_flags)\n        );\n    }\n}\n\n/// Mask IRQs on the executing core.\n#[inline(always)]\npub fn local_irq_mask() {\n    unsafe {\n        asm!(\n            \"msr DAIFSet, {arg}\",\n            arg = const daif_bits::IRQ,\n            options(nomem, nostack, preserves_flags)\n        );\n    }\n}\n\n/// Mask IRQs on the executing core and return the previously saved interrupt mask bits (DAIF).\n#[inline(always)]\npub fn local_irq_mask_save() -> u64 {\n    let saved = DAIF.get();\n    local_irq_mask();\n\n    saved\n}\n\n/// Restore the interrupt mask bits (DAIF) using the callee's argument.\n///\n/// # Invariant\n///\n/// - No sanity checks on the input.\n#[inline(always)]\npub fn local_irq_restore(saved: u64) {\n    DAIF.set(saved);\n}\n\n/// Print the AArch64 exceptions status.\n#[rustfmt::skip]\npub fn print_state() {\n    use crate::info;\n\n    let to_mask_str = |x| -> _ {\n        if x { \"Masked\" } else { \"Unmasked\" }\n    };\n\n    info!(\"      Debug:  {}\", to_mask_str(is_masked::<Debug>()));\n    info!(\"      SError: {}\", to_mask_str(is_masked::<SError>()));\n    info!(\"      IRQ:    {}\", to_mask_str(is_masked::<IRQ>()));\n    info!(\"      FIQ:    {}\", to_mask_str(is_masked::<FIQ>()));\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/_arch/aarch64/exception.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural synchronous and asynchronous exception handling.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::exception::arch_exception\n\nuse crate::{exception, memory, symbols};\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::{arch::global_asm, cell::UnsafeCell, fmt};\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    registers::InMemoryRegister,\n};\n\n// Assembly counterpart to this file.\nglobal_asm!(\n    include_str!(\"exception.s\"),\n    CONST_ESR_EL1_EC_SHIFT = const 26,\n    CONST_ESR_EL1_EC_VALUE_SVC64 = const 0x15\n);\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Wrapper structs for memory copies of registers.\n#[repr(transparent)]\nstruct SpsrEL1(InMemoryRegister<u64, SPSR_EL1::Register>);\nstruct EsrEL1(InMemoryRegister<u64, ESR_EL1::Register>);\n\n/// The exception context as it is stored on the stack on exception entry.\n#[repr(C)]\nstruct ExceptionContext {\n    /// General Purpose Registers.\n    gpr: [u64; 30],\n\n    /// The link register, aka x30.\n    lr: u64,\n\n    /// Exception link register. The program counter at the time the exception happened.\n    elr_el1: u64,\n\n    /// Saved program status.\n    spsr_el1: SpsrEL1,\n\n    /// Exception syndrome register.\n    esr_el1: EsrEL1,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Prints verbose information about the exception and then panics.\nfn default_exception_handler(exc: &ExceptionContext) {\n    panic!(\n        \"CPU Exception!\\n\\n\\\n        {}\",\n        exc\n    );\n}\n\n//------------------------------------------------------------------------------\n// Current, EL0\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn current_el0_synchronous(_e: &mut ExceptionContext) {\n    panic!(\"Should not be here. Use of SP_EL0 in EL1 is not supported.\")\n}\n\n#[no_mangle]\nextern \"C\" fn current_el0_irq(_e: &mut ExceptionContext) {\n    panic!(\"Should not be here. Use of SP_EL0 in EL1 is not supported.\")\n}\n\n#[no_mangle]\nextern \"C\" fn current_el0_serror(_e: &mut ExceptionContext) {\n    panic!(\"Should not be here. Use of SP_EL0 in EL1 is not supported.\")\n}\n\n//------------------------------------------------------------------------------\n// Current, ELx\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn current_elx_synchronous(e: &mut ExceptionContext) {\n    #[cfg(feature = \"test_build\")]\n    {\n        const TEST_SVC_ID: u64 = 0x1337;\n\n        if let Some(ESR_EL1::EC::Value::SVC64) = e.esr_el1.exception_class() {\n            if e.esr_el1.iss() == TEST_SVC_ID {\n                return;\n            }\n        }\n    }\n\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn current_elx_irq(_e: &mut ExceptionContext) {\n    let token = unsafe { &exception::asynchronous::IRQContext::new() };\n    exception::asynchronous::irq_manager().handle_pending_irqs(token);\n}\n\n#[no_mangle]\nextern \"C\" fn current_elx_serror(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n//------------------------------------------------------------------------------\n// Lower, AArch64\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn lower_aarch64_synchronous(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch64_irq(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch64_serror(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n//------------------------------------------------------------------------------\n// Lower, AArch32\n//------------------------------------------------------------------------------\n\n#[no_mangle]\nextern \"C\" fn lower_aarch32_synchronous(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch32_irq(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n#[no_mangle]\nextern \"C\" fn lower_aarch32_serror(e: &mut ExceptionContext) {\n    default_exception_handler(e);\n}\n\n//------------------------------------------------------------------------------\n// Misc\n//------------------------------------------------------------------------------\n\n/// Human readable SPSR_EL1.\n#[rustfmt::skip]\nimpl fmt::Display for SpsrEL1 {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        // Raw value.\n        writeln!(f, \"SPSR_EL1: {:#010x}\", self.0.get())?;\n\n        let to_flag_str = |x| -> _ {\n            if x { \"Set\" } else { \"Not set\" }\n         };\n\n        writeln!(f, \"      Flags:\")?;\n        writeln!(f, \"            Negative (N): {}\", to_flag_str(self.0.is_set(SPSR_EL1::N)))?;\n        writeln!(f, \"            Zero     (Z): {}\", to_flag_str(self.0.is_set(SPSR_EL1::Z)))?;\n        writeln!(f, \"            Carry    (C): {}\", to_flag_str(self.0.is_set(SPSR_EL1::C)))?;\n        writeln!(f, \"            Overflow (V): {}\", to_flag_str(self.0.is_set(SPSR_EL1::V)))?;\n\n        let to_mask_str = |x| -> _ {\n            if x { \"Masked\" } else { \"Unmasked\" }\n        };\n\n        writeln!(f, \"      Exception handling state:\")?;\n        writeln!(f, \"            Debug  (D): {}\", to_mask_str(self.0.is_set(SPSR_EL1::D)))?;\n        writeln!(f, \"            SError (A): {}\", to_mask_str(self.0.is_set(SPSR_EL1::A)))?;\n        writeln!(f, \"            IRQ    (I): {}\", to_mask_str(self.0.is_set(SPSR_EL1::I)))?;\n        writeln!(f, \"            FIQ    (F): {}\", to_mask_str(self.0.is_set(SPSR_EL1::F)))?;\n\n        write!(f, \"      Illegal Execution State (IL): {}\",\n            to_flag_str(self.0.is_set(SPSR_EL1::IL))\n        )\n    }\n}\n\nimpl EsrEL1 {\n    #[inline(always)]\n    fn exception_class(&self) -> Option<ESR_EL1::EC::Value> {\n        self.0.read_as_enum(ESR_EL1::EC)\n    }\n\n    #[cfg(feature = \"test_build\")]\n    #[inline(always)]\n    fn iss(&self) -> u64 {\n        self.0.read(ESR_EL1::ISS)\n    }\n}\n\n/// Human readable ESR_EL1.\n#[rustfmt::skip]\nimpl fmt::Display for EsrEL1 {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        // Raw print of whole register.\n        writeln!(f, \"ESR_EL1: {:#010x}\", self.0.get())?;\n\n        // Raw print of exception class.\n        write!(f, \"      Exception Class         (EC) : {:#x}\", self.0.read(ESR_EL1::EC))?;\n\n        // Exception class.\n        let ec_translation = match self.exception_class() {\n            Some(ESR_EL1::EC::Value::DataAbortCurrentEL) => \"Data Abort, current EL\",\n            _ => \"N/A\",\n        };\n        writeln!(f, \" - {}\", ec_translation)?;\n\n        // Raw print of instruction specific syndrome.\n        write!(f, \"      Instr Specific Syndrome (ISS): {:#x}\", self.0.read(ESR_EL1::ISS))\n    }\n}\n\nimpl ExceptionContext {\n    #[inline(always)]\n    fn exception_class(&self) -> Option<ESR_EL1::EC::Value> {\n        self.esr_el1.exception_class()\n    }\n\n    #[inline(always)]\n    fn fault_address_valid(&self) -> bool {\n        use ESR_EL1::EC::Value::*;\n\n        match self.exception_class() {\n            None => false,\n            Some(ec) => matches!(\n                ec,\n                InstrAbortLowerEL\n                    | InstrAbortCurrentEL\n                    | PCAlignmentFault\n                    | DataAbortLowerEL\n                    | DataAbortCurrentEL\n                    | WatchpointLowerEL\n                    | WatchpointCurrentEL\n            ),\n        }\n    }\n}\n\n/// Human readable print of the exception context.\nimpl fmt::Display for ExceptionContext {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        writeln!(f, \"{}\", self.esr_el1)?;\n\n        if self.fault_address_valid() {\n            writeln!(f, \"FAR_EL1: {:#018x}\", FAR_EL1.get() as usize)?;\n        }\n\n        writeln!(f, \"{}\", self.spsr_el1)?;\n        writeln!(f, \"ELR_EL1: {:#018x}\", self.elr_el1)?;\n        writeln!(\n            f,\n            \"      Symbol: {}\",\n            match symbols::lookup_symbol(memory::Address::new(self.elr_el1 as usize)) {\n                Some(sym) => sym.name(),\n                _ => \"Symbol not found\",\n            }\n        )?;\n        writeln!(f)?;\n        writeln!(f, \"General purpose register:\")?;\n\n        #[rustfmt::skip]\n        let alternating = |x| -> _ {\n            if x % 2 == 0 { \"   \" } else { \"\\n\" }\n        };\n\n        // Print two registers per line.\n        for (i, reg) in self.gpr.iter().enumerate() {\n            write!(f, \"      x{: <2}: {: >#018x}{}\", i, reg, alternating(i))?;\n        }\n        write!(f, \"      lr : {:#018x}\", self.lr)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse crate::exception::PrivilegeLevel;\n\n/// The processing element's current privilege level.\npub fn current_privilege_level() -> (PrivilegeLevel, &'static str) {\n    let el = CurrentEL.read_as_enum(CurrentEL::EL);\n    match el {\n        Some(CurrentEL::EL::Value::EL2) => (PrivilegeLevel::Hypervisor, \"EL2\"),\n        Some(CurrentEL::EL::Value::EL1) => (PrivilegeLevel::Kernel, \"EL1\"),\n        Some(CurrentEL::EL::Value::EL0) => (PrivilegeLevel::User, \"EL0\"),\n        _ => (PrivilegeLevel::Unknown, \"Unknown\"),\n    }\n}\n\n/// Init exception handling by setting the exception vector base address register.\n///\n/// # Safety\n///\n/// - Changes the HW state of the executing core.\n/// - The vector table and the symbol `__exception_vector_table_start` from the linker script must\n///   adhere to the alignment and size constraints demanded by the ARMv8-A Architecture Reference\n///   Manual.\npub unsafe fn handling_init() {\n    // Provided by exception.S.\n    extern \"Rust\" {\n        static __exception_vector_start: UnsafeCell<()>;\n    }\n\n    VBAR_EL1.set(__exception_vector_start.get() as u64);\n\n    // Force VBAR update to complete before next instruction.\n    barrier::isb(barrier::SY);\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/_arch/aarch64/exception.s",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//--------------------------------------------------------------------------------------------------\n// Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Call the function provided by parameter `\\handler` after saving the exception context. Provide\n/// the context as the first parameter to '\\handler'.\n.macro CALL_WITH_CONTEXT handler is_lower_el is_sync\n__vector_\\handler:\n\t// Make room on the stack for the exception context.\n\tsub\tsp,  sp,  #16 * 18\n\n\t// Store all general purpose registers on the stack.\n\tstp\tx0,  x1,  [sp, #16 * 0]\n\tstp\tx2,  x3,  [sp, #16 * 1]\n\tstp\tx4,  x5,  [sp, #16 * 2]\n\tstp\tx6,  x7,  [sp, #16 * 3]\n\tstp\tx8,  x9,  [sp, #16 * 4]\n\tstp\tx10, x11, [sp, #16 * 5]\n\tstp\tx12, x13, [sp, #16 * 6]\n\tstp\tx14, x15, [sp, #16 * 7]\n\tstp\tx16, x17, [sp, #16 * 8]\n\tstp\tx18, x19, [sp, #16 * 9]\n\tstp\tx20, x21, [sp, #16 * 10]\n\tstp\tx22, x23, [sp, #16 * 11]\n\tstp\tx24, x25, [sp, #16 * 12]\n\tstp\tx26, x27, [sp, #16 * 13]\n\tstp\tx28, x29, [sp, #16 * 14]\n\n\t// Add the exception link register (ELR_EL1), saved program status (SPSR_EL1) and exception\n\t// syndrome register (ESR_EL1).\n\tmrs\tx1,  ELR_EL1\n\tmrs\tx2,  SPSR_EL1\n\tmrs\tx3,  ESR_EL1\n\n\tstp\tlr,  x1,  [sp, #16 * 15]\n\tstp\tx2,  x3,  [sp, #16 * 16]\n\n\t// Build a stack frame for backtracing.\n.if \\is_lower_el == 1\n\t// If we came from a lower EL, make it a root frame (by storing zero) so that the kernel\n\t// does not attempt to trace into userspace.\n\tstp\txzr, xzr, [sp, #16 * 17]\n.else\n\t// For normal branches, the link address points to the instruction to be executed _after_\n\t// returning from a branch. In a backtrace, we want to show the instruction that caused the\n\t// branch, though. That is why code in backtrace.rs subtracts 4 (length of one instruction)\n\t// from the link address.\n\t//\n\t// Here we have a special case, though, because ELR_EL1 is used instead of LR to build the\n\t// stack frame, so that it becomes possible to trace beyond an exception. Hence, it must be\n\t// considered that semantics for ELR_EL1 differ from case to case.\n\t//\n\t// Unless an \"exception generating instruction\" was executed, ELR_EL1 already points to the\n\t// the correct instruction, and hence the subtraction by 4 in backtrace.rs would yield wrong\n\t// results. To cover for this, 4 is added to ELR_EL1 below unless the cause of exception was\n\t// an SVC instruction. BRK and HLT are \"exception generating instructions\" as well, but they\n\t// are not expected and therefore left out for now.\n\t//\n\t// For reference: Search for \"preferred exception return address\" in the Architecture\n\t// Reference Manual for ARMv8-A.\n.if \\is_sync == 1\n\tlsr\tw3,  w3, {CONST_ESR_EL1_EC_SHIFT}   // w3 = ESR_EL1.EC\n\tcmp\tw3,  {CONST_ESR_EL1_EC_VALUE_SVC64} // w3 == SVC64 ?\n\tb.eq\t1f\n.endif\n\tadd\tx1,  x1, #4\n1:\n\tstp\tx29, x1, [sp, #16 * 17]\n.endif\n\n\t// Set the frame pointer to the stack frame record.\n\tadd\tx29, sp, #16 * 17\n\n\t// x0 is the first argument for the function called through `\\handler`.\n\tmov\tx0,  sp\n\n\t// Call `\\handler`.\n\tbl\t\\handler\n\n\t// After returning from exception handling code, replay the saved context and return via\n\t// `eret`.\n\tb\t__exception_restore_context\n\n.size\t__vector_\\handler, . - __vector_\\handler\n.type\t__vector_\\handler, function\n.endm\n\n.macro FIQ_SUSPEND\n1:\twfe\n\tb\t1b\n.endm\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n.section .text\n\n//------------------------------------------------------------------------------\n// The exception vector table.\n//------------------------------------------------------------------------------\n\n// Align by 2^11 bytes, as demanded by ARMv8-A. Same as ALIGN(2048) in an ld script.\n.align 11\n\n// Export a symbol for the Rust code to use.\n__exception_vector_start:\n\n// Current exception level with SP_EL0.\n//\n// .org sets the offset relative to section start.\n//\n// # Safety\n//\n// - It must be ensured that `CALL_WITH_CONTEXT` <= 0x80 bytes.\n.org 0x000\n\tCALL_WITH_CONTEXT current_el0_synchronous, 0, 1\n.org 0x080\n\tCALL_WITH_CONTEXT current_el0_irq, 0, 0\n.org 0x100\n\tFIQ_SUSPEND\n.org 0x180\n\tCALL_WITH_CONTEXT current_el0_serror, 0, 0\n\n// Current exception level with SP_ELx, x > 0.\n.org 0x200\n\tCALL_WITH_CONTEXT current_elx_synchronous, 0, 1\n.org 0x280\n\tCALL_WITH_CONTEXT current_elx_irq, 0, 0\n.org 0x300\n\tFIQ_SUSPEND\n.org 0x380\n\tCALL_WITH_CONTEXT current_elx_serror, 0, 0\n\n// Lower exception level, AArch64\n.org 0x400\n\tCALL_WITH_CONTEXT lower_aarch64_synchronous, 1, 1\n.org 0x480\n\tCALL_WITH_CONTEXT lower_aarch64_irq, 1, 0\n.org 0x500\n\tFIQ_SUSPEND\n.org 0x580\n\tCALL_WITH_CONTEXT lower_aarch64_serror, 1, 0\n\n// Lower exception level, AArch32\n.org 0x600\n\tCALL_WITH_CONTEXT lower_aarch32_synchronous, 1, 0\n.org 0x680\n\tCALL_WITH_CONTEXT lower_aarch32_irq, 1, 0\n.org 0x700\n\tFIQ_SUSPEND\n.org 0x780\n\tCALL_WITH_CONTEXT lower_aarch32_serror, 1, 0\n.org 0x800\n\n//------------------------------------------------------------------------------\n// fn __exception_restore_context()\n//------------------------------------------------------------------------------\n__exception_restore_context:\n\tldr\tw19,      [sp, #16 * 16]\n\tldp\tlr,  x20, [sp, #16 * 15]\n\n\tmsr\tSPSR_EL1, x19\n\tmsr\tELR_EL1,  x20\n\n\tldp\tx0,  x1,  [sp, #16 * 0]\n\tldp\tx2,  x3,  [sp, #16 * 1]\n\tldp\tx4,  x5,  [sp, #16 * 2]\n\tldp\tx6,  x7,  [sp, #16 * 3]\n\tldp\tx8,  x9,  [sp, #16 * 4]\n\tldp\tx10, x11, [sp, #16 * 5]\n\tldp\tx12, x13, [sp, #16 * 6]\n\tldp\tx14, x15, [sp, #16 * 7]\n\tldp\tx16, x17, [sp, #16 * 8]\n\tldp\tx18, x19, [sp, #16 * 9]\n\tldp\tx20, x21, [sp, #16 * 10]\n\tldp\tx22, x23, [sp, #16 * 11]\n\tldp\tx24, x25, [sp, #16 * 12]\n\tldp\tx26, x27, [sp, #16 * 13]\n\tldp\tx28, x29, [sp, #16 * 14]\n\n\tadd\tsp,  sp,  #16 * 18\n\n\teret\n\n.size\t__exception_restore_context, . - __exception_restore_context\n.type\t__exception_restore_context, function\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/_arch/aarch64/memory/mmu/translation_table.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural translation table.\n//!\n//! Only 64 KiB granule is supported.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::memory::mmu::translation_table::arch_translation_table\n\nuse crate::{\n    bsp,\n    memory::{\n        self,\n        mmu::{\n            arch_mmu::{Granule512MiB, Granule64KiB},\n            AccessPermissions, AttributeFields, MemAttributes, MemoryRegion, PageAddress,\n        },\n        Address, Physical, Virtual,\n    },\n};\nuse core::convert;\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields,\n    registers::InMemoryRegister,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// A table descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-15.\nregister_bitfields! {u64,\n    STAGE1_TABLE_DESCRIPTOR [\n        /// Physical address of the next descriptor.\n        NEXT_LEVEL_TABLE_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16]\n\n        TYPE  OFFSET(1) NUMBITS(1) [\n            Block = 0,\n            Table = 1\n        ],\n\n        VALID OFFSET(0) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ]\n    ]\n}\n\n// A level 3 page descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-17.\nregister_bitfields! {u64,\n    STAGE1_PAGE_DESCRIPTOR [\n        /// Unprivileged execute-never.\n        UXN      OFFSET(54) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ],\n\n        /// Privileged execute-never.\n        PXN      OFFSET(53) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ],\n\n        /// Physical address of the next table descriptor (lvl2) or the page descriptor (lvl3).\n        OUTPUT_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16]\n\n        /// Access flag.\n        AF       OFFSET(10) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ],\n\n        /// Shareability field.\n        SH       OFFSET(8) NUMBITS(2) [\n            OuterShareable = 0b10,\n            InnerShareable = 0b11\n        ],\n\n        /// Access Permissions.\n        AP       OFFSET(6) NUMBITS(2) [\n            RW_EL1 = 0b00,\n            RW_EL1_EL0 = 0b01,\n            RO_EL1 = 0b10,\n            RO_EL1_EL0 = 0b11\n        ],\n\n        /// Memory attributes index into the MAIR_EL1 register.\n        AttrIndx OFFSET(2) NUMBITS(3) [],\n\n        TYPE     OFFSET(1) NUMBITS(1) [\n            Reserved_Invalid = 0,\n            Page = 1\n        ],\n\n        VALID    OFFSET(0) NUMBITS(1) [\n            False = 0,\n            True = 1\n        ]\n    ]\n}\n\n/// A table descriptor for 64 KiB aperture.\n///\n/// The output points to the next table.\n#[derive(Copy, Clone)]\n#[repr(C)]\nstruct TableDescriptor {\n    value: u64,\n}\n\n/// A page descriptor with 64 KiB aperture.\n///\n/// The output points to physical memory.\n#[derive(Copy, Clone)]\n#[repr(C)]\nstruct PageDescriptor {\n    value: u64,\n}\n\ntrait StartAddr {\n    fn virt_start_addr(&self) -> Address<Virtual>;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB\n/// aligned, so the lvl3 is put first.\n#[repr(C)]\n#[repr(align(65536))]\npub struct FixedSizeTranslationTable<const NUM_TABLES: usize, const START_FROM_TOP: bool> {\n    /// Page descriptors, covering 64 KiB windows per entry.\n    lvl3: [[PageDescriptor; 8192]; NUM_TABLES],\n\n    /// Table descriptors, covering 512 MiB windows.\n    lvl2: [TableDescriptor; NUM_TABLES],\n\n    /// Have the tables been initialized?\n    initialized: bool,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T, const N: usize> StartAddr for [T; N] {\n    fn virt_start_addr(&self) -> Address<Virtual> {\n        Address::new(self as *const _ as usize)\n    }\n}\n\nimpl TableDescriptor {\n    /// Create an instance.\n    ///\n    /// Descriptor is invalid by default.\n    pub const fn new_zeroed() -> Self {\n        Self { value: 0 }\n    }\n\n    /// Create an instance pointing to the supplied address.\n    pub fn from_next_lvl_table_addr(phys_next_lvl_table_addr: Address<Physical>) -> Self {\n        let val = InMemoryRegister::<u64, STAGE1_TABLE_DESCRIPTOR::Register>::new(0);\n\n        let shifted = phys_next_lvl_table_addr.as_usize() >> Granule64KiB::SHIFT;\n        val.write(\n            STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64)\n                + STAGE1_TABLE_DESCRIPTOR::TYPE::Table\n                + STAGE1_TABLE_DESCRIPTOR::VALID::True,\n        );\n\n        TableDescriptor { value: val.get() }\n    }\n}\n\n/// Convert the kernel's generic memory attributes to HW-specific attributes of the MMU.\nimpl convert::From<AttributeFields>\n    for tock_registers::fields::FieldValue<u64, STAGE1_PAGE_DESCRIPTOR::Register>\n{\n    fn from(attribute_fields: AttributeFields) -> Self {\n        // Memory attributes.\n        let mut desc = match attribute_fields.mem_attributes {\n            MemAttributes::CacheableDRAM => {\n                STAGE1_PAGE_DESCRIPTOR::SH::InnerShareable\n                    + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::NORMAL)\n            }\n            MemAttributes::Device => {\n                STAGE1_PAGE_DESCRIPTOR::SH::OuterShareable\n                    + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::DEVICE)\n            }\n        };\n\n        // Access Permissions.\n        desc += match attribute_fields.acc_perms {\n            AccessPermissions::ReadOnly => STAGE1_PAGE_DESCRIPTOR::AP::RO_EL1,\n            AccessPermissions::ReadWrite => STAGE1_PAGE_DESCRIPTOR::AP::RW_EL1,\n        };\n\n        // The execute-never attribute is mapped to PXN in AArch64.\n        desc += if attribute_fields.execute_never {\n            STAGE1_PAGE_DESCRIPTOR::PXN::True\n        } else {\n            STAGE1_PAGE_DESCRIPTOR::PXN::False\n        };\n\n        // Always set unprivileged exectue-never as long as userspace is not implemented yet.\n        desc += STAGE1_PAGE_DESCRIPTOR::UXN::True;\n\n        desc\n    }\n}\n\n/// Convert the HW-specific attributes of the MMU to kernel's generic memory attributes.\nimpl convert::TryFrom<InMemoryRegister<u64, STAGE1_PAGE_DESCRIPTOR::Register>> for AttributeFields {\n    type Error = &'static str;\n\n    fn try_from(\n        desc: InMemoryRegister<u64, STAGE1_PAGE_DESCRIPTOR::Register>,\n    ) -> Result<AttributeFields, Self::Error> {\n        let mem_attributes = match desc.read(STAGE1_PAGE_DESCRIPTOR::AttrIndx) {\n            memory::mmu::arch_mmu::mair::NORMAL => MemAttributes::CacheableDRAM,\n            memory::mmu::arch_mmu::mair::DEVICE => MemAttributes::Device,\n            _ => return Err(\"Unexpected memory attribute\"),\n        };\n\n        let acc_perms = match desc.read_as_enum(STAGE1_PAGE_DESCRIPTOR::AP) {\n            Some(STAGE1_PAGE_DESCRIPTOR::AP::Value::RO_EL1) => AccessPermissions::ReadOnly,\n            Some(STAGE1_PAGE_DESCRIPTOR::AP::Value::RW_EL1) => AccessPermissions::ReadWrite,\n            _ => return Err(\"Unexpected access permission\"),\n        };\n\n        let execute_never = desc.read(STAGE1_PAGE_DESCRIPTOR::PXN) > 0;\n\n        Ok(AttributeFields {\n            mem_attributes,\n            acc_perms,\n            execute_never,\n        })\n    }\n}\n\nimpl PageDescriptor {\n    /// Create an instance.\n    ///\n    /// Descriptor is invalid by default.\n    pub const fn new_zeroed() -> Self {\n        Self { value: 0 }\n    }\n\n    /// Create an instance.\n    pub fn from_output_page_addr(\n        phys_output_page_addr: PageAddress<Physical>,\n        attribute_fields: &AttributeFields,\n    ) -> Self {\n        let val = InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(0);\n\n        let shifted = phys_output_page_addr.into_inner().as_usize() >> Granule64KiB::SHIFT;\n        val.write(\n            STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted as u64)\n                + STAGE1_PAGE_DESCRIPTOR::AF::True\n                + STAGE1_PAGE_DESCRIPTOR::TYPE::Page\n                + STAGE1_PAGE_DESCRIPTOR::VALID::True\n                + (*attribute_fields).into(),\n        );\n\n        Self { value: val.get() }\n    }\n\n    /// Returns the valid bit.\n    fn is_valid(&self) -> bool {\n        InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(self.value)\n            .is_set(STAGE1_PAGE_DESCRIPTOR::VALID)\n    }\n\n    /// Returns the output page.\n    fn output_page_addr(&self) -> PageAddress<Physical> {\n        let shifted = InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(self.value)\n            .read(STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB) as usize;\n\n        PageAddress::from(shifted << Granule64KiB::SHIFT)\n    }\n\n    /// Returns the attributes.\n    fn try_attributes(&self) -> Result<AttributeFields, &'static str> {\n        InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(self.value).try_into()\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<const AS_SIZE: usize> memory::mmu::AssociatedTranslationTable\n    for memory::mmu::AddressSpace<AS_SIZE>\nwhere\n    [u8; Self::SIZE >> Granule512MiB::SHIFT]: Sized,\n{\n    type TableStartFromTop =\n        FixedSizeTranslationTable<{ Self::SIZE >> Granule512MiB::SHIFT }, true>;\n\n    type TableStartFromBottom =\n        FixedSizeTranslationTable<{ Self::SIZE >> Granule512MiB::SHIFT }, false>;\n}\n\nimpl<const NUM_TABLES: usize, const START_FROM_TOP: bool>\n    FixedSizeTranslationTable<NUM_TABLES, START_FROM_TOP>\n{\n    const START_FROM_TOP_OFFSET: Address<Virtual> =\n        Address::new((usize::MAX - (Granule512MiB::SIZE * NUM_TABLES)) + 1);\n\n    /// Create an instance.\n    #[allow(clippy::assertions_on_constants)]\n    const fn _new(for_precompute: bool) -> Self {\n        assert!(bsp::memory::mmu::KernelGranule::SIZE == Granule64KiB::SIZE);\n\n        // Can't have a zero-sized address space.\n        assert!(NUM_TABLES > 0);\n\n        Self {\n            lvl3: [[PageDescriptor::new_zeroed(); 8192]; NUM_TABLES],\n            lvl2: [TableDescriptor::new_zeroed(); NUM_TABLES],\n            initialized: for_precompute,\n        }\n    }\n\n    pub const fn new_for_precompute() -> Self {\n        Self::_new(true)\n    }\n\n    #[cfg(test)]\n    pub fn new_for_runtime() -> Self {\n        Self::_new(false)\n    }\n\n    /// Helper to calculate the lvl2 and lvl3 indices from an address.\n    #[inline(always)]\n    fn lvl2_lvl3_index_from_page_addr(\n        &self,\n        virt_page_addr: PageAddress<Virtual>,\n    ) -> Result<(usize, usize), &'static str> {\n        let mut addr = virt_page_addr.into_inner();\n\n        if START_FROM_TOP {\n            addr = addr - Self::START_FROM_TOP_OFFSET;\n        }\n\n        let lvl2_index = addr.as_usize() >> Granule512MiB::SHIFT;\n        let lvl3_index = (addr.as_usize() & Granule512MiB::MASK) >> Granule64KiB::SHIFT;\n\n        if lvl2_index > (NUM_TABLES - 1) {\n            return Err(\"Virtual page is out of bounds of translation table\");\n        }\n\n        Ok((lvl2_index, lvl3_index))\n    }\n\n    /// Returns the PageDescriptor corresponding to the supplied page address.\n    #[inline(always)]\n    fn page_descriptor_from_page_addr(\n        &self,\n        virt_page_addr: PageAddress<Virtual>,\n    ) -> Result<&PageDescriptor, &'static str> {\n        let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from_page_addr(virt_page_addr)?;\n        let desc = &self.lvl3[lvl2_index][lvl3_index];\n\n        Ok(desc)\n    }\n\n    /// Sets the PageDescriptor corresponding to the supplied page address.\n    ///\n    /// Doesn't allow overriding an already valid page.\n    #[inline(always)]\n    fn set_page_descriptor_from_page_addr(\n        &mut self,\n        virt_page_addr: PageAddress<Virtual>,\n        new_desc: &PageDescriptor,\n    ) -> Result<(), &'static str> {\n        let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from_page_addr(virt_page_addr)?;\n        let desc = &mut self.lvl3[lvl2_index][lvl3_index];\n\n        if desc.is_valid() {\n            return Err(\"Virtual page is already mapped\");\n        }\n\n        *desc = *new_desc;\n        Ok(())\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\n\nimpl<const NUM_TABLES: usize, const START_FROM_TOP: bool>\n    memory::mmu::translation_table::interface::TranslationTable\n    for FixedSizeTranslationTable<NUM_TABLES, START_FROM_TOP>\n{\n    fn init(&mut self) -> Result<(), &'static str> {\n        if self.initialized {\n            return Ok(());\n        }\n\n        // Populate the l2 entries.\n        for (lvl2_nr, lvl2_entry) in self.lvl2.iter_mut().enumerate() {\n            let virt_table_addr = self.lvl3[lvl2_nr].virt_start_addr();\n            let phys_table_addr = memory::mmu::try_kernel_virt_addr_to_phys_addr(virt_table_addr)?;\n\n            let new_desc = TableDescriptor::from_next_lvl_table_addr(phys_table_addr);\n            *lvl2_entry = new_desc;\n        }\n\n        self.initialized = true;\n\n        Ok(())\n    }\n\n    unsafe fn map_at(\n        &mut self,\n        virt_region: &MemoryRegion<Virtual>,\n        phys_region: &MemoryRegion<Physical>,\n        attr: &AttributeFields,\n    ) -> Result<(), &'static str> {\n        assert!(self.initialized, \"Translation tables not initialized\");\n\n        if virt_region.size() != phys_region.size() {\n            return Err(\"Tried to map memory regions with unequal sizes\");\n        }\n\n        if phys_region.end_exclusive_page_addr() > bsp::memory::phys_addr_space_end_exclusive_addr()\n        {\n            return Err(\"Tried to map outside of physical address space\");\n        }\n\n        let iter = phys_region.into_iter().zip(virt_region.into_iter());\n        for (phys_page_addr, virt_page_addr) in iter {\n            let new_desc = PageDescriptor::from_output_page_addr(phys_page_addr, attr);\n            let virt_page = virt_page_addr;\n\n            self.set_page_descriptor_from_page_addr(virt_page, &new_desc)?;\n        }\n\n        Ok(())\n    }\n\n    fn try_virt_page_addr_to_phys_page_addr(\n        &self,\n        virt_page_addr: PageAddress<Virtual>,\n    ) -> Result<PageAddress<Physical>, &'static str> {\n        let page_desc = self.page_descriptor_from_page_addr(virt_page_addr)?;\n\n        if !page_desc.is_valid() {\n            return Err(\"Page marked invalid\");\n        }\n\n        Ok(page_desc.output_page_addr())\n    }\n\n    fn try_page_attributes(\n        &self,\n        virt_page_addr: PageAddress<Virtual>,\n    ) -> Result<AttributeFields, &'static str> {\n        let page_desc = self.page_descriptor_from_page_addr(virt_page_addr)?;\n\n        if !page_desc.is_valid() {\n            return Err(\"Page marked invalid\");\n        }\n\n        page_desc.try_attributes()\n    }\n\n    /// Try to translate a virtual address to a physical address.\n    ///\n    /// Will only succeed if there exists a valid mapping for the input address.\n    fn try_virt_addr_to_phys_addr(\n        &self,\n        virt_addr: Address<Virtual>,\n    ) -> Result<Address<Physical>, &'static str> {\n        let virt_page = PageAddress::from(virt_addr.align_down_page());\n        let phys_page = self.try_virt_page_addr_to_phys_page_addr(virt_page)?;\n\n        Ok(phys_page.into_inner() + virt_addr.offset_into_page())\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\npub type MinSizeTranslationTable = FixedSizeTranslationTable<1, true>;\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// Check if the size of `struct TableDescriptor` is as expected.\n    #[kernel_test]\n    fn size_of_tabledescriptor_equals_64_bit() {\n        assert_eq!(\n            core::mem::size_of::<TableDescriptor>(),\n            core::mem::size_of::<u64>()\n        );\n    }\n\n    /// Check if the size of `struct PageDescriptor` is as expected.\n    #[kernel_test]\n    fn size_of_pagedescriptor_equals_64_bit() {\n        assert_eq!(\n            core::mem::size_of::<PageDescriptor>(),\n            core::mem::size_of::<u64>()\n        );\n    }\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/_arch/aarch64/memory/mmu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management Unit Driver.\n//!\n//! Only 64 KiB granule is supported.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::memory::mmu::arch_mmu\n\nuse crate::{\n    bsp, memory,\n    memory::{mmu::TranslationGranule, Address, Physical},\n};\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::intrinsics::unlikely;\nuse tock_registers::interfaces::{ReadWriteable, Readable, Writeable};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Memory Management Unit type.\nstruct MemoryManagementUnit;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub type Granule512MiB = TranslationGranule<{ 512 * 1024 * 1024 }>;\npub type Granule64KiB = TranslationGranule<{ 64 * 1024 }>;\n\n/// Constants for indexing the MAIR_EL1.\n#[allow(dead_code)]\npub mod mair {\n    pub const DEVICE: u64 = 0;\n    pub const NORMAL: u64 = 1;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic MMU: MemoryManagementUnit = MemoryManagementUnit;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<const AS_SIZE: usize> memory::mmu::AddressSpace<AS_SIZE> {\n    /// Checks for architectural restrictions.\n    pub const fn arch_address_space_size_sanity_checks() {\n        // Size must be at least one full 512 MiB table.\n        assert!((AS_SIZE % Granule512MiB::SIZE) == 0);\n\n        // Check for 48 bit virtual address size as maximum, which is supported by any ARMv8\n        // version.\n        assert!(AS_SIZE <= (1 << 48));\n    }\n}\n\nimpl MemoryManagementUnit {\n    /// Setup function for the MAIR_EL1 register.\n    #[inline(always)]\n    fn set_up_mair(&self) {\n        // Define the memory types being mapped.\n        MAIR_EL1.write(\n            // Attribute 1 - Cacheable normal DRAM.\n            MAIR_EL1::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc +\n        MAIR_EL1::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc +\n\n        // Attribute 0 - Device.\n        MAIR_EL1::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck,\n        );\n    }\n\n    /// Configure various settings of stage 1 of the EL1 translation regime.\n    #[inline(always)]\n    fn configure_translation_control(&self) {\n        let t1sz = (64 - bsp::memory::mmu::KernelVirtAddrSpace::SIZE_SHIFT) as u64;\n\n        TCR_EL1.write(\n            TCR_EL1::TBI1::Used\n                + TCR_EL1::IPS::Bits_40\n                + TCR_EL1::TG1::KiB_64\n                + TCR_EL1::SH1::Inner\n                + TCR_EL1::ORGN1::WriteBack_ReadAlloc_WriteAlloc_Cacheable\n                + TCR_EL1::IRGN1::WriteBack_ReadAlloc_WriteAlloc_Cacheable\n                + TCR_EL1::EPD1::EnableTTBR1Walks\n                + TCR_EL1::A1::TTBR1\n                + TCR_EL1::T1SZ.val(t1sz)\n                + TCR_EL1::EPD0::DisableTTBR0Walks,\n        );\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the MMU instance.\npub fn mmu() -> &'static impl memory::mmu::interface::MMU {\n    &MMU\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse memory::mmu::MMUEnableError;\n\nimpl memory::mmu::interface::MMU for MemoryManagementUnit {\n    unsafe fn enable_mmu_and_caching(\n        &self,\n        phys_tables_base_addr: Address<Physical>,\n    ) -> Result<(), MMUEnableError> {\n        if unlikely(self.is_enabled()) {\n            return Err(MMUEnableError::AlreadyEnabled);\n        }\n\n        // Fail early if translation granule is not supported.\n        if unlikely(!ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran64::Supported)) {\n            return Err(MMUEnableError::Other(\n                \"Translation granule not supported in HW\",\n            ));\n        }\n\n        // Prepare the memory attribute indirection register.\n        self.set_up_mair();\n\n        // Set the \"Translation Table Base Register\".\n        TTBR1_EL1.set_baddr(phys_tables_base_addr.as_usize() as u64);\n\n        self.configure_translation_control();\n\n        // Switch the MMU on.\n        //\n        // First, force all previous changes to be seen before the MMU is enabled.\n        barrier::isb(barrier::SY);\n\n        // Enable the MMU and turn on data and instruction caching.\n        SCTLR_EL1.modify(SCTLR_EL1::M::Enable + SCTLR_EL1::C::Cacheable + SCTLR_EL1::I::Cacheable);\n\n        // Force MMU init to complete before next instruction.\n        barrier::isb(barrier::SY);\n\n        Ok(())\n    }\n\n    #[inline(always)]\n    fn is_enabled(&self) -> bool {\n        SCTLR_EL1.matches_all(SCTLR_EL1::M::Enable)\n    }\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/_arch/aarch64/time.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural timer primitives.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::time::arch_time\n\nuse crate::{\n    bsp::{self, exception},\n    warn,\n};\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::{\n    num::{NonZeroU128, NonZeroU32, NonZeroU64},\n    ops::{Add, Div},\n    time::Duration,\n};\nuse tock_registers::interfaces::{ReadWriteable, Readable, Writeable};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nconst NANOSEC_PER_SEC: NonZeroU64 = NonZeroU64::new(1_000_000_000).unwrap();\n\n#[derive(Copy, Clone, PartialOrd, PartialEq)]\nstruct GenericTimerCounterValue(u64);\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n/// Boot assembly code overwrites this value with the value of CNTFRQ_EL0 before any Rust code is\n/// executed. This given value here is just a (safe) dummy.\n#[no_mangle]\nstatic ARCH_TIMER_COUNTER_FREQUENCY: NonZeroU32 = NonZeroU32::MIN;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nfn arch_timer_counter_frequency() -> NonZeroU32 {\n    // Read volatile is needed here to prevent the compiler from optimizing\n    // ARCH_TIMER_COUNTER_FREQUENCY away.\n    //\n    // This is safe, because all the safety requirements as stated in read_volatile()'s\n    // documentation are fulfilled.\n    unsafe { core::ptr::read_volatile(&ARCH_TIMER_COUNTER_FREQUENCY) }\n}\n\nimpl GenericTimerCounterValue {\n    pub const MAX: Self = GenericTimerCounterValue(u64::MAX);\n}\n\nimpl Add for GenericTimerCounterValue {\n    type Output = Self;\n\n    fn add(self, other: Self) -> Self {\n        GenericTimerCounterValue(self.0.wrapping_add(other.0))\n    }\n}\n\nimpl From<GenericTimerCounterValue> for Duration {\n    fn from(counter_value: GenericTimerCounterValue) -> Self {\n        if counter_value.0 == 0 {\n            return Duration::ZERO;\n        }\n\n        let frequency: NonZeroU64 = arch_timer_counter_frequency().into();\n\n        // Div<NonZeroU64> implementation for u64 cannot panic.\n        let secs = counter_value.0.div(frequency);\n\n        // This is safe, because frequency can never be greater than u32::MAX, which means the\n        // largest theoretical value for sub_second_counter_value is (u32::MAX - 1). Therefore,\n        // (sub_second_counter_value * NANOSEC_PER_SEC) cannot overflow an u64.\n        //\n        // The subsequent division ensures the result fits into u32, since the max result is smaller\n        // than NANOSEC_PER_SEC. Therefore, just cast it to u32 using `as`.\n        let sub_second_counter_value = counter_value.0 % frequency;\n        let nanos = unsafe { sub_second_counter_value.unchecked_mul(u64::from(NANOSEC_PER_SEC)) }\n            .div(frequency) as u32;\n\n        Duration::new(secs, nanos)\n    }\n}\n\nfn max_duration() -> Duration {\n    Duration::from(GenericTimerCounterValue::MAX)\n}\n\nimpl TryFrom<Duration> for GenericTimerCounterValue {\n    type Error = &'static str;\n\n    fn try_from(duration: Duration) -> Result<Self, Self::Error> {\n        if duration < resolution() {\n            return Ok(GenericTimerCounterValue(0));\n        }\n\n        if duration > max_duration() {\n            return Err(\"Conversion error. Duration too big\");\n        }\n\n        let frequency: u128 = u32::from(arch_timer_counter_frequency()) as u128;\n        let duration: u128 = duration.as_nanos();\n\n        // This is safe, because frequency can never be greater than u32::MAX, and\n        // (Duration::MAX.as_nanos() * u32::MAX) < u128::MAX.\n        let counter_value =\n            unsafe { duration.unchecked_mul(frequency) }.div(NonZeroU128::from(NANOSEC_PER_SEC));\n\n        // Since we checked above that we are <= max_duration(), just cast to u64.\n        Ok(GenericTimerCounterValue(counter_value as u64))\n    }\n}\n\n#[inline(always)]\nfn read_cntpct() -> GenericTimerCounterValue {\n    // Prevent that the counter is read ahead of time due to out-of-order execution.\n    barrier::isb(barrier::SY);\n    let cnt = CNTPCT_EL0.get();\n\n    GenericTimerCounterValue(cnt)\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The timer's resolution.\npub fn resolution() -> Duration {\n    Duration::from(GenericTimerCounterValue(1))\n}\n\n/// The uptime since power-on of the device.\n///\n/// This includes time consumed by firmware and bootloaders.\npub fn uptime() -> Duration {\n    read_cntpct().into()\n}\n\n/// Spin for a given duration.\npub fn spin_for(duration: Duration) {\n    let curr_counter_value = read_cntpct();\n\n    let counter_value_delta: GenericTimerCounterValue = match duration.try_into() {\n        Err(msg) => {\n            warn!(\"spin_for: {}. Skipping\", msg);\n            return;\n        }\n        Ok(val) => val,\n    };\n    let counter_value_target = curr_counter_value + counter_value_delta;\n\n    // Busy wait.\n    //\n    // Read CNTPCT_EL0 directly to avoid the ISB that is part of [`read_cntpct`].\n    while GenericTimerCounterValue(CNTPCT_EL0.get()) < counter_value_target {}\n}\n\n/// The associated IRQ number.\npub const fn timeout_irq() -> exception::asynchronous::IRQNumber {\n    bsp::exception::asynchronous::irq_map::ARM_NS_PHYSICAL_TIMER\n}\n\n/// Program a timer IRQ to be fired after `delay` has passed.\npub fn set_timeout_irq(due_time: Duration) {\n    let counter_value_target: GenericTimerCounterValue = match due_time.try_into() {\n        Err(msg) => {\n            warn!(\"set_timeout: {}. Skipping\", msg);\n            return;\n        }\n        Ok(val) => val,\n    };\n\n    // Set the compare value register.\n    CNTP_CVAL_EL0.set(counter_value_target.0);\n\n    // Kick off the timer.\n    CNTP_CTL_EL0.modify(CNTP_CTL_EL0::ENABLE::SET + CNTP_CTL_EL0::IMASK::CLEAR);\n}\n\n/// Conclude a pending timeout IRQ.\npub fn conclude_timeout_irq() {\n    // Disable counting. De-asserts the IRQ.\n    CNTP_CTL_EL0.modify(CNTP_CTL_EL0::ENABLE::CLEAR);\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/backtrace.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Backtracing support.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/backtrace.rs\"]\nmod arch_backtrace;\n\nuse crate::{\n    memory::{Address, Virtual},\n    symbols,\n};\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\n#[cfg(feature = \"test_build\")]\npub use arch_backtrace::{corrupt_link, corrupt_previous_frame_addr};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// A backtrace item.\n#[allow(missing_docs)]\npub enum BacktraceItem {\n    InvalidFramePointer(Address<Virtual>),\n    InvalidLink(Address<Virtual>),\n    Link(Address<Virtual>),\n}\n\n/// Pseudo-struct for printing a backtrace using its fmt::Display implementation.\npub struct Backtrace;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl fmt::Display for Backtrace {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        writeln!(f, \"Backtrace:\")?;\n        writeln!(\n            f,\n            \"      ----------------------------------------------------------------------------------------------\"\n        )?;\n        writeln!(\n            f,\n            \"          Address            Function containing address\"\n        )?;\n        writeln!(\n            f,\n            \"      ----------------------------------------------------------------------------------------------\"\n        )?;\n\n        let mut fmt_res: fmt::Result = Ok(());\n        let trace_formatter =\n            |maybe_iter: Option<&mut dyn Iterator<Item = BacktraceItem>>| match maybe_iter {\n                None => fmt_res = writeln!(f, \"ERROR! No valid stack frame found\"),\n                Some(iter) => {\n                    // Since the backtrace is printed, the first function is always\n                    // core::fmt::write. Skip 1 so it is excluded and doesn't bloat the output.\n                    for (i, backtrace_res) in iter.skip(1).enumerate() {\n                        match backtrace_res {\n                            BacktraceItem::InvalidFramePointer(addr) => {\n                                fmt_res = writeln!(\n                                    f,\n                                    \"      {:>2}. ERROR! \\\n                                    Encountered invalid frame pointer ({}) during backtrace\",\n                                    i + 1,\n                                    addr\n                                );\n                            }\n                            BacktraceItem::InvalidLink(addr) => {\n                                fmt_res = writeln!(\n                                    f,\n                                    \"      {:>2}. ERROR! \\\n                                    Link address ({}) is not contained in kernel .text section\",\n                                    i + 1,\n                                    addr\n                                );\n                            }\n                            BacktraceItem::Link(addr) => {\n                                fmt_res = writeln!(\n                                    f,\n                                    \"      {:>2}. {:016x} | {:<50}\",\n                                    i + 1,\n                                    addr.as_usize(),\n                                    match symbols::lookup_symbol(addr) {\n                                        Some(sym) => sym.name(),\n                                        _ => \"Symbol not found\",\n                                    }\n                                )\n                            }\n                        };\n\n                        if fmt_res.is_err() {\n                            break;\n                        }\n                    }\n                }\n            };\n\n        arch_backtrace::backtrace(trace_formatter);\n        fmt_res?;\n\n        writeln!(\n            f,\n            \"      ----------------------------------------------------------------------------------------------\"\n        )\n    }\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/bsp/device_driver/arm/gicv2/gicc.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GICC Driver - GIC CPU interface.\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    exception,\n    memory::{Address, Virtual},\n};\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields, register_structs,\n    registers::ReadWrite,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nregister_bitfields! {\n    u32,\n\n    /// CPU Interface Control Register\n    CTLR [\n        Enable OFFSET(0) NUMBITS(1) []\n    ],\n\n    /// Interrupt Priority Mask Register\n    PMR [\n        Priority OFFSET(0) NUMBITS(8) []\n    ],\n\n    /// Interrupt Acknowledge Register\n    IAR [\n        InterruptID OFFSET(0) NUMBITS(10) []\n    ],\n\n    /// End of Interrupt Register\n    EOIR [\n        EOIINTID OFFSET(0) NUMBITS(10) []\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    pub RegisterBlock {\n        (0x000 => CTLR: ReadWrite<u32, CTLR::Register>),\n        (0x004 => PMR: ReadWrite<u32, PMR::Register>),\n        (0x008 => _reserved1),\n        (0x00C => IAR: ReadWrite<u32, IAR::Register>),\n        (0x010 => EOIR: ReadWrite<u32, EOIR::Register>),\n        (0x014  => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the GIC CPU interface.\npub struct GICC {\n    registers: Registers,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GICC {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n        }\n    }\n\n    /// Accept interrupts of any priority.\n    ///\n    /// Quoting the GICv2 Architecture Specification:\n    ///\n    ///   \"Writing 255 to the GICC_PMR always sets it to the largest supported priority field\n    ///    value.\"\n    ///\n    /// # Safety\n    ///\n    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n    ///   of `&mut self`.\n    pub fn priority_accept_all(&self) {\n        self.registers.PMR.write(PMR::Priority.val(255)); // Comment in arch spec.\n    }\n\n    /// Enable the interface - start accepting IRQs.\n    ///\n    /// # Safety\n    ///\n    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n    ///   of `&mut self`.\n    pub fn enable(&self) {\n        self.registers.CTLR.write(CTLR::Enable::SET);\n    }\n\n    /// Extract the number of the highest-priority pending IRQ.\n    ///\n    /// Can only be called from IRQ context, which is ensured by taking an `IRQContext` token.\n    ///\n    /// # Safety\n    ///\n    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n    ///   of `&mut self`.\n    #[allow(clippy::trivially_copy_pass_by_ref)]\n    pub fn pending_irq_number<'irq_context>(\n        &self,\n        _ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) -> usize {\n        self.registers.IAR.read(IAR::InterruptID) as usize\n    }\n\n    /// Complete handling of the currently active IRQ.\n    ///\n    /// Can only be called from IRQ context, which is ensured by taking an `IRQContext` token.\n    ///\n    /// To be called after `pending_irq_number()`.\n    ///\n    /// # Safety\n    ///\n    /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead\n    ///   of `&mut self`.\n    #[allow(clippy::trivially_copy_pass_by_ref)]\n    pub fn mark_comleted<'irq_context>(\n        &self,\n        irq_number: u32,\n        _ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) {\n        self.registers.EOIR.write(EOIR::EOIINTID.val(irq_number));\n    }\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/bsp/device_driver/arm/gicv2/gicd.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GICD Driver - GIC Distributor.\n//!\n//! # Glossary\n//!   - SPI - Shared Peripheral Interrupt.\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    memory::{Address, Virtual},\n    state, synchronization,\n    synchronization::IRQSafeNullLock,\n};\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields, register_structs,\n    registers::{ReadOnly, ReadWrite},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nregister_bitfields! {\n    u32,\n\n    /// Distributor Control Register\n    CTLR [\n        Enable OFFSET(0) NUMBITS(1) []\n    ],\n\n    /// Interrupt Controller Type Register\n    TYPER [\n        ITLinesNumber OFFSET(0)  NUMBITS(5) []\n    ],\n\n    /// Interrupt Processor Targets Registers\n    ITARGETSR [\n        Offset3 OFFSET(24) NUMBITS(8) [],\n        Offset2 OFFSET(16) NUMBITS(8) [],\n        Offset1 OFFSET(8)  NUMBITS(8) [],\n        Offset0 OFFSET(0)  NUMBITS(8) []\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    SharedRegisterBlock {\n        (0x000 => CTLR: ReadWrite<u32, CTLR::Register>),\n        (0x004 => TYPER: ReadOnly<u32, TYPER::Register>),\n        (0x008 => _reserved1),\n        (0x104 => ISENABLER: [ReadWrite<u32>; 31]),\n        (0x180 => _reserved2),\n        (0x820 => ITARGETSR: [ReadWrite<u32, ITARGETSR::Register>; 248]),\n        (0xC00 => @END),\n    }\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    BankedRegisterBlock {\n        (0x000 => _reserved1),\n        (0x100 => ISENABLER: ReadWrite<u32>),\n        (0x104 => _reserved2),\n        (0x800 => ITARGETSR: [ReadOnly<u32, ITARGETSR::Register>; 8]),\n        (0x820 => @END),\n    }\n}\n\n/// Abstraction for the non-banked parts of the associated MMIO registers.\ntype SharedRegisters = MMIODerefWrapper<SharedRegisterBlock>;\n\n/// Abstraction for the banked parts of the associated MMIO registers.\ntype BankedRegisters = MMIODerefWrapper<BankedRegisterBlock>;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the GIC Distributor.\npub struct GICD {\n    /// Access to shared registers is guarded with a lock.\n    shared_registers: IRQSafeNullLock<SharedRegisters>,\n\n    /// Access to banked registers is unguarded.\n    banked_registers: BankedRegisters,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl SharedRegisters {\n    /// Return the number of IRQs that this HW implements.\n    #[inline(always)]\n    fn num_irqs(&mut self) -> usize {\n        // Query number of implemented IRQs.\n        //\n        // Refer to GICv2 Architecture Specification, Section 4.3.2.\n        ((self.TYPER.read(TYPER::ITLinesNumber) as usize) + 1) * 32\n    }\n\n    /// Return a slice of the implemented ITARGETSR.\n    #[inline(always)]\n    fn implemented_itargets_slice(&mut self) -> &[ReadWrite<u32, ITARGETSR::Register>] {\n        assert!(self.num_irqs() >= 36);\n\n        // Calculate the max index of the shared ITARGETSR array.\n        //\n        // The first 32 IRQs are private, so not included in `shared_registers`. Each ITARGETS\n        // register has four entries, so shift right by two. Subtract one because we start\n        // counting at zero.\n        let spi_itargetsr_max_index = ((self.num_irqs() - 32) >> 2) - 1;\n\n        // Rust automatically inserts slice range sanity check, i.e. max >= min.\n        &self.ITARGETSR[0..spi_itargetsr_max_index]\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl GICD {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            shared_registers: IRQSafeNullLock::new(SharedRegisters::new(mmio_start_addr)),\n            banked_registers: BankedRegisters::new(mmio_start_addr),\n        }\n    }\n\n    /// Use a banked ITARGETSR to retrieve the executing core's GIC target mask.\n    ///\n    /// Quoting the GICv2 Architecture Specification:\n    ///\n    ///   \"GICD_ITARGETSR0 to GICD_ITARGETSR7 are read-only, and each field returns a value that\n    ///    corresponds only to the processor reading the register.\"\n    fn local_gic_target_mask(&self) -> u32 {\n        self.banked_registers.ITARGETSR[0].read(ITARGETSR::Offset0)\n    }\n\n    /// Route all SPIs to the boot core and enable the distributor.\n    pub fn boot_core_init(&self) {\n        assert!(\n            state::state_manager().is_init(),\n            \"Only allowed during kernel init phase\"\n        );\n\n        // Target all SPIs to the boot core only.\n        let mask = self.local_gic_target_mask();\n\n        self.shared_registers.lock(|regs| {\n            for i in regs.implemented_itargets_slice().iter() {\n                i.write(\n                    ITARGETSR::Offset3.val(mask)\n                        + ITARGETSR::Offset2.val(mask)\n                        + ITARGETSR::Offset1.val(mask)\n                        + ITARGETSR::Offset0.val(mask),\n                );\n            }\n\n            regs.CTLR.write(CTLR::Enable::SET);\n        });\n    }\n\n    /// Enable an interrupt.\n    pub fn enable(&self, irq_num: &super::IRQNumber) {\n        let irq_num = irq_num.get();\n\n        // Each bit in the u32 enable register corresponds to one IRQ number. Shift right by 5\n        // (division by 32) and arrive at the index for the respective ISENABLER[i].\n        let enable_reg_index = irq_num >> 5;\n        let enable_bit: u32 = 1u32 << (irq_num % 32);\n\n        // Check if we are handling a private or shared IRQ.\n        match irq_num {\n            // Private.\n            0..=31 => {\n                let enable_reg = &self.banked_registers.ISENABLER;\n                enable_reg.set(enable_reg.get() | enable_bit);\n            }\n            // Shared.\n            _ => {\n                let enable_reg_index_shared = enable_reg_index - 1;\n\n                self.shared_registers.lock(|regs| {\n                    let enable_reg = &regs.ISENABLER[enable_reg_index_shared];\n                    enable_reg.set(enable_reg.get() | enable_bit);\n                });\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/bsp/device_driver/arm/gicv2.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GICv2 Driver - ARM Generic Interrupt Controller v2.\n//!\n//! The following is a collection of excerpts with useful information from\n//!   - `Programmer's Guide for ARMv8-A`\n//!   - `ARM Generic Interrupt Controller Architecture Specification`\n//!\n//! # Programmer's Guide - 10.6.1 Configuration\n//!\n//! The GIC is accessed as a memory-mapped peripheral.\n//!\n//! All cores can access the common Distributor, but the CPU interface is banked, that is, each core\n//! uses the same address to access its own private CPU interface.\n//!\n//! It is not possible for a core to access the CPU interface of another core.\n//!\n//! # Architecture Specification - 10.6.2 Initialization\n//!\n//! Both the Distributor and the CPU interfaces are disabled at reset. The GIC must be initialized\n//! after reset before it can deliver interrupts to the core.\n//!\n//! In the Distributor, software must configure the priority, target, security and enable individual\n//! interrupts. The Distributor must subsequently be enabled through its control register\n//! (GICD_CTLR). For each CPU interface, software must program the priority mask and preemption\n//! settings.\n//!\n//! Each CPU interface block itself must be enabled through its control register (GICD_CTLR). This\n//! prepares the GIC to deliver interrupts to the core.\n//!\n//! Before interrupts are expected in the core, software prepares the core to take interrupts by\n//! setting a valid interrupt vector in the vector table, and clearing interrupt mask bits in\n//! PSTATE, and setting the routing controls.\n//!\n//! The entire interrupt mechanism in the system can be disabled by disabling the Distributor.\n//! Interrupt delivery to an individual core can be disabled by disabling its CPU interface.\n//! Individual interrupts can also be disabled (or enabled) in the distributor.\n//!\n//! For an interrupt to reach the core, the individual interrupt, Distributor and CPU interface must\n//! all be enabled. The interrupt also needs to be of sufficient priority, that is, higher than the\n//! core's priority mask.\n//!\n//! # Architecture Specification - 1.4.2 Interrupt types\n//!\n//! - Peripheral interrupt\n//!     - Private Peripheral Interrupt (PPI)\n//!         - This is a peripheral interrupt that is specific to a single processor.\n//!     - Shared Peripheral Interrupt (SPI)\n//!         - This is a peripheral interrupt that the Distributor can route to any of a specified\n//!           combination of processors.\n//!\n//! - Software-generated interrupt (SGI)\n//!     - This is an interrupt generated by software writing to a GICD_SGIR register in the GIC. The\n//!       system uses SGIs for interprocessor communication.\n//!     - An SGI has edge-triggered properties. The software triggering of the interrupt is\n//!       equivalent to the edge transition of the interrupt request signal.\n//!     - When an SGI occurs in a multiprocessor implementation, the CPUID field in the Interrupt\n//!       Acknowledge Register, GICC_IAR, or the Aliased Interrupt Acknowledge Register, GICC_AIAR,\n//!       identifies the processor that requested the interrupt.\n//!\n//! # Architecture Specification - 2.2.1 Interrupt IDs\n//!\n//! Interrupts from sources are identified using ID numbers. Each CPU interface can see up to 1020\n//! interrupts. The banking of SPIs and PPIs increases the total number of interrupts supported by\n//! the Distributor.\n//!\n//! The GIC assigns interrupt ID numbers ID0-ID1019 as follows:\n//!   - Interrupt numbers 32..1019 are used for SPIs.\n//!   - Interrupt numbers 0..31 are used for interrupts that are private to a CPU interface. These\n//!     interrupts are banked in the Distributor.\n//!       - A banked interrupt is one where the Distributor can have multiple interrupts with the\n//!         same ID. A banked interrupt is identified uniquely by its ID number and its associated\n//!         CPU interface number. Of the banked interrupt IDs:\n//!           - 00..15 SGIs\n//!           - 16..31 PPIs\n\nmod gicc;\nmod gicd;\n\nuse crate::{\n    bsp::{self, device_driver::common::BoundedUsize},\n    cpu, driver, exception,\n    memory::{Address, Virtual},\n    synchronization,\n    synchronization::InitStateLock,\n};\nuse alloc::vec::Vec;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\ntype HandlerTable = Vec<Option<exception::asynchronous::IRQHandlerDescriptor<IRQNumber>>>;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Used for the associated type of trait [`exception::asynchronous::interface::IRQManager`].\npub type IRQNumber = BoundedUsize<{ GICv2::MAX_IRQ_NUMBER }>;\n\n/// Representation of the GIC.\npub struct GICv2 {\n    /// The Distributor.\n    gicd: gicd::GICD,\n\n    /// The CPU Interface.\n    gicc: gicc::GICC,\n\n    /// Stores registered IRQ handlers. Writable only during kernel init. RO afterwards.\n    handler_table: InitStateLock<HandlerTable>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GICv2 {\n    const MAX_IRQ_NUMBER: usize = 1019;\n\n    pub const COMPATIBLE: &'static str = \"GICv2 (ARM Generic Interrupt Controller v2)\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(\n        gicd_mmio_start_addr: Address<Virtual>,\n        gicc_mmio_start_addr: Address<Virtual>,\n    ) -> Self {\n        Self {\n            gicd: gicd::GICD::new(gicd_mmio_start_addr),\n            gicc: gicc::GICC::new(gicc_mmio_start_addr),\n            handler_table: InitStateLock::new(Vec::new()),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::ReadWriteEx;\n\nimpl driver::interface::DeviceDriver for GICv2 {\n    type IRQNumberType = IRQNumber;\n\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n\n    unsafe fn init(&self) -> Result<(), &'static str> {\n        self.handler_table\n            .write(|table| table.resize(IRQNumber::MAX_INCLUSIVE + 1, None));\n\n        if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() {\n            self.gicd.boot_core_init();\n        }\n\n        self.gicc.priority_accept_all();\n        self.gicc.enable();\n\n        Ok(())\n    }\n}\n\nimpl exception::asynchronous::interface::IRQManager for GICv2 {\n    type IRQNumberType = IRQNumber;\n\n    fn register_handler(\n        &self,\n        irq_handler_descriptor: exception::asynchronous::IRQHandlerDescriptor<Self::IRQNumberType>,\n    ) -> Result<(), &'static str> {\n        self.handler_table.write(|table| {\n            let irq_number = irq_handler_descriptor.number().get();\n\n            if table[irq_number].is_some() {\n                return Err(\"IRQ handler already registered\");\n            }\n\n            table[irq_number] = Some(irq_handler_descriptor);\n\n            Ok(())\n        })\n    }\n\n    fn enable(&self, irq_number: &Self::IRQNumberType) {\n        self.gicd.enable(irq_number);\n    }\n\n    fn handle_pending_irqs<'irq_context>(\n        &'irq_context self,\n        ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) {\n        // Extract the highest priority pending IRQ number from the Interrupt Acknowledge Register\n        // (IAR).\n        let irq_number = self.gicc.pending_irq_number(ic);\n\n        // Guard against spurious interrupts.\n        if irq_number > GICv2::MAX_IRQ_NUMBER {\n            return;\n        }\n\n        // Call the IRQ handler. Panic if there is none.\n        self.handler_table.read(|table| {\n            match table[irq_number] {\n                None => panic!(\"No handler registered for IRQ {}\", irq_number),\n                Some(descriptor) => {\n                    // Call the IRQ handler. Panics on failure.\n                    descriptor.handler().handle().expect(\"Error handling IRQ\");\n                }\n            }\n        });\n\n        // Signal completion of handling.\n        self.gicc.mark_comleted(irq_number as u32, ic);\n    }\n\n    fn print_handler(&self) {\n        use crate::info;\n\n        info!(\"      Peripheral handler:\");\n\n        self.handler_table.read(|table| {\n            for (i, opt) in table.iter().skip(32).enumerate() {\n                if let Some(handler) = opt {\n                    info!(\"            {: >3}. {}\", i + 32, handler.name());\n                }\n            }\n        });\n    }\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/bsp/device_driver/arm.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! ARM driver top level.\n\npub mod gicv2;\n\npub use gicv2::*;\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GPIO Driver.\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    driver,\n    exception::asynchronous::IRQNumber,\n    memory::{Address, Virtual},\n    synchronization,\n    synchronization::IRQSafeNullLock,\n};\nuse tock_registers::{\n    interfaces::{ReadWriteable, Writeable},\n    register_bitfields, register_structs,\n    registers::ReadWrite,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// GPIO registers.\n//\n// Descriptions taken from\n// - https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf\n// - https://datasheets.raspberrypi.org/bcm2711/bcm2711-peripherals.pdf\nregister_bitfields! {\n    u32,\n\n    /// GPIO Function Select 1\n    GPFSEL1 [\n        /// Pin 15\n        FSEL15 OFFSET(15) NUMBITS(3) [\n            Input = 0b000,\n            Output = 0b001,\n            AltFunc0 = 0b100  // PL011 UART RX\n\n        ],\n\n        /// Pin 14\n        FSEL14 OFFSET(12) NUMBITS(3) [\n            Input = 0b000,\n            Output = 0b001,\n            AltFunc0 = 0b100  // PL011 UART TX\n        ]\n    ],\n\n    /// GPIO Pull-up/down Register\n    ///\n    /// BCM2837 only.\n    GPPUD [\n        /// Controls the actuation of the internal pull-up/down control line to ALL the GPIO pins.\n        PUD OFFSET(0) NUMBITS(2) [\n            Off = 0b00,\n            PullDown = 0b01,\n            PullUp = 0b10\n        ]\n    ],\n\n    /// GPIO Pull-up/down Clock Register 0\n    ///\n    /// BCM2837 only.\n    GPPUDCLK0 [\n        /// Pin 15\n        PUDCLK15 OFFSET(15) NUMBITS(1) [\n            NoEffect = 0,\n            AssertClock = 1\n        ],\n\n        /// Pin 14\n        PUDCLK14 OFFSET(14) NUMBITS(1) [\n            NoEffect = 0,\n            AssertClock = 1\n        ]\n    ],\n\n    /// GPIO Pull-up / Pull-down Register 0\n    ///\n    /// BCM2711 only.\n    GPIO_PUP_PDN_CNTRL_REG0 [\n        /// Pin 15\n        GPIO_PUP_PDN_CNTRL15 OFFSET(30) NUMBITS(2) [\n            NoResistor = 0b00,\n            PullUp = 0b01\n        ],\n\n        /// Pin 14\n        GPIO_PUP_PDN_CNTRL14 OFFSET(28) NUMBITS(2) [\n            NoResistor = 0b00,\n            PullUp = 0b01\n        ]\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    RegisterBlock {\n        (0x00 => _reserved1),\n        (0x04 => GPFSEL1: ReadWrite<u32, GPFSEL1::Register>),\n        (0x08 => _reserved2),\n        (0x94 => GPPUD: ReadWrite<u32, GPPUD::Register>),\n        (0x98 => GPPUDCLK0: ReadWrite<u32, GPPUDCLK0::Register>),\n        (0x9C => _reserved3),\n        (0xE4 => GPIO_PUP_PDN_CNTRL_REG0: ReadWrite<u32, GPIO_PUP_PDN_CNTRL_REG0::Register>),\n        (0xE8 => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\nstruct GPIOInner {\n    registers: Registers,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the GPIO HW.\npub struct GPIO {\n    inner: IRQSafeNullLock<GPIOInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GPIOInner {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n        }\n    }\n\n    /// Disable pull-up/down on pins 14 and 15.\n    #[cfg(feature = \"bsp_rpi3\")]\n    fn disable_pud_14_15_bcm2837(&mut self) {\n        use crate::time;\n        use core::time::Duration;\n\n        // The Linux 2837 GPIO driver waits 1 µs between the steps.\n        const DELAY: Duration = Duration::from_micros(1);\n\n        self.registers.GPPUD.write(GPPUD::PUD::Off);\n        time::time_manager().spin_for(DELAY);\n\n        self.registers\n            .GPPUDCLK0\n            .write(GPPUDCLK0::PUDCLK15::AssertClock + GPPUDCLK0::PUDCLK14::AssertClock);\n        time::time_manager().spin_for(DELAY);\n\n        self.registers.GPPUD.write(GPPUD::PUD::Off);\n        self.registers.GPPUDCLK0.set(0);\n    }\n\n    /// Disable pull-up/down on pins 14 and 15.\n    #[cfg(feature = \"bsp_rpi4\")]\n    fn disable_pud_14_15_bcm2711(&mut self) {\n        self.registers.GPIO_PUP_PDN_CNTRL_REG0.write(\n            GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL15::PullUp\n                + GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL14::PullUp,\n        );\n    }\n\n    /// Map PL011 UART as standard output.\n    ///\n    /// TX to pin 14\n    /// RX to pin 15\n    pub fn map_pl011_uart(&mut self) {\n        // Select the UART on pins 14 and 15.\n        self.registers\n            .GPFSEL1\n            .modify(GPFSEL1::FSEL15::AltFunc0 + GPFSEL1::FSEL14::AltFunc0);\n\n        // Disable pull-up/down on pins 14 and 15.\n        #[cfg(feature = \"bsp_rpi3\")]\n        self.disable_pud_14_15_bcm2837();\n\n        #[cfg(feature = \"bsp_rpi4\")]\n        self.disable_pud_14_15_bcm2711();\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GPIO {\n    pub const COMPATIBLE: &'static str = \"BCM GPIO\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            inner: IRQSafeNullLock::new(GPIOInner::new(mmio_start_addr)),\n        }\n    }\n\n    /// Concurrency safe version of `GPIOInner.map_pl011_uart()`\n    pub fn map_pl011_uart(&self) {\n        self.inner.lock(|inner| inner.map_pl011_uart())\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl driver::interface::DeviceDriver for GPIO {\n    type IRQNumberType = IRQNumber;\n\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/local_ic.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Local Interrupt Controller Driver.\n//!\n//! # Resources\n//!\n//! - <https://datasheets.raspberrypi.com/bcm2836/bcm2836-peripherals.pdf>\n\nuse super::{LocalIRQ, PendingIRQs};\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    exception,\n    memory::{Address, Virtual},\n    synchronization,\n    synchronization::{IRQSafeNullLock, InitStateLock},\n};\nuse alloc::vec::Vec;\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_structs,\n    registers::{ReadOnly, WriteOnly},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    WORegisterBlock {\n        (0x00 => _reserved1),\n        (0x40 => CORE0_TIMER_INTERRUPT_CONTROL: WriteOnly<u32>),\n        (0x44 => @END),\n    }\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    RORegisterBlock {\n        (0x00 => _reserved1),\n        (0x60 => CORE0_INTERRUPT_SOURCE: ReadOnly<u32>),\n        (0x64 => @END),\n    }\n}\n\n/// Abstraction for the WriteOnly parts of the associated MMIO registers.\ntype WriteOnlyRegisters = MMIODerefWrapper<WORegisterBlock>;\n\n/// Abstraction for the ReadOnly parts of the associated MMIO registers.\ntype ReadOnlyRegisters = MMIODerefWrapper<RORegisterBlock>;\n\ntype HandlerTable = Vec<Option<exception::asynchronous::IRQHandlerDescriptor<LocalIRQ>>>;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the peripheral interrupt controller.\npub struct LocalIC {\n    /// Access to write registers is guarded with a lock.\n    wo_registers: IRQSafeNullLock<WriteOnlyRegisters>,\n\n    /// Register read access is unguarded.\n    ro_registers: ReadOnlyRegisters,\n\n    /// Stores registered IRQ handlers. Writable only during kernel init. RO afterwards.\n    handler_table: InitStateLock<HandlerTable>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl LocalIC {\n    // See datasheet.\n    const PERIPH_IRQ_MASK: u32 = (1 << 8);\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            wo_registers: IRQSafeNullLock::new(WriteOnlyRegisters::new(mmio_start_addr)),\n            ro_registers: ReadOnlyRegisters::new(mmio_start_addr),\n            handler_table: InitStateLock::new(Vec::new()),\n        }\n    }\n\n    /// Called by the kernel to bring up the device.\n    pub fn init(&self) {\n        self.handler_table\n            .write(|table| table.resize(LocalIRQ::MAX_INCLUSIVE + 1, None));\n    }\n\n    /// Query the list of pending IRQs.\n    fn pending_irqs(&self) -> PendingIRQs {\n        // Ignore the indicator bit for a peripheral IRQ.\n        PendingIRQs::new(\n            (self.ro_registers.CORE0_INTERRUPT_SOURCE.get() & !Self::PERIPH_IRQ_MASK).into(),\n        )\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::{Mutex, ReadWriteEx};\n\nimpl exception::asynchronous::interface::IRQManager for LocalIC {\n    type IRQNumberType = LocalIRQ;\n\n    fn register_handler(\n        &self,\n        irq_handler_descriptor: exception::asynchronous::IRQHandlerDescriptor<Self::IRQNumberType>,\n    ) -> Result<(), &'static str> {\n        self.handler_table.write(|table| {\n            let irq_number = irq_handler_descriptor.number().get();\n\n            if table[irq_number].is_some() {\n                return Err(\"IRQ handler already registered\");\n            }\n\n            table[irq_number] = Some(irq_handler_descriptor);\n\n            Ok(())\n        })\n    }\n\n    fn enable(&self, irq: &Self::IRQNumberType) {\n        self.wo_registers.lock(|regs| {\n            let enable_bit: u32 = 1 << (irq.get());\n\n            // Writing a 1 to a bit will set the corresponding IRQ enable bit. All other IRQ enable\n            // bits are unaffected. So we don't need read and OR'ing here.\n            regs.CORE0_TIMER_INTERRUPT_CONTROL.set(enable_bit);\n        });\n    }\n\n    fn handle_pending_irqs<'irq_context>(\n        &'irq_context self,\n        _ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) {\n        self.handler_table.read(|table| {\n            for irq_number in self.pending_irqs() {\n                match table[irq_number] {\n                    None => panic!(\"No handler registered for IRQ {}\", irq_number),\n                    Some(descriptor) => {\n                        // Call the IRQ handler. Panics on failure.\n                        descriptor.handler().handle().expect(\"Error handling IRQ\");\n                    }\n                }\n            }\n        })\n    }\n\n    fn print_handler(&self) {\n        use crate::info;\n\n        info!(\"      Local handler:\");\n\n        self.handler_table.read(|table| {\n            for (i, opt) in table.iter().enumerate() {\n                if let Some(handler) = opt {\n                    info!(\"            {: >3}. {}\", i, handler.name());\n                }\n            }\n        });\n    }\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Peripheral Interrupt Controller Driver.\n//!\n//! # Resources\n//!\n//! - <https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf>\n\nuse super::{PendingIRQs, PeripheralIRQ};\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    exception,\n    memory::{Address, Virtual},\n    synchronization,\n    synchronization::{IRQSafeNullLock, InitStateLock},\n};\nuse alloc::vec::Vec;\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_structs,\n    registers::{ReadOnly, WriteOnly},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    WORegisterBlock {\n        (0x00 => _reserved1),\n        (0x10 => ENABLE_1: WriteOnly<u32>),\n        (0x14 => ENABLE_2: WriteOnly<u32>),\n        (0x18 => @END),\n    }\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    RORegisterBlock {\n        (0x00 => _reserved1),\n        (0x04 => PENDING_1: ReadOnly<u32>),\n        (0x08 => PENDING_2: ReadOnly<u32>),\n        (0x0c => @END),\n    }\n}\n\n/// Abstraction for the WriteOnly parts of the associated MMIO registers.\ntype WriteOnlyRegisters = MMIODerefWrapper<WORegisterBlock>;\n\n/// Abstraction for the ReadOnly parts of the associated MMIO registers.\ntype ReadOnlyRegisters = MMIODerefWrapper<RORegisterBlock>;\n\ntype HandlerTable = Vec<Option<exception::asynchronous::IRQHandlerDescriptor<PeripheralIRQ>>>;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the peripheral interrupt controller.\npub struct PeripheralIC {\n    /// Access to write registers is guarded with a lock.\n    wo_registers: IRQSafeNullLock<WriteOnlyRegisters>,\n\n    /// Register read access is unguarded.\n    ro_registers: ReadOnlyRegisters,\n\n    /// Stores registered IRQ handlers. Writable only during kernel init. RO afterwards.\n    handler_table: InitStateLock<HandlerTable>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PeripheralIC {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            wo_registers: IRQSafeNullLock::new(WriteOnlyRegisters::new(mmio_start_addr)),\n            ro_registers: ReadOnlyRegisters::new(mmio_start_addr),\n            handler_table: InitStateLock::new(Vec::new()),\n        }\n    }\n\n    /// Called by the kernel to bring up the device.\n    pub fn init(&self) {\n        self.handler_table\n            .write(|table| table.resize(PeripheralIRQ::MAX_INCLUSIVE + 1, None));\n    }\n\n    /// Query the list of pending IRQs.\n    fn pending_irqs(&self) -> PendingIRQs {\n        let pending_mask: u64 = (u64::from(self.ro_registers.PENDING_2.get()) << 32)\n            | u64::from(self.ro_registers.PENDING_1.get());\n\n        PendingIRQs::new(pending_mask)\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::{Mutex, ReadWriteEx};\n\nimpl exception::asynchronous::interface::IRQManager for PeripheralIC {\n    type IRQNumberType = PeripheralIRQ;\n\n    fn register_handler(\n        &self,\n        irq_handler_descriptor: exception::asynchronous::IRQHandlerDescriptor<Self::IRQNumberType>,\n    ) -> Result<(), &'static str> {\n        self.handler_table.write(|table| {\n            let irq_number = irq_handler_descriptor.number().get();\n\n            if table[irq_number].is_some() {\n                return Err(\"IRQ handler already registered\");\n            }\n\n            table[irq_number] = Some(irq_handler_descriptor);\n\n            Ok(())\n        })\n    }\n\n    fn enable(&self, irq: &Self::IRQNumberType) {\n        self.wo_registers.lock(|regs| {\n            let enable_reg = if irq.get() <= 31 {\n                &regs.ENABLE_1\n            } else {\n                &regs.ENABLE_2\n            };\n\n            let enable_bit: u32 = 1 << (irq.get() % 32);\n\n            // Writing a 1 to a bit will set the corresponding IRQ enable bit. All other IRQ enable\n            // bits are unaffected. So we don't need read and OR'ing here.\n            enable_reg.set(enable_bit);\n        });\n    }\n\n    fn handle_pending_irqs<'irq_context>(\n        &'irq_context self,\n        _ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) {\n        self.handler_table.read(|table| {\n            for irq_number in self.pending_irqs() {\n                match table[irq_number] {\n                    None => panic!(\"No handler registered for IRQ {}\", irq_number),\n                    Some(descriptor) => {\n                        // Call the IRQ handler. Panics on failure.\n                        descriptor.handler().handle().expect(\"Error handling IRQ\");\n                    }\n                }\n            }\n        })\n    }\n\n    fn print_handler(&self) {\n        use crate::info;\n\n        info!(\"      Peripheral handler:\");\n\n        self.handler_table.read(|table| {\n            for (i, opt) in table.iter().enumerate() {\n                if let Some(handler) = opt {\n                    info!(\"            {: >3}. {}\", i, handler.name());\n                }\n            }\n        });\n    }\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Interrupt Controller Driver.\n\nmod local_ic;\nmod peripheral_ic;\n\nuse crate::{\n    bsp::device_driver::common::BoundedUsize,\n    driver,\n    exception::{self, asynchronous::IRQHandlerDescriptor},\n    memory::{Address, Virtual},\n};\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Wrapper struct for a bitmask indicating pending IRQ numbers.\nstruct PendingIRQs {\n    bitmask: u64,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub type LocalIRQ = BoundedUsize<{ InterruptController::MAX_LOCAL_IRQ_NUMBER }>;\npub type PeripheralIRQ = BoundedUsize<{ InterruptController::MAX_PERIPHERAL_IRQ_NUMBER }>;\n\n/// Used for the associated type of trait [`exception::asynchronous::interface::IRQManager`].\n#[derive(Copy, Clone)]\n#[allow(missing_docs)]\npub enum IRQNumber {\n    Local(LocalIRQ),\n    Peripheral(PeripheralIRQ),\n}\n\n/// Representation of the Interrupt Controller.\npub struct InterruptController {\n    local: local_ic::LocalIC,\n    periph: peripheral_ic::PeripheralIC,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PendingIRQs {\n    pub fn new(bitmask: u64) -> Self {\n        Self { bitmask }\n    }\n}\n\nimpl Iterator for PendingIRQs {\n    type Item = usize;\n\n    fn next(&mut self) -> Option<Self::Item> {\n        if self.bitmask == 0 {\n            return None;\n        }\n\n        let next = self.bitmask.trailing_zeros() as usize;\n        self.bitmask &= self.bitmask.wrapping_sub(1);\n        Some(next)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl fmt::Display for IRQNumber {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        match self {\n            Self::Local(number) => write!(f, \"Local({})\", number),\n            Self::Peripheral(number) => write!(f, \"Peripheral({})\", number),\n        }\n    }\n}\n\nimpl InterruptController {\n    // Restrict to 3 for now. This makes the code for local_ic.rs more straight forward.\n    const MAX_LOCAL_IRQ_NUMBER: usize = 3;\n    const MAX_PERIPHERAL_IRQ_NUMBER: usize = 63;\n\n    pub const COMPATIBLE: &'static str = \"BCM Interrupt Controller\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(\n        local_mmio_start_addr: Address<Virtual>,\n        periph_mmio_start_addr: Address<Virtual>,\n    ) -> Self {\n        Self {\n            local: local_ic::LocalIC::new(local_mmio_start_addr),\n            periph: peripheral_ic::PeripheralIC::new(periph_mmio_start_addr),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\n\nimpl driver::interface::DeviceDriver for InterruptController {\n    type IRQNumberType = IRQNumber;\n\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n\n    unsafe fn init(&self) -> Result<(), &'static str> {\n        self.local.init();\n        self.periph.init();\n\n        Ok(())\n    }\n}\n\nimpl exception::asynchronous::interface::IRQManager for InterruptController {\n    type IRQNumberType = IRQNumber;\n\n    fn register_handler(\n        &self,\n        irq_handler_descriptor: exception::asynchronous::IRQHandlerDescriptor<Self::IRQNumberType>,\n    ) -> Result<(), &'static str> {\n        match irq_handler_descriptor.number() {\n            IRQNumber::Local(lirq) => {\n                let local_descriptor = IRQHandlerDescriptor::new(\n                    lirq,\n                    irq_handler_descriptor.name(),\n                    irq_handler_descriptor.handler(),\n                );\n\n                self.local.register_handler(local_descriptor)\n            }\n            IRQNumber::Peripheral(pirq) => {\n                let periph_descriptor = IRQHandlerDescriptor::new(\n                    pirq,\n                    irq_handler_descriptor.name(),\n                    irq_handler_descriptor.handler(),\n                );\n\n                self.periph.register_handler(periph_descriptor)\n            }\n        }\n    }\n\n    fn enable(&self, irq: &Self::IRQNumberType) {\n        match irq {\n            IRQNumber::Local(lirq) => self.local.enable(lirq),\n            IRQNumber::Peripheral(pirq) => self.periph.enable(pirq),\n        }\n    }\n\n    fn handle_pending_irqs<'irq_context>(\n        &'irq_context self,\n        ic: &exception::asynchronous::IRQContext<'irq_context>,\n    ) {\n        self.local.handle_pending_irqs(ic);\n        self.periph.handle_pending_irqs(ic)\n    }\n\n    fn print_handler(&self) {\n        self.local.print_handler();\n        self.periph.print_handler();\n    }\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! PL011 UART driver.\n//!\n//! # Resources\n//!\n//! - <https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf>\n//! - <https://developer.arm.com/documentation/ddi0183/latest>\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper,\n    console, cpu, driver,\n    exception::{self, asynchronous::IRQNumber},\n    memory::{Address, Virtual},\n    synchronization,\n    synchronization::IRQSafeNullLock,\n};\nuse core::fmt;\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields, register_structs,\n    registers::{ReadOnly, ReadWrite, WriteOnly},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// PL011 UART registers.\n//\n// Descriptions taken from \"PrimeCell UART (PL011) Technical Reference Manual\" r1p5.\nregister_bitfields! {\n    u32,\n\n    /// Flag Register.\n    FR [\n        /// Transmit FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n        /// Line Control Register, LCR_H.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the transmit holding register is empty.\n        /// - If the FIFO is enabled, the TXFE bit is set when the transmit FIFO is empty.\n        /// - This bit does not indicate if there is data in the transmit shift register.\n        TXFE OFFSET(7) NUMBITS(1) [],\n\n        /// Transmit FIFO full. The meaning of this bit depends on the state of the FEN bit in the\n        /// LCR_H Register.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the transmit holding register is full.\n        /// - If the FIFO is enabled, the TXFF bit is set when the transmit FIFO is full.\n        TXFF OFFSET(5) NUMBITS(1) [],\n\n        /// Receive FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n        /// LCR_H Register.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the receive holding register is empty.\n        /// - If the FIFO is enabled, the RXFE bit is set when the receive FIFO is empty.\n        RXFE OFFSET(4) NUMBITS(1) [],\n\n        /// UART busy. If this bit is set to 1, the UART is busy transmitting data. This bit remains\n        /// set until the complete byte, including all the stop bits, has been sent from the shift\n        /// register.\n        ///\n        /// This bit is set as soon as the transmit FIFO becomes non-empty, regardless of whether\n        /// the UART is enabled or not.\n        BUSY OFFSET(3) NUMBITS(1) []\n    ],\n\n    /// Integer Baud Rate Divisor.\n    IBRD [\n        /// The integer baud rate divisor.\n        BAUD_DIVINT OFFSET(0) NUMBITS(16) []\n    ],\n\n    /// Fractional Baud Rate Divisor.\n    FBRD [\n        ///  The fractional baud rate divisor.\n        BAUD_DIVFRAC OFFSET(0) NUMBITS(6) []\n    ],\n\n    /// Line Control Register.\n    LCR_H [\n        /// Word length. These bits indicate the number of data bits transmitted or received in a\n        /// frame.\n        #[allow(clippy::enum_variant_names)]\n        WLEN OFFSET(5) NUMBITS(2) [\n            FiveBit = 0b00,\n            SixBit = 0b01,\n            SevenBit = 0b10,\n            EightBit = 0b11\n        ],\n\n        /// Enable FIFOs:\n        ///\n        /// 0 = FIFOs are disabled (character mode) that is, the FIFOs become 1-byte-deep holding\n        /// registers.\n        ///\n        /// 1 = Transmit and receive FIFO buffers are enabled (FIFO mode).\n        FEN  OFFSET(4) NUMBITS(1) [\n            FifosDisabled = 0,\n            FifosEnabled = 1\n        ]\n    ],\n\n    /// Control Register.\n    CR [\n        /// Receive enable. If this bit is set to 1, the receive section of the UART is enabled.\n        /// Data reception occurs for either UART signals or SIR signals depending on the setting of\n        /// the SIREN bit. When the UART is disabled in the middle of reception, it completes the\n        /// current character before stopping.\n        RXE OFFSET(9) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// Transmit enable. If this bit is set to 1, the transmit section of the UART is enabled.\n        /// Data transmission occurs for either UART signals, or SIR signals depending on the\n        /// setting of the SIREN bit. When the UART is disabled in the middle of transmission, it\n        /// completes the current character before stopping.\n        TXE OFFSET(8) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// UART enable:\n        ///\n        /// 0 = UART is disabled. If the UART is disabled in the middle of transmission or\n        /// reception, it completes the current character before stopping.\n        ///\n        /// 1 = The UART is enabled. Data transmission and reception occurs for either UART signals\n        /// or SIR signals depending on the setting of the SIREN bit\n        UARTEN OFFSET(0) NUMBITS(1) [\n            /// If the UART is disabled in the middle of transmission or reception, it completes the\n            /// current character before stopping.\n            Disabled = 0,\n            Enabled = 1\n        ]\n    ],\n\n    /// Interrupt FIFO Level Select Register.\n    IFLS [\n        /// Receive interrupt FIFO level select. The trigger points for the receive interrupt are as\n        /// follows.\n        RXIFLSEL OFFSET(3) NUMBITS(5) [\n            OneEigth = 0b000,\n            OneQuarter = 0b001,\n            OneHalf = 0b010,\n            ThreeQuarters = 0b011,\n            SevenEights = 0b100\n        ]\n    ],\n\n    /// Interrupt Mask Set/Clear Register.\n    IMSC [\n        /// Receive timeout interrupt mask. A read returns the current mask for the UARTRTINTR\n        /// interrupt.\n        ///\n        /// - On a write of 1, the mask of the UARTRTINTR interrupt is set.\n        /// - A write of 0 clears the mask.\n        RTIM OFFSET(6) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// Receive interrupt mask. A read returns the current mask for the UARTRXINTR interrupt.\n        ///\n        /// - On a write of 1, the mask of the UARTRXINTR interrupt is set.\n        /// - A write of 0 clears the mask.\n        RXIM OFFSET(4) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ]\n    ],\n\n    /// Masked Interrupt Status Register.\n    MIS [\n        /// Receive timeout masked interrupt status. Returns the masked interrupt state of the\n        /// UARTRTINTR interrupt.\n        RTMIS OFFSET(6) NUMBITS(1) [],\n\n        /// Receive masked interrupt status. Returns the masked interrupt state of the UARTRXINTR\n        /// interrupt.\n        RXMIS OFFSET(4) NUMBITS(1) []\n    ],\n\n    /// Interrupt Clear Register.\n    ICR [\n        /// Meta field for all pending interrupts.\n        ALL OFFSET(0) NUMBITS(11) []\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    pub RegisterBlock {\n        (0x00 => DR: ReadWrite<u32>),\n        (0x04 => _reserved1),\n        (0x18 => FR: ReadOnly<u32, FR::Register>),\n        (0x1c => _reserved2),\n        (0x24 => IBRD: WriteOnly<u32, IBRD::Register>),\n        (0x28 => FBRD: WriteOnly<u32, FBRD::Register>),\n        (0x2c => LCR_H: WriteOnly<u32, LCR_H::Register>),\n        (0x30 => CR: WriteOnly<u32, CR::Register>),\n        (0x34 => IFLS: ReadWrite<u32, IFLS::Register>),\n        (0x38 => IMSC: ReadWrite<u32, IMSC::Register>),\n        (0x3C => _reserved3),\n        (0x40 => MIS: ReadOnly<u32, MIS::Register>),\n        (0x44 => ICR: WriteOnly<u32, ICR::Register>),\n        (0x48 => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\n#[derive(PartialEq)]\nenum BlockingMode {\n    Blocking,\n    NonBlocking,\n}\n\nstruct PL011UartInner {\n    registers: Registers,\n    chars_written: usize,\n    chars_read: usize,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the UART.\npub struct PL011Uart {\n    inner: IRQSafeNullLock<PL011UartInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PL011UartInner {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n            chars_written: 0,\n            chars_read: 0,\n        }\n    }\n\n    /// Set up baud rate and characteristics.\n    ///\n    /// This results in 8N1 and 921_600 baud.\n    ///\n    /// The calculation for the BRD is (we set the clock to 48 MHz in config.txt):\n    /// `(48_000_000 / 16) / 921_600 = 3.2552083`.\n    ///\n    /// This means the integer part is `3` and goes into the `IBRD`.\n    /// The fractional part is `0.2552083`.\n    ///\n    /// `FBRD` calculation according to the PL011 Technical Reference Manual:\n    /// `INTEGER((0.2552083 * 64) + 0.5) = 16`.\n    ///\n    /// Therefore, the generated baud rate divider is: `3 + 16/64 = 3.25`. Which results in a\n    /// genrated baud rate of `48_000_000 / (16 * 3.25) = 923_077`.\n    ///\n    /// Error = `((923_077 - 921_600) / 921_600) * 100 = 0.16%`.\n    pub fn init(&mut self) {\n        // Execution can arrive here while there are still characters queued in the TX FIFO and\n        // actively being sent out by the UART hardware. If the UART is turned off in this case,\n        // those queued characters would be lost.\n        //\n        // For example, this can happen during runtime on a call to panic!(), because panic!()\n        // initializes its own UART instance and calls init().\n        //\n        // Hence, flush first to ensure all pending characters are transmitted.\n        self.flush();\n\n        // Turn the UART off temporarily.\n        self.registers.CR.set(0);\n\n        // Clear all pending interrupts.\n        self.registers.ICR.write(ICR::ALL::CLEAR);\n\n        // From the PL011 Technical Reference Manual:\n        //\n        // The LCR_H, IBRD, and FBRD registers form the single 30-bit wide LCR Register that is\n        // updated on a single write strobe generated by a LCR_H write. So, to internally update the\n        // contents of IBRD or FBRD, a LCR_H write must always be performed at the end.\n        //\n        // Set the baud rate, 8N1 and FIFO enabled.\n        self.registers.IBRD.write(IBRD::BAUD_DIVINT.val(3));\n        self.registers.FBRD.write(FBRD::BAUD_DIVFRAC.val(16));\n        self.registers\n            .LCR_H\n            .write(LCR_H::WLEN::EightBit + LCR_H::FEN::FifosEnabled);\n\n        // Set RX FIFO fill level at 1/8.\n        self.registers.IFLS.write(IFLS::RXIFLSEL::OneEigth);\n\n        // Enable RX IRQ + RX timeout IRQ.\n        self.registers\n            .IMSC\n            .write(IMSC::RXIM::Enabled + IMSC::RTIM::Enabled);\n\n        // Turn the UART on.\n        self.registers\n            .CR\n            .write(CR::UARTEN::Enabled + CR::TXE::Enabled + CR::RXE::Enabled);\n    }\n\n    /// Send a character.\n    fn write_char(&mut self, c: char) {\n        // Spin while TX FIFO full is set, waiting for an empty slot.\n        while self.registers.FR.matches_all(FR::TXFF::SET) {\n            cpu::nop();\n        }\n\n        // Write the character to the buffer.\n        self.registers.DR.set(c as u32);\n\n        self.chars_written += 1;\n    }\n\n    /// Send a slice of characters.\n    fn write_array(&mut self, a: &[char]) {\n        for c in a {\n            self.write_char(*c);\n        }\n    }\n\n    /// Block execution until the last buffered character has been physically put on the TX wire.\n    fn flush(&self) {\n        // Spin until the busy bit is cleared.\n        while self.registers.FR.matches_all(FR::BUSY::SET) {\n            cpu::nop();\n        }\n    }\n\n    /// Retrieve a character.\n    fn read_char_converting(&mut self, blocking_mode: BlockingMode) -> Option<char> {\n        // If RX FIFO is empty,\n        if self.registers.FR.matches_all(FR::RXFE::SET) {\n            // immediately return in non-blocking mode.\n            if blocking_mode == BlockingMode::NonBlocking {\n                return None;\n            }\n\n            // Otherwise, wait until a char was received.\n            while self.registers.FR.matches_all(FR::RXFE::SET) {\n                cpu::nop();\n            }\n        }\n\n        // Read one character.\n        let mut ret = self.registers.DR.get() as u8 as char;\n\n        // Convert carrige return to newline.\n        if ret == '\\r' {\n            ret = '\\n'\n        }\n\n        // Update statistics.\n        self.chars_read += 1;\n\n        Some(ret)\n    }\n}\n\n/// Implementing `core::fmt::Write` enables usage of the `format_args!` macros, which in turn are\n/// used to implement the `kernel`'s `print!` and `println!` macros. By implementing `write_str()`,\n/// we get `write_fmt()` automatically.\n///\n/// The function takes an `&mut self`, so it must be implemented for the inner struct.\n///\n/// See [`src/print.rs`].\n///\n/// [`src/print.rs`]: ../../print/index.html\nimpl fmt::Write for PL011UartInner {\n    fn write_str(&mut self, s: &str) -> fmt::Result {\n        for c in s.chars() {\n            self.write_char(c);\n        }\n\n        Ok(())\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PL011Uart {\n    pub const COMPATIBLE: &'static str = \"BCM PL011 UART\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: Address<Virtual>) -> Self {\n        Self {\n            inner: IRQSafeNullLock::new(PL011UartInner::new(mmio_start_addr)),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl driver::interface::DeviceDriver for PL011Uart {\n    type IRQNumberType = IRQNumber;\n\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n\n    unsafe fn init(&self) -> Result<(), &'static str> {\n        self.inner.lock(|inner| inner.init());\n\n        Ok(())\n    }\n\n    fn register_and_enable_irq_handler(\n        &'static self,\n        irq_number: &Self::IRQNumberType,\n    ) -> Result<(), &'static str> {\n        use exception::asynchronous::{irq_manager, IRQHandlerDescriptor};\n\n        let descriptor = IRQHandlerDescriptor::new(*irq_number, Self::COMPATIBLE, self);\n\n        irq_manager().register_handler(descriptor)?;\n        irq_manager().enable(irq_number);\n\n        Ok(())\n    }\n}\n\nimpl console::interface::Write for PL011Uart {\n    /// Passthrough of `args` to the `core::fmt::Write` implementation, but guarded by a Mutex to\n    /// serialize access.\n    fn write_char(&self, c: char) {\n        self.inner.lock(|inner| inner.write_char(c));\n    }\n\n    fn write_array(&self, a: &[char]) {\n        self.inner.lock(|inner| inner.write_array(a));\n    }\n\n    fn write_fmt(&self, args: core::fmt::Arguments) -> fmt::Result {\n        // Fully qualified syntax for the call to `core::fmt::Write::write_fmt()` to increase\n        // readability.\n        self.inner.lock(|inner| fmt::Write::write_fmt(inner, args))\n    }\n\n    fn flush(&self) {\n        // Spin until TX FIFO empty is set.\n        self.inner.lock(|inner| inner.flush());\n    }\n}\n\nimpl console::interface::Read for PL011Uart {\n    fn read_char(&self) -> char {\n        self.inner\n            .lock(|inner| inner.read_char_converting(BlockingMode::Blocking).unwrap())\n    }\n\n    fn clear_rx(&self) {\n        // Read from the RX FIFO until it is indicating empty.\n        while self\n            .inner\n            .lock(|inner| inner.read_char_converting(BlockingMode::NonBlocking))\n            .is_some()\n        {}\n    }\n}\n\nimpl console::interface::Statistics for PL011Uart {\n    fn chars_written(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_written)\n    }\n\n    fn chars_read(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_read)\n    }\n}\n\nimpl console::interface::All for PL011Uart {}\n\nimpl exception::asynchronous::interface::IRQHandler for PL011Uart {\n    fn handle(&self) -> Result<(), &'static str> {\n        self.inner.lock(|inner| {\n            let pending = inner.registers.MIS.extract();\n\n            // Clear all pending IRQs.\n            inner.registers.ICR.write(ICR::ALL::CLEAR);\n\n            // Check for any kind of RX interrupt.\n            if pending.matches_any(MIS::RXMIS::SET + MIS::RTMIS::SET) {\n                // Echo any received characters.\n                while let Some(c) = inner.read_char_converting(BlockingMode::NonBlocking) {\n                    inner.write_char(c)\n                }\n            }\n        });\n\n        Ok(())\n    }\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/bsp/device_driver/bcm.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BCM driver top level.\n\nmod bcm2xxx_gpio;\n#[cfg(feature = \"bsp_rpi3\")]\nmod bcm2xxx_interrupt_controller;\nmod bcm2xxx_pl011_uart;\n\npub use bcm2xxx_gpio::*;\n#[cfg(feature = \"bsp_rpi3\")]\npub use bcm2xxx_interrupt_controller::*;\npub use bcm2xxx_pl011_uart::*;\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/bsp/device_driver/common.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Common device driver code.\n\nuse crate::memory::{Address, Virtual};\nuse core::{fmt, marker::PhantomData, ops};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct MMIODerefWrapper<T> {\n    start_addr: Address<Virtual>,\n    phantom: PhantomData<fn() -> T>,\n}\n\n/// A wrapper type for usize with integrated range bound check.\n#[derive(Copy, Clone)]\npub struct BoundedUsize<const MAX_INCLUSIVE: usize>(usize);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T> MMIODerefWrapper<T> {\n    /// Create an instance.\n    pub const unsafe fn new(start_addr: Address<Virtual>) -> Self {\n        Self {\n            start_addr,\n            phantom: PhantomData,\n        }\n    }\n}\n\nimpl<T> ops::Deref for MMIODerefWrapper<T> {\n    type Target = T;\n\n    fn deref(&self) -> &Self::Target {\n        unsafe { &*(self.start_addr.as_usize() as *const _) }\n    }\n}\n\nimpl<const MAX_INCLUSIVE: usize> BoundedUsize<{ MAX_INCLUSIVE }> {\n    pub const MAX_INCLUSIVE: usize = MAX_INCLUSIVE;\n\n    /// Creates a new instance if number <= MAX_INCLUSIVE.\n    pub const fn new(number: usize) -> Self {\n        assert!(number <= MAX_INCLUSIVE);\n\n        Self(number)\n    }\n\n    /// Return the wrapped number.\n    pub const fn get(self) -> usize {\n        self.0\n    }\n}\n\nimpl<const MAX_INCLUSIVE: usize> fmt::Display for BoundedUsize<{ MAX_INCLUSIVE }> {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        write!(f, \"{}\", self.0)\n    }\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/bsp/device_driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Device driver.\n\n#[cfg(feature = \"bsp_rpi4\")]\nmod arm;\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod bcm;\nmod common;\n\n#[cfg(feature = \"bsp_rpi4\")]\npub use arm::*;\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use bcm::*;\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/bsp/raspberrypi/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Processor code.\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Used by `arch` code to find the early boot core.\n#[no_mangle]\n#[link_section = \".text._start_arguments\"]\npub static BOOT_CORE_ID: u64 = 0;\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/bsp/raspberrypi/driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP driver support.\n\nuse super::{exception, memory::map::mmio};\nuse crate::{\n    bsp::device_driver,\n    console, driver as generic_driver,\n    exception::{self as generic_exception},\n    memory,\n    memory::mmu::MMIODescriptor,\n};\nuse core::{\n    mem::MaybeUninit,\n    sync::atomic::{AtomicBool, Ordering},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic mut PL011_UART: MaybeUninit<device_driver::PL011Uart> = MaybeUninit::uninit();\nstatic mut GPIO: MaybeUninit<device_driver::GPIO> = MaybeUninit::uninit();\n\n#[cfg(feature = \"bsp_rpi3\")]\nstatic mut INTERRUPT_CONTROLLER: MaybeUninit<device_driver::InterruptController> =\n    MaybeUninit::uninit();\n\n#[cfg(feature = \"bsp_rpi4\")]\nstatic mut INTERRUPT_CONTROLLER: MaybeUninit<device_driver::GICv2> = MaybeUninit::uninit();\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// This must be called only after successful init of the memory subsystem.\nunsafe fn instantiate_uart() -> Result<(), &'static str> {\n    let mmio_descriptor = MMIODescriptor::new(mmio::PL011_UART_START, mmio::PL011_UART_SIZE);\n    let virt_addr =\n        memory::mmu::kernel_map_mmio(device_driver::PL011Uart::COMPATIBLE, &mmio_descriptor)?;\n\n    PL011_UART.write(device_driver::PL011Uart::new(virt_addr));\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the UART driver.\nunsafe fn post_init_uart() -> Result<(), &'static str> {\n    console::register_console(PL011_UART.assume_init_ref());\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the memory subsystem.\nunsafe fn instantiate_gpio() -> Result<(), &'static str> {\n    let mmio_descriptor = MMIODescriptor::new(mmio::GPIO_START, mmio::GPIO_SIZE);\n    let virt_addr =\n        memory::mmu::kernel_map_mmio(device_driver::GPIO::COMPATIBLE, &mmio_descriptor)?;\n\n    GPIO.write(device_driver::GPIO::new(virt_addr));\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the GPIO driver.\nunsafe fn post_init_gpio() -> Result<(), &'static str> {\n    GPIO.assume_init_ref().map_pl011_uart();\n    Ok(())\n}\n\n/// This must be called only after successful init of the memory subsystem.\n#[cfg(feature = \"bsp_rpi3\")]\nunsafe fn instantiate_interrupt_controller() -> Result<(), &'static str> {\n    let local_mmio_descriptor = MMIODescriptor::new(mmio::LOCAL_IC_START, mmio::LOCAL_IC_SIZE);\n    let local_virt_addr = memory::mmu::kernel_map_mmio(\n        device_driver::InterruptController::COMPATIBLE,\n        &local_mmio_descriptor,\n    )?;\n\n    let periph_mmio_descriptor =\n        MMIODescriptor::new(mmio::PERIPHERAL_IC_START, mmio::PERIPHERAL_IC_SIZE);\n    let periph_virt_addr = memory::mmu::kernel_map_mmio(\n        device_driver::InterruptController::COMPATIBLE,\n        &periph_mmio_descriptor,\n    )?;\n\n    INTERRUPT_CONTROLLER.write(device_driver::InterruptController::new(\n        local_virt_addr,\n        periph_virt_addr,\n    ));\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the memory subsystem.\n#[cfg(feature = \"bsp_rpi4\")]\nunsafe fn instantiate_interrupt_controller() -> Result<(), &'static str> {\n    let gicd_mmio_descriptor = MMIODescriptor::new(mmio::GICD_START, mmio::GICD_SIZE);\n    let gicd_virt_addr = memory::mmu::kernel_map_mmio(\"GICv2 GICD\", &gicd_mmio_descriptor)?;\n\n    let gicc_mmio_descriptor = MMIODescriptor::new(mmio::GICC_START, mmio::GICC_SIZE);\n    let gicc_virt_addr = memory::mmu::kernel_map_mmio(\"GICV2 GICC\", &gicc_mmio_descriptor)?;\n\n    INTERRUPT_CONTROLLER.write(device_driver::GICv2::new(gicd_virt_addr, gicc_virt_addr));\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the interrupt controller driver.\nunsafe fn post_init_interrupt_controller() -> Result<(), &'static str> {\n    generic_exception::asynchronous::register_irq_manager(INTERRUPT_CONTROLLER.assume_init_ref());\n\n    Ok(())\n}\n\n/// Function needs to ensure that driver registration happens only after correct instantiation.\nunsafe fn driver_uart() -> Result<(), &'static str> {\n    instantiate_uart()?;\n\n    let uart_descriptor = generic_driver::DeviceDriverDescriptor::new(\n        PL011_UART.assume_init_ref(),\n        Some(post_init_uart),\n        Some(exception::asynchronous::irq_map::PL011_UART),\n    );\n    generic_driver::driver_manager().register_driver(uart_descriptor);\n\n    Ok(())\n}\n\n/// Function needs to ensure that driver registration happens only after correct instantiation.\nunsafe fn driver_gpio() -> Result<(), &'static str> {\n    instantiate_gpio()?;\n\n    let gpio_descriptor = generic_driver::DeviceDriverDescriptor::new(\n        GPIO.assume_init_ref(),\n        Some(post_init_gpio),\n        None,\n    );\n    generic_driver::driver_manager().register_driver(gpio_descriptor);\n\n    Ok(())\n}\n\n/// Function needs to ensure that driver registration happens only after correct instantiation.\nunsafe fn driver_interrupt_controller() -> Result<(), &'static str> {\n    instantiate_interrupt_controller()?;\n\n    let interrupt_controller_descriptor = generic_driver::DeviceDriverDescriptor::new(\n        INTERRUPT_CONTROLLER.assume_init_ref(),\n        Some(post_init_interrupt_controller),\n        None,\n    );\n    generic_driver::driver_manager().register_driver(interrupt_controller_descriptor);\n\n    Ok(())\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Initialize the driver subsystem.\n///\n/// # Safety\n///\n/// See child function calls.\npub unsafe fn init() -> Result<(), &'static str> {\n    static INIT_DONE: AtomicBool = AtomicBool::new(false);\n    if INIT_DONE.load(Ordering::Relaxed) {\n        return Err(\"Init already done\");\n    }\n\n    driver_uart()?;\n    driver_gpio()?;\n    driver_interrupt_controller()?;\n\n    INIT_DONE.store(true, Ordering::Relaxed);\n    Ok(())\n}\n\n/// Minimal code needed to bring up the console in QEMU (for testing only). This is often less steps\n/// than on real hardware due to QEMU's abstractions.\n#[cfg(feature = \"test_build\")]\npub fn qemu_bring_up_console() {\n    use crate::cpu;\n\n    unsafe {\n        instantiate_uart().unwrap_or_else(|_| cpu::qemu_exit_failure());\n        console::register_console(PL011_UART.assume_init_ref());\n    };\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/bsp/raspberrypi/exception/asynchronous.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP asynchronous exception handling.\n\nuse crate::bsp;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Export for reuse in generic asynchronous.rs.\npub use bsp::device_driver::IRQNumber;\n\n/// The IRQ map.\n#[cfg(feature = \"bsp_rpi3\")]\npub mod irq_map {\n    use super::bsp::device_driver::{IRQNumber, LocalIRQ, PeripheralIRQ};\n\n    /// The non-secure physical timer IRQ number.\n    pub const ARM_NS_PHYSICAL_TIMER: IRQNumber = IRQNumber::Local(LocalIRQ::new(1));\n\n    pub(in crate::bsp) const PL011_UART: IRQNumber = IRQNumber::Peripheral(PeripheralIRQ::new(57));\n}\n\n/// The IRQ map.\n#[cfg(feature = \"bsp_rpi4\")]\npub mod irq_map {\n    use super::bsp::device_driver::IRQNumber;\n\n    /// The non-secure physical timer IRQ number.\n    pub const ARM_NS_PHYSICAL_TIMER: IRQNumber = IRQNumber::new(30);\n\n    pub(in crate::bsp) const PL011_UART: IRQNumber = IRQNumber::new(153);\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/bsp/raspberrypi/exception.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP synchronous and asynchronous exception handling.\n\npub mod asynchronous;\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/bsp/raspberrypi/kernel.ld",
    "content": "/* SPDX-License-Identifier: MIT OR Apache-2.0\n *\n * Copyright (c) 2018-2022 Andre Richter <andre.o.richter@gmail.com>\n */\n\nINCLUDE kernel_virt_addr_space_size.ld;\n\nPAGE_SIZE = 64K;\nPAGE_MASK = PAGE_SIZE - 1;\n\n/* The kernel's virtual address range will be:\n *\n * [END_ADDRESS_INCLUSIVE, START_ADDRESS]\n * [u64::MAX             , (u64::MAX - __kernel_virt_addr_space_size) + 1]\n */\n__kernel_virt_start_addr = ((0xffffffffffffffff - __kernel_virt_addr_space_size) + 1);\n\n__rpi_phys_dram_start_addr = 0;\n\n/* The physical address at which the the kernel binary will be loaded by the Raspberry's firmware */\n__rpi_phys_binary_load_addr = 0x80000;\n\n\nENTRY(__rpi_phys_binary_load_addr)\n\n/* Flags:\n *     4 == R\n *     5 == RX\n *     6 == RW\n *\n * Segments are marked PT_LOAD below so that the ELF file provides virtual and physical addresses.\n * It doesn't mean all of them need actually be loaded.\n */\nPHDRS\n{\n    segment_code            PT_LOAD FLAGS(5);\n    segment_data            PT_LOAD FLAGS(6);\n    segment_heap            PT_LOAD FLAGS(6);\n    segment_boot_core_stack PT_LOAD FLAGS(6);\n}\n\nSECTIONS\n{\n    . =  __kernel_virt_start_addr;\n\n    ASSERT((. & PAGE_MASK) == 0, \"Start of address space is not page aligned\")\n\n    /***********************************************************************************************\n    * Code + RO Data + Global Offset Table\n    ***********************************************************************************************/\n    __code_start = .;\n    .text : AT(__rpi_phys_binary_load_addr)\n    {\n        KEEP(*(.text._start))\n        *(.text._start_arguments) /* Constants (or statics in Rust speak) read by _start(). */\n        *(.text._start_rust)      /* The Rust entry point */\n        *(.text*)                 /* Everything else */\n    } :segment_code\n\n    .rodata         : ALIGN(8) { *(.rodata*) } :segment_code\n    .kernel_symbols : ALIGN(8) {\n        __kernel_symbols_start = .;\n        . += 32 * 1024;\n    } :segment_code\n\n    . = ALIGN(PAGE_SIZE);\n    __code_end_exclusive = .;\n\n    /***********************************************************************************************\n    * Data + BSS\n    ***********************************************************************************************/\n    __data_start = .;\n    .data : { *(.data*) } :segment_data\n\n    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */\n    .bss (NOLOAD) : ALIGN(16)\n    {\n        __bss_start = .;\n        *(.bss*);\n        . = ALIGN(16);\n        __bss_end_exclusive = .;\n    } :segment_data\n\n    . = ALIGN(PAGE_SIZE);\n    __data_end_exclusive = .;\n\n    /***********************************************************************************************\n    * Heap\n    ***********************************************************************************************/\n    __heap_start = .;\n    .heap (NOLOAD) :\n    {\n        . += 16 * 1024 * 1024;\n    } :segment_heap\n    __heap_end_exclusive = .;\n\n    ASSERT((. & PAGE_MASK) == 0, \"Heap is not page aligned\")\n\n    /***********************************************************************************************\n    * MMIO Remap Reserved\n    ***********************************************************************************************/\n    __mmio_remap_start = .;\n    . += 8 * 1024 * 1024;\n    __mmio_remap_end_exclusive = .;\n\n    ASSERT((. & PAGE_MASK) == 0, \"MMIO remap reservation is not page aligned\")\n\n    /***********************************************************************************************\n    * Guard Page\n    ***********************************************************************************************/\n    . += PAGE_SIZE;\n\n    /***********************************************************************************************\n    * Boot Core Stack\n    ***********************************************************************************************/\n    .boot_core_stack (NOLOAD) : AT(__rpi_phys_dram_start_addr)\n    {\n        __boot_core_stack_start = .;         /*   ^             */\n                                             /*   | stack       */\n        . += __rpi_phys_binary_load_addr;    /*   | growth      */\n                                             /*   | direction   */\n        __boot_core_stack_end_exclusive = .; /*   |             */\n    } :segment_boot_core_stack\n\n    ASSERT((. & PAGE_MASK) == 0, \"End of boot core stack is not page aligned\")\n\n    /***********************************************************************************************\n    * Misc\n    ***********************************************************************************************/\n    .got : { *(.got*) }\n    ASSERT(SIZEOF(.got) == 0, \"Relocation support not expected\")\n\n    /DISCARD/ : { *(.comment*) }\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/bsp/raspberrypi/kernel_virt_addr_space_size.ld",
    "content": "__kernel_virt_addr_space_size = 1024 * 1024 * 1024\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/bsp/raspberrypi/memory/mmu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Memory Management Unit.\n\nuse crate::{\n    memory::{\n        mmu::{\n            self as generic_mmu, AddressSpace, AssociatedTranslationTable, AttributeFields,\n            MemoryRegion, PageAddress, TranslationGranule,\n        },\n        Physical, Virtual,\n    },\n    synchronization::InitStateLock,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\ntype KernelTranslationTable =\n    <KernelVirtAddrSpace as AssociatedTranslationTable>::TableStartFromTop;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// The translation granule chosen by this BSP. This will be used everywhere else in the kernel to\n/// derive respective data structures and their sizes. For example, the `crate::memory::mmu::Page`.\npub type KernelGranule = TranslationGranule<{ 64 * 1024 }>;\n\n/// The kernel's virtual address space defined by this BSP.\npub type KernelVirtAddrSpace = AddressSpace<{ kernel_virt_addr_space_size() }>;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n/// The kernel translation tables.\n///\n/// It is mandatory that InitStateLock is transparent.\n///\n/// That is, `size_of(InitStateLock<KernelTranslationTable>) == size_of(KernelTranslationTable)`.\n/// There is a unit tests that checks this porperty.\n#[link_section = \".data\"]\n#[no_mangle]\nstatic KERNEL_TABLES: InitStateLock<KernelTranslationTable> =\n    InitStateLock::new(KernelTranslationTable::new_for_precompute());\n\n/// This value is needed during early boot for MMU setup.\n///\n/// This will be patched to the correct value by the \"translation table tool\" after linking. This\n/// given value here is just a dummy.\n#[link_section = \".text._start_arguments\"]\n#[no_mangle]\nstatic PHYS_KERNEL_TABLES_BASE_ADDR: u64 = 0xCCCCAAAAFFFFEEEE;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// This is a hack for retrieving the value for the kernel's virtual address space size as a\n/// constant from a common place, since it is needed as a compile-time/link-time constant in both,\n/// the linker script and the Rust sources.\n#[allow(clippy::needless_late_init)]\nconst fn kernel_virt_addr_space_size() -> usize {\n    let __kernel_virt_addr_space_size;\n\n    include!(\"../kernel_virt_addr_space_size.ld\");\n\n    __kernel_virt_addr_space_size\n}\n\n/// Helper function for calculating the number of pages the given parameter spans.\nconst fn size_to_num_pages(size: usize) -> usize {\n    assert!(size > 0);\n    assert!(size % KernelGranule::SIZE == 0);\n\n    size >> KernelGranule::SHIFT\n}\n\n/// The data pages of the kernel binary.\nfn virt_data_region() -> MemoryRegion<Virtual> {\n    let num_pages = size_to_num_pages(super::data_size());\n\n    let start_page_addr = super::virt_data_start();\n    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n\n    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n}\n\n// There is no reason to expect the following conversions to fail, since they were generated offline\n// by the `translation table tool`. If it doesn't work, a panic due to the unwraps is justified.\nfn kernel_virt_to_phys_region(virt_region: MemoryRegion<Virtual>) -> MemoryRegion<Physical> {\n    let phys_start_page_addr =\n        generic_mmu::try_kernel_virt_page_addr_to_phys_page_addr(virt_region.start_page_addr())\n            .unwrap();\n\n    let phys_end_exclusive_page_addr = phys_start_page_addr\n        .checked_offset(virt_region.num_pages() as isize)\n        .unwrap();\n\n    MemoryRegion::new(phys_start_page_addr, phys_end_exclusive_page_addr)\n}\n\nfn kernel_page_attributes(virt_page_addr: PageAddress<Virtual>) -> AttributeFields {\n    generic_mmu::try_kernel_page_attributes(virt_page_addr).unwrap()\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The code pages of the kernel binary.\npub fn virt_code_region() -> MemoryRegion<Virtual> {\n    let num_pages = size_to_num_pages(super::code_size());\n\n    let start_page_addr = super::virt_code_start();\n    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n\n    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n}\n\n/// The heap pages.\npub fn virt_heap_region() -> MemoryRegion<Virtual> {\n    let num_pages = size_to_num_pages(super::heap_size());\n\n    let start_page_addr = super::virt_heap_start();\n    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n\n    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n}\n\n/// The boot core stack pages.\npub fn virt_boot_core_stack_region() -> MemoryRegion<Virtual> {\n    let num_pages = size_to_num_pages(super::boot_core_stack_size());\n\n    let start_page_addr = super::virt_boot_core_stack_start();\n    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n\n    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n}\n\n/// Return a reference to the kernel's translation tables.\npub fn kernel_translation_tables() -> &'static InitStateLock<KernelTranslationTable> {\n    &KERNEL_TABLES\n}\n\n/// The MMIO remap pages.\npub fn virt_mmio_remap_region() -> MemoryRegion<Virtual> {\n    let num_pages = size_to_num_pages(super::mmio_remap_size());\n\n    let start_page_addr = super::virt_mmio_remap_start();\n    let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap();\n\n    MemoryRegion::new(start_page_addr, end_exclusive_page_addr)\n}\n\n/// Add mapping records for the kernel binary.\n///\n/// The actual translation table entries for the kernel binary are generated using the offline\n/// `translation table tool` and patched into the kernel binary. This function just adds the mapping\n/// record entries.\npub fn kernel_add_mapping_records_for_precomputed() {\n    let virt_code_region = virt_code_region();\n    generic_mmu::kernel_add_mapping_record(\n        \"Kernel code and RO data\",\n        &virt_code_region,\n        &kernel_virt_to_phys_region(virt_code_region),\n        &kernel_page_attributes(virt_code_region.start_page_addr()),\n    );\n\n    let virt_data_region = virt_data_region();\n    generic_mmu::kernel_add_mapping_record(\n        \"Kernel data and bss\",\n        &virt_data_region,\n        &kernel_virt_to_phys_region(virt_data_region),\n        &kernel_page_attributes(virt_data_region.start_page_addr()),\n    );\n\n    let virt_heap_region = virt_heap_region();\n    generic_mmu::kernel_add_mapping_record(\n        \"Kernel heap\",\n        &virt_heap_region,\n        &kernel_virt_to_phys_region(virt_heap_region),\n        &kernel_page_attributes(virt_heap_region.start_page_addr()),\n    );\n\n    let virt_boot_core_stack_region = virt_boot_core_stack_region();\n    generic_mmu::kernel_add_mapping_record(\n        \"Kernel boot-core stack\",\n        &virt_boot_core_stack_region,\n        &kernel_virt_to_phys_region(virt_boot_core_stack_region),\n        &kernel_page_attributes(virt_boot_core_stack_region.start_page_addr()),\n    );\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/bsp/raspberrypi/memory.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Memory Management.\n//!\n//! The physical memory layout.\n//!\n//! The Raspberry's firmware copies the kernel binary to 0x8_0000. The preceding region will be used\n//! as the boot core's stack.\n//!\n//! +---------------------------------------+\n//! |                                       | boot_core_stack_start @ 0x0\n//! |                                       |                                ^\n//! | Boot-core Stack                       |                                | stack\n//! |                                       |                                | growth\n//! |                                       |                                | direction\n//! +---------------------------------------+\n//! |                                       | code_start @ 0x8_0000 == boot_core_stack_end_exclusive\n//! | .text                                 |\n//! | .rodata                               |\n//! | .got                                  |\n//! | .kernel_symbols                       |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       | data_start == code_end_exclusive\n//! | .data                                 |\n//! | .bss                                  |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       | heap_start == data_end_exclusive\n//! | .heap                                 |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       | heap_end_exclusive\n//! |                                       |\n//!\n//!\n//!\n//!\n//!\n//! The virtual memory layout is as follows:\n//!\n//! +---------------------------------------+\n//! |                                       | code_start @ __kernel_virt_start_addr\n//! | .text                                 |\n//! | .rodata                               |\n//! | .got                                  |\n//! | .kernel_symbols                       |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       | data_start == code_end_exclusive\n//! | .data                                 |\n//! | .bss                                  |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       | heap_start == data_end_exclusive\n//! | .heap                                 |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       |  mmio_remap_start == heap_end_exclusive\n//! | VA region for MMIO remapping          |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       |  mmio_remap_end_exclusive\n//! | Unmapped guard page                   |\n//! |                                       |\n//! +---------------------------------------+\n//! |                                       | boot_core_stack_start\n//! |                                       |                                ^\n//! | Boot-core Stack                       |                                | stack\n//! |                                       |                                | growth\n//! |                                       |                                | direction\n//! +---------------------------------------+\n//! |                                       | boot_core_stack_end_exclusive\n//! |                                       |\npub mod mmu;\n\nuse crate::memory::{mmu::PageAddress, Address, Physical, Virtual};\nuse core::cell::UnsafeCell;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Symbols from the linker script.\nextern \"Rust\" {\n    static __code_start: UnsafeCell<()>;\n    static __code_end_exclusive: UnsafeCell<()>;\n\n    static __data_start: UnsafeCell<()>;\n    static __data_end_exclusive: UnsafeCell<()>;\n\n    static __heap_start: UnsafeCell<()>;\n    static __heap_end_exclusive: UnsafeCell<()>;\n\n    static __mmio_remap_start: UnsafeCell<()>;\n    static __mmio_remap_end_exclusive: UnsafeCell<()>;\n\n    static __boot_core_stack_start: UnsafeCell<()>;\n    static __boot_core_stack_end_exclusive: UnsafeCell<()>;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// The board's physical memory map.\n#[rustfmt::skip]\npub(super) mod map {\n    use super::*;\n\n    /// Physical devices.\n    #[cfg(feature = \"bsp_rpi3\")]\n    pub mod mmio {\n        use super::*;\n\n        pub const PERIPHERAL_IC_START: Address<Physical> = Address::new(0x3F00_B200);\n        pub const PERIPHERAL_IC_SIZE:  usize             =              0x24;\n\n        pub const GPIO_START:          Address<Physical> = Address::new(0x3F20_0000);\n        pub const GPIO_SIZE:           usize             =              0xA0;\n\n        pub const PL011_UART_START:    Address<Physical> = Address::new(0x3F20_1000);\n        pub const PL011_UART_SIZE:     usize             =              0x48;\n\n        pub const LOCAL_IC_START:      Address<Physical> = Address::new(0x4000_0000);\n        pub const LOCAL_IC_SIZE:       usize             =              0x100;\n\n        pub const END:                 Address<Physical> = Address::new(0x4001_0000);\n    }\n\n    /// Physical devices.\n    #[cfg(feature = \"bsp_rpi4\")]\n    pub mod mmio {\n        use super::*;\n\n        pub const GPIO_START:       Address<Physical> = Address::new(0xFE20_0000);\n        pub const GPIO_SIZE:        usize             =              0xA0;\n\n        pub const PL011_UART_START: Address<Physical> = Address::new(0xFE20_1000);\n        pub const PL011_UART_SIZE:  usize             =              0x48;\n\n        pub const GICD_START:       Address<Physical> = Address::new(0xFF84_1000);\n        pub const GICD_SIZE:        usize             =              0x824;\n\n        pub const GICC_START:       Address<Physical> = Address::new(0xFF84_2000);\n        pub const GICC_SIZE:        usize             =              0x14;\n\n        pub const END:              Address<Physical> = Address::new(0xFF85_0000);\n    }\n\n    pub const END: Address<Physical> = mmio::END;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Start page address of the code segment.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn virt_code_start() -> PageAddress<Virtual> {\n    PageAddress::from(unsafe { __code_start.get() as usize })\n}\n\n/// Size of the code segment.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn code_size() -> usize {\n    unsafe { (__code_end_exclusive.get() as usize) - (__code_start.get() as usize) }\n}\n\n/// Start page address of the data segment.\n#[inline(always)]\nfn virt_data_start() -> PageAddress<Virtual> {\n    PageAddress::from(unsafe { __data_start.get() as usize })\n}\n\n/// Size of the data segment.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn data_size() -> usize {\n    unsafe { (__data_end_exclusive.get() as usize) - (__data_start.get() as usize) }\n}\n\n/// Start page address of the heap segment.\n#[inline(always)]\nfn virt_heap_start() -> PageAddress<Virtual> {\n    PageAddress::from(unsafe { __heap_start.get() as usize })\n}\n\n/// Size of the heap segment.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn heap_size() -> usize {\n    unsafe { (__heap_end_exclusive.get() as usize) - (__heap_start.get() as usize) }\n}\n\n/// Start page address of the MMIO remap reservation.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn virt_mmio_remap_start() -> PageAddress<Virtual> {\n    PageAddress::from(unsafe { __mmio_remap_start.get() as usize })\n}\n\n/// Size of the MMIO remap reservation.\n///\n/// # Safety\n///\n/// - Value is provided by the linker script and must be trusted as-is.\n#[inline(always)]\nfn mmio_remap_size() -> usize {\n    unsafe { (__mmio_remap_end_exclusive.get() as usize) - (__mmio_remap_start.get() as usize) }\n}\n\n/// Start page address of the boot core's stack.\n#[inline(always)]\nfn virt_boot_core_stack_start() -> PageAddress<Virtual> {\n    PageAddress::from(unsafe { __boot_core_stack_start.get() as usize })\n}\n\n/// Size of the boot core's stack.\n#[inline(always)]\nfn boot_core_stack_size() -> usize {\n    unsafe {\n        (__boot_core_stack_end_exclusive.get() as usize) - (__boot_core_stack_start.get() as usize)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Exclusive end address of the physical address space.\n#[inline(always)]\npub fn phys_addr_space_end_exclusive_addr() -> PageAddress<Physical> {\n    PageAddress::from(map::END)\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/bsp/raspberrypi.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Top-level BSP file for the Raspberry Pi 3 and 4.\n\npub mod cpu;\npub mod driver;\npub mod exception;\npub mod memory;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Board identification.\npub fn board_name() -> &'static str {\n    #[cfg(feature = \"bsp_rpi3\")]\n    {\n        \"Raspberry Pi 3\"\n    }\n\n    #[cfg(feature = \"bsp_rpi4\")]\n    {\n        \"Raspberry Pi 4\"\n    }\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/bsp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Conditional reexporting of Board Support Packages.\n\nmod device_driver;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod raspberrypi;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use raspberrypi::*;\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/common.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! General purpose code.\n\n/// Check if a value is aligned to a given size.\n#[inline(always)]\npub const fn is_aligned(value: usize, alignment: usize) -> bool {\n    assert!(alignment.is_power_of_two());\n\n    (value & (alignment - 1)) == 0\n}\n\n/// Align down.\n#[inline(always)]\npub const fn align_down(value: usize, alignment: usize) -> usize {\n    assert!(alignment.is_power_of_two());\n\n    value & !(alignment - 1)\n}\n\n/// Align up.\n#[inline(always)]\npub const fn align_up(value: usize, alignment: usize) -> usize {\n    assert!(alignment.is_power_of_two());\n\n    (value + alignment - 1) & !(alignment - 1)\n}\n\n/// Convert a size into human readable format.\npub const fn size_human_readable_ceil(size: usize) -> (usize, &'static str) {\n    const KIB: usize = 1024;\n    const MIB: usize = 1024 * 1024;\n    const GIB: usize = 1024 * 1024 * 1024;\n\n    if (size / GIB) > 0 {\n        (size.div_ceil(GIB), \"GiB\")\n    } else if (size / MIB) > 0 {\n        (size.div_ceil(MIB), \"MiB\")\n    } else if (size / KIB) > 0 {\n        (size.div_ceil(KIB), \"KiB\")\n    } else {\n        (size, \"Byte\")\n    }\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/console/buffer_console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A console that buffers input during the init phase.\n\nuse super::interface;\nuse crate::{console, info, synchronization, synchronization::InitStateLock};\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nconst BUF_SIZE: usize = 1024 * 64;\n\npub struct BufferConsoleInner {\n    buf: [char; BUF_SIZE],\n    write_ptr: usize,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct BufferConsole {\n    inner: InitStateLock<BufferConsoleInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\npub static BUFFER_CONSOLE: BufferConsole = BufferConsole {\n    inner: InitStateLock::new(BufferConsoleInner {\n        // Use the null character, so this lands in .bss and does not waste space in the binary.\n        buf: ['\\0'; BUF_SIZE],\n        write_ptr: 0,\n    }),\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl BufferConsoleInner {\n    fn write_char(&mut self, c: char) {\n        if self.write_ptr < (BUF_SIZE - 1) {\n            self.buf[self.write_ptr] = c;\n            self.write_ptr += 1;\n        }\n    }\n}\n\nimpl fmt::Write for BufferConsoleInner {\n    fn write_str(&mut self, s: &str) -> fmt::Result {\n        for c in s.chars() {\n            self.write_char(c);\n        }\n\n        Ok(())\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::interface::ReadWriteEx;\n\nimpl BufferConsole {\n    /// Dump the buffer.\n    ///\n    /// # Invariant\n    ///\n    /// It is expected that this is only called when self != crate::console::console().\n    pub fn dump(&self) {\n        self.inner.read(|inner| {\n            console::console().write_array(&inner.buf[0..inner.write_ptr]);\n\n            if inner.write_ptr == (BUF_SIZE - 1) {\n                info!(\"Pre-UART buffer overflowed\");\n            } else if inner.write_ptr > 0 {\n                info!(\"End of pre-UART buffer\")\n            }\n        });\n    }\n}\n\nimpl interface::Write for BufferConsole {\n    fn write_char(&self, c: char) {\n        self.inner.write(|inner| inner.write_char(c));\n    }\n\n    fn write_array(&self, _a: &[char]) {}\n\n    fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result {\n        self.inner.write(|inner| fmt::Write::write_fmt(inner, args))\n    }\n\n    fn flush(&self) {}\n}\n\nimpl interface::Read for BufferConsole {\n    fn clear_rx(&self) {}\n}\n\nimpl interface::Statistics for BufferConsole {}\nimpl interface::All for BufferConsole {}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! System console.\n\nmod buffer_console;\n\nuse crate::synchronization;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Console interfaces.\npub mod interface {\n    use core::fmt;\n\n    /// Console write functions.\n    pub trait Write {\n        /// Write a single character.\n        fn write_char(&self, c: char);\n\n        /// Write a slice of characters.\n        fn write_array(&self, a: &[char]);\n\n        /// Write a Rust format string.\n        fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result;\n\n        /// Block until the last buffered character has been physically put on the TX wire.\n        fn flush(&self);\n    }\n\n    /// Console read functions.\n    pub trait Read {\n        /// Read a single character.\n        fn read_char(&self) -> char {\n            ' '\n        }\n\n        /// Clear RX buffers, if any.\n        fn clear_rx(&self);\n    }\n\n    /// Console statistics.\n    pub trait Statistics {\n        /// Return the number of characters written.\n        fn chars_written(&self) -> usize {\n            0\n        }\n\n        /// Return the number of characters read.\n        fn chars_read(&self) -> usize {\n            0\n        }\n    }\n\n    /// Trait alias for a full-fledged console.\n    pub trait All: Write + Read + Statistics {}\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic CUR_CONSOLE: InitStateLock<&'static (dyn interface::All + Sync)> =\n    InitStateLock::new(&buffer_console::BUFFER_CONSOLE);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::{interface::ReadWriteEx, InitStateLock};\n\n/// Register a new console.\npub fn register_console(new_console: &'static (dyn interface::All + Sync)) {\n    CUR_CONSOLE.write(|con| *con = new_console);\n\n    static FIRST_SWITCH: InitStateLock<bool> = InitStateLock::new(true);\n    FIRST_SWITCH.write(|first| {\n        if *first {\n            *first = false;\n\n            buffer_console::BUFFER_CONSOLE.dump();\n        }\n    });\n}\n\n/// Return a reference to the currently registered console.\n///\n/// This is the global console used by all printing macros.\npub fn console() -> &'static dyn interface::All {\n    CUR_CONSOLE.read(|con| *con)\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Boot code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/cpu/boot.rs\"]\nmod arch_boot;\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/cpu/smp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Symmetric multiprocessing.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/cpu/smp.rs\"]\nmod arch_smp;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_smp::core_id;\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Processor code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/cpu.rs\"]\nmod arch_cpu;\n\nmod boot;\n\npub mod smp;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_cpu::{nop, wait_forever};\n\n#[cfg(feature = \"test_build\")]\npub use arch_cpu::{qemu_exit_failure, qemu_exit_success};\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Driver support.\n\nuse crate::{\n    exception, info,\n    synchronization::{interface::ReadWriteEx, InitStateLock},\n};\nuse alloc::vec::Vec;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Driver interfaces.\npub mod interface {\n    /// Device Driver functions.\n    pub trait DeviceDriver {\n        /// Different interrupt controllers might use different types for IRQ number.\n        type IRQNumberType: super::fmt::Display;\n\n        /// Return a compatibility string for identifying the driver.\n        fn compatible(&self) -> &'static str;\n\n        /// Called by the kernel to bring up the device.\n        ///\n        /// # Safety\n        ///\n        /// - During init, drivers might do stuff with system-wide impact.\n        unsafe fn init(&self) -> Result<(), &'static str> {\n            Ok(())\n        }\n\n        /// Called by the kernel to register and enable the device's IRQ handler.\n        ///\n        /// Rust's type system will prevent a call to this function unless the calling instance\n        /// itself has static lifetime.\n        fn register_and_enable_irq_handler(\n            &'static self,\n            irq_number: &Self::IRQNumberType,\n        ) -> Result<(), &'static str> {\n            panic!(\n                \"Attempt to enable IRQ {} for device {}, but driver does not support this\",\n                irq_number,\n                self.compatible()\n            )\n        }\n    }\n}\n\n/// Tpye to be used as an optional callback after a driver's init() has run.\npub type DeviceDriverPostInitCallback = unsafe fn() -> Result<(), &'static str>;\n\n/// A descriptor for device drivers.\npub struct DeviceDriverDescriptor<T>\nwhere\n    T: 'static,\n{\n    device_driver: &'static (dyn interface::DeviceDriver<IRQNumberType = T> + Sync),\n    post_init_callback: Option<DeviceDriverPostInitCallback>,\n    irq_number: Option<T>,\n}\n\n/// Provides device driver management functions.\npub struct DriverManager<T>\nwhere\n    T: 'static,\n{\n    descriptors: InitStateLock<Vec<DeviceDriverDescriptor<T>>>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic DRIVER_MANAGER: DriverManager<exception::asynchronous::IRQNumber> = DriverManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T> DeviceDriverDescriptor<T> {\n    /// Create an instance.\n    pub fn new(\n        device_driver: &'static (dyn interface::DeviceDriver<IRQNumberType = T> + Sync),\n        post_init_callback: Option<DeviceDriverPostInitCallback>,\n        irq_number: Option<T>,\n    ) -> Self {\n        Self {\n            device_driver,\n            post_init_callback,\n            irq_number,\n        }\n    }\n}\n\n/// Return a reference to the global DriverManager.\npub fn driver_manager() -> &'static DriverManager<exception::asynchronous::IRQNumber> {\n    &DRIVER_MANAGER\n}\n\nimpl<T> DriverManager<T>\nwhere\n    T: fmt::Display,\n{\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            descriptors: InitStateLock::new(Vec::new()),\n        }\n    }\n\n    /// Register a device driver with the kernel.\n    pub fn register_driver(&self, descriptor: DeviceDriverDescriptor<T>) {\n        self.descriptors\n            .write(|descriptors| descriptors.push(descriptor));\n    }\n\n    /// Fully initialize all drivers and their interrupts handlers.\n    ///\n    /// # Safety\n    ///\n    /// - During init, drivers might do stuff with system-wide impact.\n    pub unsafe fn init_drivers_and_irqs(&self) {\n        self.descriptors.read(|descriptors| {\n            for descriptor in descriptors {\n                // 1. Initialize driver.\n                if let Err(x) = descriptor.device_driver.init() {\n                    panic!(\n                        \"Error initializing driver: {}: {}\",\n                        descriptor.device_driver.compatible(),\n                        x\n                    );\n                }\n\n                // 2. Call corresponding post init callback.\n                if let Some(callback) = &descriptor.post_init_callback {\n                    if let Err(x) = callback() {\n                        panic!(\n                            \"Error during driver post-init callback: {}: {}\",\n                            descriptor.device_driver.compatible(),\n                            x\n                        );\n                    }\n                }\n            }\n\n            // 3. After all post-init callbacks were done, the interrupt controller should be\n            //    registered and functional. So let drivers register with it now.\n            for descriptor in descriptors {\n                if let Some(irq_number) = &descriptor.irq_number {\n                    if let Err(x) = descriptor\n                        .device_driver\n                        .register_and_enable_irq_handler(irq_number)\n                    {\n                        panic!(\n                            \"Error during driver interrupt handler registration: {}: {}\",\n                            descriptor.device_driver.compatible(),\n                            x\n                        );\n                    }\n                }\n            }\n        })\n    }\n\n    /// Enumerate all registered device drivers.\n    pub fn enumerate(&self) {\n        self.descriptors.read(|descriptors| {\n            for (i, desc) in descriptors.iter().enumerate() {\n                info!(\"      {}. {}\", i + 1, desc.device_driver.compatible());\n            }\n        });\n    }\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/exception/asynchronous/null_irq_manager.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Null IRQ Manager.\n\nuse super::{interface, IRQContext, IRQHandlerDescriptor};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct NullIRQManager;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\npub static NULL_IRQ_MANAGER: NullIRQManager = NullIRQManager {};\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl interface::IRQManager for NullIRQManager {\n    type IRQNumberType = super::IRQNumber;\n\n    fn register_handler(\n        &self,\n        _descriptor: IRQHandlerDescriptor<Self::IRQNumberType>,\n    ) -> Result<(), &'static str> {\n        panic!(\"No IRQ Manager registered yet\");\n    }\n\n    fn enable(&self, _irq_number: &Self::IRQNumberType) {\n        panic!(\"No IRQ Manager registered yet\");\n    }\n\n    fn handle_pending_irqs<'irq_context>(&'irq_context self, _ic: &IRQContext<'irq_context>) {\n        panic!(\"No IRQ Manager registered yet\");\n    }\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/exception/asynchronous.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Asynchronous exception handling.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/exception/asynchronous.rs\"]\nmod arch_asynchronous;\nmod null_irq_manager;\n\nuse crate::{bsp, synchronization};\nuse core::marker::PhantomData;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_asynchronous::{\n    is_local_irq_masked, local_irq_mask, local_irq_mask_save, local_irq_restore, local_irq_unmask,\n    print_state,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Interrupt number as defined by the BSP.\npub type IRQNumber = bsp::exception::asynchronous::IRQNumber;\n\n/// Interrupt descriptor.\n#[derive(Copy, Clone)]\npub struct IRQHandlerDescriptor<T>\nwhere\n    T: Copy,\n{\n    /// The IRQ number.\n    number: T,\n\n    /// Descriptive name.\n    name: &'static str,\n\n    /// Reference to handler trait object.\n    handler: &'static (dyn interface::IRQHandler + Sync),\n}\n\n/// IRQContext token.\n///\n/// An instance of this type indicates that the local core is currently executing in IRQ\n/// context, aka executing an interrupt vector or subcalls of it.\n///\n/// Concept and implementation derived from the `CriticalSection` introduced in\n/// <https://github.com/rust-embedded/bare-metal>\n#[derive(Clone, Copy)]\npub struct IRQContext<'irq_context> {\n    _0: PhantomData<&'irq_context ()>,\n}\n\n/// Asynchronous exception handling interfaces.\npub mod interface {\n\n    /// Implemented by types that handle IRQs.\n    pub trait IRQHandler {\n        /// Called when the corresponding interrupt is asserted.\n        fn handle(&self) -> Result<(), &'static str>;\n    }\n\n    /// IRQ management functions.\n    ///\n    /// The `BSP` is supposed to supply one global instance. Typically implemented by the\n    /// platform's interrupt controller.\n    pub trait IRQManager {\n        /// The IRQ number type depends on the implementation.\n        type IRQNumberType: Copy;\n\n        /// Register a handler.\n        fn register_handler(\n            &self,\n            irq_handler_descriptor: super::IRQHandlerDescriptor<Self::IRQNumberType>,\n        ) -> Result<(), &'static str>;\n\n        /// Enable an interrupt in the controller.\n        fn enable(&self, irq_number: &Self::IRQNumberType);\n\n        /// Handle pending interrupts.\n        ///\n        /// This function is called directly from the CPU's IRQ exception vector. On AArch64,\n        /// this means that the respective CPU core has disabled exception handling.\n        /// This function can therefore not be preempted and runs start to finish.\n        ///\n        /// Takes an IRQContext token to ensure it can only be called from IRQ context.\n        #[allow(clippy::trivially_copy_pass_by_ref)]\n        fn handle_pending_irqs<'irq_context>(\n            &'irq_context self,\n            ic: &super::IRQContext<'irq_context>,\n        );\n\n        /// Print list of registered handlers.\n        fn print_handler(&self) {}\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic CUR_IRQ_MANAGER: InitStateLock<\n    &'static (dyn interface::IRQManager<IRQNumberType = IRQNumber> + Sync),\n> = InitStateLock::new(&null_irq_manager::NULL_IRQ_MANAGER);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::{interface::ReadWriteEx, InitStateLock};\n\nimpl<T> IRQHandlerDescriptor<T>\nwhere\n    T: Copy,\n{\n    /// Create an instance.\n    pub const fn new(\n        number: T,\n        name: &'static str,\n        handler: &'static (dyn interface::IRQHandler + Sync),\n    ) -> Self {\n        Self {\n            number,\n            name,\n            handler,\n        }\n    }\n\n    /// Return the number.\n    pub const fn number(&self) -> T {\n        self.number\n    }\n\n    /// Return the name.\n    pub const fn name(&self) -> &'static str {\n        self.name\n    }\n\n    /// Return the handler.\n    pub const fn handler(&self) -> &'static (dyn interface::IRQHandler + Sync) {\n        self.handler\n    }\n}\n\nimpl<'irq_context> IRQContext<'irq_context> {\n    /// Creates an IRQContext token.\n    ///\n    /// # Safety\n    ///\n    /// - This must only be called when the current core is in an interrupt context and will not\n    ///   live beyond the end of it. That is, creation is allowed in interrupt vector functions. For\n    ///   example, in the ARMv8-A case, in `extern \"C\" fn current_elx_irq()`.\n    /// - Note that the lifetime `'irq_context` of the returned instance is unconstrained. User code\n    ///   must not be able to influence the lifetime picked for this type, since that might cause it\n    ///   to be inferred to `'static`.\n    #[inline(always)]\n    pub unsafe fn new() -> Self {\n        IRQContext { _0: PhantomData }\n    }\n}\n\n/// Executes the provided closure while IRQs are masked on the executing core.\n///\n/// While the function temporarily changes the HW state of the executing core, it restores it to the\n/// previous state before returning, so this is deemed safe.\n#[inline(always)]\npub fn exec_with_irq_masked<T>(f: impl FnOnce() -> T) -> T {\n    let saved = local_irq_mask_save();\n    let ret = f();\n    local_irq_restore(saved);\n\n    ret\n}\n\n/// Register a new IRQ manager.\npub fn register_irq_manager(\n    new_manager: &'static (dyn interface::IRQManager<IRQNumberType = IRQNumber> + Sync),\n) {\n    CUR_IRQ_MANAGER.write(|manager| *manager = new_manager);\n}\n\n/// Return a reference to the currently registered IRQ manager.\n///\n/// This is the IRQ manager used by the architectural interrupt handling code.\npub fn irq_manager() -> &'static dyn interface::IRQManager<IRQNumberType = IRQNumber> {\n    CUR_IRQ_MANAGER.read(|manager| *manager)\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/exception.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Synchronous and asynchronous exception handling.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/exception.rs\"]\nmod arch_exception;\n\npub mod asynchronous;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_exception::{current_privilege_level, handling_init};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Kernel privilege levels.\n#[allow(missing_docs)]\n#[derive(Eq, PartialEq)]\npub enum PrivilegeLevel {\n    User,\n    Kernel,\n    Hypervisor,\n    Unknown,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// Libkernel unit tests must execute in kernel mode.\n    #[kernel_test]\n    fn test_runner_executes_in_kernel_mode() {\n        let (level, _) = current_privilege_level();\n\n        assert!(level == PrivilegeLevel::Kernel)\n    }\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/lib.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n// Rust embedded logo for `make doc`.\n#![doc(\n    html_logo_url = \"https://raw.githubusercontent.com/rust-embedded/wg/master/assets/logo/ewg-logo-blue-white-on-transparent.png\"\n)]\n\n//! The `kernel` library.\n//!\n//! Used to compose the final kernel binary.\n//!\n//! # Code organization and architecture\n//!\n//! The code is divided into different *modules*, each representing a typical **subsystem** of the\n//! `kernel`. Top-level module files of subsystems reside directly in the `src` folder. For example,\n//! `src/memory.rs` contains code that is concerned with all things memory management.\n//!\n//! ## Visibility of processor architecture code\n//!\n//! Some of the `kernel`'s subsystems depend on low-level code that is specific to the target\n//! processor architecture. For each supported processor architecture, there exists a subfolder in\n//! `src/_arch`, for example, `src/_arch/aarch64`.\n//!\n//! The architecture folders mirror the subsystem modules laid out in `src`. For example,\n//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go\n//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in\n//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic\n//! module's name prefixed with `arch_`.\n//!\n//! For example, this is the top of `src/memory/mmu.rs`:\n//!\n//! ```\n//! #[cfg(target_arch = \"aarch64\")]\n//! #[path = \"../_arch/aarch64/memory/mmu.rs\"]\n//! mod arch_mmu;\n//! ```\n//!\n//! Often times, items from the `arch_ module` will be publicly reexported by the parent module.\n//! This way, each architecture specific module can provide its implementation of an item, while the\n//! caller must not be concerned which architecture has been conditionally compiled.\n//!\n//! ## BSP code\n//!\n//! `BSP` stands for Board Support Package. `BSP` code is organized under `src/bsp.rs` and contains\n//! target board specific definitions and functions. These are things such as the board's memory map\n//! or instances of drivers for devices that are featured on the respective board.\n//!\n//! Just like processor architecture code, the `BSP` code's module structure tries to mirror the\n//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is\n//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`.\n//!\n//! ## Kernel interfaces\n//!\n//! Both `arch` and `bsp` contain code that is conditionally compiled depending on the actual target\n//! and board for which the kernel is compiled. For example, the `interrupt controller` hardware of\n//! the `Raspberry Pi 3` and the `Raspberry Pi 4` is different, but we want the rest of the `kernel`\n//! code to play nicely with any of the two without much hassle.\n//!\n//! In order to provide a clean abstraction between `arch`, `bsp` and `generic kernel code`,\n//! `interface` traits are provided *whenever possible* and *where it makes sense*. They are defined\n//! in the respective subsystem module and help to enforce the idiom of *program to an interface,\n//! not an implementation*. For example, there will be a common IRQ handling interface which the two\n//! different interrupt controller `drivers` of both Raspberrys will implement, and only export the\n//! interface to the rest of the `kernel`.\n//!\n//! ```\n//!         +-------------------+\n//!         | Interface (Trait) |\n//!         |                   |\n//!         +--+-------------+--+\n//!            ^             ^\n//!            |             |\n//!            |             |\n//! +----------+--+       +--+----------+\n//! | kernel code |       |  bsp code   |\n//! |             |       |  arch code  |\n//! +-------------+       +-------------+\n//! ```\n//!\n//! # Summary\n//!\n//! For a logical `kernel` subsystem, corresponding code can be distributed over several physical\n//! locations. Here is an example for the **memory** subsystem:\n//!\n//! - `src/memory.rs` and `src/memory/**/*`\n//!   - Common code that is agnostic of target processor architecture and `BSP` characteristics.\n//!     - Example: A function to zero a chunk of memory.\n//!   - Interfaces for the memory subsystem that are implemented by `arch` or `BSP` code.\n//!     - Example: An `MMU` interface that defines `MMU` function prototypes.\n//! - `src/bsp/__board_name__/memory.rs` and `src/bsp/__board_name__/memory/**/*`\n//!   - `BSP` specific code.\n//!   - Example: The board's memory map (physical addresses of DRAM and MMIO devices).\n//! - `src/_arch/__arch_name__/memory.rs` and `src/_arch/__arch_name__/memory/**/*`\n//!   - Processor architecture specific code.\n//!   - Example: Implementation of the `MMU` interface for the `__arch_name__` processor\n//!     architecture.\n//!\n//! From a namespace perspective, **memory** subsystem code lives in:\n//!\n//! - `crate::memory::*`\n//! - `crate::bsp::memory::*`\n//!\n//! # Boot flow\n//!\n//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.\n//!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.\n//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.\n\n#![allow(clippy::upper_case_acronyms)]\n#![allow(incomplete_features)]\n#![feature(alloc_error_handler)]\n#![feature(asm_const)]\n#![feature(const_option)]\n#![feature(core_intrinsics)]\n#![feature(format_args_nl)]\n#![feature(generic_const_exprs)]\n#![feature(int_roundings)]\n#![feature(is_sorted)]\n#![feature(linkage)]\n#![feature(nonzero_min_max)]\n#![feature(panic_info_message)]\n#![feature(step_trait)]\n#![feature(trait_alias)]\n#![feature(unchecked_math)]\n#![no_std]\n// Testing\n#![cfg_attr(test, no_main)]\n#![feature(custom_test_frameworks)]\n#![reexport_test_harness_main = \"test_main\"]\n#![test_runner(crate::test_runner)]\n\nextern crate alloc;\n\nmod panic_wait;\nmod synchronization;\n\npub mod backtrace;\npub mod bsp;\npub mod common;\npub mod console;\npub mod cpu;\npub mod driver;\npub mod exception;\npub mod memory;\npub mod print;\npub mod state;\npub mod symbols;\npub mod time;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Version string.\npub fn version() -> &'static str {\n    concat!(\n        env!(\"CARGO_PKG_NAME\"),\n        \" version \",\n        env!(\"CARGO_PKG_VERSION\")\n    )\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n/// The default runner for unit tests.\npub fn test_runner(tests: &[&test_types::UnitTest]) {\n    // This line will be printed as the test header.\n    println!(\"Running {} tests\", tests.len());\n\n    for (i, test) in tests.iter().enumerate() {\n        print!(\"{:>3}. {:.<58}\", i + 1, test.name);\n\n        // Run the actual test.\n        (test.test_func)();\n\n        // Failed tests call panic!(). Execution reaches here only if the test has passed.\n        println!(\"[ok]\")\n    }\n}\n\n/// The `kernel_init()` for unit tests.\n#[cfg(test)]\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    test_main();\n\n    cpu::qemu_exit_success()\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/main.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n// Rust embedded logo for `make doc`.\n#![doc(\n    html_logo_url = \"https://raw.githubusercontent.com/rust-embedded/wg/master/assets/logo/ewg-logo-blue-white-on-transparent.png\"\n)]\n\n//! The `kernel` binary.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\nextern crate alloc;\n\nuse libkernel::{bsp, cpu, driver, exception, info, memory, state, time};\n\n/// Early init code.\n///\n/// When this code runs, virtual memory is already enabled.\n///\n/// # Safety\n///\n/// - Only a single core must be active and running this function.\n/// - Printing will not work until the respective driver's MMIO is remapped.\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n\n    // Initialize the timer subsystem.\n    if let Err(x) = time::init() {\n        panic!(\"Error initializing timer subsystem: {}\", x);\n    }\n\n    // Initialize the BSP driver subsystem.\n    if let Err(x) = bsp::driver::init() {\n        panic!(\"Error initializing BSP driver subsystem: {}\", x);\n    }\n\n    // Initialize all device drivers.\n    driver::driver_manager().init_drivers_and_irqs();\n\n    bsp::memory::mmu::kernel_add_mapping_records_for_precomputed();\n\n    // Unmask interrupts on the boot CPU core.\n    exception::asynchronous::local_irq_unmask();\n\n    // Announce conclusion of the kernel_init() phase.\n    state::state_manager().transition_to_single_core_main();\n\n    // Transition from unsafe to safe.\n    kernel_main()\n}\n\n/// The main function running after the early init.\nfn kernel_main() -> ! {\n    use alloc::boxed::Box;\n    use core::time::Duration;\n\n    info!(\"{}\", libkernel::version());\n    info!(\"Booting on: {}\", bsp::board_name());\n\n    info!(\"MMU online:\");\n    memory::mmu::kernel_print_mappings();\n\n    let (_, privilege_level) = exception::current_privilege_level();\n    info!(\"Current privilege level: {}\", privilege_level);\n\n    info!(\"Exception handling state:\");\n    exception::asynchronous::print_state();\n\n    info!(\n        \"Architectural timer resolution: {} ns\",\n        time::time_manager().resolution().as_nanos()\n    );\n\n    info!(\"Drivers loaded:\");\n    driver::driver_manager().enumerate();\n\n    info!(\"Registered IRQ handlers:\");\n    exception::asynchronous::irq_manager().print_handler();\n\n    info!(\"Kernel heap:\");\n    memory::heap_alloc::kernel_heap_allocator().print_usage();\n\n    time::time_manager().set_timeout_once(Duration::from_secs(5), Box::new(|| info!(\"Once 5\")));\n    time::time_manager().set_timeout_once(Duration::from_secs(3), Box::new(|| info!(\"Once 2\")));\n    time::time_manager()\n        .set_timeout_periodic(Duration::from_secs(1), Box::new(|| info!(\"Periodic 1 sec\")));\n\n    info!(\"Echoing input now\");\n    cpu::wait_forever();\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/memory/heap_alloc.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Heap allocation.\n\nuse crate::{\n    backtrace, bsp, common, debug, info,\n    memory::{Address, Virtual},\n    synchronization,\n    synchronization::IRQSafeNullLock,\n    warn,\n};\nuse alloc::alloc::{GlobalAlloc, Layout};\nuse core::sync::atomic::{AtomicBool, Ordering};\nuse linked_list_allocator::Heap as LinkedListHeap;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// A heap allocator that can be lazyily initialized.\npub struct HeapAllocator {\n    inner: IRQSafeNullLock<LinkedListHeap>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n#[global_allocator]\nstatic KERNEL_HEAP_ALLOCATOR: HeapAllocator = HeapAllocator::new();\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n#[inline(always)]\nfn debug_print_alloc_dealloc(operation: &'static str, ptr: *mut u8, layout: Layout) {\n    let size = layout.size();\n    let (size_h, size_unit) = common::size_human_readable_ceil(size);\n    let addr = Address::<Virtual>::new(ptr as usize);\n\n    debug!(\n        \"Kernel Heap: {}\\n      \\\n        Size:     {:#x} ({} {})\\n      \\\n        Start:    {}\\n      \\\n        End excl: {}\\n\\n      \\\n        {}\",\n        operation,\n        size,\n        size_h,\n        size_unit,\n        addr,\n        addr + size,\n        backtrace::Backtrace\n    );\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\n#[alloc_error_handler]\nfn alloc_error_handler(layout: Layout) -> ! {\n    panic!(\"Allocation error: {:?}\", layout)\n}\n\n/// Return a reference to the kernel's heap allocator.\npub fn kernel_heap_allocator() -> &'static HeapAllocator {\n    &KERNEL_HEAP_ALLOCATOR\n}\n\nimpl HeapAllocator {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            inner: IRQSafeNullLock::new(LinkedListHeap::empty()),\n        }\n    }\n\n    /// Print the current heap usage.\n    pub fn print_usage(&self) {\n        let (used, free) = KERNEL_HEAP_ALLOCATOR\n            .inner\n            .lock(|inner| (inner.used(), inner.free()));\n\n        if used >= 1024 {\n            let (used_h, used_unit) = common::size_human_readable_ceil(used);\n            info!(\"      Used: {} Byte ({} {})\", used, used_h, used_unit);\n        } else {\n            info!(\"      Used: {} Byte\", used);\n        }\n\n        if free >= 1024 {\n            let (free_h, free_unit) = common::size_human_readable_ceil(free);\n            info!(\"      Free: {} Byte ({} {})\", free, free_h, free_unit);\n        } else {\n            info!(\"      Free: {} Byte\", free);\n        }\n    }\n}\n\nunsafe impl GlobalAlloc for HeapAllocator {\n    unsafe fn alloc(&self, layout: Layout) -> *mut u8 {\n        let result = KERNEL_HEAP_ALLOCATOR\n            .inner\n            .lock(|inner| inner.allocate_first_fit(layout).ok());\n\n        match result {\n            None => core::ptr::null_mut(),\n            Some(allocation) => {\n                let ptr = allocation.as_ptr();\n\n                debug_print_alloc_dealloc(\"Allocation\", ptr, layout);\n\n                ptr\n            }\n        }\n    }\n\n    unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {\n        KERNEL_HEAP_ALLOCATOR\n            .inner\n            .lock(|inner| inner.deallocate(core::ptr::NonNull::new_unchecked(ptr), layout));\n\n        debug_print_alloc_dealloc(\"Free\", ptr, layout);\n    }\n}\n\n/// Query the BSP for the heap region and initialize the kernel's heap allocator with it.\npub fn kernel_init_heap_allocator() {\n    static INIT_DONE: AtomicBool = AtomicBool::new(false);\n    if INIT_DONE.load(Ordering::Relaxed) {\n        warn!(\"Already initialized\");\n        return;\n    }\n\n    let region = bsp::memory::mmu::virt_heap_region();\n\n    KERNEL_HEAP_ALLOCATOR.inner.lock(|inner| unsafe {\n        inner.init(region.start_addr().as_usize() as *mut u8, region.size())\n    });\n\n    INIT_DONE.store(true, Ordering::Relaxed);\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/memory/mmu/mapping_record.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A record of mapped pages.\n\nuse super::{\n    AccessPermissions, Address, AttributeFields, MMIODescriptor, MemAttributes, MemoryRegion,\n    Physical, Virtual,\n};\nuse crate::{bsp, common, info, synchronization, synchronization::InitStateLock};\nuse alloc::{vec, vec::Vec};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Type describing a virtual memory mapping.\n#[allow(missing_docs)]\nstruct MappingRecordEntry {\n    pub users: Vec<&'static str>,\n    pub phys_start_addr: Address<Physical>,\n    pub virt_start_addr: Address<Virtual>,\n    pub num_pages: usize,\n    pub attribute_fields: AttributeFields,\n}\n\nstruct MappingRecord {\n    inner: Vec<MappingRecordEntry>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic KERNEL_MAPPING_RECORD: InitStateLock<MappingRecord> =\n    InitStateLock::new(MappingRecord::new());\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl MappingRecordEntry {\n    pub fn new(\n        name: &'static str,\n        virt_region: &MemoryRegion<Virtual>,\n        phys_region: &MemoryRegion<Physical>,\n        attr: &AttributeFields,\n    ) -> Self {\n        Self {\n            users: vec![name],\n            phys_start_addr: phys_region.start_addr(),\n            virt_start_addr: virt_region.start_addr(),\n            num_pages: phys_region.num_pages(),\n            attribute_fields: *attr,\n        }\n    }\n\n    pub fn add_user(&mut self, user: &'static str) {\n        self.users.push(user);\n    }\n}\n\nimpl MappingRecord {\n    pub const fn new() -> Self {\n        Self { inner: Vec::new() }\n    }\n\n    fn sort(&mut self) {\n        if !self.inner.is_sorted_by_key(|item| item.virt_start_addr) {\n            self.inner.sort_unstable_by_key(|item| item.virt_start_addr)\n        }\n    }\n\n    fn find_duplicate(\n        &mut self,\n        phys_region: &MemoryRegion<Physical>,\n    ) -> Option<&mut MappingRecordEntry> {\n        self.inner\n            .iter_mut()\n            .filter(|x| x.attribute_fields.mem_attributes == MemAttributes::Device)\n            .find(|x| {\n                if x.phys_start_addr != phys_region.start_addr() {\n                    return false;\n                }\n\n                if x.num_pages != phys_region.num_pages() {\n                    return false;\n                }\n\n                true\n            })\n    }\n\n    pub fn add(\n        &mut self,\n        name: &'static str,\n        virt_region: &MemoryRegion<Virtual>,\n        phys_region: &MemoryRegion<Physical>,\n        attr: &AttributeFields,\n    ) {\n        self.inner.push(MappingRecordEntry::new(\n            name,\n            virt_region,\n            phys_region,\n            attr,\n        ));\n\n        self.sort();\n    }\n\n    pub fn print(&self) {\n        info!(\"      -------------------------------------------------------------------------------------------------------------------------------------------\");\n        info!(\n            \"      {:^44}     {:^30}   {:^7}   {:^9}   {:^35}\",\n            \"Virtual\", \"Physical\", \"Size\", \"Attr\", \"Entity\"\n        );\n        info!(\"      -------------------------------------------------------------------------------------------------------------------------------------------\");\n\n        for i in self.inner.iter() {\n            let size = i.num_pages * bsp::memory::mmu::KernelGranule::SIZE;\n            let virt_start = i.virt_start_addr;\n            let virt_end_inclusive = virt_start + (size - 1);\n            let phys_start = i.phys_start_addr;\n            let phys_end_inclusive = phys_start + (size - 1);\n\n            let (size, unit) = common::size_human_readable_ceil(size);\n\n            let attr = match i.attribute_fields.mem_attributes {\n                MemAttributes::CacheableDRAM => \"C\",\n                MemAttributes::Device => \"Dev\",\n            };\n\n            let acc_p = match i.attribute_fields.acc_perms {\n                AccessPermissions::ReadOnly => \"RO\",\n                AccessPermissions::ReadWrite => \"RW\",\n            };\n\n            let xn = if i.attribute_fields.execute_never {\n                \"XN\"\n            } else {\n                \"X\"\n            };\n\n            info!(\n                \"      {}..{} --> {}..{} | {:>3} {} | {:<3} {} {:<2} | {}\",\n                virt_start,\n                virt_end_inclusive,\n                phys_start,\n                phys_end_inclusive,\n                size,\n                unit,\n                attr,\n                acc_p,\n                xn,\n                i.users[0]\n            );\n\n            for k in &i.users[1..] {\n                info!(\n                        \"                                                                                                            | {}\",\n                        k\n                    );\n            }\n        }\n\n        info!(\"      -------------------------------------------------------------------------------------------------------------------------------------------\");\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::interface::ReadWriteEx;\n\n/// Add an entry to the mapping info record.\npub fn kernel_add(\n    name: &'static str,\n    virt_region: &MemoryRegion<Virtual>,\n    phys_region: &MemoryRegion<Physical>,\n    attr: &AttributeFields,\n) {\n    KERNEL_MAPPING_RECORD.write(|mr| mr.add(name, virt_region, phys_region, attr))\n}\n\npub fn kernel_find_and_insert_mmio_duplicate(\n    mmio_descriptor: &MMIODescriptor,\n    new_user: &'static str,\n) -> Option<Address<Virtual>> {\n    let phys_region: MemoryRegion<Physical> = (*mmio_descriptor).into();\n\n    KERNEL_MAPPING_RECORD.write(|mr| {\n        let dup = mr.find_duplicate(&phys_region)?;\n\n        dup.add_user(new_user);\n\n        Some(dup.virt_start_addr)\n    })\n}\n\n/// Human-readable print of all recorded kernel mappings.\npub fn kernel_print() {\n    KERNEL_MAPPING_RECORD.read(|mr| mr.print());\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/memory/mmu/page_alloc.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Page allocation.\n\nuse super::MemoryRegion;\nuse crate::{\n    memory::{AddressType, Virtual},\n    synchronization::IRQSafeNullLock,\n    warn,\n};\nuse core::num::NonZeroUsize;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// A page allocator that can be lazyily initialized.\npub struct PageAllocator<ATYPE: AddressType> {\n    pool: Option<MemoryRegion<ATYPE>>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic KERNEL_MMIO_VA_ALLOCATOR: IRQSafeNullLock<PageAllocator<Virtual>> =\n    IRQSafeNullLock::new(PageAllocator::new());\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the kernel's MMIO virtual address allocator.\npub fn kernel_mmio_va_allocator() -> &'static IRQSafeNullLock<PageAllocator<Virtual>> {\n    &KERNEL_MMIO_VA_ALLOCATOR\n}\n\nimpl<ATYPE: AddressType> PageAllocator<ATYPE> {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self { pool: None }\n    }\n\n    /// Initialize the allocator.\n    pub fn init(&mut self, pool: MemoryRegion<ATYPE>) {\n        if self.pool.is_some() {\n            warn!(\"Already initialized\");\n            return;\n        }\n\n        self.pool = Some(pool);\n    }\n\n    /// Allocate a number of pages.\n    pub fn alloc(\n        &mut self,\n        num_requested_pages: NonZeroUsize,\n    ) -> Result<MemoryRegion<ATYPE>, &'static str> {\n        if self.pool.is_none() {\n            return Err(\"Allocator not initialized\");\n        }\n\n        self.pool\n            .as_mut()\n            .unwrap()\n            .take_first_n_pages(num_requested_pages)\n    }\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/memory/mmu/translation_table.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Translation table.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../../_arch/aarch64/memory/mmu/translation_table.rs\"]\nmod arch_translation_table;\n\nuse super::{AttributeFields, MemoryRegion};\nuse crate::memory::{Address, Physical, Virtual};\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\n#[cfg(target_arch = \"aarch64\")]\npub use arch_translation_table::FixedSizeTranslationTable;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Translation table interfaces.\npub mod interface {\n    use crate::memory::mmu::PageAddress;\n\n    use super::*;\n\n    /// Translation table operations.\n    pub trait TranslationTable {\n        /// Anything that needs to run before any of the other provided functions can be used.\n        ///\n        /// # Safety\n        ///\n        /// - Implementor must ensure that this function can run only once or is harmless if invoked\n        ///   multiple times.\n        fn init(&mut self) -> Result<(), &'static str>;\n\n        /// Map the given virtual memory region to the given physical memory region.\n        ///\n        /// # Safety\n        ///\n        /// - Using wrong attributes can cause multiple issues of different nature in the system.\n        /// - It is not required that the architectural implementation prevents aliasing. That is,\n        ///   mapping to the same physical memory using multiple virtual addresses, which would\n        ///   break Rust's ownership assumptions. This should be protected against in the kernel's\n        ///   generic MMU code.\n        unsafe fn map_at(\n            &mut self,\n            virt_region: &MemoryRegion<Virtual>,\n            phys_region: &MemoryRegion<Physical>,\n            attr: &AttributeFields,\n        ) -> Result<(), &'static str>;\n\n        /// Try to translate a virtual page address to a physical page address.\n        ///\n        /// Will only succeed if there exists a valid mapping for the input page.\n        fn try_virt_page_addr_to_phys_page_addr(\n            &self,\n            virt_page_addr: PageAddress<Virtual>,\n        ) -> Result<PageAddress<Physical>, &'static str>;\n\n        /// Try to get the attributes of a page.\n        ///\n        /// Will only succeed if there exists a valid mapping for the input page.\n        fn try_page_attributes(\n            &self,\n            virt_page_addr: PageAddress<Virtual>,\n        ) -> Result<AttributeFields, &'static str>;\n\n        /// Try to translate a virtual address to a physical address.\n        ///\n        /// Will only succeed if there exists a valid mapping for the input address.\n        fn try_virt_addr_to_phys_addr(\n            &self,\n            virt_addr: Address<Virtual>,\n        ) -> Result<Address<Physical>, &'static str>;\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use crate::memory::mmu::{AccessPermissions, MemAttributes, PageAddress};\n    use arch_translation_table::MinSizeTranslationTable;\n    use interface::TranslationTable;\n    use test_macros::kernel_test;\n\n    /// Sanity checks for the TranslationTable implementation.\n    #[kernel_test]\n    fn translationtable_implementation_sanity() {\n        // This will occupy a lot of space on the stack.\n        let mut tables = MinSizeTranslationTable::new_for_runtime();\n\n        assert_eq!(tables.init(), Ok(()));\n\n        let virt_end_exclusive_page_addr: PageAddress<Virtual> = PageAddress::MAX;\n        let virt_start_page_addr: PageAddress<Virtual> =\n            virt_end_exclusive_page_addr.checked_offset(-5).unwrap();\n\n        let phys_start_page_addr: PageAddress<Physical> = PageAddress::from(0);\n        let phys_end_exclusive_page_addr: PageAddress<Physical> =\n            phys_start_page_addr.checked_offset(5).unwrap();\n\n        let virt_region = MemoryRegion::new(virt_start_page_addr, virt_end_exclusive_page_addr);\n        let phys_region = MemoryRegion::new(phys_start_page_addr, phys_end_exclusive_page_addr);\n\n        let attr = AttributeFields {\n            mem_attributes: MemAttributes::CacheableDRAM,\n            acc_perms: AccessPermissions::ReadWrite,\n            execute_never: true,\n        };\n\n        unsafe { assert_eq!(tables.map_at(&virt_region, &phys_region, &attr), Ok(())) };\n\n        assert_eq!(\n            tables.try_virt_page_addr_to_phys_page_addr(virt_start_page_addr),\n            Ok(phys_start_page_addr)\n        );\n\n        assert_eq!(\n            tables.try_page_attributes(virt_start_page_addr.checked_offset(-1).unwrap()),\n            Err(\"Page marked invalid\")\n        );\n\n        assert_eq!(tables.try_page_attributes(virt_start_page_addr), Ok(attr));\n\n        let virt_addr = virt_start_page_addr.into_inner() + 0x100;\n        let phys_addr = phys_start_page_addr.into_inner() + 0x100;\n        assert_eq!(tables.try_virt_addr_to_phys_addr(virt_addr), Ok(phys_addr));\n    }\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/memory/mmu/types.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management Unit types.\n\nuse crate::{\n    bsp, common,\n    memory::{Address, AddressType, Physical},\n};\nuse core::{convert::From, iter::Step, num::NonZeroUsize, ops::Range};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// A wrapper type around [Address] that ensures page alignment.\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub struct PageAddress<ATYPE: AddressType> {\n    inner: Address<ATYPE>,\n}\n\n/// A type that describes a region of memory in quantities of pages.\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub struct MemoryRegion<ATYPE: AddressType> {\n    start: PageAddress<ATYPE>,\n    end_exclusive: PageAddress<ATYPE>,\n}\n\n/// Architecture agnostic memory attributes.\n#[allow(missing_docs)]\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub enum MemAttributes {\n    CacheableDRAM,\n    Device,\n}\n\n/// Architecture agnostic access permissions.\n#[allow(missing_docs)]\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub enum AccessPermissions {\n    ReadOnly,\n    ReadWrite,\n}\n\n/// Collection of memory attributes.\n#[allow(missing_docs)]\n#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)]\npub struct AttributeFields {\n    pub mem_attributes: MemAttributes,\n    pub acc_perms: AccessPermissions,\n    pub execute_never: bool,\n}\n\n/// An MMIO descriptor for use in device drivers.\n#[derive(Copy, Clone)]\npub struct MMIODescriptor {\n    start_addr: Address<Physical>,\n    end_addr_exclusive: Address<Physical>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n//------------------------------------------------------------------------------\n// PageAddress\n//------------------------------------------------------------------------------\nimpl<ATYPE: AddressType> PageAddress<ATYPE> {\n    /// The largest value that can be represented by this type.\n    pub const MAX: Self = PageAddress {\n        inner: Address::new(usize::MAX).align_down_page(),\n    };\n\n    /// Unwraps the value.\n    pub fn into_inner(self) -> Address<ATYPE> {\n        self.inner\n    }\n\n    /// Calculates the offset from the page address.\n    ///\n    /// `count` is in units of [PageAddress]. For example, a count of 2 means `result = self + 2 *\n    /// page_size`.\n    pub fn checked_offset(self, count: isize) -> Option<Self> {\n        if count == 0 {\n            return Some(self);\n        }\n\n        let delta = count\n            .unsigned_abs()\n            .checked_mul(bsp::memory::mmu::KernelGranule::SIZE)?;\n        let result = if count.is_positive() {\n            self.inner.as_usize().checked_add(delta)?\n        } else {\n            self.inner.as_usize().checked_sub(delta)?\n        };\n\n        Some(Self {\n            inner: Address::new(result),\n        })\n    }\n}\n\nimpl<ATYPE: AddressType> From<usize> for PageAddress<ATYPE> {\n    fn from(addr: usize) -> Self {\n        assert!(\n            common::is_aligned(addr, bsp::memory::mmu::KernelGranule::SIZE),\n            \"Input usize not page aligned\"\n        );\n\n        Self {\n            inner: Address::new(addr),\n        }\n    }\n}\n\nimpl<ATYPE: AddressType> From<Address<ATYPE>> for PageAddress<ATYPE> {\n    fn from(addr: Address<ATYPE>) -> Self {\n        assert!(addr.is_page_aligned(), \"Input Address not page aligned\");\n\n        Self { inner: addr }\n    }\n}\n\nimpl<ATYPE: AddressType> Step for PageAddress<ATYPE> {\n    fn steps_between(start: &Self, end: &Self) -> Option<usize> {\n        if start > end {\n            return None;\n        }\n\n        // Since start <= end, do unchecked arithmetic.\n        Some(\n            (end.inner.as_usize() - start.inner.as_usize())\n                >> bsp::memory::mmu::KernelGranule::SHIFT,\n        )\n    }\n\n    fn forward_checked(start: Self, count: usize) -> Option<Self> {\n        start.checked_offset(count as isize)\n    }\n\n    fn backward_checked(start: Self, count: usize) -> Option<Self> {\n        start.checked_offset(-(count as isize))\n    }\n}\n\n//------------------------------------------------------------------------------\n// MemoryRegion\n//------------------------------------------------------------------------------\nimpl<ATYPE: AddressType> MemoryRegion<ATYPE> {\n    /// Create an instance.\n    pub fn new(start: PageAddress<ATYPE>, end_exclusive: PageAddress<ATYPE>) -> Self {\n        assert!(start <= end_exclusive);\n\n        Self {\n            start,\n            end_exclusive,\n        }\n    }\n\n    fn as_range(&self) -> Range<PageAddress<ATYPE>> {\n        self.into_iter()\n    }\n\n    /// Returns the start page address.\n    pub fn start_page_addr(&self) -> PageAddress<ATYPE> {\n        self.start\n    }\n\n    /// Returns the start address.\n    pub fn start_addr(&self) -> Address<ATYPE> {\n        self.start.into_inner()\n    }\n\n    /// Returns the exclusive end page address.\n    pub fn end_exclusive_page_addr(&self) -> PageAddress<ATYPE> {\n        self.end_exclusive\n    }\n\n    /// Returns the exclusive end page address.\n    pub fn end_inclusive_page_addr(&self) -> PageAddress<ATYPE> {\n        self.end_exclusive.checked_offset(-1).unwrap()\n    }\n\n    /// Checks if self contains an address.\n    pub fn contains(&self, addr: Address<ATYPE>) -> bool {\n        let page_addr = PageAddress::from(addr.align_down_page());\n        self.as_range().contains(&page_addr)\n    }\n\n    /// Checks if there is an overlap with another memory region.\n    pub fn overlaps(&self, other_region: &Self) -> bool {\n        let self_range = self.as_range();\n\n        self_range.contains(&other_region.start_page_addr())\n            || self_range.contains(&other_region.end_inclusive_page_addr())\n    }\n\n    /// Returns the number of pages contained in this region.\n    pub fn num_pages(&self) -> usize {\n        PageAddress::steps_between(&self.start, &self.end_exclusive).unwrap()\n    }\n\n    /// Returns the size in bytes of this region.\n    pub fn size(&self) -> usize {\n        // Invariant: start <= end_exclusive, so do unchecked arithmetic.\n        let end_exclusive = self.end_exclusive.into_inner().as_usize();\n        let start = self.start.into_inner().as_usize();\n\n        end_exclusive - start\n    }\n\n    /// Splits the MemoryRegion like:\n    ///\n    /// --------------------------------------------------------------------------------\n    /// |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |\n    /// --------------------------------------------------------------------------------\n    ///   ^                               ^                                       ^\n    ///   |                               |                                       |\n    ///   left_start     left_end_exclusive                                       |\n    ///                                                                           |\n    ///                                   ^                                       |\n    ///                                   |                                       |\n    ///                                   right_start           right_end_exclusive\n    ///\n    /// Left region is returned to the caller. Right region is the new region for this struct.\n    pub fn take_first_n_pages(&mut self, num_pages: NonZeroUsize) -> Result<Self, &'static str> {\n        let count: usize = num_pages.into();\n\n        let left_end_exclusive = self.start.checked_offset(count as isize);\n        let left_end_exclusive = match left_end_exclusive {\n            None => return Err(\"Overflow while calculating left_end_exclusive\"),\n            Some(x) => x,\n        };\n\n        if left_end_exclusive > self.end_exclusive {\n            return Err(\"Not enough free pages\");\n        }\n\n        let allocation = Self {\n            start: self.start,\n            end_exclusive: left_end_exclusive,\n        };\n        self.start = left_end_exclusive;\n\n        Ok(allocation)\n    }\n}\n\nimpl<ATYPE: AddressType> IntoIterator for MemoryRegion<ATYPE> {\n    type Item = PageAddress<ATYPE>;\n    type IntoIter = Range<Self::Item>;\n\n    fn into_iter(self) -> Self::IntoIter {\n        Range {\n            start: self.start,\n            end: self.end_exclusive,\n        }\n    }\n}\n\nimpl From<MMIODescriptor> for MemoryRegion<Physical> {\n    fn from(desc: MMIODescriptor) -> Self {\n        let start = PageAddress::from(desc.start_addr.align_down_page());\n        let end_exclusive = PageAddress::from(desc.end_addr_exclusive().align_up_page());\n\n        Self {\n            start,\n            end_exclusive,\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// MMIODescriptor\n//------------------------------------------------------------------------------\n\nimpl MMIODescriptor {\n    /// Create an instance.\n    pub const fn new(start_addr: Address<Physical>, size: usize) -> Self {\n        assert!(size > 0);\n        let end_addr_exclusive = Address::new(start_addr.as_usize() + size);\n\n        Self {\n            start_addr,\n            end_addr_exclusive,\n        }\n    }\n\n    /// Return the start address.\n    pub const fn start_addr(&self) -> Address<Physical> {\n        self.start_addr\n    }\n\n    /// Return the exclusive end address.\n    pub fn end_addr_exclusive(&self) -> Address<Physical> {\n        self.end_addr_exclusive\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use crate::memory::Virtual;\n    use test_macros::kernel_test;\n\n    /// Sanity of [PageAddress] methods.\n    #[kernel_test]\n    fn pageaddress_type_method_sanity() {\n        let page_addr: PageAddress<Virtual> =\n            PageAddress::from(bsp::memory::mmu::KernelGranule::SIZE * 2);\n\n        assert_eq!(\n            page_addr.checked_offset(-2),\n            Some(PageAddress::<Virtual>::from(0))\n        );\n\n        assert_eq!(\n            page_addr.checked_offset(2),\n            Some(PageAddress::<Virtual>::from(\n                bsp::memory::mmu::KernelGranule::SIZE * 4\n            ))\n        );\n\n        assert_eq!(\n            PageAddress::<Virtual>::from(0).checked_offset(0),\n            Some(PageAddress::<Virtual>::from(0))\n        );\n        assert_eq!(PageAddress::<Virtual>::from(0).checked_offset(-1), None);\n\n        let max_page_addr = Address::<Virtual>::new(usize::MAX).align_down_page();\n        assert_eq!(\n            PageAddress::<Virtual>::from(max_page_addr).checked_offset(1),\n            None\n        );\n\n        let zero = PageAddress::<Virtual>::from(0);\n        let three = PageAddress::<Virtual>::from(bsp::memory::mmu::KernelGranule::SIZE * 3);\n        assert_eq!(PageAddress::steps_between(&zero, &three), Some(3));\n    }\n\n    /// Sanity of [MemoryRegion] methods.\n    #[kernel_test]\n    fn memoryregion_type_method_sanity() {\n        let zero = PageAddress::<Virtual>::from(0);\n        let zero_region = MemoryRegion::new(zero, zero);\n        assert_eq!(zero_region.num_pages(), 0);\n        assert_eq!(zero_region.size(), 0);\n\n        let one = PageAddress::<Virtual>::from(bsp::memory::mmu::KernelGranule::SIZE);\n        let one_region = MemoryRegion::new(zero, one);\n        assert_eq!(one_region.num_pages(), 1);\n        assert_eq!(one_region.size(), bsp::memory::mmu::KernelGranule::SIZE);\n\n        let three = PageAddress::<Virtual>::from(bsp::memory::mmu::KernelGranule::SIZE * 3);\n        let mut three_region = MemoryRegion::new(zero, three);\n        assert!(three_region.contains(zero.into_inner()));\n        assert!(!three_region.contains(three.into_inner()));\n        assert!(three_region.overlaps(&one_region));\n\n        let allocation = three_region\n            .take_first_n_pages(NonZeroUsize::new(2).unwrap())\n            .unwrap();\n        assert_eq!(allocation.num_pages(), 2);\n        assert_eq!(three_region.num_pages(), 1);\n\n        for (i, alloc) in allocation.into_iter().enumerate() {\n            assert_eq!(\n                alloc.into_inner().as_usize(),\n                i * bsp::memory::mmu::KernelGranule::SIZE\n            );\n        }\n    }\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/memory/mmu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management Unit.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/memory/mmu.rs\"]\nmod arch_mmu;\n\nmod mapping_record;\nmod page_alloc;\nmod translation_table;\nmod types;\n\nuse crate::{\n    bsp,\n    memory::{Address, Physical, Virtual},\n    synchronization::{self, interface::Mutex},\n};\nuse core::{fmt, num::NonZeroUsize};\n\npub use types::*;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// MMU enable errors variants.\n#[allow(missing_docs)]\n#[derive(Debug)]\npub enum MMUEnableError {\n    AlreadyEnabled,\n    Other(&'static str),\n}\n\n/// Memory Management interfaces.\npub mod interface {\n    use super::*;\n\n    /// MMU functions.\n    pub trait MMU {\n        /// Turns on the MMU for the first time and enables data and instruction caching.\n        ///\n        /// # Safety\n        ///\n        /// - Changes the HW's global state.\n        unsafe fn enable_mmu_and_caching(\n            &self,\n            phys_tables_base_addr: Address<Physical>,\n        ) -> Result<(), MMUEnableError>;\n\n        /// Returns true if the MMU is enabled, false otherwise.\n        fn is_enabled(&self) -> bool;\n    }\n}\n\n/// Describes the characteristics of a translation granule.\npub struct TranslationGranule<const GRANULE_SIZE: usize>;\n\n/// Describes properties of an address space.\npub struct AddressSpace<const AS_SIZE: usize>;\n\n/// Intended to be implemented for [`AddressSpace`].\npub trait AssociatedTranslationTable {\n    /// A translation table whose address range is:\n    ///\n    /// [u64::MAX, (u64::MAX - AS_SIZE) + 1]\n    type TableStartFromTop;\n\n    /// A translation table whose address range is:\n    ///\n    /// [AS_SIZE - 1, 0]\n    type TableStartFromBottom;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\nuse interface::MMU;\nuse synchronization::interface::ReadWriteEx;\nuse translation_table::interface::TranslationTable;\n\n/// Map a region in the kernel's translation tables.\n///\n/// No input checks done, input is passed through to the architectural implementation.\n///\n/// # Safety\n///\n/// - See `map_at()`.\n/// - Does not prevent aliasing.\nunsafe fn kernel_map_at_unchecked(\n    name: &'static str,\n    virt_region: &MemoryRegion<Virtual>,\n    phys_region: &MemoryRegion<Physical>,\n    attr: &AttributeFields,\n) -> Result<(), &'static str> {\n    bsp::memory::mmu::kernel_translation_tables()\n        .write(|tables| tables.map_at(virt_region, phys_region, attr))?;\n\n    kernel_add_mapping_record(name, virt_region, phys_region, attr);\n\n    Ok(())\n}\n\n/// Try to translate a kernel virtual address to a physical address.\n///\n/// Will only succeed if there exists a valid mapping for the input address.\nfn try_kernel_virt_addr_to_phys_addr(\n    virt_addr: Address<Virtual>,\n) -> Result<Address<Physical>, &'static str> {\n    bsp::memory::mmu::kernel_translation_tables()\n        .read(|tables| tables.try_virt_addr_to_phys_addr(virt_addr))\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl fmt::Display for MMUEnableError {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        match self {\n            MMUEnableError::AlreadyEnabled => write!(f, \"MMU is already enabled\"),\n            MMUEnableError::Other(x) => write!(f, \"{}\", x),\n        }\n    }\n}\n\nimpl<const GRANULE_SIZE: usize> TranslationGranule<GRANULE_SIZE> {\n    /// The granule's size.\n    pub const SIZE: usize = Self::size_checked();\n\n    /// The granule's mask.\n    pub const MASK: usize = Self::SIZE - 1;\n\n    /// The granule's shift, aka log2(size).\n    pub const SHIFT: usize = Self::SIZE.trailing_zeros() as usize;\n\n    const fn size_checked() -> usize {\n        assert!(GRANULE_SIZE.is_power_of_two());\n\n        GRANULE_SIZE\n    }\n}\n\nimpl<const AS_SIZE: usize> AddressSpace<AS_SIZE> {\n    /// The address space size.\n    pub const SIZE: usize = Self::size_checked();\n\n    /// The address space shift, aka log2(size).\n    pub const SIZE_SHIFT: usize = Self::SIZE.trailing_zeros() as usize;\n\n    const fn size_checked() -> usize {\n        assert!(AS_SIZE.is_power_of_two());\n\n        // Check for architectural restrictions as well.\n        Self::arch_address_space_size_sanity_checks();\n\n        AS_SIZE\n    }\n}\n\n/// Query the BSP for the reserved virtual addresses for MMIO remapping and initialize the kernel's\n/// MMIO VA allocator with it.\npub fn kernel_init_mmio_va_allocator() {\n    let region = bsp::memory::mmu::virt_mmio_remap_region();\n\n    page_alloc::kernel_mmio_va_allocator().lock(|allocator| allocator.init(region));\n}\n\n/// Add an entry to the mapping info record.\npub fn kernel_add_mapping_record(\n    name: &'static str,\n    virt_region: &MemoryRegion<Virtual>,\n    phys_region: &MemoryRegion<Physical>,\n    attr: &AttributeFields,\n) {\n    mapping_record::kernel_add(name, virt_region, phys_region, attr);\n}\n\n/// MMIO remapping in the kernel translation tables.\n///\n/// Typically used by device drivers.\n///\n/// # Safety\n///\n/// - Same as `kernel_map_at_unchecked()`, minus the aliasing part.\npub unsafe fn kernel_map_mmio(\n    name: &'static str,\n    mmio_descriptor: &MMIODescriptor,\n) -> Result<Address<Virtual>, &'static str> {\n    let phys_region = MemoryRegion::from(*mmio_descriptor);\n    let offset_into_start_page = mmio_descriptor.start_addr().offset_into_page();\n\n    // Check if an identical region has been mapped for another driver. If so, reuse it.\n    let virt_addr = if let Some(addr) =\n        mapping_record::kernel_find_and_insert_mmio_duplicate(mmio_descriptor, name)\n    {\n        addr\n    // Otherwise, allocate a new region and map it.\n    } else {\n        let num_pages = match NonZeroUsize::new(phys_region.num_pages()) {\n            None => return Err(\"Requested 0 pages\"),\n            Some(x) => x,\n        };\n\n        let virt_region =\n            page_alloc::kernel_mmio_va_allocator().lock(|allocator| allocator.alloc(num_pages))?;\n\n        kernel_map_at_unchecked(\n            name,\n            &virt_region,\n            &phys_region,\n            &AttributeFields {\n                mem_attributes: MemAttributes::Device,\n                acc_perms: AccessPermissions::ReadWrite,\n                execute_never: true,\n            },\n        )?;\n\n        virt_region.start_addr()\n    };\n\n    Ok(virt_addr + offset_into_start_page)\n}\n\n/// Try to translate a kernel virtual page address to a physical page address.\n///\n/// Will only succeed if there exists a valid mapping for the input page.\npub fn try_kernel_virt_page_addr_to_phys_page_addr(\n    virt_page_addr: PageAddress<Virtual>,\n) -> Result<PageAddress<Physical>, &'static str> {\n    bsp::memory::mmu::kernel_translation_tables()\n        .read(|tables| tables.try_virt_page_addr_to_phys_page_addr(virt_page_addr))\n}\n\n/// Try to get the attributes of a kernel page.\n///\n/// Will only succeed if there exists a valid mapping for the input page.\npub fn try_kernel_page_attributes(\n    virt_page_addr: PageAddress<Virtual>,\n) -> Result<AttributeFields, &'static str> {\n    bsp::memory::mmu::kernel_translation_tables()\n        .read(|tables| tables.try_page_attributes(virt_page_addr))\n}\n\n/// Human-readable print of all recorded kernel mappings.\npub fn kernel_print_mappings() {\n    mapping_record::kernel_print()\n}\n\n/// Enable the MMU and data + instruction caching.\n///\n/// # Safety\n///\n/// - Crucial function during kernel init. Changes the the complete memory view of the processor.\n#[inline(always)]\npub unsafe fn enable_mmu_and_caching(\n    phys_tables_base_addr: Address<Physical>,\n) -> Result<(), MMUEnableError> {\n    arch_mmu::mmu().enable_mmu_and_caching(phys_tables_base_addr)\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/memory.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Memory Management.\n\npub mod heap_alloc;\npub mod mmu;\n\nuse crate::{bsp, common};\nuse core::{\n    fmt,\n    marker::PhantomData,\n    ops::{Add, Sub},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Metadata trait for marking the type of an address.\npub trait AddressType: Copy + Clone + PartialOrd + PartialEq + Ord + Eq {}\n\n/// Zero-sized type to mark a physical address.\n#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq)]\npub enum Physical {}\n\n/// Zero-sized type to mark a virtual address.\n#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq)]\npub enum Virtual {}\n\n/// Generic address type.\n#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq)]\npub struct Address<ATYPE: AddressType> {\n    value: usize,\n    _address_type: PhantomData<fn() -> ATYPE>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl AddressType for Physical {}\nimpl AddressType for Virtual {}\n\nimpl<ATYPE: AddressType> Address<ATYPE> {\n    /// Create an instance.\n    pub const fn new(value: usize) -> Self {\n        Self {\n            value,\n            _address_type: PhantomData,\n        }\n    }\n\n    /// Convert to usize.\n    pub const fn as_usize(self) -> usize {\n        self.value\n    }\n\n    /// Align down to page size.\n    #[must_use]\n    pub const fn align_down_page(self) -> Self {\n        let aligned = common::align_down(self.value, bsp::memory::mmu::KernelGranule::SIZE);\n\n        Self::new(aligned)\n    }\n\n    /// Align up to page size.\n    #[must_use]\n    pub const fn align_up_page(self) -> Self {\n        let aligned = common::align_up(self.value, bsp::memory::mmu::KernelGranule::SIZE);\n\n        Self::new(aligned)\n    }\n\n    /// Checks if the address is page aligned.\n    pub const fn is_page_aligned(&self) -> bool {\n        common::is_aligned(self.value, bsp::memory::mmu::KernelGranule::SIZE)\n    }\n\n    /// Return the address' offset into the corresponding page.\n    pub const fn offset_into_page(&self) -> usize {\n        self.value & bsp::memory::mmu::KernelGranule::MASK\n    }\n}\n\nimpl<ATYPE: AddressType> Add<usize> for Address<ATYPE> {\n    type Output = Self;\n\n    #[inline(always)]\n    fn add(self, rhs: usize) -> Self::Output {\n        match self.value.checked_add(rhs) {\n            None => panic!(\"Overflow on Address::add\"),\n            Some(x) => Self::new(x),\n        }\n    }\n}\n\nimpl<ATYPE: AddressType> Sub<usize> for Address<ATYPE> {\n    type Output = Self;\n\n    #[inline(always)]\n    fn sub(self, rhs: usize) -> Self::Output {\n        match self.value.checked_sub(rhs) {\n            None => panic!(\"Overflow on Address::sub\"),\n            Some(x) => Self::new(x),\n        }\n    }\n}\n\nimpl<ATYPE: AddressType> Sub<Address<ATYPE>> for Address<ATYPE> {\n    type Output = Self;\n\n    #[inline(always)]\n    fn sub(self, rhs: Address<ATYPE>) -> Self::Output {\n        match self.value.checked_sub(rhs.value) {\n            None => panic!(\"Overflow on Address::sub\"),\n            Some(x) => Self::new(x),\n        }\n    }\n}\n\nimpl Address<Virtual> {\n    /// Checks if the address is part of the boot core stack region.\n    pub fn is_valid_stack_addr(&self) -> bool {\n        bsp::memory::mmu::virt_boot_core_stack_region().contains(*self)\n    }\n\n    /// Checks if the address is part of the kernel code region.\n    pub fn is_valid_code_addr(&self) -> bool {\n        bsp::memory::mmu::virt_code_region().contains(*self)\n    }\n}\n\nimpl fmt::Display for Address<Physical> {\n    // Don't expect to see physical addresses greater than 40 bit.\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        let q3: u8 = ((self.value >> 32) & 0xff) as u8;\n        let q2: u16 = ((self.value >> 16) & 0xffff) as u16;\n        let q1: u16 = (self.value & 0xffff) as u16;\n\n        write!(f, \"0x\")?;\n        write!(f, \"{:02x}_\", q3)?;\n        write!(f, \"{:04x}_\", q2)?;\n        write!(f, \"{:04x}\", q1)\n    }\n}\n\nimpl fmt::Display for Address<Virtual> {\n    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n        let q4: u16 = ((self.value >> 48) & 0xffff) as u16;\n        let q3: u16 = ((self.value >> 32) & 0xffff) as u16;\n        let q2: u16 = ((self.value >> 16) & 0xffff) as u16;\n        let q1: u16 = (self.value & 0xffff) as u16;\n\n        write!(f, \"0x\")?;\n        write!(f, \"{:04x}_\", q4)?;\n        write!(f, \"{:04x}_\", q3)?;\n        write!(f, \"{:04x}_\", q2)?;\n        write!(f, \"{:04x}\", q1)\n    }\n}\n\n/// Initialize the memory subsystem.\npub fn init() {\n    mmu::kernel_init_mmio_va_allocator();\n    heap_alloc::kernel_init_heap_allocator();\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// Sanity of [Address] methods.\n    #[kernel_test]\n    fn address_type_method_sanity() {\n        let addr = Address::<Virtual>::new(bsp::memory::mmu::KernelGranule::SIZE + 100);\n\n        assert_eq!(\n            addr.align_down_page().as_usize(),\n            bsp::memory::mmu::KernelGranule::SIZE\n        );\n\n        assert_eq!(\n            addr.align_up_page().as_usize(),\n            bsp::memory::mmu::KernelGranule::SIZE * 2\n        );\n\n        assert!(!addr.is_page_aligned());\n\n        assert_eq!(addr.offset_into_page(), 100);\n    }\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/panic_wait.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A panic handler that infinitely waits.\n\nuse crate::{backtrace, cpu, exception, println};\nuse core::panic::PanicInfo;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// The point of exit for `libkernel`.\n///\n/// It is linked weakly, so that the integration tests can overload its standard behavior.\n#[linkage = \"weak\"]\n#[no_mangle]\nfn _panic_exit() -> ! {\n    #[cfg(not(feature = \"test_build\"))]\n    {\n        cpu::wait_forever()\n    }\n\n    #[cfg(feature = \"test_build\")]\n    {\n        cpu::qemu_exit_failure()\n    }\n}\n\n/// Stop immediately if called a second time.\n///\n/// # Note\n///\n/// Using atomics here relieves us from needing to use `unsafe` for the static variable.\n///\n/// On `AArch64`, which is the only implemented architecture at the time of writing this,\n/// [`AtomicBool::load`] and [`AtomicBool::store`] are lowered to ordinary load and store\n/// instructions. They are therefore safe to use even with MMU + caching deactivated.\n///\n/// [`AtomicBool::load`]: core::sync::atomic::AtomicBool::load\n/// [`AtomicBool::store`]: core::sync::atomic::AtomicBool::store\nfn panic_prevent_reenter() {\n    use core::sync::atomic::{AtomicBool, Ordering};\n\n    #[cfg(not(target_arch = \"aarch64\"))]\n    compile_error!(\"Add the target_arch to above's check if the following code is safe to use\");\n\n    static PANIC_IN_PROGRESS: AtomicBool = AtomicBool::new(false);\n\n    if !PANIC_IN_PROGRESS.load(Ordering::Relaxed) {\n        PANIC_IN_PROGRESS.store(true, Ordering::Relaxed);\n\n        return;\n    }\n\n    _panic_exit()\n}\n\n#[panic_handler]\nfn panic(info: &PanicInfo) -> ! {\n    exception::asynchronous::local_irq_mask();\n\n    // Protect against panic infinite loops if any of the following code panics itself.\n    panic_prevent_reenter();\n\n    let timestamp = crate::time::time_manager().uptime();\n    let (location, line, column) = match info.location() {\n        Some(loc) => (loc.file(), loc.line(), loc.column()),\n        _ => (\"???\", 0, 0),\n    };\n\n    println!(\n        \"[  {:>3}.{:06}] Kernel panic!\\n\\n\\\n        Panic location:\\n      File '{}', line {}, column {}\\n\\n\\\n        {}\\n\\n\\\n        {}\",\n        timestamp.as_secs(),\n        timestamp.subsec_micros(),\n        location,\n        line,\n        column,\n        info.message().unwrap_or(&format_args!(\"\")),\n        backtrace::Backtrace\n    );\n\n    _panic_exit()\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/print.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Printing.\n\nuse crate::console;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n#[doc(hidden)]\npub fn _print(args: fmt::Arguments) {\n    console::console().write_fmt(args).unwrap();\n}\n\n/// Prints without a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! print {\n    ($($arg:tt)*) => ($crate::print::_print(format_args!($($arg)*)));\n}\n\n/// Prints with a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! println {\n    () => ($crate::print!(\"\\n\"));\n    ($($arg:tt)*) => ({\n        $crate::print::_print(format_args_nl!($($arg)*));\n    })\n}\n\n/// Prints an info, with a newline.\n#[macro_export]\nmacro_rules! info {\n    ($string:expr) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[  {:>3}.{:06}] \", $string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n        ));\n    });\n    ($format_string:expr, $($arg:tt)*) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[  {:>3}.{:06}] \", $format_string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n            $($arg)*\n        ));\n    })\n}\n\n/// Prints a warning, with a newline.\n#[macro_export]\nmacro_rules! warn {\n    ($string:expr) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[W {:>3}.{:06}] \", $string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n        ));\n    });\n    ($format_string:expr, $($arg:tt)*) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[W {:>3}.{:06}] \", $format_string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n            $($arg)*\n        ));\n    })\n}\n\n/// Debug print, with a newline.\n#[macro_export]\nmacro_rules! debug {\n    ($string:expr) => ({\n        if cfg!(feature = \"debug_prints\") {\n            let timestamp = $crate::time::time_manager().uptime();\n\n            $crate::print::_print(format_args_nl!(\n                concat!(\"<[>D {:>3}.{:06}> \", $string),\n                timestamp.as_secs(),\n                timestamp.subsec_micros(),\n            ));\n        }\n    });\n    ($format_string:expr, $($arg:tt)*) => ({\n        if cfg!(feature = \"debug_prints\") {\n            let timestamp = $crate::time::time_manager().uptime();\n\n            $crate::print::_print(format_args_nl!(\n                concat!(\"<D {:>3}.{:06}> \", $format_string),\n                timestamp.as_secs(),\n                timestamp.subsec_micros(),\n                $($arg)*\n            ));\n        }\n    })\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/state.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! State information about the kernel itself.\n\nuse core::sync::atomic::{AtomicU8, Ordering};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Different stages in the kernel execution.\n#[derive(Copy, Clone, Eq, PartialEq)]\nenum State {\n    /// The kernel starts booting in this state.\n    Init,\n\n    /// The kernel transitions to this state when jumping to `kernel_main()` (at the end of\n    /// `kernel_init()`, after all init calls are done).\n    SingleCoreMain,\n\n    /// The kernel transitions to this state when it boots the secondary cores, aka switches\n    /// exectution mode to symmetric multiprocessing (SMP).\n    MultiCoreMain,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Maintains the kernel state and state transitions.\npub struct StateManager(AtomicU8);\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic STATE_MANAGER: StateManager = StateManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the global StateManager.\npub fn state_manager() -> &'static StateManager {\n    &STATE_MANAGER\n}\n\nimpl StateManager {\n    const INIT: u8 = 0;\n    const SINGLE_CORE_MAIN: u8 = 1;\n    const MULTI_CORE_MAIN: u8 = 2;\n\n    /// Create a new instance.\n    pub const fn new() -> Self {\n        Self(AtomicU8::new(Self::INIT))\n    }\n\n    /// Return the current state.\n    fn state(&self) -> State {\n        let state = self.0.load(Ordering::Acquire);\n\n        match state {\n            Self::INIT => State::Init,\n            Self::SINGLE_CORE_MAIN => State::SingleCoreMain,\n            Self::MULTI_CORE_MAIN => State::MultiCoreMain,\n            _ => panic!(\"Invalid KERNEL_STATE\"),\n        }\n    }\n\n    /// Return if the kernel is init state.\n    pub fn is_init(&self) -> bool {\n        self.state() == State::Init\n    }\n\n    /// Transition from Init to SingleCoreMain.\n    pub fn transition_to_single_core_main(&self) {\n        if self\n            .0\n            .compare_exchange(\n                Self::INIT,\n                Self::SINGLE_CORE_MAIN,\n                Ordering::Acquire,\n                Ordering::Relaxed,\n            )\n            .is_err()\n        {\n            panic!(\"transition_to_single_core_main() called while state != Init\");\n        }\n    }\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/symbols.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Debug symbol support.\n\nuse crate::memory::{Address, Virtual};\nuse core::{cell::UnsafeCell, slice};\nuse debug_symbol_types::Symbol;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Symbol from the linker script.\nextern \"Rust\" {\n    static __kernel_symbols_start: UnsafeCell<()>;\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n/// This will be patched to the correct value by the \"kernel symbols tool\" after linking. This given\n/// value here is just a (safe) dummy.\n#[no_mangle]\nstatic NUM_KERNEL_SYMBOLS: u64 = 0;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nfn kernel_symbol_section_virt_start_addr() -> Address<Virtual> {\n    Address::new(unsafe { __kernel_symbols_start.get() as usize })\n}\n\nfn num_kernel_symbols() -> usize {\n    unsafe {\n        // Read volatile is needed here to prevent the compiler from optimizing NUM_KERNEL_SYMBOLS\n        // away.\n        core::ptr::read_volatile(&NUM_KERNEL_SYMBOLS as *const u64) as usize\n    }\n}\n\nfn kernel_symbols_slice() -> &'static [Symbol] {\n    let ptr = kernel_symbol_section_virt_start_addr().as_usize() as *const Symbol;\n\n    unsafe { slice::from_raw_parts(ptr, num_kernel_symbols()) }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Retrieve the symbol corresponding to a virtual address, if any.\npub fn lookup_symbol(addr: Address<Virtual>) -> Option<&'static Symbol> {\n    kernel_symbols_slice()\n        .iter()\n        .find(|&i| i.contains(addr.as_usize()))\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// Sanity of symbols module.\n    #[kernel_test]\n    fn symbols_sanity() {\n        let first_sym = lookup_symbol(Address::new(\n            crate::common::is_aligned as *const usize as usize,\n        ))\n        .unwrap()\n        .name();\n\n        assert_eq!(first_sym, \"libkernel::common::is_aligned\");\n\n        let second_sym = lookup_symbol(Address::new(crate::version as *const usize as usize))\n            .unwrap()\n            .name();\n\n        assert_eq!(second_sym, \"libkernel::version\");\n    }\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/synchronization.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Synchronization primitives.\n//!\n//! # Resources\n//!\n//!   - <https://doc.rust-lang.org/book/ch16-04-extensible-concurrency-sync-and-send.html>\n//!   - <https://stackoverflow.com/questions/59428096/understanding-the-send-trait>\n//!   - <https://doc.rust-lang.org/std/cell/index.html>\n\nuse core::cell::UnsafeCell;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Synchronization interfaces.\npub mod interface {\n\n    /// Any object implementing this trait guarantees exclusive access to the data wrapped within\n    /// the Mutex for the duration of the provided closure.\n    pub trait Mutex {\n        /// The type of the data that is wrapped by this mutex.\n        type Data;\n\n        /// Locks the mutex and grants the closure temporary mutable access to the wrapped data.\n        fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R;\n    }\n\n    /// A reader-writer exclusion type.\n    ///\n    /// The implementing object allows either a number of readers or at most one writer at any point\n    /// in time.\n    pub trait ReadWriteEx {\n        /// The type of encapsulated data.\n        type Data;\n\n        /// Grants temporary mutable access to the encapsulated data.\n        fn write<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R;\n\n        /// Grants temporary immutable access to the encapsulated data.\n        fn read<'a, R>(&'a self, f: impl FnOnce(&'a Self::Data) -> R) -> R;\n    }\n}\n\n/// A pseudo-lock for teaching purposes.\n///\n/// In contrast to a real Mutex implementation, does not protect against concurrent access from\n/// other cores to the contained data. This part is preserved for later lessons.\n///\n/// The lock will only be used as long as it is safe to do so, i.e. as long as the kernel is\n/// executing on a single core.\npub struct IRQSafeNullLock<T>\nwhere\n    T: ?Sized,\n{\n    data: UnsafeCell<T>,\n}\n\n/// A pseudo-lock that is RW during the single-core kernel init phase and RO afterwards.\n///\n/// Intended to encapsulate data that is populated during kernel init when no concurrency exists.\npub struct InitStateLock<T>\nwhere\n    T: ?Sized,\n{\n    data: UnsafeCell<T>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nunsafe impl<T> Send for IRQSafeNullLock<T> where T: ?Sized + Send {}\nunsafe impl<T> Sync for IRQSafeNullLock<T> where T: ?Sized + Send {}\n\nimpl<T> IRQSafeNullLock<T> {\n    /// Create an instance.\n    pub const fn new(data: T) -> Self {\n        Self {\n            data: UnsafeCell::new(data),\n        }\n    }\n}\n\nunsafe impl<T> Send for InitStateLock<T> where T: ?Sized + Send {}\nunsafe impl<T> Sync for InitStateLock<T> where T: ?Sized + Send {}\n\nimpl<T> InitStateLock<T> {\n    /// Create an instance.\n    pub const fn new(data: T) -> Self {\n        Self {\n            data: UnsafeCell::new(data),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse crate::{exception, state};\n\nimpl<T> interface::Mutex for IRQSafeNullLock<T> {\n    type Data = T;\n\n    fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R {\n        // In a real lock, there would be code encapsulating this line that ensures that this\n        // mutable reference will ever only be given out once at a time.\n        let data = unsafe { &mut *self.data.get() };\n\n        // Execute the closure while IRQs are masked.\n        exception::asynchronous::exec_with_irq_masked(|| f(data))\n    }\n}\n\nimpl<T> interface::ReadWriteEx for InitStateLock<T> {\n    type Data = T;\n\n    fn write<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R {\n        assert!(\n            state::state_manager().is_init(),\n            \"InitStateLock::write called after kernel init phase\"\n        );\n        assert!(\n            !exception::asynchronous::is_local_irq_masked(),\n            \"InitStateLock::write called with IRQs unmasked\"\n        );\n\n        let data = unsafe { &mut *self.data.get() };\n\n        f(data)\n    }\n\n    fn read<'a, R>(&'a self, f: impl FnOnce(&'a Self::Data) -> R) -> R {\n        let data = unsafe { &*self.data.get() };\n\n        f(data)\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use test_macros::kernel_test;\n\n    /// InitStateLock must be transparent.\n    #[kernel_test]\n    fn init_state_lock_is_transparent() {\n        use core::mem::size_of;\n\n        assert_eq!(size_of::<InitStateLock<u64>>(), size_of::<u64>());\n    }\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/src/time.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Timer primitives.\n//!\n//! # Resources\n//!\n//! - <https://stackoverflow.com/questions/41081240/idiomatic-callbacks-in-rust>\n//! - <https://doc.rust-lang.org/stable/std/panic/fn.set_hook.html>\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/time.rs\"]\nmod arch_time;\n\nuse crate::{\n    driver, exception,\n    exception::asynchronous::IRQNumber,\n    synchronization::{interface::Mutex, IRQSafeNullLock},\n    warn,\n};\nuse alloc::{boxed::Box, vec::Vec};\nuse core::{\n    sync::atomic::{AtomicBool, Ordering},\n    time::Duration,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nstruct Timeout {\n    due_time: Duration,\n    period: Option<Duration>,\n    callback: TimeoutCallback,\n}\n\nstruct OrderedTimeoutQueue {\n    // Can be replaced with a BinaryHeap once it's new() becomes const.\n    inner: Vec<Timeout>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// The callback type used by timer IRQs.\npub type TimeoutCallback = Box<dyn Fn() + Send>;\n\n/// Provides time management functions.\npub struct TimeManager {\n    queue: IRQSafeNullLock<OrderedTimeoutQueue>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic TIME_MANAGER: TimeManager = TimeManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl Timeout {\n    pub fn is_periodic(&self) -> bool {\n        self.period.is_some()\n    }\n\n    pub fn refresh(&mut self) {\n        if let Some(delay) = self.period {\n            self.due_time += delay;\n        }\n    }\n}\n\nimpl OrderedTimeoutQueue {\n    pub const fn new() -> Self {\n        Self { inner: Vec::new() }\n    }\n\n    pub fn push(&mut self, timeout: Timeout) {\n        self.inner.push(timeout);\n\n        // Note reverse compare order so that earliest expiring item is at end of vec. We do this so\n        // that we can use Vec::pop below to retrieve the item that is next due.\n        self.inner.sort_by(|a, b| b.due_time.cmp(&a.due_time));\n    }\n\n    pub fn peek_next_due_time(&self) -> Option<Duration> {\n        let timeout = self.inner.last()?;\n\n        Some(timeout.due_time)\n    }\n\n    pub fn pop(&mut self) -> Option<Timeout> {\n        self.inner.pop()\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the global TimeManager.\npub fn time_manager() -> &'static TimeManager {\n    &TIME_MANAGER\n}\n\nimpl TimeManager {\n    /// Compatibility string.\n    pub const COMPATIBLE: &'static str = \"ARM Architectural Timer\";\n\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            queue: IRQSafeNullLock::new(OrderedTimeoutQueue::new()),\n        }\n    }\n\n    /// The timer's resolution.\n    pub fn resolution(&self) -> Duration {\n        arch_time::resolution()\n    }\n\n    /// The uptime since power-on of the device.\n    ///\n    /// This includes time consumed by firmware and bootloaders.\n    pub fn uptime(&self) -> Duration {\n        arch_time::uptime()\n    }\n\n    /// Spin for a given duration.\n    pub fn spin_for(&self, duration: Duration) {\n        arch_time::spin_for(duration)\n    }\n\n    /// Set a timeout.\n    fn set_timeout(&self, timeout: Timeout) {\n        self.queue.lock(|queue| {\n            queue.push(timeout);\n\n            arch_time::set_timeout_irq(queue.peek_next_due_time().unwrap());\n        });\n    }\n\n    /// Set a one-shot timeout.\n    pub fn set_timeout_once(&self, delay: Duration, callback: TimeoutCallback) {\n        let timeout = Timeout {\n            due_time: self.uptime() + delay,\n            period: None,\n            callback,\n        };\n\n        self.set_timeout(timeout);\n    }\n\n    /// Set a periodic timeout.\n    pub fn set_timeout_periodic(&self, delay: Duration, callback: TimeoutCallback) {\n        let timeout = Timeout {\n            due_time: self.uptime() + delay,\n            period: Some(delay),\n            callback,\n        };\n\n        self.set_timeout(timeout);\n    }\n}\n\n/// Initialize the timer subsystem.\npub fn init() -> Result<(), &'static str> {\n    static INIT_DONE: AtomicBool = AtomicBool::new(false);\n    if INIT_DONE.load(Ordering::Relaxed) {\n        return Err(\"Init already done\");\n    }\n\n    let timer_descriptor =\n        driver::DeviceDriverDescriptor::new(time_manager(), None, Some(arch_time::timeout_irq()));\n    driver::driver_manager().register_driver(timer_descriptor);\n\n    INIT_DONE.store(true, Ordering::Relaxed);\n    Ok(())\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\n\nimpl driver::interface::DeviceDriver for TimeManager {\n    type IRQNumberType = IRQNumber;\n\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n\n    fn register_and_enable_irq_handler(\n        &'static self,\n        irq_number: &Self::IRQNumberType,\n    ) -> Result<(), &'static str> {\n        use exception::asynchronous::{irq_manager, IRQHandlerDescriptor};\n\n        let descriptor = IRQHandlerDescriptor::new(*irq_number, Self::COMPATIBLE, self);\n\n        irq_manager().register_handler(descriptor)?;\n        irq_manager().enable(irq_number);\n\n        Ok(())\n    }\n}\n\nimpl exception::asynchronous::interface::IRQHandler for TimeManager {\n    fn handle(&self) -> Result<(), &'static str> {\n        arch_time::conclude_timeout_irq();\n\n        let maybe_timeout: Option<Timeout> = self.queue.lock(|queue| {\n            let next_due_time = queue.peek_next_due_time()?;\n            if next_due_time > self.uptime() {\n                return None;\n            }\n\n            let mut timeout = queue.pop().unwrap();\n\n            // Refresh as early as possible to prevent drift.\n            if timeout.is_periodic() {\n                timeout.refresh();\n            }\n\n            Some(timeout)\n        });\n\n        let timeout = match maybe_timeout {\n            None => {\n                warn!(\"Spurious timeout IRQ\");\n                return Ok(());\n            }\n            Some(t) => t,\n        };\n\n        // Important: Call the callback while not holding any lock, because the callback might\n        // attempt to modify data that is protected by a lock (in particular, the timeout queue\n        // itself).\n        (timeout.callback)();\n\n        self.queue.lock(|queue| {\n            if timeout.is_periodic() {\n                // There might be some overhead involved in the periodic path, because the timeout\n                // item is first popped from the underlying Vec and then pushed back again. It could\n                // be faster to keep the item in the queue and find a way to work with a reference\n                // to it.\n                //\n                // We are not going this route on purpose, though. It allows to keep the code simple\n                // and the focus on the high-level concepts.\n                queue.push(timeout);\n            };\n\n            if let Some(due_time) = queue.peek_next_due_time() {\n                arch_time::set_timeout_irq(due_time);\n            }\n        });\n\n        Ok(())\n    }\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/tests/00_console_sanity.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'console_io_test'\n\n# Verify sending and receiving works as expected.\nclass TxRxHandshakeTest < SubtestBase\n    def name\n        'Transmit and Receive handshake'\n    end\n\n    def run(qemu_out, qemu_in)\n        qemu_in.write_nonblock('ABC')\n        expect_or_raise(qemu_out, 'OK1234')\n    end\nend\n\n# Check for correct TX statistics implementation. Depends on test 1 being run first.\nclass TxStatisticsTest < SubtestBase\n    def name\n        'Transmit statistics'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, '6')\n    end\nend\n\n# Check for correct RX statistics implementation. Depends on test 1 being run first.\nclass RxStatisticsTest < SubtestBase\n    def name\n        'Receive statistics'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, '3')\n    end\nend\n\n## -------------------------------------------------------------------------------------------------\n## Test registration\n## -------------------------------------------------------------------------------------------------\ndef subtest_collection\n    [TxRxHandshakeTest.new, TxStatisticsTest.new, RxStatisticsTest.new]\nend\n"
  },
  {
    "path": "20_timer_callbacks/kernel/tests/00_console_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Console sanity tests - RX, TX and statistics.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Console tests should time out on the I/O harness in case of panic.\nmod panic_wait_forever;\n\nuse libkernel::{bsp, console, cpu, exception, memory, print};\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    use console::console;\n\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    // Handshake\n    assert_eq!(console().read_char(), 'A');\n    assert_eq!(console().read_char(), 'B');\n    assert_eq!(console().read_char(), 'C');\n    print!(\"OK1234\");\n\n    // 6\n    print!(\"{}\", console().chars_written());\n\n    // 3\n    print!(\"{}\", console().chars_read());\n\n    // The QEMU process running this test will be closed by the I/O test harness.\n    cpu::wait_forever();\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/tests/01_timer_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Timer sanity tests.\n\n#![feature(custom_test_frameworks)]\n#![no_main]\n#![no_std]\n#![reexport_test_harness_main = \"test_main\"]\n#![test_runner(libkernel::test_runner)]\n\nuse core::time::Duration;\nuse libkernel::{bsp, cpu, exception, memory, time};\nuse test_macros::kernel_test;\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    // Depending on CPU arch, some timer bring-up code could go here. Not needed for the RPi.\n\n    test_main();\n\n    cpu::qemu_exit_success()\n}\n\n/// Simple check that the timer is running.\n#[kernel_test]\nfn timer_is_counting() {\n    assert!(time::time_manager().uptime().as_nanos() > 0)\n}\n\n/// Timer resolution must be sufficient.\n#[kernel_test]\nfn timer_resolution_is_sufficient() {\n    assert!(time::time_manager().resolution().as_nanos() > 0);\n    assert!(time::time_manager().resolution().as_nanos() < 100)\n}\n\n/// Sanity check spin_for() implementation.\n#[kernel_test]\nfn spin_accuracy_check_1_second() {\n    let t1 = time::time_manager().uptime();\n    time::time_manager().spin_for(Duration::from_secs(1));\n    let t2 = time::time_manager().uptime();\n\n    assert_eq!((t2 - t1).as_secs(), 1)\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/tests/02_exception_sync_page_fault.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Page faults must result in synchronous exceptions.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Overwrites libkernel's `panic_wait::_panic_exit()` so that it returns a \"success\" code.\n///\n/// In this test, reaching the panic is a success, because it is called from the synchronous\n/// exception handler, which is what this test wants to achieve.\n///\n/// It also means that this integration test can not use any other code that calls panic!() directly\n/// or indirectly.\nmod panic_exit_success;\n\nuse libkernel::{bsp, cpu, exception, info, memory, println};\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    // This line will be printed as the test header.\n    println!(\"Testing synchronous exception handling by causing a page fault\");\n\n    info!(\"Writing to bottom of address space to address 1 GiB...\");\n    let big_addr: u64 = 1024 * 1024 * 1024;\n    core::ptr::read_volatile(big_addr as *mut u64);\n\n    // If execution reaches here, the memory access above did not cause a page fault exception.\n    cpu::qemu_exit_failure()\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/tests/03_exception_restore_sanity.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'console_io_test'\n\n# Verify that exception restore works.\nclass ExceptionRestoreTest < SubtestBase\n    def name\n        'Exception restore'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, 'Back from system call!')\n    end\nend\n\n## -------------------------------------------------------------------------------------------------\n## Test registration\n## -------------------------------------------------------------------------------------------------\ndef subtest_collection\n    [ExceptionRestoreTest.new]\nend\n"
  },
  {
    "path": "20_timer_callbacks/kernel/tests/03_exception_restore_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A simple sanity test to see if exception restore code works.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Console tests should time out on the I/O harness in case of panic.\nmod panic_wait_forever;\n\nuse core::arch::asm;\nuse libkernel::{bsp, cpu, exception, info, memory, println};\n\n#[inline(never)]\nfn nested_system_call() {\n    #[cfg(target_arch = \"aarch64\")]\n    unsafe {\n        asm!(\"svc #0x1337\", options(nomem, nostack, preserves_flags));\n    }\n\n    #[cfg(not(target_arch = \"aarch64\"))]\n    {\n        info!(\"Not supported yet\");\n        cpu::wait_forever();\n    }\n}\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    // This line will be printed as the test header.\n    println!(\"Testing exception restore\");\n\n    info!(\"Making a dummy system call\");\n\n    // Calling this inside a function indirectly tests if the link register is restored properly.\n    nested_system_call();\n\n    info!(\"Back from system call!\");\n\n    // The QEMU process running this test will be closed by the I/O test harness.\n    cpu::wait_forever();\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/tests/04_exception_irq_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! IRQ handling sanity tests.\n\n#![feature(custom_test_frameworks)]\n#![no_main]\n#![no_std]\n#![reexport_test_harness_main = \"test_main\"]\n#![test_runner(libkernel::test_runner)]\n\nuse libkernel::{bsp, cpu, exception, memory};\nuse test_macros::kernel_test;\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    exception::handling_init();\n    exception::asynchronous::local_irq_unmask();\n\n    test_main();\n\n    cpu::qemu_exit_success()\n}\n\n/// Check that IRQ masking works.\n#[kernel_test]\nfn local_irq_mask_works() {\n    // Precondition: IRQs are unmasked.\n    assert!(exception::asynchronous::is_local_irq_masked());\n\n    exception::asynchronous::local_irq_mask();\n    assert!(!exception::asynchronous::is_local_irq_masked());\n\n    // Restore earlier state.\n    exception::asynchronous::local_irq_unmask();\n}\n\n/// Check that IRQ unmasking works.\n#[kernel_test]\nfn local_irq_unmask_works() {\n    // Precondition: IRQs are masked.\n    exception::asynchronous::local_irq_mask();\n    assert!(!exception::asynchronous::is_local_irq_masked());\n\n    exception::asynchronous::local_irq_unmask();\n    assert!(exception::asynchronous::is_local_irq_masked());\n}\n\n/// Check that IRQ mask save is saving \"something\".\n#[kernel_test]\nfn local_irq_mask_save_works() {\n    // Precondition: IRQs are unmasked.\n    assert!(exception::asynchronous::is_local_irq_masked());\n\n    let first = exception::asynchronous::local_irq_mask_save();\n    assert!(!exception::asynchronous::is_local_irq_masked());\n\n    let second = exception::asynchronous::local_irq_mask_save();\n    assert_ne!(first, second);\n\n    exception::asynchronous::local_irq_restore(first);\n    assert!(exception::asynchronous::is_local_irq_masked());\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/tests/05_backtrace_sanity.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'console_io_test'\n\n# Verify that panic produces a backtrace.\nclass PanicBacktraceTest < SubtestBase\n    def name\n        'Panic produces backtrace'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, 'Kernel panic!')\n        expect_or_raise(qemu_out, 'Backtrace:')\n    end\nend\n\n# Verify backtrace correctness.\nclass BacktraceCorrectnessTest < SubtestBase\n    def name\n        'Backtrace is correct'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, '| core::panicking::panic')\n        expect_or_raise(qemu_out, '| _05_backtrace_sanity::nested')\n        expect_or_raise(qemu_out, '| kernel_init')\n    end\nend\n\n## -------------------------------------------------------------------------------------------------\n## Test registration\n## -------------------------------------------------------------------------------------------------\ndef subtest_collection\n    [PanicBacktraceTest.new, BacktraceCorrectnessTest.new]\nend\n"
  },
  {
    "path": "20_timer_callbacks/kernel/tests/05_backtrace_sanity.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Test if backtracing code detects an invalid frame pointer.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Console tests should time out on the I/O harness in case of panic.\nmod panic_wait_forever;\n\nuse libkernel::{bsp, cpu, exception, memory};\n\n#[inline(never)]\nfn nested() {\n    panic!()\n}\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    nested();\n\n    // The QEMU process running this test will be closed by the I/O test harness.\n    cpu::wait_forever()\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/tests/06_backtrace_invalid_frame.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'console_io_test'\n\n# Test detection of invalid frame pointers.\nclass InvalidFramePointerTest < SubtestBase\n    def name\n        'Detect invalid frame pointer'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out,\n                        /Encountered invalid frame pointer \\(.*\\) during backtrace/)\n    end\nend\n\n## -------------------------------------------------------------------------------------------------\n## Test registration\n## -------------------------------------------------------------------------------------------------\ndef subtest_collection\n    [InvalidFramePointerTest.new]\nend\n"
  },
  {
    "path": "20_timer_callbacks/kernel/tests/06_backtrace_invalid_frame.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Test if backtracing code detects an invalid frame pointer.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Console tests should time out on the I/O harness in case of panic.\nmod panic_wait_forever;\n\nuse libkernel::{backtrace, bsp, cpu, exception, memory};\n\n#[inline(never)]\nfn nested() {\n    unsafe { backtrace::corrupt_previous_frame_addr() };\n\n    panic!()\n}\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    nested();\n\n    // The QEMU process running this test will be closed by the I/O test harness.\n    cpu::wait_forever()\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/tests/07_backtrace_invalid_link.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'console_io_test'\n\n# Test detection of invalid link.\nclass InvalidLinkTest < SubtestBase\n    def name\n        'Detect invalid link'\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, /Link address \\(.*\\) is not contained in kernel .text section/)\n    end\nend\n\n## -------------------------------------------------------------------------------------------------\n## Test registration\n## -------------------------------------------------------------------------------------------------\ndef subtest_collection\n    [InvalidLinkTest.new]\nend\n"
  },
  {
    "path": "20_timer_callbacks/kernel/tests/07_backtrace_invalid_link.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Test if backtracing code detects an invalid link.\n\n#![feature(format_args_nl)]\n#![no_main]\n#![no_std]\n\n/// Console tests should time out on the I/O harness in case of panic.\nmod panic_wait_forever;\n\nuse libkernel::{backtrace, bsp, cpu, exception, memory};\n\n#[inline(never)]\nfn nested_2() -> &'static str {\n    unsafe { backtrace::corrupt_link() };\n    libkernel::println!(\"{}\", libkernel::backtrace::Backtrace);\n    \"foo\"\n}\n\n#[inline(never)]\nfn nested_1() {\n    libkernel::println!(\"{}\", nested_2())\n}\n\n#[no_mangle]\nunsafe fn kernel_init() -> ! {\n    exception::handling_init();\n    memory::init();\n    bsp::driver::qemu_bring_up_console();\n\n    nested_1();\n\n    // The QEMU process running this test will be closed by the I/O test harness.\n    cpu::wait_forever()\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/tests/boot_test_string.rb",
    "content": "# frozen_string_literal: true\n\nEXPECTED_PRINT = 'Once 5'\n"
  },
  {
    "path": "20_timer_callbacks/kernel/tests/panic_exit_success/mod.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n/// Overwrites libkernel's `panic_wait::_panic_exit()` with the QEMU-exit version.\n#[no_mangle]\nfn _panic_exit() -> ! {\n    libkernel::cpu::qemu_exit_success()\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel/tests/panic_wait_forever/mod.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n/// Overwrites libkernel's `panic_wait::_panic_exit()` with wait_forever.\n#[no_mangle]\nfn _panic_exit() -> ! {\n    libkernel::cpu::wait_forever()\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel_symbols/Cargo.toml",
    "content": "[package]\nname = \"kernel_symbols\"\nversion = \"0.1.0\"\nedition = \"2021\"\n\n[features]\ndefault = []\ngenerated_symbols_available = []\n\n##--------------------------------------------------------------------------------------------------\n## Dependencies\n##--------------------------------------------------------------------------------------------------\n\n[dependencies]\ndebug-symbol-types = { path = \"../libraries/debug-symbol-types\" }\n"
  },
  {
    "path": "20_timer_callbacks/kernel_symbols/build.rs",
    "content": "use std::{env, path::Path};\n\nfn main() {\n    if let Ok(path) = env::var(\"KERNEL_SYMBOLS_DEMANGLED_RS\") {\n        if Path::new(&path).exists() {\n            println!(\"cargo:rustc-cfg=feature=\\\"generated_symbols_available\\\"\")\n        }\n    }\n\n    println!(\n        \"cargo:rerun-if-changed={}\",\n        Path::new(\"kernel_symbols.ld\").display()\n    );\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel_symbols/kernel_symbols.ld",
    "content": "/* SPDX-License-Identifier: MIT OR Apache-2.0\n *\n * Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n */\n\nSECTIONS\n{\n    .rodata : {\n        ASSERT(. > 0xffffffff00000000, \"Expected higher half address\")\n\n        KEEP(*(.rodata.symbol_desc*))\n        . = ALIGN(8);\n        *(.rodata*)\n    }\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel_symbols/src/main.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Generation of kernel symbols.\n\n#![no_std]\n#![no_main]\n\n#[cfg(feature = \"generated_symbols_available\")]\ninclude!(env!(\"KERNEL_SYMBOLS_DEMANGLED_RS\"));\n\n#[panic_handler]\nfn panic(_info: &core::panic::PanicInfo) -> ! {\n    unimplemented!()\n}\n"
  },
  {
    "path": "20_timer_callbacks/kernel_symbols.mk",
    "content": "## SPDX-License-Identifier: MIT OR Apache-2.0\n##\n## Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\ninclude ../common/format.mk\ninclude ../common/docker.mk\n\n##--------------------------------------------------------------------------------------------------\n## Check for input variables that need be exported by the calling Makefile\n##--------------------------------------------------------------------------------------------------\nifndef KERNEL_SYMBOLS_TOOL_PATH\n$(error KERNEL_SYMBOLS_TOOL_PATH is not set)\nendif\n\nifndef TARGET\n$(error TARGET is not set)\nendif\n\nifndef KERNEL_SYMBOLS_INPUT_ELF\n$(error KERNEL_SYMBOLS_INPUT_ELF is not set)\nendif\n\nifndef KERNEL_SYMBOLS_OUTPUT_ELF\n$(error KERNEL_SYMBOLS_OUTPUT_ELF is not set)\nendif\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets and Prerequisites\n##--------------------------------------------------------------------------------------------------\nKERNEL_SYMBOLS_MANIFEST      = kernel_symbols/Cargo.toml\nKERNEL_SYMBOLS_LINKER_SCRIPT = kernel_symbols/kernel_symbols.ld\n\nKERNEL_SYMBOLS_RS           = $(KERNEL_SYMBOLS_INPUT_ELF)_symbols.rs\nKERNEL_SYMBOLS_DEMANGLED_RS = $(shell pwd)/$(KERNEL_SYMBOLS_INPUT_ELF)_symbols_demangled.rs\n\nKERNEL_SYMBOLS_ELF      = target/$(TARGET)/release/kernel_symbols\nKERNEL_SYMBOLS_STRIPPED = target/$(TARGET)/release/kernel_symbols_stripped\n\n# Export for build.rs of kernel_symbols crate.\nexport KERNEL_SYMBOLS_DEMANGLED_RS\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Command building blocks\n##--------------------------------------------------------------------------------------------------\nGET_SYMBOLS_SECTION_VIRT_ADDR = $(DOCKER_TOOLS) $(EXEC_SYMBOLS_TOOL) \\\n    --get_symbols_section_virt_addr $(KERNEL_SYMBOLS_OUTPUT_ELF)\n\nRUSTFLAGS = -C link-arg=--script=$(KERNEL_SYMBOLS_LINKER_SCRIPT) \\\n    -C link-arg=--section-start=.rodata=$$($(GET_SYMBOLS_SECTION_VIRT_ADDR))\n\nRUSTFLAGS_PEDANTIC = $(RUSTFLAGS) \\\n    -D warnings                   \\\n    -D missing_docs\n\nCOMPILER_ARGS = --target=$(TARGET) \\\n    --release\n\nRUSTC_CMD   = cargo rustc $(COMPILER_ARGS) --manifest-path $(KERNEL_SYMBOLS_MANIFEST)\nOBJCOPY_CMD = rust-objcopy \\\n    --strip-all            \\\n    -O binary\n\nEXEC_SYMBOLS_TOOL  = ruby $(KERNEL_SYMBOLS_TOOL_PATH)/main.rb\n\n##------------------------------------------------------------------------------\n## Dockerization\n##------------------------------------------------------------------------------\nDOCKER_CMD = docker run -t --rm -v $(shell pwd):/work/tutorial -w /work/tutorial\n\n# DOCKER_IMAGE defined in include file (see top of this file).\nDOCKER_TOOLS = $(DOCKER_CMD) $(DOCKER_IMAGE)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: all symbols measure_time_start measure_time_finish\n\nall: measure_time_start symbols measure_time_finish\n\nsymbols:\n\t@cp $(KERNEL_SYMBOLS_INPUT_ELF) $(KERNEL_SYMBOLS_OUTPUT_ELF)\n\n\t@$(DOCKER_TOOLS) $(EXEC_SYMBOLS_TOOL) --gen_symbols $(KERNEL_SYMBOLS_OUTPUT_ELF) \\\n                $(KERNEL_SYMBOLS_RS)\n\n\t$(call color_progress_prefix, \"Demangling\")\n\t@echo Symbol names\n\t@cat $(KERNEL_SYMBOLS_RS) | rustfilt > $(KERNEL_SYMBOLS_DEMANGLED_RS)\n\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(RUSTC_CMD)\n\n\t$(call color_progress_prefix, \"Stripping\")\n\t@echo Symbols ELF file\n\t@$(OBJCOPY_CMD) $(KERNEL_SYMBOLS_ELF) $(KERNEL_SYMBOLS_STRIPPED)\n\n\t@$(DOCKER_TOOLS) $(EXEC_SYMBOLS_TOOL) --patch_data $(KERNEL_SYMBOLS_OUTPUT_ELF) \\\n                $(KERNEL_SYMBOLS_STRIPPED)\n\n# Note: The following is the only _trivial_ way I could think of that works out of the box on both\n# Linux and macOS. Since macOS does not have the %N nanosecond format string option, the\n# resolution is restricted to whole seconds.\nmeasure_time_start:\n\t@date +%s > /tmp/kernel_symbols_start.date\n\nmeasure_time_finish:\n\t@date +%s > /tmp/kernel_symbols_end.date\n\n\t$(call color_progress_prefix, \"Finished\")\n\t@echo \"in $$((`cat /tmp/kernel_symbols_end.date` - `cat /tmp/kernel_symbols_start.date`)).0s\"\n\n\t@rm /tmp/kernel_symbols_end.date /tmp/kernel_symbols_start.date\n"
  },
  {
    "path": "20_timer_callbacks/libraries/debug-symbol-types/Cargo.toml",
    "content": "[package]\nname = \"debug-symbol-types\"\nversion = \"0.1.0\"\nedition = \"2021\"\n"
  },
  {
    "path": "20_timer_callbacks/libraries/debug-symbol-types/src/lib.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Types for implementing debug symbol support.\n\n#![no_std]\n\nuse core::ops::Range;\n\n/// A symbol containing a size.\n#[repr(C)]\n#[derive(Clone)]\npub struct Symbol {\n    addr_range: Range<usize>,\n    name: &'static str,\n}\n\nimpl Symbol {\n    /// Create an instance.\n    pub const fn new(start: usize, size: usize, name: &'static str) -> Symbol {\n        Symbol {\n            addr_range: Range {\n                start,\n                end: start + size,\n            },\n            name,\n        }\n    }\n\n    /// Returns true if addr is contained in the range.\n    pub fn contains(&self, addr: usize) -> bool {\n        self.addr_range.contains(&addr)\n    }\n\n    /// Returns the symbol's name.\n    pub fn name(&self) -> &'static str {\n        self.name\n    }\n\n    /// Returns the symbol's size.\n    pub fn size(&self) -> usize {\n        self.addr_range.end - self.addr_range.start\n    }\n}\n"
  },
  {
    "path": "20_timer_callbacks/libraries/test-macros/Cargo.toml",
    "content": "[package]\nname = \"test-macros\"\nversion = \"0.1.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n\n[lib]\nproc-macro = true\n\n[dependencies]\nproc-macro2 = \"1.x\"\nquote = \"1.x\"\nsyn = { version = \"1.x\", features = [\"full\"] }\ntest-types = { path = \"../test-types\" }\n"
  },
  {
    "path": "20_timer_callbacks/libraries/test-macros/src/lib.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\nuse proc_macro::TokenStream;\nuse proc_macro2::Span;\nuse quote::quote;\nuse syn::{parse_macro_input, Ident, ItemFn};\n\n#[proc_macro_attribute]\npub fn kernel_test(_attr: TokenStream, input: TokenStream) -> TokenStream {\n    let f = parse_macro_input!(input as ItemFn);\n\n    let test_name = &format!(\"{}\", f.sig.ident);\n    let test_ident = Ident::new(\n        &format!(\"{}_TEST_CONTAINER\", f.sig.ident.to_string().to_uppercase()),\n        Span::call_site(),\n    );\n    let test_code_block = f.block;\n\n    quote!(\n        #[test_case]\n        const #test_ident: test_types::UnitTest = test_types::UnitTest {\n            name: #test_name,\n            test_func: || #test_code_block,\n        };\n    )\n    .into()\n}\n"
  },
  {
    "path": "20_timer_callbacks/libraries/test-types/Cargo.toml",
    "content": "[package]\nname = \"test-types\"\nversion = \"0.1.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n"
  },
  {
    "path": "20_timer_callbacks/libraries/test-types/src/lib.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Types for the `custom_test_frameworks` implementation.\n\n#![no_std]\n\n/// Unit test container.\npub struct UnitTest {\n    /// Name of the test.\n    pub name: &'static str,\n\n    /// Function pointer to the test.\n    pub test_func: fn(),\n}\n"
  },
  {
    "path": "20_timer_callbacks/tools/kernel_symbols_tool/cmds.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\ndef generate_symbols(kernel_elf, output_file)\n    File.open(output_file, 'w') do |file|\n        header = <<~HEREDOC\n            use debug_symbol_types::Symbol;\n\n            # [no_mangle]\n            # [link_section = \".rodata.symbol_desc\"]\n            static KERNEL_SYMBOLS: [Symbol; #{kernel_elf.num_symbols}] = [\n        HEREDOC\n\n        file.write(header)\n        kernel_elf.symbols.each do |sym|\n            value = sym.header.st_value\n            size = sym.header.st_size\n            name = sym.name\n\n            file.write(\"    Symbol::new(#{value}, #{size}, \\\"#{name}\\\"),\\n\")\n        end\n        file.write(\"];\\n\")\n    end\nend\n\ndef get_symbols_section_virt_addr(kernel_elf)\n    kernel_elf.kernel_symbols_section_virt_addr\nend\n\ndef patch_symbol_data(kernel_elf, symbols_blob_path)\n    symbols_blob = File.binread(symbols_blob_path)\n\n    raise if symbols_blob.size > kernel_elf.kernel_symbols_section_size\n\n    File.binwrite(kernel_elf.path, File.binread(symbols_blob_path),\n                  kernel_elf.kernel_symbols_section_offset_in_file)\nend\n\ndef patch_num_symbols(kernel_elf)\n    num_packed = [kernel_elf.num_symbols].pack('Q<*') # \"Q\" == uint64_t, \"<\" == little endian\n    File.binwrite(kernel_elf.path, num_packed, kernel_elf.num_kernel_symbols_offset_in_file)\nend\n"
  },
  {
    "path": "20_timer_callbacks/tools/kernel_symbols_tool/kernel_elf.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n# KernelELF\nclass KernelELF\n    attr_reader :path\n\n    def initialize(kernel_elf_path, kernel_symbols_section, num_kernel_symbols)\n        @elf = ELFTools::ELFFile.new(File.open(kernel_elf_path))\n        @symtab_section = @elf.section_by_name('.symtab')\n\n        @path = kernel_elf_path\n        fetch_values(kernel_symbols_section, num_kernel_symbols)\n    end\n\n    private\n\n    def fetch_values(kernel_symbols_section, num_kernel_symbols)\n        sym = @symtab_section.symbol_by_name(num_kernel_symbols)\n        raise \"Symbol \\\"#{num_kernel_symbols}\\\" not found\" if sym.nil?\n\n        @num_kernel_symbols = sym\n\n        section = @elf.section_by_name(kernel_symbols_section)\n        raise \"Section \\\"#{kernel_symbols_section}\\\" not found\" if section.nil?\n\n        @kernel_symbols_section = section\n    end\n\n    def num_kernel_symbols_virt_addr\n        @num_kernel_symbols.header.st_value\n    end\n\n    def segment_containing_virt_addr(virt_addr)\n        @elf.each_segments do |segment|\n            return segment if segment.vma_in?(virt_addr)\n        end\n    end\n\n    def virt_addr_to_file_offset(virt_addr)\n        segment = segment_containing_virt_addr(virt_addr)\n        segment.vma_to_offset(virt_addr)\n    end\n\n    public\n\n    def symbols\n        non_zero_symbols = @symtab_section.symbols.reject { |sym| sym.header.st_size.zero? }\n        non_zero_symbols.sort_by { |sym| sym.header.st_value }\n    end\n\n    def num_symbols\n        symbols.size\n    end\n\n    def kernel_symbols_section_virt_addr\n        @kernel_symbols_section.header.sh_addr.to_i\n    end\n\n    def kernel_symbols_section_size\n        @kernel_symbols_section.header.sh_size.to_i\n    end\n\n    def kernel_symbols_section_offset_in_file\n        virt_addr_to_file_offset(kernel_symbols_section_virt_addr)\n    end\n\n    def num_kernel_symbols_offset_in_file\n        virt_addr_to_file_offset(num_kernel_symbols_virt_addr)\n    end\nend\n"
  },
  {
    "path": "20_timer_callbacks/tools/kernel_symbols_tool/main.rb",
    "content": "#!/usr/bin/env ruby\n# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'rubygems'\nrequire 'bundler/setup'\nrequire 'colorize'\nrequire 'elftools'\n\nrequire_relative 'kernel_elf'\nrequire_relative 'cmds'\n\nKERNEL_SYMBOLS_SECTION = '.kernel_symbols'\nNUM_KERNEL_SYMBOLS = 'NUM_KERNEL_SYMBOLS'\n\ncmd = ARGV[0]\n\nkernel_elf_path = ARGV[1]\nkernel_elf = KernelELF.new(kernel_elf_path, KERNEL_SYMBOLS_SECTION, NUM_KERNEL_SYMBOLS)\n\ncase cmd\nwhen '--gen_symbols'\n    output_file = ARGV[2]\n\n    print 'Generating'.rjust(12).green.bold\n    puts ' Symbols source file'\n\n    generate_symbols(kernel_elf, output_file)\nwhen '--get_symbols_section_virt_addr'\n    addr = get_symbols_section_virt_addr(kernel_elf)\n\n    puts \"0x#{addr.to_s(16)}\"\nwhen '--patch_data'\n    symbols_blob_path = ARGV[2]\n    num_symbols = kernel_elf.num_symbols\n\n    print 'Patching'.rjust(12).green.bold\n    puts \" Symbols blob and number of symbols (#{num_symbols}) into ELF\"\n\n    patch_symbol_data(kernel_elf, symbols_blob_path)\n    patch_num_symbols(kernel_elf)\nelse\n    raise\nend\n"
  },
  {
    "path": "20_timer_callbacks/tools/translation_table_tool/arch.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n# Bitfield manipulation.\nclass BitField\n    def initialize\n        @value = 0\n    end\n\n    def self.attr_bitfield(name, offset, num_bits)\n        define_method(\"#{name}=\") do |bits|\n            mask = (2**num_bits) - 1\n\n            raise \"Input out of range: #{name} = 0x#{bits.to_s(16)}\" if (bits & ~mask).positive?\n\n            # Clear bitfield\n            @value &= ~(mask << offset)\n\n            # Set it\n            @value |= (bits << offset)\n        end\n    end\n\n    def to_i\n        @value\n    end\n\n    def size_in_byte\n        8\n    end\nend\n\n# An array class that knows its memory location.\nclass CArray < Array\n    attr_reader :phys_start_addr\n\n    def initialize(phys_start_addr, size, &block)\n        @phys_start_addr = phys_start_addr\n\n        super(size, &block)\n    end\n\n    def size_in_byte\n        inject(0) { |sum, n| sum + n.size_in_byte }\n    end\nend\n\n#---------------------------------------------------------------------------------------------------\n# Arch::\n#---------------------------------------------------------------------------------------------------\nmodule Arch\n#---------------------------------------------------------------------------------------------------\n# Arch::ARMv8\n#---------------------------------------------------------------------------------------------------\nmodule ARMv8\n# ARMv8 Table Descriptor.\nclass Stage1TableDescriptor < BitField\n    module NextLevelTableAddr\n        OFFSET = 16\n        NUMBITS = 32\n    end\n\n    module Type\n        OFFSET = 1\n        NUMBITS = 1\n\n        BLOCK = 0\n        TABLE = 1\n    end\n\n    module Valid\n        OFFSET = 0\n        NUMBITS = 1\n\n        FALSE = 0\n        TRUE = 1\n    end\n\n    attr_bitfield(:__next_level_table_addr, NextLevelTableAddr::OFFSET, NextLevelTableAddr::NUMBITS)\n    attr_bitfield(:type, Type::OFFSET, Type::NUMBITS)\n    attr_bitfield(:valid, Valid::OFFSET, Valid::NUMBITS)\n\n    def next_level_table_addr=(addr)\n        addr >>= Granule64KiB::SHIFT\n\n        self.__next_level_table_addr = addr\n    end\n\n    private :__next_level_table_addr=\nend\n\n# ARMv8 level 3 page descriptor.\nclass Stage1PageDescriptor < BitField\n    module UXN\n        OFFSET = 54\n        NUMBITS = 1\n\n        FALSE = 0\n        TRUE = 1\n    end\n\n    module PXN\n        OFFSET = 53\n        NUMBITS = 1\n\n        FALSE = 0\n        TRUE = 1\n    end\n\n    module OutputAddr\n        OFFSET = 16\n        NUMBITS = 32\n    end\n\n    module AF\n        OFFSET = 10\n        NUMBITS = 1\n\n        FALSE = 0\n        TRUE = 1\n    end\n\n    module SH\n        OFFSET = 8\n        NUMBITS = 2\n\n        INNER_SHAREABLE = 0b11\n    end\n\n    module AP\n        OFFSET = 6\n        NUMBITS = 2\n\n        RW_EL1 = 0b00\n        RO_EL1 = 0b10\n    end\n\n    module AttrIndx\n        OFFSET = 2\n        NUMBITS = 3\n    end\n\n    module Type\n        OFFSET = 1\n        NUMBITS = 1\n\n        RESERVED_INVALID = 0\n        PAGE = 1\n    end\n\n    module Valid\n        OFFSET = 0\n        NUMBITS = 1\n\n        FALSE = 0\n        TRUE = 1\n    end\n\n    attr_bitfield(:uxn, UXN::OFFSET, UXN::NUMBITS)\n    attr_bitfield(:pxn, PXN::OFFSET, PXN::NUMBITS)\n    attr_bitfield(:__output_addr, OutputAddr::OFFSET, OutputAddr::NUMBITS)\n    attr_bitfield(:af, AF::OFFSET, AF::NUMBITS)\n    attr_bitfield(:sh, SH::OFFSET, SH::NUMBITS)\n    attr_bitfield(:ap, AP::OFFSET, AP::NUMBITS)\n    attr_bitfield(:attr_indx, AttrIndx::OFFSET, AttrIndx::NUMBITS)\n    attr_bitfield(:type, Type::OFFSET, Type::NUMBITS)\n    attr_bitfield(:valid, Valid::OFFSET, Valid::NUMBITS)\n\n    def output_addr=(addr)\n        addr >>= Granule64KiB::SHIFT\n\n        self.__output_addr = addr\n    end\n\n    private :__output_addr=\nend\n\n# Translation table representing the structure defined in translation_table.rs.\nclass TranslationTable\n    module MAIR\n        NORMAL = 1\n    end\n\n    def initialize\n        do_sanity_checks\n\n        num_lvl2_tables = BSP.kernel_virt_addr_space_size >> Granule512MiB::SHIFT\n\n        @lvl3 = new_lvl3(num_lvl2_tables, BSP.phys_addr_of_kernel_tables)\n\n        @lvl2_phys_start_addr = @lvl3.phys_start_addr + @lvl3.size_in_byte\n        @lvl2 = new_lvl2(num_lvl2_tables, @lvl2_phys_start_addr)\n\n        populate_lvl2_entries\n    end\n\n    def map_at(virt_region, phys_region, attributes)\n        return if virt_region.empty?\n\n        raise if virt_region.size != phys_region.size\n        raise if phys_region.last > BSP.phys_addr_space_end_page\n\n        virt_region.zip(phys_region).each do |virt_page, phys_page|\n            desc = page_descriptor_from(virt_page)\n            set_lvl3_entry(desc, phys_page, attributes)\n        end\n    end\n\n    def to_binary\n        data = @lvl3.flatten.map(&:to_i) + @lvl2.map(&:to_i)\n        data.pack('Q<*') # \"Q\" == uint64_t, \"<\" == little endian\n    end\n\n    def phys_tables_base_addr_binary\n        [@lvl2_phys_start_addr].pack('Q<*') # \"Q\" == uint64_t, \"<\" == little endian\n    end\n\n    def phys_tables_base_addr\n        @lvl2_phys_start_addr\n    end\n\n    private\n\n    def do_sanity_checks\n        raise unless BSP.kernel_granule::SIZE == Granule64KiB::SIZE\n        raise unless (BSP.kernel_virt_addr_space_size % Granule512MiB::SIZE).zero?\n    end\n\n    def new_lvl3(num_lvl2_tables, start_addr)\n        CArray.new(start_addr, num_lvl2_tables) do\n            temp = CArray.new(start_addr, 8192) do\n                Stage1PageDescriptor.new\n            end\n            start_addr += temp.size_in_byte\n\n            temp\n        end\n    end\n\n    def new_lvl2(num_lvl2_tables, start_addr)\n        CArray.new(start_addr, num_lvl2_tables) do\n            Stage1TableDescriptor.new\n        end\n    end\n\n    def populate_lvl2_entries\n        @lvl2.each_with_index do |descriptor, i|\n            descriptor.next_level_table_addr = @lvl3[i].phys_start_addr\n            descriptor.type = Stage1TableDescriptor::Type::TABLE\n            descriptor.valid = Stage1TableDescriptor::Valid::TRUE\n        end\n    end\n\n    def lvl2_lvl3_index_from(addr)\n        addr -= BSP.kernel_virt_start_addr\n\n        lvl2_index = addr >> Granule512MiB::SHIFT\n        lvl3_index = (addr & Granule512MiB::MASK) >> Granule64KiB::SHIFT\n\n        raise unless lvl2_index < @lvl2.size\n\n        [lvl2_index, lvl3_index]\n    end\n\n    def page_descriptor_from(virt_addr)\n        lvl2_index, lvl3_index = lvl2_lvl3_index_from(virt_addr)\n\n        @lvl3[lvl2_index][lvl3_index]\n    end\n\n    # rubocop:disable Metrics/MethodLength\n    def set_attributes(desc, attributes)\n        case attributes.mem_attributes\n        when :CacheableDRAM\n            desc.sh = Stage1PageDescriptor::SH::INNER_SHAREABLE\n            desc.attr_indx = MAIR::NORMAL\n        else\n            raise 'Invalid input'\n        end\n\n        desc.ap = case attributes.acc_perms\n                  when :ReadOnly\n                      Stage1PageDescriptor::AP::RO_EL1\n                  when :ReadWrite\n                      Stage1PageDescriptor::AP::RW_EL1\n                  else\n                      raise 'Invalid input'\n\n                  end\n\n        desc.pxn = if attributes.execute_never\n                       Stage1PageDescriptor::PXN::TRUE\n                   else\n                       Stage1PageDescriptor::PXN::FALSE\n                   end\n\n        desc.uxn = Stage1PageDescriptor::UXN::TRUE\n    end\n    # rubocop:enable Metrics/MethodLength\n\n    def set_lvl3_entry(desc, output_addr, attributes)\n        desc.output_addr = output_addr\n        desc.af = Stage1PageDescriptor::AF::TRUE\n        desc.type = Stage1PageDescriptor::Type::PAGE\n        desc.valid = Stage1PageDescriptor::Valid::TRUE\n\n        set_attributes(desc, attributes)\n    end\nend\nend\nend\n"
  },
  {
    "path": "20_timer_callbacks/tools/translation_table_tool/bsp.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n# Raspberry Pi 3 + 4\nclass RaspberryPi\n    attr_reader :kernel_granule, :kernel_virt_addr_space_size, :kernel_virt_start_addr\n\n    MEMORY_SRC = File.read('kernel/src/bsp/raspberrypi/memory.rs').split(\"\\n\")\n\n    def initialize\n        @kernel_granule = Granule64KiB\n\n        @kernel_virt_addr_space_size = KERNEL_ELF.symbol_value('__kernel_virt_addr_space_size')\n        @kernel_virt_start_addr = KERNEL_ELF.symbol_value('__kernel_virt_start_addr')\n\n        @virt_addr_of_kernel_tables = KERNEL_ELF.symbol_value('KERNEL_TABLES')\n        @virt_addr_of_phys_kernel_tables_base_addr = KERNEL_ELF.symbol_value(\n            'PHYS_KERNEL_TABLES_BASE_ADDR'\n        )\n    end\n\n    def phys_addr_of_kernel_tables\n        KERNEL_ELF.virt_to_phys(@virt_addr_of_kernel_tables)\n    end\n\n    def kernel_tables_offset_in_file\n        KERNEL_ELF.virt_addr_to_file_offset(@virt_addr_of_kernel_tables)\n    end\n\n    def phys_kernel_tables_base_addr_offset_in_file\n        KERNEL_ELF.virt_addr_to_file_offset(@virt_addr_of_phys_kernel_tables_base_addr)\n    end\n\n    def phys_addr_space_end_page\n        x = MEMORY_SRC.grep(/pub const END/)\n        x = case BSP_TYPE\n            when :rpi3\n                x[0]\n            when :rpi4\n                x[1]\n            else\n                raise\n            end\n\n        # Extract the hex literal with underscores like 0x0123_abcd.\n        x = x.scan(/0x[\\h_]*/)[0]\n\n        # Further remove x and _ and convert to int.\n        x.scan(/\\h+/).join.to_i(16)\n    end\nend\n"
  },
  {
    "path": "20_timer_callbacks/tools/translation_table_tool/generic.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\nmodule Granule64KiB\n    SIZE = 64 * 1024\n    SHIFT = Math.log2(SIZE).to_i\nend\n\nmodule Granule512MiB\n    SIZE = 512 * 1024 * 1024\n    SHIFT = Math.log2(SIZE).to_i\n    MASK = SIZE - 1\nend\n\n# Monkey-patch Integer with some helper functions.\nclass Integer\n    def power_of_two?\n        self[0].zero?\n    end\n\n    def aligned?(alignment)\n        raise unless alignment.power_of_two?\n\n        (self & (alignment - 1)).zero?\n    end\n\n    def align_up(alignment)\n        raise unless alignment.power_of_two?\n\n        (self + alignment - 1) & ~(alignment - 1)\n    end\n\n    def to_hex_underscore(with_leading_zeros: false)\n        fmt = with_leading_zeros ? '%016x' : '%x'\n        value = format(fmt, self).to_s.reverse.scan(/.{4}|.+/).join('_').reverse\n\n        format('0x%s', value)\n    end\nend\n\n# An array where each value is the start address of a Page.\nclass MemoryRegion < Array\n    def initialize(start_addr, size, granule_size)\n        raise unless start_addr.aligned?(granule_size)\n        raise unless size.positive?\n        raise unless (size % granule_size).zero?\n\n        num_pages = size / granule_size\n        super(num_pages) do |i|\n            (i * granule_size) + start_addr\n        end\n    end\nend\n\n# Collection of memory attributes.\nclass AttributeFields\n    attr_reader :mem_attributes, :acc_perms, :execute_never\n\n    def initialize(mem_attributes, acc_perms, execute_never)\n        @mem_attributes = mem_attributes\n        @acc_perms = acc_perms\n        @execute_never = execute_never\n    end\n\n    def to_s\n        x = case @mem_attributes\n            when :CacheableDRAM\n                'C'\n            else\n                '?'\n            end\n\n        y = case @acc_perms\n            when :ReadWrite\n                'RW'\n            when :ReadOnly\n                'RO'\n            else\n                '??'\n            end\n\n        z = @execute_never ? 'XN' : 'X '\n\n        \"#{x} #{y} #{z}\"\n    end\nend\n\n# A container that describes a virt-to-phys region mapping.\nclass MappingDescriptor\n    @max_section_name_length = 'Sections'.length\n\n    class << self\n        attr_accessor :max_section_name_length\n\n        def update_max_section_name_length(length)\n            @max_section_name_length = [@max_section_name_length, length].max\n        end\n    end\n\n    attr_reader :name, :virt_region, :phys_region, :attributes\n\n    def initialize(name, virt_region, phys_region, attributes)\n        @name = name\n        @virt_region = virt_region\n        @phys_region = phys_region\n        @attributes = attributes\n    end\n\n    def size_human_readable(size)\n        if size >= (1024 * 1024)\n            \"#{(size / (1024 * 1024)).to_s.rjust(3)} MiB\"\n        elsif size >= 1024\n            \"#{(size / 1024).to_s.rjust(3)} KiB\"\n        else\n            raise\n        end\n    end\n\n    def to_s\n        name = @name.ljust(self.class.max_section_name_length)\n        virt_start = @virt_region.first.to_hex_underscore(with_leading_zeros: true)\n        phys_start = @phys_region.first.to_hex_underscore(with_leading_zeros: true)\n        size = size_human_readable(@virt_region.size * 65_536)\n\n        \"#{name} | #{virt_start} | #{phys_start} | #{size} | #{@attributes}\"\n    end\n\n    def self.print_divider\n        print '             '\n        print '-' * max_section_name_length\n        puts '--------------------------------------------------------------------'\n    end\n\n    def self.print_header\n        print_divider\n        print '             '\n        print 'Sections'.center(max_section_name_length)\n        print '   '\n        print 'Virt Start Addr'.center(21)\n        print '   '\n        print 'Phys Start Addr'.center(21)\n        print '   '\n        print 'Size'.center(7)\n        print '   '\n        print 'Attr'.center(7)\n        puts\n        print_divider\n    end\nend\n\ndef kernel_map_binary\n    mapping_descriptors = KERNEL_ELF.generate_mapping_descriptors\n\n    # Generate_mapping_descriptors updates the header being printed with this call. So it must come\n    # afterwards.\n    MappingDescriptor.print_header\n\n    mapping_descriptors.each do |i|\n        print 'Generating'.rjust(12).green.bold\n        print ' '\n        puts i\n\n        TRANSLATION_TABLES.map_at(i.virt_region, i.phys_region, i.attributes)\n    end\n\n    MappingDescriptor.print_divider\nend\n\ndef kernel_patch_tables(kernel_elf_path)\n    print 'Patching'.rjust(12).green.bold\n    print ' Kernel table struct at ELF file offset '\n    puts BSP.kernel_tables_offset_in_file.to_hex_underscore\n\n    File.binwrite(kernel_elf_path, TRANSLATION_TABLES.to_binary, BSP.kernel_tables_offset_in_file)\nend\n\ndef kernel_patch_base_addr(kernel_elf_path)\n    print 'Patching'.rjust(12).green.bold\n    print ' Kernel tables physical base address start argument to value '\n    print TRANSLATION_TABLES.phys_tables_base_addr.to_hex_underscore\n    print ' at ELF file offset '\n    puts BSP.phys_kernel_tables_base_addr_offset_in_file.to_hex_underscore\n\n    File.binwrite(kernel_elf_path, TRANSLATION_TABLES.phys_tables_base_addr_binary,\n                  BSP.phys_kernel_tables_base_addr_offset_in_file)\nend\n"
  },
  {
    "path": "20_timer_callbacks/tools/translation_table_tool/kernel_elf.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n# KernelELF\nclass KernelELF\n    SECTION_FLAG_ALLOC = 2\n\n    def initialize(kernel_elf_path)\n        @elf = ELFTools::ELFFile.new(File.open(kernel_elf_path))\n        @symtab_section = @elf.section_by_name('.symtab')\n    end\n\n    def machine\n        @elf.machine.to_sym\n    end\n\n    def symbol_value(symbol_name)\n        @symtab_section.symbol_by_name(symbol_name).header.st_value\n    end\n\n    def segment_containing_virt_addr(virt_addr)\n        @elf.each_segments do |segment|\n            return segment if segment.vma_in?(virt_addr)\n        end\n    end\n\n    def virt_to_phys(virt_addr)\n        segment = segment_containing_virt_addr(virt_addr)\n        translation_offset = segment.header.p_vaddr - segment.header.p_paddr\n\n        virt_addr - translation_offset\n    end\n\n    def virt_addr_to_file_offset(virt_addr)\n        segment = segment_containing_virt_addr(virt_addr)\n        segment.vma_to_offset(virt_addr)\n    end\n\n    def sections_in_segment(segment)\n        head = segment.mem_head\n        tail = segment.mem_tail\n\n        sections = @elf.each_sections.select do |section|\n            file_offset = section.header.sh_addr\n            flags = section.header.sh_flags\n\n            file_offset >= head && file_offset < tail && (flags & SECTION_FLAG_ALLOC != 0)\n        end\n\n        sections.map(&:name).join(' ')\n    end\n\n    def select_load_segments\n        @elf.each_segments.select do |segment|\n            segment.instance_of?(ELFTools::Segments::LoadSegment)\n        end\n    end\n\n    def segment_get_acc_perms(segment)\n        if segment.readable? && segment.writable?\n            :ReadWrite\n        elsif segment.readable?\n            :ReadOnly\n        else\n            :Invalid\n        end\n    end\n\n    def update_max_section_name_length(descriptors)\n        MappingDescriptor.update_max_section_name_length(descriptors.map { |i| i.name.size }.max)\n    end\n\n    def generate_mapping_descriptors\n        descriptors = select_load_segments.map do |segment|\n            # Assume each segment is page aligned.\n            size = segment.mem_size.align_up(BSP.kernel_granule::SIZE)\n            virt_start_addr = segment.header.p_vaddr\n            phys_start_addr = segment.header.p_paddr\n            acc_perms = segment_get_acc_perms(segment)\n            execute_never = !segment.executable?\n            section_names = sections_in_segment(segment)\n\n            virt_region = MemoryRegion.new(virt_start_addr, size, BSP.kernel_granule::SIZE)\n            phys_region = MemoryRegion.new(phys_start_addr, size, BSP.kernel_granule::SIZE)\n            attributes = AttributeFields.new(:CacheableDRAM, acc_perms, execute_never)\n\n            MappingDescriptor.new(section_names, virt_region, phys_region, attributes)\n        end\n\n        update_max_section_name_length(descriptors)\n        descriptors\n    end\nend\n"
  },
  {
    "path": "20_timer_callbacks/tools/translation_table_tool/main.rb",
    "content": "#!/usr/bin/env ruby\n# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'rubygems'\nrequire 'bundler/setup'\nrequire 'colorize'\nrequire 'elftools'\n\nrequire_relative 'generic'\nrequire_relative 'kernel_elf'\nrequire_relative 'bsp'\nrequire_relative 'arch'\n\nBSP_TYPE = ARGV[0].to_sym\nkernel_elf_path = ARGV[1]\n\nstart = Time.now\n\nKERNEL_ELF = KernelELF.new(kernel_elf_path)\n\nBSP = case BSP_TYPE\n      when :rpi3, :rpi4\n          RaspberryPi.new\n      else\n          raise\n      end\n\nTRANSLATION_TABLES = case KERNEL_ELF.machine\n                     when :AArch64\n                         Arch::ARMv8::TranslationTable.new\n                     else\n                         raise\n                     end\n\nkernel_map_binary\nkernel_patch_tables(kernel_elf_path)\nkernel_patch_base_addr(kernel_elf_path)\n\nelapsed = Time.now - start\n\nprint 'Finished'.rjust(12).green.bold\nputs \" in #{elapsed.round(2)}s\"\n"
  },
  {
    "path": "Gemfile",
    "content": "# frozen_string_literal: true\n\nsource 'https://rubygems.org'\n\ngem 'colorize'\ngem 'elftools'\n\ngroup :uart do\n    gem 'ruby-progressbar'\n    gem 'serialport'\nend\n\ngroup :development do\n    gem 'rubocop', '>= 1.38.0', require: false\nend\n"
  },
  {
    "path": "LICENSE-APACHE",
    "content": "                              Apache License\n                        Version 2.0, January 2004\n                     http://www.apache.org/licenses/\n\nTERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n1. Definitions.\n\n   \"License\" shall mean the terms and conditions for use, reproduction,\n   and distribution as defined by Sections 1 through 9 of this document.\n\n   \"Licensor\" shall mean the copyright owner or entity authorized by\n   the copyright owner that is granting the License.\n\n   \"Legal Entity\" shall mean the union of the acting entity and all\n   other entities that control, are controlled by, or are under common\n   control with that entity. For the purposes of this definition,\n   \"control\" means (i) the power, direct or indirect, to cause the\n   direction or management of such entity, whether by contract or\n   otherwise, or (ii) ownership of fifty percent (50%) or more of the\n   outstanding shares, or (iii) beneficial ownership of such entity.\n\n   \"You\" (or \"Your\") shall mean an individual or Legal Entity\n   exercising permissions granted by this License.\n\n   \"Source\" form shall mean the preferred form for making modifications,\n   including but not limited to software source code, documentation\n   source, and configuration files.\n\n   \"Object\" form shall mean any form resulting from mechanical\n   transformation or translation of a Source form, including but\n   not limited to compiled object code, generated documentation,\n   and conversions to other media types.\n\n   \"Work\" shall mean the work of authorship, whether in Source or\n   Object form, made available under the License, as indicated by a\n   copyright notice that is included in or attached to the work\n   (an example is provided in the Appendix below).\n\n   \"Derivative Works\" shall mean any work, whether in Source or Object\n   form, that is based on (or derived from) the Work and for which the\n   editorial revisions, annotations, elaborations, or other modifications\n   represent, as a whole, an original work of authorship. For the purposes\n   of this License, Derivative Works shall not include works that remain\n   separable from, or merely link (or bind by name) to the interfaces of,\n   the Work and Derivative Works thereof.\n\n   \"Contribution\" shall mean any work of authorship, including\n   the original version of the Work and any modifications or additions\n   to that Work or Derivative Works thereof, that is intentionally\n   submitted to Licensor for inclusion in the Work by the copyright owner\n   or by an individual or Legal Entity authorized to submit on behalf of\n   the copyright owner. For the purposes of this definition, \"submitted\"\n   means any form of electronic, verbal, or written communication sent\n   to the Licensor or its representatives, including but not limited to\n   communication on electronic mailing lists, source code control systems,\n   and issue tracking systems that are managed by, or on behalf of, the\n   Licensor for the purpose of discussing and improving the Work, but\n   excluding communication that is conspicuously marked or otherwise\n   designated in writing by the copyright owner as \"Not a Contribution.\"\n\n   \"Contributor\" shall mean Licensor and any individual or Legal Entity\n   on behalf of whom a Contribution has been received by Licensor and\n   subsequently incorporated within the Work.\n\n2. Grant of Copyright License. Subject to the terms and conditions of\n   this License, each Contributor hereby grants to You a perpetual,\n   worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n   copyright license to reproduce, prepare Derivative Works of,\n   publicly display, publicly perform, sublicense, and distribute the\n   Work and such Derivative Works in Source or Object form.\n\n3. Grant of Patent License. Subject to the terms and conditions of\n   this License, each Contributor hereby grants to You a perpetual,\n   worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n   (except as stated in this section) patent license to make, have made,\n   use, offer to sell, sell, import, and otherwise transfer the Work,\n   where such license applies only to those patent claims licensable\n   by such Contributor that are necessarily infringed by their\n   Contribution(s) alone or by combination of their Contribution(s)\n   with the Work to which such Contribution(s) was submitted. If You\n   institute patent litigation against any entity (including a\n   cross-claim or counterclaim in a lawsuit) alleging that the Work\n   or a Contribution incorporated within the Work constitutes direct\n   or contributory patent infringement, then any patent licenses\n   granted to You under this License for that Work shall terminate\n   as of the date such litigation is filed.\n\n4. Redistribution. You may reproduce and distribute copies of the\n   Work or Derivative Works thereof in any medium, with or without\n   modifications, and in Source or Object form, provided that You\n   meet the following conditions:\n\n   (a) You must give any other recipients of the Work or\n       Derivative Works a copy of this License; and\n\n   (b) You must cause any modified files to carry prominent notices\n       stating that You changed the files; and\n\n   (c) You must retain, in the Source form of any Derivative Works\n       that You distribute, all copyright, patent, trademark, and\n       attribution notices from the Source form of the Work,\n       excluding those notices that do not pertain to any part of\n       the Derivative Works; and\n\n   (d) If the Work includes a \"NOTICE\" text file as part of its\n       distribution, then any Derivative Works that You distribute must\n       include a readable copy of the attribution notices contained\n       within such NOTICE file, excluding those notices that do not\n       pertain to any part of the Derivative Works, in at least one\n       of the following places: within a NOTICE text file distributed\n       as part of the Derivative Works; within the Source form or\n       documentation, if provided along with the Derivative Works; or,\n       within a display generated by the Derivative Works, if and\n       wherever such third-party notices normally appear. The contents\n       of the NOTICE file are for informational purposes only and\n       do not modify the License. You may add Your own attribution\n       notices within Derivative Works that You distribute, alongside\n       or as an addendum to the NOTICE text from the Work, provided\n       that such additional attribution notices cannot be construed\n       as modifying the License.\n\n   You may add Your own copyright statement to Your modifications and\n   may provide additional or different license terms and conditions\n   for use, reproduction, or distribution of Your modifications, or\n   for any such Derivative Works as a whole, provided Your use,\n   reproduction, and distribution of the Work otherwise complies with\n   the conditions stated in this License.\n\n5. Submission of Contributions. Unless You explicitly state otherwise,\n   any Contribution intentionally submitted for inclusion in the Work\n   by You to the Licensor shall be under the terms and conditions of\n   this License, without any additional terms or conditions.\n   Notwithstanding the above, nothing herein shall supersede or modify\n   the terms of any separate license agreement you may have executed\n   with Licensor regarding such Contributions.\n\n6. Trademarks. This License does not grant permission to use the trade\n   names, trademarks, service marks, or product names of the Licensor,\n   except as required for reasonable and customary use in describing the\n   origin of the Work and reproducing the content of the NOTICE file.\n\n7. Disclaimer of Warranty. Unless required by applicable law or\n   agreed to in writing, Licensor provides the Work (and each\n   Contributor provides its Contributions) on an \"AS IS\" BASIS,\n   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n   implied, including, without limitation, any warranties or conditions\n   of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n   PARTICULAR PURPOSE. You are solely responsible for determining the\n   appropriateness of using or redistributing the Work and assume any\n   risks associated with Your exercise of permissions under this License.\n\n8. Limitation of Liability. In no event and under no legal theory,\n   whether in tort (including negligence), contract, or otherwise,\n   unless required by applicable law (such as deliberate and grossly\n   negligent acts) or agreed to in writing, shall any Contributor be\n   liable to You for damages, including any direct, indirect, special,\n   incidental, or consequential damages of any character arising as a\n   result of this License or out of the use or inability to use the\n   Work (including but not limited to damages for loss of goodwill,\n   work stoppage, computer failure or malfunction, or any and all\n   other commercial damages or losses), even if such Contributor\n   has been advised of the possibility of such damages.\n\n9. Accepting Warranty or Additional Liability. While redistributing\n   the Work or Derivative Works thereof, You may choose to offer,\n   and charge a fee for, acceptance of support, warranty, indemnity,\n   or other liability obligations and/or rights consistent with this\n   License. However, in accepting such obligations, You may act only\n   on Your own behalf and on Your sole responsibility, not on behalf\n   of any other Contributor, and only if You agree to indemnify,\n   defend, and hold each Contributor harmless for any liability\n   incurred by, or claims asserted against, such Contributor by reason\n   of your accepting any such warranty or additional liability.\n\nEND OF TERMS AND CONDITIONS\n\nAPPENDIX: How to apply the Apache License to your work.\n\n   To apply the Apache License to your work, attach the following\n   boilerplate notice, with the fields enclosed by brackets \"[]\"\n   replaced with your own identifying information. (Don't include\n   the brackets!)  The text should be enclosed in the appropriate\n   comment syntax for the file format. We also recommend that a\n   file or class name and description of purpose be included on the\n   same \"printed page\" as the copyright notice for easier\n   identification within third-party archives.\n\nCopyright [yyyy] [name of copyright owner]\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n\thttp://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n"
  },
  {
    "path": "LICENSE-MIT",
    "content": "MIT License\n\nCopyright (C) 2018-2023 by the respective authors\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n"
  },
  {
    "path": "README.CN.md",
    "content": "# Rust 开发树莓派操作系统教程\n\n![](https://github.com/rust-embedded/rust-raspberrypi-OS-tutorials/workflows/BSP-RPi3/badge.svg) ![](https://github.com/rust-embedded/rust-raspberrypi-OS-tutorials/workflows/BSP-RPi4/badge.svg) ![](https://github.com/rust-embedded/rust-raspberrypi-OS-tutorials/workflows/Unit-Tests/badge.svg) ![](https://github.com/rust-embedded/rust-raspberrypi-OS-tutorials/workflows/Integration-Tests/badge.svg) ![](https://img.shields.io/badge/License-MIT%20OR%20Apache--2.0-blue)\n\n<br/>\n\n<img src=\"doc/header.jpg\" height=\"379\"> <img src=\"doc/minipush_demo_frontpage.gif\" height=\"379\">\n\n## ℹ️ 介绍\n\n该系列教程适用于 ARM64 位[ARMv8-A 架构]的业余操作系统开发者。该教程会从零开始，一步步地指导你如何开发一个[功能健全的]\n嵌入式操作系统的内核。包含了实现一般操作系统的任务，例如开发串口控制台，设置虚拟内存和处理硬件异常。\n同时利用 Rust 的特性来提供系统的安全和速度。\n\n祝你玩得开心！\n\n_带上我最诚挚的问候,<br>Andre ([@andre-richter])_\n\n[armv8-a 架构]: https://developer.arm.com/products/architecture/cpu-architecture/a-profile/docs\n[功能健全的]: https://en.wikipedia.org/wiki/Monolithic_kernel\n[@andre-richter]: https://github.com/andre-richter\n\n## 📑 教程结构\n\n- 每篇教程都包含一个独立可引导的二进制内核文件。\n- 每篇新的教程都在之前的基础上扩展。\n- 每篇教程的指南里面都有一个简短的章节来总结新增的代码和功能，也会展示源代码的区别，方便检查和同步。\n- 部分教程中有除了`tl;dr`章节外还有非常详细、具体的介绍。长期计划是所有的教程都会有详细的文字说明。但是目前只有那些我认为教程的`tl;dr`和`diff`还不够详细的章节会详细说明。\n- 教程中所用的代码可以在**树莓派 3 和 4**上运行。\n  - 教程的第一到五章是基础内容，只能运行在`QEMU`上。\n  - 到了[第六章]时(06_drivers_gpio_uart)，你可以在树莓派上加载和运行内核并通过`UART`来观察输出结果。\n- 虽然这些教程是以树莓派 3 和 4 为试验对象，但代码是模块化的，所以应该容易移植到其他 CPU 架构的开发板上。\n  - 我希望会有人有机会去实现**RISC-V**架构的代码。\n- 我推荐使用[Visual Studio Code],配置[Rust Analyzer]插件开发代码。\n- 除了文本教程之外，也可以用`make doc`命令利用网页的优势来浏览代码。\n\n### `make doc` 的输出\n\n![make doc](doc/make_doc.png)\n\n[Visual Studio Code]: https://code.visualstudio.com\n[Rust Analyzer]: https://rust-analyzer.github.io\n\n## 🛠 系统要求\n\n本教程主要是面向**Linux**发行版的。理论上，文中大部分内容在其他类Unix系统诸如**macOS**也能正常工作，但请注意，只是理论上。\n\n### 🚀 tl;dr 版本\n\n1. [安装 Docker][install_docker]。\n2. **(仅限Linux)** 确保您的用户帐户在 [docker group] 中。\n3. 准备Rust工具链。其中大部分将在首次使用时通过[rust-toolchain.toml](rust-toolchain.toml)文件进行处理。我们要做的是：\n   1. 如果你已经安装了一个版本的Rust:\n      ```bash\n      cargo install cargo-binutils rustfilt\n      ```\n\n   1. 如果你想要全新安装:\n      ```bash\n      curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh\n\n      source $HOME/.cargo/env\n      cargo install cargo-binutils rustfilt\n      ```\n\n4. 如果你使用 `Visual Studio Code`，我强烈推荐你安装[Rust Analyzer 扩展]。\n5. **(仅限macOS)** 安装一些`Ruby` gems。\n\n这是作者最后一次在`macOS Monterey`上用`Ruby 3.0.2`版本进行测试。如果您正在使用`rbenv`，那么相应的`.ruby-version`文件已经就位。\n如果你从未听说过`rbenv`，请尝试使用[这个指南](https://stackoverflow.com/a/68118750)。\n\n在存储库根文件夹中运行此操作：\n\n```bash\nbundle config set --local path '.vendor/bundle'\nbundle config set --local without 'development'\nbundle install\n```\n\n[docker group]: https://docs.docker.com/engine/install/linux-postinstall/\n[Rust Analyzer 扩展]: https://marketplace.visualstudio.com/items?itemName=matklad.rust-analyzer\n\n\n## 🧰 长期版本: 消除工具链烦恼\n\n这个系列的教程会着重关注用户体验的友好性。因此，我尽量消除嵌入式开发中的最大痛点：`Toolchain hassle`。\n\nRust内置的交叉编译支持在这方面帮了我们大忙。我们只需要使用`rustup`安装目标工具链就可以在`x86`宿主机上交叉编译支持树莓派的目标文件。然而，除了Rust编译器，我们还需要更多的工具。例如：\nRust本身在这方面已经起到了很大的作用，因为它内置了对交叉编译的支持。从`x86`宿主机到树莓派的`AArch64`架构的交叉编译所需的一切都将由`rustup`自动安装。然而，除了Rust编译器，我们还将使用更多的工具。例如：\n\n- 用于在我们的宿主系统上模拟我们内核运行环境的`QEMU`。\n- 一个叫`Minipush`的自制工具，可以通过`UART`将内核加载到树莓派上。\n- 用于调式目标文件的`OpenOCD`和`GDB`。\n\n在你的宿主机上安装/编译正确版本的上述工具很可能会遇到很多麻烦。举个例子，你的发行版也许并不会提供我们需要的最新版本的软件包。又或者你在编译这些工具时会遇到一些很难处理的依赖问题。\n\n这也是为什么我们要尽可能使用[Docker][install_docker]的原因。我们提供了一个已经预装了所有需要的工具及依赖的容器，当需要的时候它就会被自动拉取。如果你想要了解更多有关Docker和这个容器的细节，请查看本仓库下的[docker](docker) 文件夹。\n\n[install_docker]: https://docs.docker.com/get-docker/\n\n## 📟 USB 串行输出\n\n由于教程中开发的内核是在真实的硬件上运行的，因此强烈建议您使用 USB 串行调试线来进行试验。连接调试线后，树莓派需要通过额外电源供电。\n\n- 您可以在[\\[1\\]] [\\[2\\]]中或者[淘宝]上找到USB转串口线，但许多其他线材也可以工作。理想情况下，您的线材应基于`CP2102`芯片。\n- 您将其连接到`GND`和GPIO引脚`14/15`，如下所示。\n- [教程5](05_drivers_gpio_uart/README.CN.md)是这个设备第一次需要使用的地方。查看它了解如何准备SD卡以从中启动自制内核的说明。\n- 从[教程6](06_uart_chainloader/README.CN.md)开始，在树莓派上启动内核变得非常舒适。在本教程中开发了一个所谓的`chainloader`，。\n  这将是您暂时需要在SD卡上手动复制的最后一个文件。这将使您能够在通过`UART`按需引导期间加载教程内核。\n\n![UART wiring diagram](doc/wiring.png)\n\n[\\[1\\]]: https://www.amazon.de/dp/B0757FQ5CX/ref=cm_sw_r_tw_dp_U_x_ozGRDbVTJAG4Q\n[\\[2\\]]: https://www.adafruit.com/product/954\n[淘宝]: https://www.taobao.com/\n\n## 🙌 致谢\n\n这个教程最初是由[Zoltan Baldaszti](https://github.com/bztsrc)的[项目](https://github.com/bztsrc/raspi3-tutorial)衍生出来的，感谢它给我开了一个头。\n\n## License\n\nLicensed under either of\n\n- Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)\n- MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)\n\nat your option.\n\n### 贡献\n\n除非您明确声明，否则有意提交给您的任何贡献（包括 Apache-2.0 许可中定义的）均应按上述双重许可，且无任何附加条款或条件。\n"
  },
  {
    "path": "README.ES.md",
    "content": "# Tutoriales de desarrollo de Sistemas Operativos en Rust con la Raspberry Pi\n\n![](https://github.com/rust-embedded/rust-raspberrypi-OS-tutorials/workflows/BSP-RPi3/badge.svg) ![](https://github.com/rust-embedded/rust-raspberrypi-OS-tutorials/workflows/BSP-RPi4/badge.svg) ![](https://github.com/rust-embedded/rust-raspberrypi-OS-tutorials/workflows/Unit-Tests/badge.svg) ![](https://github.com/rust-embedded/rust-raspberrypi-OS-tutorials/workflows/Integration-Tests/badge.svg) ![](https://img.shields.io/badge/License-MIT%20OR%20Apache--2.0-blue)\n\n<br/>\n\n<img src=\"doc/header.jpg\" height=\"372\"> <img src=\"doc/minipush_demo_frontpage.gif\" height=\"372\">\n\n## ℹ️ Introducción\n\nEsto es una serie de tutoriales para los desarrolladores aficionados a los Sistemas Operativos (OS) \nque se están adentrando a la nueva arquitectura ARM de 64 bits [ARMv8-A\narchitecture]. Los tutoriales darán una guía paso a paso en cómo escribir un Sistema Operativo \n[monolítico] desde cero.\nEstos tutoriales cubren la implementación común de diferentes tareas de Sistemas Operativos, como \nescribir en una terminal serie, configurar la memoria virtual y manejar excepciones de hardware (HW). \nTodo mientras usamos la seguridad y velocidad que `Rust` nos proporciona.\n\n¡Divértanse!\n\n_Atentamente, <br>Andre ([@andre-richter])_\n\nP.S.: Para otros lenguajes, por favor busquen los diferentes archivos README. Por ejemplo, [`README.CN.md`](README.CN.md) o [`README.ES.md`](README.ES.md). Muchas gracias a nuestros\n[traductores](#traducciones-de-este-repositorio) 🙌.\n\n[ARMv8-A architecture]: https://developer.arm.com/products/architecture/cpu-architecture/a-profile/docs\n[monolítico]: https://en.wikipedia.org/wiki/Monolithic_kernel\n[@andre-richter]: https://github.com/andre-richter\n\n## 📑 Estructura\n\n- Cada tutorial contiene un solo binario arrancable correspondiente al núcleo.\n- Cada tutorial nuevo extiende el tutorial anterior.\n- Cada tutorial tendrá un `README` y cada `README` tendrá un pequeña sección de [`tl;dr`](https://es.wikipedia.org/wiki/TL;DR) en donde se dará una pequeña perspectiva general de los cambios y se mostrará el código fuente `diff` del tutorial anterior para que se puedan inspeccionar los cambios/adiciones que han ocurrido.\n  - Algunos tutoriales además de tener un `tl;dr` también tendrán una sección en la que se dará una explicación con todo lujo de detalle.\n     El plan a largo plazo es que cada tutorial tenga una buena explicación además del `tl;dr` y el `diff`; pero por el momento los únicos tutoriales\n    que gozan de una son los tutoriales en los que creo que el `tl;dr` y el `diff` no son suficientes para comprender lo que está pasando.\n- El código que se escribió en este tutorial soporta y corre en la **Raspberry Pi 3** y en la **Raspberry 4**\n  - Del tutorial 1 hasta el 5 son tutoriales \"preparatorios\", por lo que este código solo tendrá sentido ejecutarlo en [`QEMU`](https://www.qemu.org/).\n  - Cuando llegues al [tutorial 5](05_drivers_gpio_uart) podrás comenzar a cargar y a ejecutar el núcleo en una\n    Raspeberry de verdad, y observar la salida serie (`UART`).\n- Aunque la Raspberry Pi 3 y 4 son las principales placas este código está escrito en un estilo modular,\n  lo que permite una fácil portabilidad a otras arquitecturas de CPU o/y placas.\n  - Me encantaría si alguien intentase adaptar este código en una arquitectura **RISC-V**.\n- Para la edición recomiendo [Visual Studio Code] con [Rust Analyzer].\n- En adición al texto que aparece en los tutoriales también sería recomendable revisar \n  el comando `make doc` en cada tutorial. Este comando te deja navegar el código documentado de una manera cómoda.\n\n### Salida del comando `make doc`\n\n![make doc](doc/make_doc.png)\n\n[Visual Studio Code]: https://code.visualstudio.com\n[Rust Analyzer]: https://rust-analyzer.github.io\n\n## 🛠 Requisitos del sistema\n\nEstos tutoriales están dirigidos principalmente a distribuciones de **Linux**. \nMuchas de las cosas vistas aquí también funcionan en **macOS**, pero esto solo es _experimental_.\n\n### 🚀 La versión tl;dr\n\n1. [Instala Docker Desktop][install_docker].\n\n2. (**Solo para Linux**) Asegúrate de que la cuenta de tu usuario está en el [grupo `docker`][docker group].\n\n3. Prepara la `Rust` toolchain. La mayor parte se hará automáticamente durante el primer uso del archivo [rust-toolchain.toml](rust-toolchain.toml). \n   Todo lo que nos queda hacer a nosotros es: \n   \n   i. Si ya tienes una versión de Rust instalada:\n   \n   ```bash\n   cargo install cargo-binutils rustfilt\n   ```\n   \n   ii. Si necesitas instalar Rust desde cero:\n   \n   ```bash\n   curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh\n   \n   source $HOME/.cargo/env\n   cargo install cargo-binutils rustfilt\n   ```\n\n4. En caso de que uses `Visual Studio Code`, recomiendo que instales la extensión [Rust Analyzer extension].\n\n5. (**Solo para macOS**) Instala algunas `Ruby` gems.\n   \n   Ejecuta esto en la carpeta root del repositorio:\n   \n   ```bash\n   bundle install --path .vendor/bundle --without development\n   ```\n\n[docker group]: https://docs.docker.com/engine/install/linux-postinstall/\n[Rust Analyzer extension]: https://marketplace.visualstudio.com/items?itemName=matklad.rust-analyzer\n\n### 🧰 Más detalles: Eliminando Lios con Toolchains\n\nEsta serie trata de enfocarse lo máximo posible en tener una experiencia agradable para el usuario.\nPor lo tanto, se han dirigido muchos esfuerzos a eliminar la parte más difícil del desarrollo de\nlos sistemas incorporados (embedded) tanto como se pudo.\n\nRust por sí mismo ya ayuda mucho, porque tiene integrado el soporte para compilación cruzada.\nTodo lo que necesitamos para compilar desde una máquina con una arquitectura `x86` a una Raspberry Pi\ncon arquitectura `AArch64` será automáticamente instalado por `rustup`. Sin embargo, además de usar\nel compilador de Rust, también usaremos algunas otras herramientas, entre las cuales están:\n\n- `QEMU` para emular nuestro núcleo en nuestra máquina principal.\n- Una herramienta llamada `Minipush` para cargar el núcleo en una Raspberry Pi cuando queramos usando `UART`.\n- `OpenOCD` y `GDB` para hacer depuración (\"debugging\") en la máquina a instalar.\n\nHay muchas cosas que pueden salir mal mientras instalamos y/o compilamos las versiones correctas de cada\nherramienta en tu máquina. Por ejemplo, tu distribución de Linux tal vez podría no proporcionar las versiones más\nrecientes de paquetes que se necesiten. O tal vez te falten algunas dependencias para la compilar estas herramientas.\n\nEsta es la razón por la cual usaremos [Docker][install_docker] mientras sea posible. Te\nestamos proporcionando un contenedor que tiene todas las herramientas o dependencias preinstaladas.\nSi quieres saber más acerca de Docker y revisar el contenedor proporcionado, por favor revisa la carpeta\n[docker](docker) del repositorio.\n\n[install_docker]: https://docs.docker.com/get-docker/\n\n## 📟 Puerto Serie USB\n\nYa que el núcleo desarrollado en este tutorial se ejecuta en hardware real, se recomienda que tengas un adaptador de puerto serie USB cable para sentir la experiencia completa.\n\n- Puedes encontrar estos cables que deberían funcionar sin ningún problema en [\\[1\\]] [\\[2\\]], pero\n  hay muchos otros que pueden funcionar. Idealmente, tu cable estaría basado en el chip `CP2102`.\n- Lo conectas a los pines `GND` y `GPIO` `14/15` como se muestra en la parte inferior.  \n- [Tutorial 5](05_drivers_gpio_uart) es la primera vez en la que lo vas usar. Revisa las instrucciones\n  sobre cómo preparar una tarjeta SD para arrancar en tu núcleo desde ahí.\n- Empezando con el [tutorial 6](06_uart_chainloader), arrancar núcleos en tu Raspberry Pi comienza a ser\n  más fácil. En este tutorial se desarrolla un `chainloader`, que será el último archivo que necesitarás\n  copiar de manera manual a la tarjeta SD por el momento. Esto te permitirá cargar los núcleos de los tutoriales\n  durante el arranque usando `UART`.\n\n![UART wiring diagram](doc/wiring.png)\n\n[\\[1\\]]: https://www.amazon.de/dp/B0757FQ5CX/ref=cm_sw_r_tw_dp_U_x_ozGRDbVTJAG4Q\n[\\[2\\]]: https://www.adafruit.com/product/954\n\n## 🙌 Agradecimientos\n\nLa versión original de estos tutoriales empezó como un fork de los increíbles \n[tutoriales de programación en hardware en la RPi3](https://github.com/bztsrc/raspi3-tutorial) en `C`\nde [Zoltan Baldaszti](https://github.com/bztsrc). ¡Gracias por darme un punto de partida!\n\n### Traducciones de este repositorio\n\n- **Chino:**\n  - [@colachg] y [@readlnh].\n  - Necesitan actualizaciones.\n- **Español:**\n  - [@zanezhub].\n  - En el futuro habrán tutoriales traducidos al español. \n\n[@colachg]: https://github.com/colachg\n[@readlnh]: https://github.com/readlnh\n[@zanezhub]: https://github.com/zanezhub\n\n## Licencia\n\nEste proyecto está licenciado por cualquiera de las siguientes licencias como alguna de tus dos opciones\n\n- Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) o http://www.apache.org/licenses/LICENSE-2.0)\n- MIT license ([LICENSE-MIT](LICENSE-MIT) o http://opensource.org/licenses/MIT)\n\n### Contribución\n\nA menos de que lo menciones, cualquier contribución enviada por ti para su inclusión en este trabajo,\ntal como se define en la licencia Apache-2.0, deberá tener doble licencia como se muestra en la parte superior, sin ningún\ncambio de términos o condiciones.\n"
  },
  {
    "path": "README.md",
    "content": "# Operating System development tutorials in Rust on the Raspberry Pi\n\n![](https://github.com/rust-embedded/rust-raspberrypi-OS-tutorials/workflows/BSP-RPi3/badge.svg) ![](https://github.com/rust-embedded/rust-raspberrypi-OS-tutorials/workflows/BSP-RPi4/badge.svg) ![](https://github.com/rust-embedded/rust-raspberrypi-OS-tutorials/workflows/Unit-Tests/badge.svg) ![](https://github.com/rust-embedded/rust-raspberrypi-OS-tutorials/workflows/Integration-Tests/badge.svg) ![](https://img.shields.io/badge/License-MIT%20OR%20Apache--2.0-blue)\n\n<br/>\n\n<img src=\"doc/header.jpg\" height=\"372\"> <img src=\"doc/minipush_demo_frontpage.gif\" height=\"372\">\n\n## ℹ️ Introduction\n\nThis is a tutorial series for hobby OS developers who are new to ARM's 64 bit [ARMv8-A\narchitecture]. The tutorials will give a guided, step-by-step tour of how to write a [monolithic]\nOperating System `kernel` for an `embedded system` from scratch. They cover implementation of common\nOperating Systems tasks, like writing to the serial console, setting up virtual memory and handling\nHW exceptions. All while leveraging `Rust`'s unique features to provide for safety and speed.\n\nHave fun!\n\n_Best regards,<br>Andre ([@andre-richter])_\n\nP.S.: For other languages, please look out for alternative README files. For example,\n[`README.CN.md`](README.CN.md) or [`README.ES.md`](README.ES.md). Many thanks to our\n[translators](#translations-of-this-repository) 🙌.\n\n[ARMv8-A architecture]: https://developer.arm.com/products/architecture/cpu-architecture/a-profile/docs\n[monolithic]: https://en.wikipedia.org/wiki/Monolithic_kernel\n[@andre-richter]: https://github.com/andre-richter\n\n## 📑 Organization\n\n- Each tutorial contains a stand-alone, bootable `kernel` binary.\n- Each new tutorial extends the previous one.\n- Each tutorial `README` will have a short `tl;dr` section giving a brief overview of the additions,\n  and show the source code `diff` to the previous tutorial, so that you can conveniently inspect the\n  changes/additions.\n    - Some tutorials have a full-fledged, detailed text in addition to the `tl;dr` section. The\n      long-term plan is that all tutorials get a full text, but for now this is exclusive to\n      tutorials where I think that `tl;dr` and `diff` are not enough to get the idea.\n- The code written in these tutorials supports and runs on the **Raspberry Pi 3** and the\n  **Raspberry Pi 4**.\n  - Tutorials 1 till 5 are groundwork code which only makes sense to run in `QEMU`.\n  - Starting with [tutorial 5](05_drivers_gpio_uart), you can load and run the kernel on the real\n    Raspberrys and observe output over `UART`.\n- Although the Raspberry Pi 3 and 4 are the main target boards, the code is written in a modular\n  fashion which allows for easy porting to other CPU architectures and/or boards.\n  - I would really love if someone takes a shot at a **RISC-V** implementation!\n- For editing, I recommend [Visual Studio Code] with [Rust Analyzer].\n- In addition to the tutorial text, also check out the `make doc` command in each tutorial. It lets\n  you browse the extensively documented code in a convenient way.\n\n### Output of `make doc`\n\n![make doc](doc/make_doc.png)\n\n[Visual Studio Code]: https://code.visualstudio.com\n[Rust Analyzer]: https://rust-analyzer.github.io\n\n## 🛠 System Requirements\n\nThe tutorials are primarily targeted at **Linux**-based distributions. Most stuff will also work on **macOS**, but this is only _experimental_.\n\n### 🚀 The tl;dr Version\n\n1. [Install Docker Engine][install_docker].\n1. (**Linux only**) Ensure your user account is in the [docker group].\n1. Prepare the `Rust` toolchain. Most of it will be handled on first use through the\n   [rust-toolchain.toml](rust-toolchain.toml) file. What's left for us to do is:\n   1. If you already have a version of Rust installed:\n      ```bash\n      cargo install cargo-binutils rustfilt\n      ```\n\n   1. If you need to install Rust from scratch:\n      ```bash\n      curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh\n\n      source $HOME/.cargo/env\n      cargo install cargo-binutils rustfilt\n      ```\n\n1. In case you use `Visual Studio Code`, I strongly recommend installing the [Rust Analyzer extension].\n1. (**macOS only**) Install a few `Ruby` gems.\n\n  This was last tested by the author with Ruby version `3.0.2` on `macOS Monterey`. If you are using\n  `rbenv`, the respective `.ruby-version` file is already in place. If you never heard of `rbenv`,\n  try using [this little guide](https://stackoverflow.com/a/68118750).\n\n   Run this in the repository root folder:\n\n   ```bash\n   bundle config set --local path '.vendor/bundle'\n   bundle config set --local without 'development'\n   bundle install\n   ```\n\n[docker group]: https://docs.docker.com/engine/install/linux-postinstall/\n[Rust Analyzer extension]: https://marketplace.visualstudio.com/items?itemName=matklad.rust-analyzer\n\n### 🧰 More Details: Eliminating Toolchain Hassle\n\nThis series tries to put a strong focus on user friendliness. Therefore, efforts were made to\neliminate the biggest painpoint in embedded development as much as possible: `Toolchain hassle`.\n\nRust itself is already helping a lot in that regard, because it has built-in support for\ncross-compilation. All that we need for cross-compiling from an `x86` host to the Raspberry Pi's\n`AArch64` architecture will be automatically installed by `rustup`. However, besides the Rust\ncompiler, we will use some more tools. Among others:\n\n- `QEMU` to emulate our kernel on the host system.\n- A self-made tool called `Minipush` to load a kernel onto the Raspberry Pi on-demand over `UART`.\n- `OpenOCD` and `GDB` for debugging on the target.\n\nThere is a lot that can go wrong while installing and/or compiling the correct version of each tool\non your host machine. For example, your distribution might not provide the latest version that is\nneeded. Or you are missing some hard-to-get dependencies for the compilation of one of these tools.\n\nThis is why we will make use of [Docker][install_docker] whenever possible. We are providing an\naccompanying container that has all the needed tools or dependencies pre-installed, and it gets\npulled in automagically once it is needed. If you want to know more about Docker and peek at the\nprovided container, please refer to the repository's [docker](docker) folder.\n\n[install_docker]: https://docs.docker.com/engine/install/#server\n\n## 📟 USB Serial Output\n\nSince the kernel developed in the tutorials runs on the real hardware, it is highly recommended to\nget a USB serial cable to get the full experience.\n\n- You can find USB-to-serial cables that should work right away at [\\[1\\]] [\\[2\\]], but many others\n  will work too. Ideally, your cable is based on the `CP2102` chip.\n- You connect it to `GND` and GPIO pins `14/15` as shown below.\n- [Tutorial 5](05_drivers_gpio_uart) is the first where you can use it. Check it out for\n  instructions on how to prepare the SD card to boot your self-made kernel from it.\n- Starting with [tutorial 6](06_uart_chainloader), booting kernels on your Raspberry is getting\n  _really_ comfortable. In this tutorial, a so-called `chainloader` is developed, which will be the\n  last file you need to manually copy on the SD card for a while. It will enable you to load the\n  tutorial kernels during boot on demand over `UART`.\n\n![UART wiring diagram](doc/wiring.png)\n\n[\\[1\\]]: https://www.amazon.de/dp/B0757FQ5CX/ref=cm_sw_r_tw_dp_U_x_ozGRDbVTJAG4Q\n[\\[2\\]]: https://www.adafruit.com/product/954\n\n## 🙌 Acknowledgements\n\nThe original version of the tutorials started out as a fork of [Zoltan\nBaldaszti](https://github.com/bztsrc)'s awesome [tutorials on bare metal programming on\nRPi3](https://github.com/bztsrc/raspi3-tutorial) in `C`. Thanks for giving me a head start!\n\n### Translations of this repository\n\n - **Chinese**\n   - [@colachg] and [@readlnh].\n   - Need updating.\n - **Spanish**\n   -  [@zanezhub].\n   -  In the future there'll be tutorials translated to spanish.\n\n[@colachg]: https://github.com/colachg\n[@readlnh]: https://github.com/readlnh\n[@zanezhub]: https://github.com/zanezhub\n\n## License\n\nLicensed under either of\n\n- Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or <https://www.apache.org/licenses/LICENSE-2.0>)\n- MIT license ([LICENSE-MIT](LICENSE-MIT) or <https://opensource.org/licenses/MIT>)\n\nat your option.\n\n### Contribution\n\nUnless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the\nwork by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any\nadditional terms or conditions.\n"
  },
  {
    "path": "SPONSORING.md",
    "content": "# 💖 Sponsoring the Developers\n\nThis repository is the result of the combined efforts of a large number of developers - in terms of\nwriting the source code, but also in performing code reviews, writing documentation, and handling\nissues and PRs. Some of these developers received a salary or other payment from a third-party for\ntheir contributions. Others receive no payment of any kind, and simply hope that others find their\ncontributions useful.\n\nIf you would like to support any of the developers, please take a moment to [click through to their\nprofiles] to see if they accept sponsorship and/or charitable gifts made in their name.\n\n[click through to their profiles]: https://github.com/rust-embedded/rust-raspberrypi-OS-tutorials/graphs/contributors\n"
  },
  {
    "path": "X1_JTAG_boot/.vscode/settings.json",
    "content": "{\n    \"editor.formatOnSave\": true,\n    \"editor.rulers\": [100],\n    \"rust-analyzer.cargo.target\": \"aarch64-unknown-none-softfloat\",\n    \"rust-analyzer.cargo.features\": [\"bsp_rpi3\"],\n    \"rust-analyzer.checkOnSave.allTargets\": false,\n    \"rust-analyzer.checkOnSave.extraArgs\": [\"--bins\"],\n    \"rust-analyzer.lens.debug\": false,\n    \"rust-analyzer.lens.run\": false\n}\n"
  },
  {
    "path": "X1_JTAG_boot/Cargo.toml",
    "content": "[package]\nname = \"mingo\"\nversion = \"0.8.0\"\nauthors = [\"Andre Richter <andre.o.richter@gmail.com>\"]\nedition = \"2021\"\n\n[profile.release]\nlto = true\n\n[features]\ndefault = []\nbsp_rpi3 = [\"tock-registers\"]\nbsp_rpi4 = [\"tock-registers\"]\n\n[[bin]]\nname = \"kernel\"\npath = \"src/main.rs\"\n\n##--------------------------------------------------------------------------------------------------\n## Dependencies\n##--------------------------------------------------------------------------------------------------\n\n[dependencies]\n\n# Optional dependencies\ntock-registers = { version = \"0.8.x\", default-features = false, features = [\"register_types\"], optional = true }\n\n# Platform specific dependencies\n[target.'cfg(target_arch = \"aarch64\")'.dependencies]\naarch64-cpu = { version = \"9.x.x\" }\n"
  },
  {
    "path": "X1_JTAG_boot/Makefile",
    "content": "## SPDX-License-Identifier: MIT OR Apache-2.0\n##\n## Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\ninclude ../common/docker.mk\ninclude ../common/format.mk\ninclude ../common/operating_system.mk\n\n##--------------------------------------------------------------------------------------------------\n## Optional, user-provided configuration values\n##--------------------------------------------------------------------------------------------------\n\n# Default to the RPi3.\nBSP ?= rpi3\n\n# Default to a serial device name that is common in Linux.\nDEV_SERIAL ?= /dev/ttyUSB0\n\n\n\n##--------------------------------------------------------------------------------------------------\n## BSP-specific configuration values\n##--------------------------------------------------------------------------------------------------\nQEMU_MISSING_STRING = \"This board is not yet supported for QEMU.\"\n\nifeq ($(BSP),rpi3)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE = raspi3\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    LD_SCRIPT_PATH    = $(shell pwd)/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a53\nelse ifeq ($(BSP),rpi4)\n    TARGET            = aarch64-unknown-none-softfloat\n    KERNEL_BIN        = kernel8.img\n    QEMU_BINARY       = qemu-system-aarch64\n    QEMU_MACHINE_TYPE =\n    QEMU_RELEASE_ARGS = -serial stdio -display none\n    OBJDUMP_BINARY    = aarch64-none-elf-objdump\n    NM_BINARY         = aarch64-none-elf-nm\n    READELF_BINARY    = aarch64-none-elf-readelf\n    LD_SCRIPT_PATH    = $(shell pwd)/src/bsp/raspberrypi\n    RUSTC_MISC_ARGS   = -C target-cpu=cortex-a72\nendif\n\n# Export for build.rs.\nexport LD_SCRIPT_PATH\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets and Prerequisites\n##--------------------------------------------------------------------------------------------------\nKERNEL_MANIFEST      = Cargo.toml\nKERNEL_LINKER_SCRIPT = kernel.ld\nLAST_BUILD_CONFIG    = target/$(BSP).build_config\n\nKERNEL_ELF      = target/$(TARGET)/release/kernel\n# This parses cargo's dep-info file.\n# https://doc.rust-lang.org/cargo/guide/build-cache.html#dep-info-files\nKERNEL_ELF_DEPS = $(filter-out %: ,$(file < $(KERNEL_ELF).d)) $(KERNEL_MANIFEST) $(LAST_BUILD_CONFIG)\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Command building blocks\n##--------------------------------------------------------------------------------------------------\nRUSTFLAGS = $(RUSTC_MISC_ARGS)                   \\\n    -C link-arg=--library-path=$(LD_SCRIPT_PATH) \\\n    -C link-arg=--script=$(KERNEL_LINKER_SCRIPT)\n\nRUSTFLAGS_PEDANTIC = $(RUSTFLAGS) \\\n    -D warnings                   \\\n    -D missing_docs\n\nFEATURES      = --features bsp_$(BSP)\nCOMPILER_ARGS = --target=$(TARGET) \\\n    $(FEATURES)                    \\\n    --release\n\nRUSTC_CMD   = cargo rustc $(COMPILER_ARGS)\nDOC_CMD     = cargo doc $(COMPILER_ARGS)\nCLIPPY_CMD  = cargo clippy $(COMPILER_ARGS)\nOBJCOPY_CMD = rust-objcopy \\\n    --strip-all            \\\n    -O binary\n\nEXEC_QEMU          = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE)\nEXEC_TEST_DISPATCH = ruby ../common/tests/dispatch.rb\nEXEC_MINIPUSH      = ruby ../common/serial/minipush.rb\n\n##------------------------------------------------------------------------------\n## Dockerization\n##------------------------------------------------------------------------------\nDOCKER_CMD            = docker run -t --rm -v $(shell pwd):/work/tutorial -w /work/tutorial\nDOCKER_CMD_INTERACT   = $(DOCKER_CMD) -i\nDOCKER_ARG_DIR_COMMON = -v $(shell pwd)/../common:/work/common\nDOCKER_ARG_DEV        = --privileged -v /dev:/dev\n\n# DOCKER_IMAGE defined in include file (see top of this file).\nDOCKER_QEMU  = $(DOCKER_CMD_INTERACT) $(DOCKER_IMAGE)\nDOCKER_TOOLS = $(DOCKER_CMD) $(DOCKER_IMAGE)\nDOCKER_TEST  = $(DOCKER_CMD) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\n\n# Dockerize commands, which require USB device passthrough, only on Linux.\nifeq ($(shell uname -s),Linux)\n    DOCKER_CMD_DEV = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_DEV)\n\n    DOCKER_CHAINBOOT = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE)\nendif\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: all doc qemu chainboot clippy clean readelf objdump nm check\n\nall: $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Save the configuration as a file, so make understands if it changed.\n##------------------------------------------------------------------------------\n$(LAST_BUILD_CONFIG):\n\t@rm -f target/*.build_config\n\t@mkdir -p target\n\t@touch $(LAST_BUILD_CONFIG)\n\n##------------------------------------------------------------------------------\n## Compile the kernel ELF\n##------------------------------------------------------------------------------\n$(KERNEL_ELF): $(KERNEL_ELF_DEPS)\n\t$(call color_header, \"Compiling kernel ELF - $(BSP)\")\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(RUSTC_CMD)\n\n##------------------------------------------------------------------------------\n## Generate the stripped kernel binary\n##------------------------------------------------------------------------------\n$(KERNEL_BIN): $(KERNEL_ELF)\n\t$(call color_header, \"Generating stripped binary\")\n\t@$(OBJCOPY_CMD) $(KERNEL_ELF) $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Name\")\n\t@echo $(KERNEL_BIN)\n\t$(call color_progress_prefix, \"Size\")\n\t$(call disk_usage_KiB, $(KERNEL_BIN))\n\n##------------------------------------------------------------------------------\n## Generate the documentation\n##------------------------------------------------------------------------------\ndoc:\n\t$(call color_header, \"Generating docs\")\n\t@$(DOC_CMD) --document-private-items --open\n\n##------------------------------------------------------------------------------\n## Run the kernel in QEMU\n##------------------------------------------------------------------------------\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\nqemu:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\nqemu: $(KERNEL_BIN)\n\t$(call color_header, \"Launching QEMU\")\n\t@$(DOCKER_QEMU) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\nendif\n\n##------------------------------------------------------------------------------\n## Push the kernel to the real HW target\n##------------------------------------------------------------------------------\nchainboot: $(KERNEL_BIN)\n\t@$(DOCKER_CHAINBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run clippy\n##------------------------------------------------------------------------------\nclippy:\n\t@RUSTFLAGS=\"$(RUSTFLAGS_PEDANTIC)\" $(CLIPPY_CMD)\n\n##------------------------------------------------------------------------------\n## Clean\n##------------------------------------------------------------------------------\nclean:\n\trm -rf target $(KERNEL_BIN)\n\n##------------------------------------------------------------------------------\n## Run readelf\n##------------------------------------------------------------------------------\nreadelf: $(KERNEL_ELF)\n\t$(call color_header, \"Launching readelf\")\n\t@$(DOCKER_TOOLS) $(READELF_BINARY) --headers $(KERNEL_ELF)\n\n##------------------------------------------------------------------------------\n## Run objdump\n##------------------------------------------------------------------------------\nobjdump: $(KERNEL_ELF)\n\t$(call color_header, \"Launching objdump\")\n\t@$(DOCKER_TOOLS) $(OBJDUMP_BINARY) --disassemble --demangle \\\n                --section .text   \\\n                --section .rodata \\\n                $(KERNEL_ELF) | rustfilt\n\n##------------------------------------------------------------------------------\n## Run nm\n##------------------------------------------------------------------------------\nnm: $(KERNEL_ELF)\n\t$(call color_header, \"Launching nm\")\n\t@$(DOCKER_TOOLS) $(NM_BINARY) --demangle --print-size $(KERNEL_ELF) | sort | rustfilt\n\n\n\n##--------------------------------------------------------------------------------------------------\n## Testing targets\n##--------------------------------------------------------------------------------------------------\n.PHONY: test test_boot\n\nifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board.\n\ntest_boot test:\n\t$(call color_header, \"$(QEMU_MISSING_STRING)\")\n\nelse # QEMU is supported.\n\n##------------------------------------------------------------------------------\n## Run boot test\n##------------------------------------------------------------------------------\ntest_boot: $(KERNEL_BIN)\n\t$(call color_header, \"Boot test - $(BSP)\")\n\t@$(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN)\n\ntest: test_boot\n\nendif\n"
  },
  {
    "path": "X1_JTAG_boot/README.md",
    "content": "# Xtra 1 - JTAG boot\n\nNot much is happening here. The binary just patiently waits for a `JTAG` debugger to connect.\n"
  },
  {
    "path": "X1_JTAG_boot/build.rs",
    "content": "use std::{env, fs, process};\n\nfn main() {\n    let ld_script_path = match env::var(\"LD_SCRIPT_PATH\") {\n        Ok(var) => var,\n        _ => process::exit(0),\n    };\n\n    let files = fs::read_dir(ld_script_path).unwrap();\n    files\n        .filter_map(Result::ok)\n        .filter(|d| {\n            if let Some(e) = d.path().extension() {\n                e == \"ld\"\n            } else {\n                false\n            }\n        })\n        .for_each(|f| println!(\"cargo:rerun-if-changed={}\", f.path().display()));\n}\n"
  },
  {
    "path": "X1_JTAG_boot/src/_arch/aarch64/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural boot code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::boot::arch_boot\n\nuse core::arch::global_asm;\n\n// Assembly counterpart to this file.\nglobal_asm!(\n    include_str!(\"boot.s\"),\n    CONST_CORE_ID_MASK = const 0b11\n);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The Rust entry of the `kernel` binary.\n///\n/// The function is called from the assembly `_start` function.\n#[no_mangle]\npub unsafe fn _start_rust() -> ! {\n    crate::kernel_init()\n}\n"
  },
  {
    "path": "X1_JTAG_boot/src/_arch/aarch64/cpu/boot.s",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//--------------------------------------------------------------------------------------------------\n// Definitions\n//--------------------------------------------------------------------------------------------------\n\n// Load the address of a symbol into a register, PC-relative.\n//\n// The symbol must lie within +/- 4 GiB of the Program Counter.\n//\n// # Resources\n//\n// - https://sourceware.org/binutils/docs-2.36/as/AArch64_002dRelocations.html\n.macro ADR_REL register, symbol\n\tadrp\t\\register, \\symbol\n\tadd\t\\register, \\register, #:lo12:\\symbol\n.endm\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n.section .text._start\n\n//------------------------------------------------------------------------------\n// fn _start()\n//------------------------------------------------------------------------------\n_start:\n\t// Only proceed on the boot core. Park it otherwise.\n\tmrs\tx1, MPIDR_EL1\n\tand\tx1, x1, {CONST_CORE_ID_MASK}\n\tldr\tx2, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs\n\tcmp\tx1, x2\n\tb.ne\t.L_parking_loop\n\n\t// If execution reaches here, it is the boot core.\n\n\t// Initialize DRAM.\n\tADR_REL\tx0, __bss_start\n\tADR_REL x1, __bss_end_exclusive\n\n.L_bss_init_loop:\n\tcmp\tx0, x1\n\tb.eq\t.L_prepare_rust\n\tstp\txzr, xzr, [x0], #16\n\tb\t.L_bss_init_loop\n\n\t// Prepare the jump to Rust code.\n.L_prepare_rust:\n\t// Set the stack pointer.\n\tADR_REL\tx0, __boot_core_stack_end_exclusive\n\tmov\tsp, x0\n\n\t// Read the CPU's timer counter frequency and store it in ARCH_TIMER_COUNTER_FREQUENCY.\n\t// Abort if the frequency read back as 0.\n\tADR_REL\tx1, ARCH_TIMER_COUNTER_FREQUENCY // provided by aarch64/time.rs\n\tmrs\tx2, CNTFRQ_EL0\n\tcmp\tx2, xzr\n\tb.eq\t.L_parking_loop\n\tstr\tw2, [x1]\n\n\t// Jump to Rust code.\n\tb\t_start_rust\n\n\t// Infinitely wait for events (aka \"park the core\").\n.L_parking_loop:\n\twfe\n\tb\t.L_parking_loop\n\n.size\t_start, . - _start\n.type\t_start, function\n.global\t_start\n"
  },
  {
    "path": "X1_JTAG_boot/src/_arch/aarch64/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural processor code.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::cpu::arch_cpu\n\nuse aarch64_cpu::asm;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\npub use asm::nop;\n\n/// Pause execution on the core.\n#[inline(always)]\npub fn wait_forever() -> ! {\n    loop {\n        asm::wfe()\n    }\n}\n"
  },
  {
    "path": "X1_JTAG_boot/src/_arch/aarch64/time.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Architectural timer primitives.\n//!\n//! # Orientation\n//!\n//! Since arch modules are imported into generic modules using the path attribute, the path of this\n//! file is:\n//!\n//! crate::time::arch_time\n\n#[cfg(feature = \"bsp_rpi3\")]\nuse crate::warn;\nuse aarch64_cpu::{asm::barrier, registers::*};\nuse core::{\n    num::{NonZeroU128, NonZeroU32, NonZeroU64},\n    ops::{Add, Div},\n    time::Duration,\n};\nuse tock_registers::interfaces::Readable;\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nconst NANOSEC_PER_SEC: NonZeroU64 = NonZeroU64::new(1_000_000_000).unwrap();\n\n#[derive(Copy, Clone, PartialOrd, PartialEq)]\nstruct GenericTimerCounterValue(u64);\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n/// Boot assembly code overwrites this value with the value of CNTFRQ_EL0 before any Rust code is\n/// executed. This given value here is just a (safe) dummy.\n#[no_mangle]\nstatic ARCH_TIMER_COUNTER_FREQUENCY: NonZeroU32 = NonZeroU32::MIN;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nfn arch_timer_counter_frequency() -> NonZeroU32 {\n    // Read volatile is needed here to prevent the compiler from optimizing\n    // ARCH_TIMER_COUNTER_FREQUENCY away.\n    //\n    // This is safe, because all the safety requirements as stated in read_volatile()'s\n    // documentation are fulfilled.\n    unsafe { core::ptr::read_volatile(&ARCH_TIMER_COUNTER_FREQUENCY) }\n}\n\nimpl GenericTimerCounterValue {\n    pub const MAX: Self = GenericTimerCounterValue(u64::MAX);\n}\n\nimpl Add for GenericTimerCounterValue {\n    type Output = Self;\n\n    fn add(self, other: Self) -> Self {\n        GenericTimerCounterValue(self.0.wrapping_add(other.0))\n    }\n}\n\nimpl From<GenericTimerCounterValue> for Duration {\n    fn from(counter_value: GenericTimerCounterValue) -> Self {\n        if counter_value.0 == 0 {\n            return Duration::ZERO;\n        }\n\n        let frequency: NonZeroU64 = arch_timer_counter_frequency().into();\n\n        // Div<NonZeroU64> implementation for u64 cannot panic.\n        let secs = counter_value.0.div(frequency);\n\n        // This is safe, because frequency can never be greater than u32::MAX, which means the\n        // largest theoretical value for sub_second_counter_value is (u32::MAX - 1). Therefore,\n        // (sub_second_counter_value * NANOSEC_PER_SEC) cannot overflow an u64.\n        //\n        // The subsequent division ensures the result fits into u32, since the max result is smaller\n        // than NANOSEC_PER_SEC. Therefore, just cast it to u32 using `as`.\n        let sub_second_counter_value = counter_value.0 % frequency;\n        let nanos = unsafe { sub_second_counter_value.unchecked_mul(u64::from(NANOSEC_PER_SEC)) }\n            .div(frequency) as u32;\n\n        Duration::new(secs, nanos)\n    }\n}\n\nfn max_duration() -> Duration {\n    Duration::from(GenericTimerCounterValue::MAX)\n}\n\nimpl TryFrom<Duration> for GenericTimerCounterValue {\n    type Error = &'static str;\n\n    fn try_from(duration: Duration) -> Result<Self, Self::Error> {\n        if duration < resolution() {\n            return Ok(GenericTimerCounterValue(0));\n        }\n\n        if duration > max_duration() {\n            return Err(\"Conversion error. Duration too big\");\n        }\n\n        let frequency: u128 = u32::from(arch_timer_counter_frequency()) as u128;\n        let duration: u128 = duration.as_nanos();\n\n        // This is safe, because frequency can never be greater than u32::MAX, and\n        // (Duration::MAX.as_nanos() * u32::MAX) < u128::MAX.\n        let counter_value =\n            unsafe { duration.unchecked_mul(frequency) }.div(NonZeroU128::from(NANOSEC_PER_SEC));\n\n        // Since we checked above that we are <= max_duration(), just cast to u64.\n        Ok(GenericTimerCounterValue(counter_value as u64))\n    }\n}\n\n#[inline(always)]\nfn read_cntpct() -> GenericTimerCounterValue {\n    // Prevent that the counter is read ahead of time due to out-of-order execution.\n    barrier::isb(barrier::SY);\n    let cnt = CNTPCT_EL0.get();\n\n    GenericTimerCounterValue(cnt)\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// The timer's resolution.\npub fn resolution() -> Duration {\n    Duration::from(GenericTimerCounterValue(1))\n}\n\n/// The uptime since power-on of the device.\n///\n/// This includes time consumed by firmware and bootloaders.\npub fn uptime() -> Duration {\n    read_cntpct().into()\n}\n\n/// Spin for a given duration.\n#[cfg(feature = \"bsp_rpi3\")]\npub fn spin_for(duration: Duration) {\n    let curr_counter_value = read_cntpct();\n\n    let counter_value_delta: GenericTimerCounterValue = match duration.try_into() {\n        Err(msg) => {\n            warn!(\"spin_for: {}. Skipping\", msg);\n            return;\n        }\n        Ok(val) => val,\n    };\n    let counter_value_target = curr_counter_value + counter_value_delta;\n\n    // Busy wait.\n    //\n    // Read CNTPCT_EL0 directly to avoid the ISB that is part of [`read_cntpct`].\n    while GenericTimerCounterValue(CNTPCT_EL0.get()) < counter_value_target {}\n}\n"
  },
  {
    "path": "X1_JTAG_boot/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! GPIO Driver.\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper, driver, synchronization,\n    synchronization::NullLock,\n};\nuse tock_registers::{\n    interfaces::{ReadWriteable, Writeable},\n    register_bitfields, register_structs,\n    registers::ReadWrite,\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// GPIO registers.\n//\n// Descriptions taken from\n// - https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf\n// - https://datasheets.raspberrypi.org/bcm2711/bcm2711-peripherals.pdf\nregister_bitfields! {\n    u32,\n\n    /// GPIO Function Select 1\n    GPFSEL1 [\n        /// Pin 15\n        FSEL15 OFFSET(15) NUMBITS(3) [\n            Input = 0b000,\n            Output = 0b001,\n            AltFunc0 = 0b100  // PL011 UART RX\n\n        ],\n\n        /// Pin 14\n        FSEL14 OFFSET(12) NUMBITS(3) [\n            Input = 0b000,\n            Output = 0b001,\n            AltFunc0 = 0b100  // PL011 UART TX\n        ]\n    ],\n\n    /// GPIO Pull-up/down Register\n    ///\n    /// BCM2837 only.\n    GPPUD [\n        /// Controls the actuation of the internal pull-up/down control line to ALL the GPIO pins.\n        PUD OFFSET(0) NUMBITS(2) [\n            Off = 0b00,\n            PullDown = 0b01,\n            PullUp = 0b10\n        ]\n    ],\n\n    /// GPIO Pull-up/down Clock Register 0\n    ///\n    /// BCM2837 only.\n    GPPUDCLK0 [\n        /// Pin 15\n        PUDCLK15 OFFSET(15) NUMBITS(1) [\n            NoEffect = 0,\n            AssertClock = 1\n        ],\n\n        /// Pin 14\n        PUDCLK14 OFFSET(14) NUMBITS(1) [\n            NoEffect = 0,\n            AssertClock = 1\n        ]\n    ],\n\n    /// GPIO Pull-up / Pull-down Register 0\n    ///\n    /// BCM2711 only.\n    GPIO_PUP_PDN_CNTRL_REG0 [\n        /// Pin 15\n        GPIO_PUP_PDN_CNTRL15 OFFSET(30) NUMBITS(2) [\n            NoResistor = 0b00,\n            PullUp = 0b01\n        ],\n\n        /// Pin 14\n        GPIO_PUP_PDN_CNTRL14 OFFSET(28) NUMBITS(2) [\n            NoResistor = 0b00,\n            PullUp = 0b01\n        ]\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    RegisterBlock {\n        (0x00 => _reserved1),\n        (0x04 => GPFSEL1: ReadWrite<u32, GPFSEL1::Register>),\n        (0x08 => _reserved2),\n        (0x94 => GPPUD: ReadWrite<u32, GPPUD::Register>),\n        (0x98 => GPPUDCLK0: ReadWrite<u32, GPPUDCLK0::Register>),\n        (0x9C => _reserved3),\n        (0xE4 => GPIO_PUP_PDN_CNTRL_REG0: ReadWrite<u32, GPIO_PUP_PDN_CNTRL_REG0::Register>),\n        (0xE8 => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\nstruct GPIOInner {\n    registers: Registers,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the GPIO HW.\npub struct GPIO {\n    inner: NullLock<GPIOInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GPIOInner {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n        }\n    }\n\n    /// Disable pull-up/down on pins 14 and 15.\n    #[cfg(feature = \"bsp_rpi3\")]\n    fn disable_pud_14_15_bcm2837(&mut self) {\n        use crate::time;\n        use core::time::Duration;\n\n        // The Linux 2837 GPIO driver waits 1 µs between the steps.\n        const DELAY: Duration = Duration::from_micros(1);\n\n        self.registers.GPPUD.write(GPPUD::PUD::Off);\n        time::time_manager().spin_for(DELAY);\n\n        self.registers\n            .GPPUDCLK0\n            .write(GPPUDCLK0::PUDCLK15::AssertClock + GPPUDCLK0::PUDCLK14::AssertClock);\n        time::time_manager().spin_for(DELAY);\n\n        self.registers.GPPUD.write(GPPUD::PUD::Off);\n        self.registers.GPPUDCLK0.set(0);\n    }\n\n    /// Disable pull-up/down on pins 14 and 15.\n    #[cfg(feature = \"bsp_rpi4\")]\n    fn disable_pud_14_15_bcm2711(&mut self) {\n        self.registers.GPIO_PUP_PDN_CNTRL_REG0.write(\n            GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL15::PullUp\n                + GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL14::PullUp,\n        );\n    }\n\n    /// Map PL011 UART as standard output.\n    ///\n    /// TX to pin 14\n    /// RX to pin 15\n    pub fn map_pl011_uart(&mut self) {\n        // Select the UART on pins 14 and 15.\n        self.registers\n            .GPFSEL1\n            .modify(GPFSEL1::FSEL15::AltFunc0 + GPFSEL1::FSEL14::AltFunc0);\n\n        // Disable pull-up/down on pins 14 and 15.\n        #[cfg(feature = \"bsp_rpi3\")]\n        self.disable_pud_14_15_bcm2837();\n\n        #[cfg(feature = \"bsp_rpi4\")]\n        self.disable_pud_14_15_bcm2711();\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl GPIO {\n    pub const COMPATIBLE: &'static str = \"BCM GPIO\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            inner: NullLock::new(GPIOInner::new(mmio_start_addr)),\n        }\n    }\n\n    /// Concurrency safe version of `GPIOInner.map_pl011_uart()`\n    pub fn map_pl011_uart(&self) {\n        self.inner.lock(|inner| inner.map_pl011_uart())\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl driver::interface::DeviceDriver for GPIO {\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n}\n"
  },
  {
    "path": "X1_JTAG_boot/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! PL011 UART driver.\n//!\n//! # Resources\n//!\n//! - <https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf>\n//! - <https://developer.arm.com/documentation/ddi0183/latest>\n\nuse crate::{\n    bsp::device_driver::common::MMIODerefWrapper, console, cpu, driver, synchronization,\n    synchronization::NullLock,\n};\nuse core::fmt;\nuse tock_registers::{\n    interfaces::{Readable, Writeable},\n    register_bitfields, register_structs,\n    registers::{ReadOnly, ReadWrite, WriteOnly},\n};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n// PL011 UART registers.\n//\n// Descriptions taken from \"PrimeCell UART (PL011) Technical Reference Manual\" r1p5.\nregister_bitfields! {\n    u32,\n\n    /// Flag Register.\n    FR [\n        /// Transmit FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n        /// Line Control Register, LCR_H.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the transmit holding register is empty.\n        /// - If the FIFO is enabled, the TXFE bit is set when the transmit FIFO is empty.\n        /// - This bit does not indicate if there is data in the transmit shift register.\n        TXFE OFFSET(7) NUMBITS(1) [],\n\n        /// Transmit FIFO full. The meaning of this bit depends on the state of the FEN bit in the\n        /// LCR_H Register.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the transmit holding register is full.\n        /// - If the FIFO is enabled, the TXFF bit is set when the transmit FIFO is full.\n        TXFF OFFSET(5) NUMBITS(1) [],\n\n        /// Receive FIFO empty. The meaning of this bit depends on the state of the FEN bit in the\n        /// LCR_H Register.\n        ///\n        /// - If the FIFO is disabled, this bit is set when the receive holding register is empty.\n        /// - If the FIFO is enabled, the RXFE bit is set when the receive FIFO is empty.\n        RXFE OFFSET(4) NUMBITS(1) [],\n\n        /// UART busy. If this bit is set to 1, the UART is busy transmitting data. This bit remains\n        /// set until the complete byte, including all the stop bits, has been sent from the shift\n        /// register.\n        ///\n        /// This bit is set as soon as the transmit FIFO becomes non-empty, regardless of whether\n        /// the UART is enabled or not.\n        BUSY OFFSET(3) NUMBITS(1) []\n    ],\n\n    /// Integer Baud Rate Divisor.\n    IBRD [\n        /// The integer baud rate divisor.\n        BAUD_DIVINT OFFSET(0) NUMBITS(16) []\n    ],\n\n    /// Fractional Baud Rate Divisor.\n    FBRD [\n        ///  The fractional baud rate divisor.\n        BAUD_DIVFRAC OFFSET(0) NUMBITS(6) []\n    ],\n\n    /// Line Control Register.\n    LCR_H [\n        /// Word length. These bits indicate the number of data bits transmitted or received in a\n        /// frame.\n        #[allow(clippy::enum_variant_names)]\n        WLEN OFFSET(5) NUMBITS(2) [\n            FiveBit = 0b00,\n            SixBit = 0b01,\n            SevenBit = 0b10,\n            EightBit = 0b11\n        ],\n\n        /// Enable FIFOs:\n        ///\n        /// 0 = FIFOs are disabled (character mode) that is, the FIFOs become 1-byte-deep holding\n        /// registers.\n        ///\n        /// 1 = Transmit and receive FIFO buffers are enabled (FIFO mode).\n        FEN  OFFSET(4) NUMBITS(1) [\n            FifosDisabled = 0,\n            FifosEnabled = 1\n        ]\n    ],\n\n    /// Control Register.\n    CR [\n        /// Receive enable. If this bit is set to 1, the receive section of the UART is enabled.\n        /// Data reception occurs for either UART signals or SIR signals depending on the setting of\n        /// the SIREN bit. When the UART is disabled in the middle of reception, it completes the\n        /// current character before stopping.\n        RXE OFFSET(9) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// Transmit enable. If this bit is set to 1, the transmit section of the UART is enabled.\n        /// Data transmission occurs for either UART signals, or SIR signals depending on the\n        /// setting of the SIREN bit. When the UART is disabled in the middle of transmission, it\n        /// completes the current character before stopping.\n        TXE OFFSET(8) NUMBITS(1) [\n            Disabled = 0,\n            Enabled = 1\n        ],\n\n        /// UART enable:\n        ///\n        /// 0 = UART is disabled. If the UART is disabled in the middle of transmission or\n        /// reception, it completes the current character before stopping.\n        ///\n        /// 1 = The UART is enabled. Data transmission and reception occurs for either UART signals\n        /// or SIR signals depending on the setting of the SIREN bit\n        UARTEN OFFSET(0) NUMBITS(1) [\n            /// If the UART is disabled in the middle of transmission or reception, it completes the\n            /// current character before stopping.\n            Disabled = 0,\n            Enabled = 1\n        ]\n    ],\n\n    /// Interrupt Clear Register.\n    ICR [\n        /// Meta field for all pending interrupts.\n        ALL OFFSET(0) NUMBITS(11) []\n    ]\n}\n\nregister_structs! {\n    #[allow(non_snake_case)]\n    pub RegisterBlock {\n        (0x00 => DR: ReadWrite<u32>),\n        (0x04 => _reserved1),\n        (0x18 => FR: ReadOnly<u32, FR::Register>),\n        (0x1c => _reserved2),\n        (0x24 => IBRD: WriteOnly<u32, IBRD::Register>),\n        (0x28 => FBRD: WriteOnly<u32, FBRD::Register>),\n        (0x2c => LCR_H: WriteOnly<u32, LCR_H::Register>),\n        (0x30 => CR: WriteOnly<u32, CR::Register>),\n        (0x34 => _reserved3),\n        (0x44 => ICR: WriteOnly<u32, ICR::Register>),\n        (0x48 => @END),\n    }\n}\n\n/// Abstraction for the associated MMIO registers.\ntype Registers = MMIODerefWrapper<RegisterBlock>;\n\n#[derive(PartialEq)]\nenum BlockingMode {\n    Blocking,\n    NonBlocking,\n}\n\nstruct PL011UartInner {\n    registers: Registers,\n    chars_written: usize,\n    chars_read: usize,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Representation of the UART.\npub struct PL011Uart {\n    inner: NullLock<PL011UartInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PL011UartInner {\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            registers: Registers::new(mmio_start_addr),\n            chars_written: 0,\n            chars_read: 0,\n        }\n    }\n\n    /// Set up baud rate and characteristics.\n    ///\n    /// This results in 8N1 and 921_600 baud.\n    ///\n    /// The calculation for the BRD is (we set the clock to 48 MHz in config.txt):\n    /// `(48_000_000 / 16) / 921_600 = 3.2552083`.\n    ///\n    /// This means the integer part is `3` and goes into the `IBRD`.\n    /// The fractional part is `0.2552083`.\n    ///\n    /// `FBRD` calculation according to the PL011 Technical Reference Manual:\n    /// `INTEGER((0.2552083 * 64) + 0.5) = 16`.\n    ///\n    /// Therefore, the generated baud rate divider is: `3 + 16/64 = 3.25`. Which results in a\n    /// genrated baud rate of `48_000_000 / (16 * 3.25) = 923_077`.\n    ///\n    /// Error = `((923_077 - 921_600) / 921_600) * 100 = 0.16%`.\n    pub fn init(&mut self) {\n        // Execution can arrive here while there are still characters queued in the TX FIFO and\n        // actively being sent out by the UART hardware. If the UART is turned off in this case,\n        // those queued characters would be lost.\n        //\n        // For example, this can happen during runtime on a call to panic!(), because panic!()\n        // initializes its own UART instance and calls init().\n        //\n        // Hence, flush first to ensure all pending characters are transmitted.\n        self.flush();\n\n        // Turn the UART off temporarily.\n        self.registers.CR.set(0);\n\n        // Clear all pending interrupts.\n        self.registers.ICR.write(ICR::ALL::CLEAR);\n\n        // From the PL011 Technical Reference Manual:\n        //\n        // The LCR_H, IBRD, and FBRD registers form the single 30-bit wide LCR Register that is\n        // updated on a single write strobe generated by a LCR_H write. So, to internally update the\n        // contents of IBRD or FBRD, a LCR_H write must always be performed at the end.\n        //\n        // Set the baud rate, 8N1 and FIFO enabled.\n        self.registers.IBRD.write(IBRD::BAUD_DIVINT.val(3));\n        self.registers.FBRD.write(FBRD::BAUD_DIVFRAC.val(16));\n        self.registers\n            .LCR_H\n            .write(LCR_H::WLEN::EightBit + LCR_H::FEN::FifosEnabled);\n\n        // Turn the UART on.\n        self.registers\n            .CR\n            .write(CR::UARTEN::Enabled + CR::TXE::Enabled + CR::RXE::Enabled);\n    }\n\n    /// Send a character.\n    fn write_char(&mut self, c: char) {\n        // Spin while TX FIFO full is set, waiting for an empty slot.\n        while self.registers.FR.matches_all(FR::TXFF::SET) {\n            cpu::nop();\n        }\n\n        // Write the character to the buffer.\n        self.registers.DR.set(c as u32);\n\n        self.chars_written += 1;\n    }\n\n    /// Block execution until the last buffered character has been physically put on the TX wire.\n    fn flush(&self) {\n        // Spin until the busy bit is cleared.\n        while self.registers.FR.matches_all(FR::BUSY::SET) {\n            cpu::nop();\n        }\n    }\n\n    /// Retrieve a character.\n    fn read_char_converting(&mut self, blocking_mode: BlockingMode) -> Option<char> {\n        // If RX FIFO is empty,\n        if self.registers.FR.matches_all(FR::RXFE::SET) {\n            // immediately return in non-blocking mode.\n            if blocking_mode == BlockingMode::NonBlocking {\n                return None;\n            }\n\n            // Otherwise, wait until a char was received.\n            while self.registers.FR.matches_all(FR::RXFE::SET) {\n                cpu::nop();\n            }\n        }\n\n        // Read one character.\n        let mut ret = self.registers.DR.get() as u8 as char;\n\n        // Convert carrige return to newline.\n        if ret == '\\r' {\n            ret = '\\n'\n        }\n\n        // Update statistics.\n        self.chars_read += 1;\n\n        Some(ret)\n    }\n}\n\n/// Implementing `core::fmt::Write` enables usage of the `format_args!` macros, which in turn are\n/// used to implement the `kernel`'s `print!` and `println!` macros. By implementing `write_str()`,\n/// we get `write_fmt()` automatically.\n///\n/// The function takes an `&mut self`, so it must be implemented for the inner struct.\n///\n/// See [`src/print.rs`].\n///\n/// [`src/print.rs`]: ../../print/index.html\nimpl fmt::Write for PL011UartInner {\n    fn write_str(&mut self, s: &str) -> fmt::Result {\n        for c in s.chars() {\n            self.write_char(c);\n        }\n\n        Ok(())\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl PL011Uart {\n    pub const COMPATIBLE: &'static str = \"BCM PL011 UART\";\n\n    /// Create an instance.\n    ///\n    /// # Safety\n    ///\n    /// - The user must ensure to provide a correct MMIO start address.\n    pub const unsafe fn new(mmio_start_addr: usize) -> Self {\n        Self {\n            inner: NullLock::new(PL011UartInner::new(mmio_start_addr)),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\nimpl driver::interface::DeviceDriver for PL011Uart {\n    fn compatible(&self) -> &'static str {\n        Self::COMPATIBLE\n    }\n\n    unsafe fn init(&self) -> Result<(), &'static str> {\n        self.inner.lock(|inner| inner.init());\n\n        Ok(())\n    }\n}\n\nimpl console::interface::Write for PL011Uart {\n    /// Passthrough of `args` to the `core::fmt::Write` implementation, but guarded by a Mutex to\n    /// serialize access.\n    fn write_char(&self, c: char) {\n        self.inner.lock(|inner| inner.write_char(c));\n    }\n\n    fn write_fmt(&self, args: core::fmt::Arguments) -> fmt::Result {\n        // Fully qualified syntax for the call to `core::fmt::Write::write_fmt()` to increase\n        // readability.\n        self.inner.lock(|inner| fmt::Write::write_fmt(inner, args))\n    }\n\n    fn flush(&self) {\n        // Spin until TX FIFO empty is set.\n        self.inner.lock(|inner| inner.flush());\n    }\n}\n\nimpl console::interface::Read for PL011Uart {\n    fn read_char(&self) -> char {\n        self.inner\n            .lock(|inner| inner.read_char_converting(BlockingMode::Blocking).unwrap())\n    }\n\n    fn clear_rx(&self) {\n        // Read from the RX FIFO until it is indicating empty.\n        while self\n            .inner\n            .lock(|inner| inner.read_char_converting(BlockingMode::NonBlocking))\n            .is_some()\n        {}\n    }\n}\n\nimpl console::interface::Statistics for PL011Uart {\n    fn chars_written(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_written)\n    }\n\n    fn chars_read(&self) -> usize {\n        self.inner.lock(|inner| inner.chars_read)\n    }\n}\n\nimpl console::interface::All for PL011Uart {}\n"
  },
  {
    "path": "X1_JTAG_boot/src/bsp/device_driver/bcm.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BCM driver top level.\n\nmod bcm2xxx_gpio;\nmod bcm2xxx_pl011_uart;\n\npub use bcm2xxx_gpio::*;\npub use bcm2xxx_pl011_uart::*;\n"
  },
  {
    "path": "X1_JTAG_boot/src/bsp/device_driver/common.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Common device driver code.\n\nuse core::{marker::PhantomData, ops};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct MMIODerefWrapper<T> {\n    start_addr: usize,\n    phantom: PhantomData<fn() -> T>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl<T> MMIODerefWrapper<T> {\n    /// Create an instance.\n    pub const unsafe fn new(start_addr: usize) -> Self {\n        Self {\n            start_addr,\n            phantom: PhantomData,\n        }\n    }\n}\n\nimpl<T> ops::Deref for MMIODerefWrapper<T> {\n    type Target = T;\n\n    fn deref(&self) -> &Self::Target {\n        unsafe { &*(self.start_addr as *const _) }\n    }\n}\n"
  },
  {
    "path": "X1_JTAG_boot/src/bsp/device_driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Device driver.\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod bcm;\nmod common;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use bcm::*;\n"
  },
  {
    "path": "X1_JTAG_boot/src/bsp/raspberrypi/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Processor code.\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Used by `arch` code to find the early boot core.\n#[no_mangle]\n#[link_section = \".text._start_arguments\"]\npub static BOOT_CORE_ID: u64 = 0;\n"
  },
  {
    "path": "X1_JTAG_boot/src/bsp/raspberrypi/driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP driver support.\n\nuse super::memory::map::mmio;\nuse crate::{bsp::device_driver, console, driver as generic_driver};\nuse core::sync::atomic::{AtomicBool, Ordering};\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic PL011_UART: device_driver::PL011Uart =\n    unsafe { device_driver::PL011Uart::new(mmio::PL011_UART_START) };\nstatic GPIO: device_driver::GPIO = unsafe { device_driver::GPIO::new(mmio::GPIO_START) };\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// This must be called only after successful init of the UART driver.\nfn post_init_uart() -> Result<(), &'static str> {\n    console::register_console(&PL011_UART);\n\n    Ok(())\n}\n\n/// This must be called only after successful init of the GPIO driver.\nfn post_init_gpio() -> Result<(), &'static str> {\n    GPIO.map_pl011_uart();\n    Ok(())\n}\n\nfn driver_uart() -> Result<(), &'static str> {\n    let uart_descriptor =\n        generic_driver::DeviceDriverDescriptor::new(&PL011_UART, Some(post_init_uart));\n    generic_driver::driver_manager().register_driver(uart_descriptor);\n\n    Ok(())\n}\n\nfn driver_gpio() -> Result<(), &'static str> {\n    let gpio_descriptor = generic_driver::DeviceDriverDescriptor::new(&GPIO, Some(post_init_gpio));\n    generic_driver::driver_manager().register_driver(gpio_descriptor);\n\n    Ok(())\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Initialize the driver subsystem.\n///\n/// # Safety\n///\n/// See child function calls.\npub unsafe fn init() -> Result<(), &'static str> {\n    static INIT_DONE: AtomicBool = AtomicBool::new(false);\n    if INIT_DONE.load(Ordering::Relaxed) {\n        return Err(\"Init already done\");\n    }\n\n    driver_uart()?;\n    driver_gpio()?;\n\n    INIT_DONE.store(true, Ordering::Relaxed);\n    Ok(())\n}\n"
  },
  {
    "path": "X1_JTAG_boot/src/bsp/raspberrypi/kernel.ld",
    "content": "/* SPDX-License-Identifier: MIT OR Apache-2.0\n *\n * Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n */\n\n__rpi_phys_dram_start_addr = 0;\n\n/* The physical address at which the the kernel binary will be loaded by the Raspberry's firmware */\n__rpi_phys_binary_load_addr = 0x80000;\n\n\nENTRY(__rpi_phys_binary_load_addr)\n\n/* Flags:\n *     4 == R\n *     5 == RX\n *     6 == RW\n *\n * Segments are marked PT_LOAD below so that the ELF file provides virtual and physical addresses.\n * It doesn't mean all of them need actually be loaded.\n */\nPHDRS\n{\n    segment_boot_core_stack PT_LOAD FLAGS(6);\n    segment_code            PT_LOAD FLAGS(5);\n    segment_data            PT_LOAD FLAGS(6);\n}\n\nSECTIONS\n{\n    . =  __rpi_phys_dram_start_addr;\n\n    /***********************************************************************************************\n    * Boot Core Stack\n    ***********************************************************************************************/\n    .boot_core_stack (NOLOAD) :\n    {\n                                             /*   ^             */\n                                             /*   | stack       */\n        . += __rpi_phys_binary_load_addr;    /*   | growth      */\n                                             /*   | direction   */\n        __boot_core_stack_end_exclusive = .; /*   |             */\n    } :segment_boot_core_stack\n\n    /***********************************************************************************************\n    * Code + RO Data + Global Offset Table\n    ***********************************************************************************************/\n    .text :\n    {\n        KEEP(*(.text._start))\n        *(.text._start_arguments) /* Constants (or statics in Rust speak) read by _start(). */\n        *(.text._start_rust)      /* The Rust entry point */\n        *(.text*)                 /* Everything else */\n    } :segment_code\n\n    .rodata : ALIGN(8) { *(.rodata*) } :segment_code\n\n    /***********************************************************************************************\n    * Data + BSS\n    ***********************************************************************************************/\n    .data : { *(.data*) } :segment_data\n\n    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */\n    .bss (NOLOAD) : ALIGN(16)\n    {\n        __bss_start = .;\n        *(.bss*);\n        . = ALIGN(16);\n        __bss_end_exclusive = .;\n    } :segment_data\n\n    /***********************************************************************************************\n    * Misc\n    ***********************************************************************************************/\n    .got : { *(.got*) }\n    ASSERT(SIZEOF(.got) == 0, \"Relocation support not expected\")\n\n    /DISCARD/ : { *(.comment*) }\n}\n"
  },
  {
    "path": "X1_JTAG_boot/src/bsp/raspberrypi/memory.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! BSP Memory Management.\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// The board's physical memory map.\n#[rustfmt::skip]\npub(super) mod map {\n\n    pub const GPIO_OFFSET:         usize = 0x0020_0000;\n    pub const UART_OFFSET:         usize = 0x0020_1000;\n\n    /// Physical devices.\n    #[cfg(feature = \"bsp_rpi3\")]\n    pub mod mmio {\n        use super::*;\n\n        pub const START:            usize =         0x3F00_0000;\n        pub const GPIO_START:       usize = START + GPIO_OFFSET;\n        pub const PL011_UART_START: usize = START + UART_OFFSET;\n    }\n\n    /// Physical devices.\n    #[cfg(feature = \"bsp_rpi4\")]\n    pub mod mmio {\n        use super::*;\n\n        pub const START:            usize =         0xFE00_0000;\n        pub const GPIO_START:       usize = START + GPIO_OFFSET;\n        pub const PL011_UART_START: usize = START + UART_OFFSET;\n    }\n}\n"
  },
  {
    "path": "X1_JTAG_boot/src/bsp/raspberrypi.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Top-level BSP file for the Raspberry Pi 3 and 4.\n\npub mod cpu;\npub mod driver;\npub mod memory;\n"
  },
  {
    "path": "X1_JTAG_boot/src/bsp.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Conditional reexporting of Board Support Packages.\n\nmod device_driver;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\nmod raspberrypi;\n\n#[cfg(any(feature = \"bsp_rpi3\", feature = \"bsp_rpi4\"))]\npub use raspberrypi::*;\n"
  },
  {
    "path": "X1_JTAG_boot/src/console/null_console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2022-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Null console.\n\nuse super::interface;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\npub struct NullConsole;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\npub static NULL_CONSOLE: NullConsole = NullConsole {};\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl interface::Write for NullConsole {\n    fn write_char(&self, _c: char) {}\n\n    fn write_fmt(&self, _args: fmt::Arguments) -> fmt::Result {\n        fmt::Result::Ok(())\n    }\n\n    fn flush(&self) {}\n}\n\nimpl interface::Read for NullConsole {\n    fn clear_rx(&self) {}\n}\n\nimpl interface::Statistics for NullConsole {}\nimpl interface::All for NullConsole {}\n"
  },
  {
    "path": "X1_JTAG_boot/src/console.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! System console.\n\nmod null_console;\n\nuse crate::synchronization::{self, NullLock};\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Console interfaces.\npub mod interface {\n    use core::fmt;\n\n    /// Console write functions.\n    pub trait Write {\n        /// Write a single character.\n        fn write_char(&self, c: char);\n\n        /// Write a Rust format string.\n        fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result;\n\n        /// Block until the last buffered character has been physically put on the TX wire.\n        fn flush(&self);\n    }\n\n    /// Console read functions.\n    pub trait Read {\n        /// Read a single character.\n        fn read_char(&self) -> char {\n            ' '\n        }\n\n        /// Clear RX buffers, if any.\n        fn clear_rx(&self);\n    }\n\n    /// Console statistics.\n    pub trait Statistics {\n        /// Return the number of characters written.\n        fn chars_written(&self) -> usize {\n            0\n        }\n\n        /// Return the number of characters read.\n        fn chars_read(&self) -> usize {\n            0\n        }\n    }\n\n    /// Trait alias for a full-fledged console.\n    pub trait All: Write + Read + Statistics {}\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic CUR_CONSOLE: NullLock<&'static (dyn interface::All + Sync)> =\n    NullLock::new(&null_console::NULL_CONSOLE);\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\nuse synchronization::interface::Mutex;\n\n/// Register a new console.\npub fn register_console(new_console: &'static (dyn interface::All + Sync)) {\n    CUR_CONSOLE.lock(|con| *con = new_console);\n}\n\n/// Return a reference to the currently registered console.\n///\n/// This is the global console used by all printing macros.\npub fn console() -> &'static dyn interface::All {\n    CUR_CONSOLE.lock(|con| *con)\n}\n"
  },
  {
    "path": "X1_JTAG_boot/src/cpu/boot.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Boot code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"../_arch/aarch64/cpu/boot.rs\"]\nmod arch_boot;\n"
  },
  {
    "path": "X1_JTAG_boot/src/cpu.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Processor code.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/cpu.rs\"]\nmod arch_cpu;\n\nmod boot;\n\n//--------------------------------------------------------------------------------------------------\n// Architectural Public Reexports\n//--------------------------------------------------------------------------------------------------\npub use arch_cpu::{nop, wait_forever};\n"
  },
  {
    "path": "X1_JTAG_boot/src/driver.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Driver support.\n\nuse crate::synchronization::{interface::Mutex, NullLock};\n\n//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\nconst NUM_DRIVERS: usize = 5;\n\nstruct DriverManagerInner {\n    next_index: usize,\n    descriptors: [Option<DeviceDriverDescriptor>; NUM_DRIVERS],\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Driver interfaces.\npub mod interface {\n    /// Device Driver functions.\n    pub trait DeviceDriver {\n        /// Return a compatibility string for identifying the driver.\n        fn compatible(&self) -> &'static str;\n\n        /// Called by the kernel to bring up the device.\n        ///\n        /// # Safety\n        ///\n        /// - During init, drivers might do stuff with system-wide impact.\n        unsafe fn init(&self) -> Result<(), &'static str> {\n            Ok(())\n        }\n    }\n}\n\n/// Tpye to be used as an optional callback after a driver's init() has run.\npub type DeviceDriverPostInitCallback = unsafe fn() -> Result<(), &'static str>;\n\n/// A descriptor for device drivers.\n#[derive(Copy, Clone)]\npub struct DeviceDriverDescriptor {\n    device_driver: &'static (dyn interface::DeviceDriver + Sync),\n    post_init_callback: Option<DeviceDriverPostInitCallback>,\n}\n\n/// Provides device driver management functions.\npub struct DriverManager {\n    inner: NullLock<DriverManagerInner>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic DRIVER_MANAGER: DriverManager = DriverManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\nimpl DriverManagerInner {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            next_index: 0,\n            descriptors: [None; NUM_DRIVERS],\n        }\n    }\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nimpl DeviceDriverDescriptor {\n    /// Create an instance.\n    pub fn new(\n        device_driver: &'static (dyn interface::DeviceDriver + Sync),\n        post_init_callback: Option<DeviceDriverPostInitCallback>,\n    ) -> Self {\n        Self {\n            device_driver,\n            post_init_callback,\n        }\n    }\n}\n\n/// Return a reference to the global DriverManager.\npub fn driver_manager() -> &'static DriverManager {\n    &DRIVER_MANAGER\n}\n\nimpl DriverManager {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self {\n            inner: NullLock::new(DriverManagerInner::new()),\n        }\n    }\n\n    /// Register a device driver with the kernel.\n    pub fn register_driver(&self, descriptor: DeviceDriverDescriptor) {\n        self.inner.lock(|inner| {\n            inner.descriptors[inner.next_index] = Some(descriptor);\n            inner.next_index += 1;\n        })\n    }\n\n    /// Helper for iterating over registered drivers.\n    fn for_each_descriptor<'a>(&'a self, f: impl FnMut(&'a DeviceDriverDescriptor)) {\n        self.inner.lock(|inner| {\n            inner\n                .descriptors\n                .iter()\n                .filter_map(|x| x.as_ref())\n                .for_each(f)\n        })\n    }\n\n    /// Fully initialize all drivers.\n    ///\n    /// # Safety\n    ///\n    /// - During init, drivers might do stuff with system-wide impact.\n    pub unsafe fn init_drivers(&self) {\n        self.for_each_descriptor(|descriptor| {\n            // 1. Initialize driver.\n            if let Err(x) = descriptor.device_driver.init() {\n                panic!(\n                    \"Error initializing driver: {}: {}\",\n                    descriptor.device_driver.compatible(),\n                    x\n                );\n            }\n\n            // 2. Call corresponding post init callback.\n            if let Some(callback) = &descriptor.post_init_callback {\n                if let Err(x) = callback() {\n                    panic!(\n                        \"Error during driver post-init callback: {}: {}\",\n                        descriptor.device_driver.compatible(),\n                        x\n                    );\n                }\n            }\n        });\n    }\n}\n"
  },
  {
    "path": "X1_JTAG_boot/src/main.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n// Rust embedded logo for `make doc`.\n#![doc(\n    html_logo_url = \"https://raw.githubusercontent.com/rust-embedded/wg/master/assets/logo/ewg-logo-blue-white-on-transparent.png\"\n)]\n\n//! The `kernel` binary.\n//!\n//! # Code organization and architecture\n//!\n//! The code is divided into different *modules*, each representing a typical **subsystem** of the\n//! `kernel`. Top-level module files of subsystems reside directly in the `src` folder. For example,\n//! `src/memory.rs` contains code that is concerned with all things memory management.\n//!\n//! ## Visibility of processor architecture code\n//!\n//! Some of the `kernel`'s subsystems depend on low-level code that is specific to the target\n//! processor architecture. For each supported processor architecture, there exists a subfolder in\n//! `src/_arch`, for example, `src/_arch/aarch64`.\n//!\n//! The architecture folders mirror the subsystem modules laid out in `src`. For example,\n//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go\n//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in\n//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic\n//! module's name prefixed with `arch_`.\n//!\n//! For example, this is the top of `src/memory/mmu.rs`:\n//!\n//! ```\n//! #[cfg(target_arch = \"aarch64\")]\n//! #[path = \"../_arch/aarch64/memory/mmu.rs\"]\n//! mod arch_mmu;\n//! ```\n//!\n//! Often times, items from the `arch_ module` will be publicly reexported by the parent module.\n//! This way, each architecture specific module can provide its implementation of an item, while the\n//! caller must not be concerned which architecture has been conditionally compiled.\n//!\n//! ## BSP code\n//!\n//! `BSP` stands for Board Support Package. `BSP` code is organized under `src/bsp.rs` and contains\n//! target board specific definitions and functions. These are things such as the board's memory map\n//! or instances of drivers for devices that are featured on the respective board.\n//!\n//! Just like processor architecture code, the `BSP` code's module structure tries to mirror the\n//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is\n//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`.\n//!\n//! ## Kernel interfaces\n//!\n//! Both `arch` and `bsp` contain code that is conditionally compiled depending on the actual target\n//! and board for which the kernel is compiled. For example, the `interrupt controller` hardware of\n//! the `Raspberry Pi 3` and the `Raspberry Pi 4` is different, but we want the rest of the `kernel`\n//! code to play nicely with any of the two without much hassle.\n//!\n//! In order to provide a clean abstraction between `arch`, `bsp` and `generic kernel code`,\n//! `interface` traits are provided *whenever possible* and *where it makes sense*. They are defined\n//! in the respective subsystem module and help to enforce the idiom of *program to an interface,\n//! not an implementation*. For example, there will be a common IRQ handling interface which the two\n//! different interrupt controller `drivers` of both Raspberrys will implement, and only export the\n//! interface to the rest of the `kernel`.\n//!\n//! ```\n//!         +-------------------+\n//!         | Interface (Trait) |\n//!         |                   |\n//!         +--+-------------+--+\n//!            ^             ^\n//!            |             |\n//!            |             |\n//! +----------+--+       +--+----------+\n//! | kernel code |       |  bsp code   |\n//! |             |       |  arch code  |\n//! +-------------+       +-------------+\n//! ```\n//!\n//! # Summary\n//!\n//! For a logical `kernel` subsystem, corresponding code can be distributed over several physical\n//! locations. Here is an example for the **memory** subsystem:\n//!\n//! - `src/memory.rs` and `src/memory/**/*`\n//!   - Common code that is agnostic of target processor architecture and `BSP` characteristics.\n//!     - Example: A function to zero a chunk of memory.\n//!   - Interfaces for the memory subsystem that are implemented by `arch` or `BSP` code.\n//!     - Example: An `MMU` interface that defines `MMU` function prototypes.\n//! - `src/bsp/__board_name__/memory.rs` and `src/bsp/__board_name__/memory/**/*`\n//!   - `BSP` specific code.\n//!   - Example: The board's memory map (physical addresses of DRAM and MMIO devices).\n//! - `src/_arch/__arch_name__/memory.rs` and `src/_arch/__arch_name__/memory/**/*`\n//!   - Processor architecture specific code.\n//!   - Example: Implementation of the `MMU` interface for the `__arch_name__` processor\n//!     architecture.\n//!\n//! From a namespace perspective, **memory** subsystem code lives in:\n//!\n//! - `crate::memory::*`\n//! - `crate::bsp::memory::*`\n//!\n//! # Boot flow\n//!\n//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.\n//!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.\n//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.\n\n#![allow(clippy::upper_case_acronyms)]\n#![feature(asm_const)]\n#![feature(const_option)]\n#![feature(format_args_nl)]\n#![feature(nonzero_min_max)]\n#![feature(panic_info_message)]\n#![feature(trait_alias)]\n#![feature(unchecked_math)]\n#![no_main]\n#![no_std]\n\nmod bsp;\nmod console;\nmod cpu;\nmod driver;\nmod panic_wait;\nmod print;\nmod synchronization;\nmod time;\n\n/// Early init code.\n///\n/// # Safety\n///\n/// - Only a single core must be active and running this function.\n/// - The init calls in this function must appear in the correct order.\nunsafe fn kernel_init() -> ! {\n    // Initialize the BSP driver subsystem.\n    if let Err(x) = bsp::driver::init() {\n        panic!(\"Error initializing BSP driver subsystem: {}\", x);\n    }\n\n    // Initialize all device drivers.\n    driver::driver_manager().init_drivers();\n    // println! is usable from here on.\n\n    // Transition from unsafe to safe.\n    kernel_main()\n}\n\n/// The main function running after the early init.\nfn kernel_main() -> ! {\n    info!(\"Parking CPU core. Please connect over JTAG now.\");\n\n    cpu::wait_forever()\n}\n"
  },
  {
    "path": "X1_JTAG_boot/src/panic_wait.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! A panic handler that infinitely waits.\n\nuse crate::{cpu, println};\nuse core::panic::PanicInfo;\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n/// Stop immediately if called a second time.\n///\n/// # Note\n///\n/// Using atomics here relieves us from needing to use `unsafe` for the static variable.\n///\n/// On `AArch64`, which is the only implemented architecture at the time of writing this,\n/// [`AtomicBool::load`] and [`AtomicBool::store`] are lowered to ordinary load and store\n/// instructions. They are therefore safe to use even with MMU + caching deactivated.\n///\n/// [`AtomicBool::load`]: core::sync::atomic::AtomicBool::load\n/// [`AtomicBool::store`]: core::sync::atomic::AtomicBool::store\nfn panic_prevent_reenter() {\n    use core::sync::atomic::{AtomicBool, Ordering};\n\n    #[cfg(not(target_arch = \"aarch64\"))]\n    compile_error!(\"Add the target_arch to above's check if the following code is safe to use\");\n\n    static PANIC_IN_PROGRESS: AtomicBool = AtomicBool::new(false);\n\n    if !PANIC_IN_PROGRESS.load(Ordering::Relaxed) {\n        PANIC_IN_PROGRESS.store(true, Ordering::Relaxed);\n\n        return;\n    }\n\n    cpu::wait_forever()\n}\n\n#[panic_handler]\nfn panic(info: &PanicInfo) -> ! {\n    // Protect against panic infinite loops if any of the following code panics itself.\n    panic_prevent_reenter();\n\n    let timestamp = crate::time::time_manager().uptime();\n    let (location, line, column) = match info.location() {\n        Some(loc) => (loc.file(), loc.line(), loc.column()),\n        _ => (\"???\", 0, 0),\n    };\n\n    println!(\n        \"[  {:>3}.{:06}] Kernel panic!\\n\\n\\\n        Panic location:\\n      File '{}', line {}, column {}\\n\\n\\\n        {}\",\n        timestamp.as_secs(),\n        timestamp.subsec_micros(),\n        location,\n        line,\n        column,\n        info.message().unwrap_or(&format_args!(\"\")),\n    );\n\n    cpu::wait_forever()\n}\n"
  },
  {
    "path": "X1_JTAG_boot/src/print.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Printing.\n\nuse crate::console;\nuse core::fmt;\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n#[doc(hidden)]\npub fn _print(args: fmt::Arguments) {\n    console::console().write_fmt(args).unwrap();\n}\n\n/// Prints without a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! print {\n    ($($arg:tt)*) => ($crate::print::_print(format_args!($($arg)*)));\n}\n\n/// Prints with a newline.\n///\n/// Carbon copy from <https://doc.rust-lang.org/src/std/macros.rs.html>\n#[macro_export]\nmacro_rules! println {\n    () => ($crate::print!(\"\\n\"));\n    ($($arg:tt)*) => ({\n        $crate::print::_print(format_args_nl!($($arg)*));\n    })\n}\n\n/// Prints an info, with a newline.\n#[macro_export]\nmacro_rules! info {\n    ($string:expr) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[  {:>3}.{:06}] \", $string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n        ));\n    });\n    ($format_string:expr, $($arg:tt)*) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[  {:>3}.{:06}] \", $format_string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n            $($arg)*\n        ));\n    })\n}\n\n/// Prints a warning, with a newline.\n#[macro_export]\nmacro_rules! warn {\n    ($string:expr) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[W {:>3}.{:06}] \", $string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n        ));\n    });\n    ($format_string:expr, $($arg:tt)*) => ({\n        let timestamp = $crate::time::time_manager().uptime();\n\n        $crate::print::_print(format_args_nl!(\n            concat!(\"[W {:>3}.{:06}] \", $format_string),\n            timestamp.as_secs(),\n            timestamp.subsec_micros(),\n            $($arg)*\n        ));\n    })\n}\n"
  },
  {
    "path": "X1_JTAG_boot/src/synchronization.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Synchronization primitives.\n//!\n//! # Resources\n//!\n//!   - <https://doc.rust-lang.org/book/ch16-04-extensible-concurrency-sync-and-send.html>\n//!   - <https://stackoverflow.com/questions/59428096/understanding-the-send-trait>\n//!   - <https://doc.rust-lang.org/std/cell/index.html>\n\nuse core::cell::UnsafeCell;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Synchronization interfaces.\npub mod interface {\n\n    /// Any object implementing this trait guarantees exclusive access to the data wrapped within\n    /// the Mutex for the duration of the provided closure.\n    pub trait Mutex {\n        /// The type of the data that is wrapped by this mutex.\n        type Data;\n\n        /// Locks the mutex and grants the closure temporary mutable access to the wrapped data.\n        fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R;\n    }\n}\n\n/// A pseudo-lock for teaching purposes.\n///\n/// In contrast to a real Mutex implementation, does not protect against concurrent access from\n/// other cores to the contained data. This part is preserved for later lessons.\n///\n/// The lock will only be used as long as it is safe to do so, i.e. as long as the kernel is\n/// executing single-threaded, aka only running on a single core with interrupts disabled.\npub struct NullLock<T>\nwhere\n    T: ?Sized,\n{\n    data: UnsafeCell<T>,\n}\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\nunsafe impl<T> Send for NullLock<T> where T: ?Sized + Send {}\nunsafe impl<T> Sync for NullLock<T> where T: ?Sized + Send {}\n\nimpl<T> NullLock<T> {\n    /// Create an instance.\n    pub const fn new(data: T) -> Self {\n        Self {\n            data: UnsafeCell::new(data),\n        }\n    }\n}\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\n\nimpl<T> interface::Mutex for NullLock<T> {\n    type Data = T;\n\n    fn lock<'a, R>(&'a self, f: impl FnOnce(&'a mut Self::Data) -> R) -> R {\n        // In a real lock, there would be code encapsulating this line that ensures that this\n        // mutable reference will ever only be given out once at a time.\n        let data = unsafe { &mut *self.data.get() };\n\n        f(data)\n    }\n}\n"
  },
  {
    "path": "X1_JTAG_boot/src/time.rs",
    "content": "// SPDX-License-Identifier: MIT OR Apache-2.0\n//\n// Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n//! Timer primitives.\n\n#[cfg(target_arch = \"aarch64\")]\n#[path = \"_arch/aarch64/time.rs\"]\nmod arch_time;\n\nuse core::time::Duration;\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n/// Provides time management functions.\npub struct TimeManager;\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\nstatic TIME_MANAGER: TimeManager = TimeManager::new();\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n/// Return a reference to the global TimeManager.\npub fn time_manager() -> &'static TimeManager {\n    &TIME_MANAGER\n}\n\nimpl TimeManager {\n    /// Create an instance.\n    pub const fn new() -> Self {\n        Self\n    }\n\n    /// The uptime since power-on of the device.\n    ///\n    /// This includes time consumed by firmware and bootloaders.\n    pub fn uptime(&self) -> Duration {\n        arch_time::uptime()\n    }\n\n    /// Spin for a given duration.\n    #[cfg(feature = \"bsp_rpi3\")]\n    pub fn spin_for(&self, duration: Duration) {\n        arch_time::spin_for(duration)\n    }\n}\n"
  },
  {
    "path": "X1_JTAG_boot/tests/boot_test_string.rb",
    "content": "# frozen_string_literal: true\n\nEXPECTED_PRINT = 'Please connect over JTAG now'\n"
  },
  {
    "path": "X1_JTAG_boot/update.sh",
    "content": "#!/usr/bin/env bash\n\nBSP=rpi4 make\ncp kernel8.img jtag_boot_rpi4.img\nmake\ncp kernel8.img jtag_boot_rpi3.img\nrm kernel8.img\n"
  },
  {
    "path": "common/docker.mk",
    "content": "DOCKER_IMAGE := rustembedded/osdev-utils:2021.12\n"
  },
  {
    "path": "common/format.mk",
    "content": "define color_header\n    @tput setaf 6 2> /dev/null || true\n    @printf '\\n%s\\n' $(1)\n    @tput sgr0 2> /dev/null || true\nendef\n\ndefine color_progress_prefix\n    @tput setaf 2 2> /dev/null || true\n    @tput bold 2 2> /dev/null || true\n    @printf '%12s ' $(1)\n    @tput sgr0 2> /dev/null || true\nendef\n"
  },
  {
    "path": "common/operating_system.mk",
    "content": "ifeq ($(shell uname -s),Linux)\n    DU_ARGUMENTS = --block-size=1024 --apparent-size\nelse ifeq ($(shell uname -s),Darwin)\n    DU_ARGUMENTS = -k -A\nendif\n\ndefine disk_usage_KiB\n    @printf '%s KiB\\n' `du $(DU_ARGUMENTS) $(1) | cut -f1`\nendef\n"
  },
  {
    "path": "common/serial/minipush/progressbar_patch.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\n# Monkey-patch ruby-progressbar so that it supports reporting the progress in KiB instead of Byte.\n\nclass ProgressBar\n    # Add kibi version of progress\n    class Progress\n        def progress_kibi\n            @progress / 1024\n        end\n    end\n\n    module Format\n        # Add new formatting option\n        class Molecule\n            MOLECULES_EXTENDED = MOLECULES.dup\n            MOLECULES_EXTENDED[:k] = %i[progressable progress_kibi]\n\n            def initialize(letter)\n                self.key         = letter\n                self.method_name = MOLECULES_EXTENDED.fetch(key.to_sym)\n            end\n        end\n    end\nend\n"
  },
  {
    "path": "common/serial/minipush.rb",
    "content": "#!/usr/bin/env ruby\n# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire_relative 'miniterm'\nrequire 'ruby-progressbar'\nrequire_relative 'minipush/progressbar_patch'\nrequire 'timeout'\n\nclass ProtocolError < StandardError; end\n\n# The main class\nclass MiniPush < MiniTerm\n    def initialize(serial_name, payload_path)\n        super(serial_name)\n\n        @name_short = 'MP' # override\n        @payload_path = payload_path\n        @payload_size = nil\n        @payload_data = nil\n    end\n\n    private\n\n    # The three characters signaling the request token form the consecutive sequence \"\\x03\\x03\\x03\".\n    def wait_for_payload_request\n        puts \"[#{@name_short}] 🔌 Please power the target now\"\n\n        # Timeout for the request token starts after the first sign of life was received.\n        received = @target_serial.readpartial(4096)\n        Timeout.timeout(10) do\n            count = 0\n\n            loop do\n                raise ProtocolError if received.nil?\n\n                received.chars.each do |c|\n                    if c == \"\\u{3}\"\n                        count += 1\n                        return true if count == 3\n                    else\n                        # A normal character resets token counting.\n                        count = 0\n\n                        print c\n                    end\n                end\n\n                received = @target_serial.readpartial(4096)\n            end\n        end\n    end\n\n    def load_payload\n        @payload_size = File.size(@payload_path)\n        @payload_data = File.binread(@payload_path)\n    end\n\n    def send_size\n        @target_serial.print([@payload_size].pack('L<'))\n        raise ProtocolError if @target_serial.read(2) != 'OK'\n    end\n\n    def send_payload\n        pb = ProgressBar.create(\n            total: @payload_size,\n            format: \"[#{@name_short}] ⏩ Pushing %k KiB %b🦀%i %p%% %r KiB/s %a\",\n            rate_scale: ->(rate) { rate / 1024 },\n            length: 92,\n            output: $stdout\n        )\n\n        # Send in 512 byte chunks.\n        while pb.progress < pb.total\n            part = @payload_data.slice(pb.progress, 512)\n            pb.progress += @target_serial.write(part)\n        end\n    end\n\n    # override\n    def handle_reconnect(_error)\n        connection_reset\n\n        puts\n        puts \"[#{@name_short}] ⚡ \" \\\n             \"#{'Connection or protocol Error: '.light_red}\" \\\n             \"#{'Remove power and USB serial. Reinsert serial first, then power'.light_red}\"\n        sleep(1) while serial_connected?\n    end\n\n    public\n\n    # override\n    def run\n        open_serial\n        wait_for_payload_request\n        load_payload\n        send_size\n        send_payload\n        terminal\n    rescue ConnectionError, EOFError, Errno::EIO, ProtocolError, Timeout::Error => e\n        handle_reconnect(e)\n        retry\n    rescue StandardError => e\n        handle_unexpected(e)\n    ensure\n        connection_reset\n        puts\n        puts \"[#{@name_short}] Bye 👋\"\n    end\nend\n\n## -------------------------------------------------------------------------------------------------\n## Execution starts here\n## -------------------------------------------------------------------------------------------------\nif __FILE__ == $PROGRAM_NAME\n    puts\n    puts 'Minipush 1.0'.cyan\n    puts\n\n    # CTRL + C handler. Only here to suppress Ruby's default exception print.\n    trap('INT') do\n        # The `ensure` block from `MiniPush::run` will run after exit, restoring console state.\n        exit\n    end\n\n    MiniPush.new(ARGV[0], ARGV[1]).run\nend\n"
  },
  {
    "path": "common/serial/miniterm.rb",
    "content": "#!/usr/bin/env ruby\n# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'rubygems'\nrequire 'bundler/setup'\n\nrequire 'colorize'\nrequire 'io/console'\nrequire 'serialport'\n\nSERIAL_BAUD = 921_600\n\nclass ConnectionError < StandardError; end\n\n# The main class\nclass MiniTerm\n    def initialize(serial_name)\n        @name_short = 'MT'\n        @target_serial_name = serial_name\n        @target_serial = nil\n        @host_console = IO.console\n    end\n\n    private\n\n    def serial_connected?\n        File.exist?(@target_serial_name)\n    end\n\n    def wait_for_serial\n        return if serial_connected?\n\n        puts \"[#{@name_short}] ⏳ Waiting for #{@target_serial_name}\"\n        loop do\n            sleep(1)\n            break if serial_connected?\n        end\n    end\n\n    def open_serial\n        wait_for_serial\n\n        @target_serial = SerialPort.new(@target_serial_name, SERIAL_BAUD, 8, 1, SerialPort::NONE)\n\n        # Ensure all output is immediately flushed to the device.\n        @target_serial.sync = true\n    rescue Errno::EACCES => e\n        puts \"[#{@name_short}] 🚫 #{e.message} - Maybe try with 'sudo'\"\n        exit\n    else\n        puts \"[#{@name_short}] ✅ Serial connected\"\n    end\n\n    def terminal\n        @host_console.raw!\n\n        Thread.abort_on_exception = true\n        Thread.report_on_exception = false\n\n        # Receive from target and print on host console.\n        target_to_host = Thread.new do\n            loop do\n                char = @target_serial.getc\n\n                raise ConnectionError if char.nil?\n\n                # Translate incoming newline to newline + carriage return.\n                @host_console.putc(\"\\r\") if char == \"\\n\"\n                @host_console.putc(char)\n            end\n        end\n\n        # Transmit host console input to target.\n        loop do\n            c = @host_console.getc\n\n            # CTRL + C in raw mode was pressed.\n            if c == \"\\u{3}\"\n                target_to_host.kill\n                break\n            end\n\n            @target_serial.putc(c)\n        end\n    end\n\n    def connection_reset\n        @target_serial&.close\n        @target_serial = nil\n        @host_console.cooked!\n    end\n\n    # When the serial lost power or was removed during R/W operation.\n    def handle_reconnect(_error)\n        connection_reset\n\n        puts\n        puts \"[#{@name_short}] ⚡ #{'Connection Error: Reinsert the USB serial again'.light_red}\"\n    end\n\n    def handle_unexpected(error)\n        connection_reset\n\n        puts\n        puts \"[#{@name_short}] ⚡ #{\"Unexpected Error: #{error.inspect}\".light_red}\"\n    end\n\n    public\n\n    def run\n        open_serial\n        terminal\n    rescue ConnectionError, EOFError, Errno::EIO => e\n        handle_reconnect(e)\n        retry\n    rescue StandardError => e\n        handle_unexpected(e)\n    ensure\n        connection_reset\n        puts\n        puts \"[#{@name_short}] Bye 👋\"\n    end\nend\n\n## -------------------------------------------------------------------------------------------------\n## Execution starts here\n## -------------------------------------------------------------------------------------------------\nif __FILE__ == $PROGRAM_NAME\n    puts\n    puts 'Miniterm 1.0'.cyan\n    puts\n\n    # CTRL + C handler. Only here to suppress Ruby's default exception print.\n    trap('INT') do\n        # The `ensure` block from `MiniTerm::run` will run after exit, restoring console state.\n        exit\n    end\n\n    MiniTerm.new(ARGV[0]).run\nend\n"
  },
  {
    "path": "common/tests/boot_test.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire_relative 'console_io_test'\n\n# Wait for an expected print during boot.\nclass ExpectedBootPrintTest < SubtestBase\n    def initialize(expected_print)\n        super()\n        @expected_print = expected_print\n    end\n\n    def name\n        \"Checking for the string: '#{@expected_print}'\"\n    end\n\n    def run(qemu_out, _qemu_in)\n        expect_or_raise(qemu_out, @expected_print)\n    end\nend\n\n# Check for an expected string when booting the kernel in QEMU.\nclass BootTest < ConsoleIOTest\n    def initialize(qemu_cmd, expected_print)\n        subtests = [ExpectedBootPrintTest.new(expected_print)]\n\n        super(qemu_cmd, 'Boot test', subtests)\n    end\nend\n"
  },
  {
    "path": "common/tests/console_io_test.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'expect'\nrequire 'pty'\nrequire 'timeout'\nrequire_relative 'test'\n\n# Error class for when expect times out.\nclass ExpectTimeoutError < StandardError\n    def initialize(string)\n        super(\"Timeout while expecting string: #{string}\")\n    end\nend\n\n# Provide boilderplate for expecting a string and throwing an error on failure.\nclass SubtestBase\n    TIMEOUT_SECONDS = 3\n\n    def expect_or_raise(io, string, timeout = TIMEOUT_SECONDS)\n        raise ExpectTimeoutError, string if io.expect(string, timeout).nil?\n    end\nend\n\n# Monkey-patch IO so that we get access to the buffer of a previously unsuccessful expect().\nclass IO\n    def unused_buf\n        @unusedBuf\n    end\nend\n\n# A wrapper class that records characters that have been received from a PTY.\nclass PTYLoggerWrapper\n    def initialize(pty, linebreak = \"\\n\")\n        @pty = pty\n        @linebreak = linebreak\n        @log = []\n    end\n\n    def expect(pattern, timeout)\n        result = @pty.expect(pattern, timeout)\n        @log << if result.nil?\n                    @pty.unused_buf\n                else\n                    result\n                end\n\n        result\n    end\n\n    def log\n        @log.join.split(@linebreak)\n    end\nend\n\n# A test doing console I/O with the QEMU binary.\nclass ConsoleIOTest < Test\n    MAX_TIME_ALL_TESTS_SECONDS = 20\n\n    def initialize(qemu_cmd, test_name, console_subtests)\n        super()\n\n        @qemu_cmd = qemu_cmd\n        @console_subtests = console_subtests\n\n        @test_name = test_name\n        @test_description = \"Running #{@console_subtests.length} console I/O tests\"\n        @test_output = []\n        @test_error = nil\n    end\n\n    private\n\n    def format_test_name(number, name)\n        formatted_name = \"#{number.to_s.rjust(3)}. #{name}\"\n        formatted_name.ljust(63, '.')\n    end\n\n    def run_subtest(subtest, test_id, qemu_out, qemu_in)\n        @test_output << format_test_name(test_id, subtest.name)\n        subtest.run(qemu_out, qemu_in)\n        @test_output.last.concat('[ok]')\n    end\n\n    # override\n    def setup\n        qemu_out, @qemu_in = PTY.spawn(@qemu_cmd)\n        @qemu_out_wrapped = PTYLoggerWrapper.new(qemu_out)\n    end\n\n    # override\n    def finish\n        @test_output << ''\n        @test_output << 'Console log:'\n        @test_output += @qemu_out_wrapped.log.map { |line| \"  #{line}\" }\n    end\n\n    # override\n    def run_concrete_test\n        @test_error = false\n\n        Timeout.timeout(MAX_TIME_ALL_TESTS_SECONDS) do\n            @console_subtests.each_with_index do |t, i|\n                run_subtest(t, i + 1, @qemu_out_wrapped, @qemu_in)\n            end\n        end\n    rescue Errno::EIO => e\n        @test_error = \"#{e.inspect} - QEMU might have quit early\"\n    rescue Timeout::Error\n        @test_error = \"Overall time for tests exceeded (#{MAX_TIME_ALL_TESTS_SECONDS}s)\"\n    rescue StandardError => e\n        @test_error = e.inspect\n    end\nend\n"
  },
  {
    "path": "common/tests/dispatch.rb",
    "content": "#!/usr/bin/env ruby\n# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\nfile_dir = File.dirname(__FILE__)\n$LOAD_PATH.unshift(file_dir) unless $LOAD_PATH.include?(file_dir)\n\nrequire 'boot_test'\nrequire 'console_io_test'\nrequire 'exit_code_test'\n\nqemu_cmd = ARGV.join(' ')\nbinary = ARGV.last\ntest_name = binary.gsub(%r{.*deps/}, '').split('-')[0]\n\n# Check if virtual manifest (tutorial 12 or later) or not\npath_prefix = File.exist?('kernel/Cargo.toml') ? 'kernel/' : ''\n\ncase test_name\nwhen 'kernel8.img'\n    load \"#{path_prefix}tests/boot_test_string.rb\" # provides 'EXPECTED_PRINT'\n    BootTest.new(qemu_cmd, EXPECTED_PRINT).run # Doesn't return\n\nwhen 'libkernel'\n    ExitCodeTest.new(qemu_cmd, 'Kernel library unit tests').run # Doesn't return\n\nelse\n    console_test_file = \"#{path_prefix}tests/#{test_name}.rb\"\n    test_name.concat('.rs')\n    test = if File.exist?(console_test_file)\n               load console_test_file # provides 'subtest_collection'\n               ConsoleIOTest.new(qemu_cmd, test_name, subtest_collection)\n           else\n               ExitCodeTest.new(qemu_cmd, test_name)\n           end\n\n    test.run # Doesn't return\nend\n"
  },
  {
    "path": "common/tests/exit_code_test.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'English'\nrequire_relative 'test'\nrequire 'io/wait'\n\n# A test that only inspects the exit code of the QEMU binary.\nclass ExitCodeTest < Test\n    MAX_WAIT_SECS = 5\n\n    def initialize(qemu_cmd, test_name)\n        super()\n\n        @qemu_cmd = qemu_cmd\n\n        @test_name = test_name\n        @test_description = nil\n        @test_output = []\n        @test_error = nil\n    end\n\n    private\n\n    # override\n    def setup\n        @qemu_serial = IO.popen(@qemu_cmd)\n    end\n\n    # override\n    # Convert the recorded output to an array of lines, and extract the test description.\n    def finish\n        @test_output = @test_output.join.split(\"\\n\")\n        @test_description = @test_output.shift\n    end\n\n    # override\n    def run_concrete_test\n        Timeout.timeout(MAX_WAIT_SECS) do\n            @test_output << @qemu_serial.read_nonblock(1024) while @qemu_serial.wait_readable\n        end\n    rescue EOFError\n        @qemu_serial.close\n        @test_error = $CHILD_STATUS.to_i.zero? ? false : 'QEMU exit status != 0'\n    rescue Timeout::Error\n        @test_error = 'Timed out waiting for test'\n    rescue StandardError => e\n        @test_error = e.inspect\n    end\nend\n"
  },
  {
    "path": "common/tests/test.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n# Test base class.\nclass Test\n    INDENT = '         '\n\n    def initialize\n        # Template instance variables.\n        # @test_name\n        # @test_description\n        # @test_output\n        # @test_error\n    end\n\n    private\n\n    def print_border(content)\n        puts \"#{INDENT}-------------------------------------------------------------------\"\n        puts content\n        puts \"#{INDENT}-------------------------------------------------------------------\"\n    end\n\n    def print_header\n        print_border(\"#{INDENT}🦀 #{@test_description}\")\n        puts\n    end\n\n    def print_footer_error(error)\n        puts\n        print_border(\"#{INDENT}❌ Failure: #{@test_name}: #{error}\")\n        puts\n        puts\n    end\n\n    def print_footer_success\n        puts\n        print_border(\"#{INDENT}✅ Success: #{@test_name}\")\n        puts\n        puts\n    end\n\n    # Expects @test_output the be an array of lines, without '\\n'\n    def print_output\n        @test_output.each { |x| print \"#{INDENT}#{x}\\n\" }\n    end\n\n    # Template method.\n    def setup; end\n\n    # Template method.\n    def finish; end\n\n    # Template method.\n    def run_concrete_test\n        raise('Not implemented')\n    end\n\n    public\n\n    def run\n        setup\n        run_concrete_test\n        finish\n\n        print_header\n        print_output\n\n        exit_code = if @test_error\n                        print_footer_error(@test_error)\n                        false\n                    else\n                        print_footer_success\n                        true\n                    end\n\n        exit(exit_code)\n    end\nend\n"
  },
  {
    "path": "contributor_setup.sh",
    "content": "#!/usr/bin/env bash\n\ngit config core.hooksPath .githooks\n\n#\n# Ruby and Bundler\n#\nif ! command -v bundle &> /dev/null\nthen\n    echo \"'bundle' could not be found. Please install Ruby and Bundler.\"\n    exit\nfi\nbundle config set --local path '.vendor/bundle'\nbundle install\n\n#\n# NPM\n#\nif ! command -v npm &> /dev/null\nthen\n    echo \"'npm' could not be found. Please install it.\"\n    exit\nfi\nnpm install --save-dev --save-exact prettier\n\n#\n# Misspell\n#\nif ! command -v curl &> /dev/null\nthen\n    echo \"'curl' could not be found. Please install it.\"\n    exit\nfi\ncurl -L -o ./install-misspell.sh https://raw.githubusercontent.com/client9/misspell/master/install-misspell.sh\nsh ./install-misspell.sh -b .vendor\nrm install-misspell.sh\n"
  },
  {
    "path": "devtool_completion.bash",
    "content": "#!/usr/bin/env bash\n\ncomplete -W \"clean clippy copyright diff fmt fmt_check make make_xtra misspell ready_for_publish ready_for_publish_no_rust rubocop test test_boot test_integration test_unit test_xtra update\" devtool\n"
  },
  {
    "path": "doc/14_BCM_driver.drawio",
    "content": "<mxfile host=\"app.diagrams.net\" etag=\"RT3A7tpwiELOEPGuEBzN\" version=\"12.9.7\" type=\"device\"><diagram name=\"Page-1\" id=\"e7e014a7-5840-1c2e-5031-d8a46d1fe8dd\">7Vpbc+I2FP41zLSdCeM78Mgtu+k2TTakm+YpI2xhqxGWI4sA/fU9smXHNxqSjYFON8MQ6+j4WD7fuXyy6Zjj5eYTR1FwyTxMO4bmbTrmpGMYumUYHfnRvG0qsbSBmUp8Tjyl9SKYkb+xEmpKuiIejkuKgjEqSFQWuiwMsStKMsQ5W5fVFoyWrxohH9cEMxfRuvSOeCJIpYOi/DMmfqCu3NfUuufIffQ5W4Xqch3DXCR/6fQSZaaUfhwgj61LIrwR5ywU6gb+mK9CsQLxJQtZx54GQkgXDDvGOXwWUrHrM+ZTjCISd122BLEbg8r5Ai0JlQBkNkaJDbiIOe2YY86YSI+WmzGmEsAMmnQl5ztmcx9wHIp9Trh7+nX2rTd9ZtMvX26uru2/rsz1mbLyjOgKZ75KPCK2GQiJH7E0onXM0TogAs8i5MrZNcQdyAKxpDDS4VC64rx6x6nXzFEsOHvEY0YZh9mQhWBjVL+NbE2YC7wpiNRtfcJsiQXfgoqK6zPTyJBfF8NEyYJCiFi2EiKFrJ9be3EdHCjvNXtyZD/dTyYj3A8sfRyi4SSYfjuz9nAlxFkkD2UiYK7cNVPz0n0xOJaE/i2LpED6W6B5Fvq6o6eCPOYtpXHNYiIIC0FE8UJUICGUZj6HRMC6Z+NeDQ2YGTg9Ezmvgfg9eKlZ01SuyfHTnTp+ht2AXw70x+Nn1PDDGxdHqV+H8EHxNnQDDm5YxTVo4dYrfm+M9iIYSoQo8SVyLjg0CQrpSAJFcKgmlsTz5GUac6+cndV4ah1JS6siadeR1K1+E5JtAWm2nYjvzMOWsehXoDAGDUnVcxqSSu+1hoVVw2IeR2k6/fIjh3bg1pBCjtG1D5lDdkMOOVSiE5Uwc55Wkr+Mloj7JKmTUKGjDXwnztFS+ZlgUTpnFeYk3GcKOTmnwMttwpGv/idXJgUBWkrgaH10yxERuaisXrIGfiHVK8x5TZIJLm6+XqIQCCcvGJhX1UEWVWXvclhaPnKP7XJJfcFNKwi4XELG7jNb+m6z7SxZ6nLskxhQfghQCAnJf+p2uz833Y1UxiGaU1xSOZyDpW66yIcIhx40gwfCn+IjribiJBSNjiuvo1JVq6UwycW8Yqo+xUBrQZM9kCyvWRlMY0Y3XiuDhZJdLqb5zkoOYFtEXHVM0RzTUb5Pq9TztO7nxPcjmIptV5lKw57B0BqqrN5emXVOsMzmRW00vgT1C6nNVxEUVW0MMcAZpfJ0bcLJ89uroUeeG+8sc7hcofFvJQ+MJjZOI9ormysP4f7CbdpcOW4fzxcfxLrtXoUy9Ht2Qyw37X9bC+Xe66wbyuhQPhWC0Zwy91GGA4qDpFhIZ8L8eQJHUiFgVESkwqQLrh0OxtZ0JM/YEPGnPBu4Ujq6z2zB8WRTHGwLbG8n0AAR3xYsyuF9vlgYvNhMRpnRnRjHbMVdvEdFEJDQWOzB0LBXeoRWj5niPiwj/xxTJCB7S2trCgll7ppB5ynEnzPoVqqpYVX3EOmtqjOLj6Uqxiy9xn+re/zUGTVTSZzmd/odods/5Sp8jTmJAswRvRi/tdieBPE7k03k5usMLfDvK0p/k4lfoux3HLZkVyHd3mA/3knWD0K05FpvMPIKqzn49S9CAvtPJHDdU59T7ncrafE+jvof9kZHr/bGQSPPa+iNg9Z64+BNvdEjaMlC7zaQcVRoinqpKRpWvSnubmRZZyz2xbxLNnfGvPnl3a5wVjvNr79v83OO2Px6NfKVbS3e2vps81VTLbe+7EnToVjb60yrFGyv8az3EL5DhOcxuZlR52ZO9aXX3gGqGzVjh2ZndtN7tZNhZ9Cj/7PE7JbBzByIhObJpy6AQwhfq0g+pgl96ZWVYJwgeorkgskXNEIqmdqxuMbg9KiGXX+LeSSqob+HauSD/ahGJQjeX9qzIvODeRysrO/xlvawzKMUe20wjyNE6xGJiK7XiUjPfCcRMbVei0QEhi8/w0rVX35QZ07/AQ==</diagram></mxfile>"
  },
  {
    "path": "doc/14_GICv2_driver.drawio",
    "content": "<mxfile host=\"app.diagrams.net\" etag=\"i20qkP7-yfUxbQLs3tNp\" version=\"12.9.7\" type=\"device\"><diagram name=\"Page-1\" id=\"e7e014a7-5840-1c2e-5031-d8a46d1fe8dd\">7VpbU+M2FP41mWk7k4zvIY9xLiy7pUsJ3YUnRrEVW0WxjKyQ0F9fyZYd2xIQsoSGtkwGpCP5SDrfdy5y6Nij5eaUgjQ+JyHEHcsINx173LEs07GsjvgY4WMhcQzHKSQRRaGctRXM0F9QCg0pXaEQZo2JjBDMUNoUBiRJYMAaMkApWTenLQhurpqCCCqCWQCwKv2OQhYX0kFd/gmiKJYrnxhy33MQ3EWUrBK5XMeyF/lPMbwEpSo5P4tBSNYNEdywKUmYPMAf81XCVlx8ThLScScxY8IEw4415Z+FmNiLCIkwBCnKegFZcnGQ8SnTBVgiLAAodfi5Dr6IPenYI0oIK1rLzQhiAWAJTbGT6ROjlQ0oTNguD3y//zz71p88kMmXL5dfL9w/v9rrrtTyAPAKlrbKLcIeSxByO0KhxOjY/jpGDM5SEIjRNecdl8VsiXnP5E1himn7xIXVbD9jlNzBEcGE8tGEJFyHrx6j3BOkDG5qInmsU0iWkNFHPkXyumtbJfLrOk2kLK5RxHGlEEhko0rb1nS8Ia2nt6Tv3t+Mxz48iR1zlIDhOJ586zo7mJLzLBVN4QiQSnPN5LgwX8YNi5LoiqRCIOzNwLykvumZhaDivCNnXJAMMUQSLsJwwVqQIIxLm3NHgGbowr6CBh8ZeH0beC+B+CN4yVHblqap8DM9FT/L1eBXAf32+FkKfnATwLSw65B/QPaYBDHlZlhlCrT86C27a9leB0OKAEaRQC7gBs1JIQyJeBAcyoElCkOxjNb3mt7Z5tPBkXSMNpKuiqTpnOiQPBSQrsYRPSzQSRuYefcrEXv9JaARyjHm7Eo3/HduHKOQd5lwRjHm1MYE3F2JnBiT4FU6eSuSf/OVUU0AlgI4rPauKECsEjWnN7Rxu6D2CnOqSErB2eXv5yDhyZLWFMzb07ksbcv2MlgRgiqLPWUSdcO6HcRUbKGsTEpd5tNqD7NlMZfCCGUc5dsYJNwh6U+9Xu9n3WnEZJiAOYaNKe9nYDG32ORtCpOQZ5RbRO+zf3A3KUUJ0xquuY9WVG2HwtwXq4gpcx3hsxY4r99EeC3DYMEZ03opDNZCdjOYVlWh6PCSDgWyjcEcYr+qMVvxvIj7VdJ+iyjruu0oq6l3LMNVo6z5FmH27Az6VhJ3N5/TeXT5EFy709kuleOPlTt7VjsvZzzF8Bp4nsTipJXwrIGmdOl7mtLF7B8KCrVymWdpUbP88u8pVN4YNk2d4lk9jQsdzIOcI6xTqqrg9Gw0fm21cBS5ussHeMEzAwv42wrjX0lw16yyZjFnTXgJo0xfXO1Wlxxq6z5I7qrdHV2CbN0lFwvoBYESQvhI2B/MDeNtfNcz+23n1aY/TcwdHMp3dXeMFjS88BqKd2C8FyKwJEl4FQtu+HxgmsOQm5T3JBKW84pUBjeIXctgKto3os3DV9Ebb2pD48eyk/CjX1fr8k7tKdHdPpb3yueKo8FQeVmn1DQZWdEAvhTzVKTrl8Uyd1KIAUMPzTV1QEp1F4SXmLWiqd9mzUlZRpVKit3K5+pvzlqqXPdFVYw7OGSKqpxb1Tn3p5t35Kli9GFTxf/xVom3g+MLt/1jCbfmPuG26rx7uPU+Yri13zPc6l8hvo5vcyyKTNsPQRbnNxyzQTujQbvc/1u0qznycDByJn49wVcsu6mTTMu45ylc0XHLwJt6MfAsHTXsK0B4zoyulsrvwkdHJVF/Tz46pnJ5bH8LceD0b6o3/OPK/w/WhywASt1ZCpLG/TFBbMYAg+rt8VPxCvNKvN198uV8Xd+u18r/YKGhCfQnO1Yae3yNy7vbL9sLz9z+24Q9+Rs=</diagram></mxfile>"
  },
  {
    "path": "doc/14_header.drawio",
    "content": "<mxfile host=\"app.diagrams.net\" etag=\"oa282FtPgby8-42Oxups\" version=\"12.9.6\" type=\"device\"><diagram id=\"iMJBNVRyMKf7lWlc_RqV\" name=\"Page-1\">7Vxbb+I4FP41SLsPrYhNAnlsaOlWmkpsO6OdeTSJE7wNcdaYFubXr50LIbEpKU2AtlRViU8SX875zh21A4ez5S1D8fSeejjsgK637MDrDgCGMTDFh6SsUkq/10sJASNe9lBBeCS/cUbsZtQF8fC89CCnNOQkLhNdGkXY5SUaYoy+lB/zaVheNUYBVgiPLgpV6j/E49OUaufbk/S/MAmm2cqDbnZjhvJnM8J8ijz6UiLhJR/RiGc7/DFZRHwhyPc0oh3zZsq5PONVB4zEry8fvAwoDUKMYjK/dOlMkN25eGTkoxkJJYPzOZxkDrEIvOnAIaOUp1ez5RCHUkA579OdjLbcXR+e4YjXeWFJLsZXt/BhzLybMf4ds8XPmwtop9M8o3CRcTVjCV/lbGZ0EXlYztLtQOdlSjh+jJEr774IYAnalM9CMTLEpeTFqHrklG3pzQxGhji/M+eMPuEhDSkTpIhG4o6TbQgzjpdbj2qsGSjQjekMc7YSj7xoYDDdgEDPyogoE2ywfrfgnLjImPcGRhoKH53HcXJhhVweNUZRia3WfwspeWeC3KcgYfGFmzJCoKrLgskfhiWEI/YrNtQ1bGt9DaD5Z/G+uArkJ4sJzBcTJ0jXS28lkH/CimAFg3lZelqB+CQMKyQUkiASQ1cIBAu6I8VFhGZeZTdmxPPkMlq4lAG1AQrQ3YWg94Aje6GXQWBVHm5AxzA10AFtIQfs1kAceVfSXhYC8NB8mnDQUNXvjczEnmJl36JnOl7lNIZDxMlzeXodA7MVxpSIhQtJmf2yqKoymNMFc3H21qbp2zVRZR6OWIC5Mk8izvWp95cwbN42ANMs7EG3sA2GaettQ+9sG+rYhsGJGQdLgc4DmscTzJJdj4n4I7FUlZ4IaGJ5OaPeItwUSoj9DHEuiYJvyShl7U6xcRprXHjbEqkIZKAKBPS1AmlLIn3VXA9B58oRMV8sFECcSgwdJ1e3CdNqtmTbxTzho1RpA8TLVF8rmrue5l2zOMN7cftOCpYtYmkYh+JNRsMQy+1dM2Gj2YaBmBTLV4C1YQ88RuPvieFUlXU3NDYNRwdAD+GB7ypWRtyx3AGe+M2AyTDKaDKABk7G4NJUAWW3hafBbvefa3MBMej8izlf5VnPICes051eI6q5XQG2s1hl6Cv28p0BwgUo+/V+eQLq+3PcikevkTXtyJOEFU9TYp8spbt7W+LUgC4Ao3cJzBL7QNdU3Z1GekZrSVR3N1/LAQJlfEoDGqHwG5X+KeFtogoZu9CC0zLn1WBaUEZE7vR6Z4xRFQReEv5TvihsRjr6lU0jr6+Xm4NVNujow26e29Kt3AFbZCz4kwTCr70L9WB4pwICw1ZA1K8XpDelioaaebeKmSkK/SFhbqgiR0iFrTbQIIe/ysMCEclotR7l5TVr7QNz4/7mqooek8bJYdI+GCSrBqttSNZJ6cOQxPNtWdC+BTU53oidusmPLrhXwqz1o5XAzE9+BD2W3Er4Zzod81rnwhgWsSiaJEcsQNW4l9IlZQf1Umo+/ym8f//Y3r+n8PUWR5gRV64WefIvc6fi4wmzKOlluNRTSxdtJL/rmsV7khxseCbu67TPtvoQWU0lOYpooanLczSyrfru5mSrFjK+TmS3V8Bm1/XAg0O5UVgtm7btRjWllnNk11Bkt38ScURMwnL5BlaLfG0Dskat5hzX7Rt/QOPYcd3nrOrA3pHjOqBm6HcPf9+jCAVJ+fk7Q4QrnD6dBtSbtPQxO4HRhDChmspCnTAP2q5Wk9uzNPeUJtQWXA8qTTWX/WINRtMqBxXH7zACNQ9uqsXYQH9RmeLqQTYXb++Gz3LTJ9BL9H1sudpeote3J92GIhjTLuPmFHqJuXn5OM3EFOtfvJsIahRHTj3wtOzTayeCAxcQPnzRKc+vdyb4oJ3ejWUfvZ0IauT456LTAYtOuXE8JUweuJ8IahQmznWnvd3UsfuJudX9bO7/2P1EqNadzv3EfRKdE+wnQrUKdQ7tXg3tzJpuFG7JyJp3o4fuJ8Ia39s4h3YHDO1qpxstYbJf+Tr4ofuJUFPoO8d1TcUfR+8nwhrVuA8Y1x29nwjVctm5A1VHmH3NV2OP3k+EapHuLM09pdlmP1EMi/9VkHrA4r9KwJv/AQ==</diagram></mxfile>"
  },
  {
    "path": "doc/demo_PS1.txt",
    "content": "export PS1=\"\\[\\033[01;32m\\]rust@osdev\\[\\033[00m\\]:\\[\\033[01;34m\\]\\W\\[\\033[00m\\] >> \"\n"
  },
  {
    "path": "doc/source_section_divider.txt",
    "content": "//--------------------------------------------------------------------------------------------------\n// Private Definitions\n//--------------------------------------------------------------------------------------------------\n\n\n\n//--------------------------------------------------------------------------------------------------\n// Public Definitions\n//--------------------------------------------------------------------------------------------------\n\n\n\n//--------------------------------------------------------------------------------------------------\n// Global instances\n//--------------------------------------------------------------------------------------------------\n\n\n\n//--------------------------------------------------------------------------------------------------\n// Private Code\n//--------------------------------------------------------------------------------------------------\n\n\n\n//--------------------------------------------------------------------------------------------------\n// Public Code\n//--------------------------------------------------------------------------------------------------\n\n\n\n//------------------------------------------------------------------------------\n// OS Interface Code\n//------------------------------------------------------------------------------\n\n\n\n//--------------------------------------------------------------------------------------------------\n// Testing\n//--------------------------------------------------------------------------------------------------\n"
  },
  {
    "path": "docker/README.md",
    "content": "# Docker containers for easy dependency bundling\n\nTo facilitate downloading, building and running the required\ndevelopment tools, some of the tutorials in this repository use a\ncontainer software called docker. While docker specifics are\noutside of this tutorial's scope it is worth mentioning that\nhuman readable docker configuration files named \"Dockerfile\"\nexist in the sub-directories of this folder.\n\nThese files may be of interest to those who want details about\nwhat will happen when docker is invoked.\n"
  },
  {
    "path": "docker/rustembedded-osdev-utils/Dockerfile",
    "content": "## SPDX-License-Identifier: MIT OR Apache-2.0\n##\n## Copyright (c) 2017-2023 Andre Richter <andre.o.richter@gmail.com>\n## Copyright (c) 2019-2023 Nao Taco <naotaco@gmail.com>\nFROM ubuntu:20.04\n\nARG VCS_REF\nARG GCC_AARCH64=https://developer.arm.com/-/media/Files/downloads/gnu-a/10.3-2021.07/binrel/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf.tar.xz\nARG GCC_X86_64=https://developer.arm.com/-/media/Files/downloads/gnu-a/10.3-2021.07/binrel/gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz\n\nLABEL org.label-schema.vcs-ref=$VCS_REF \\\n    org.label-schema.vcs-url=\"https://github.com/rust-embedded/rust-raspberrypi-OS-tutorials\"\n\nLABEL maintainer=\"The resources team <resources@teams.rust-embedded.org>, Andre Richter <andre.o.richter@gmail.com>\"\n\n# Ruby gems\nCOPY Gemfile .\n\nRUN set -ex;                                      \\\n    tempPkgs='                                    \\\n        automake                                  \\\n        bison                                     \\\n        build-essential                           \\\n        flex                                      \\\n        git                                       \\\n        libtool                                   \\\n        ninja-build                               \\\n        pkg-config                                \\\n        wget                                      \\\n    ';                                            \\\n    apt-get update;                               \\\n    apt-get install -q -y --no-install-recommends \\\n        $tempPkgs                                 \\\n        # persistent packages\n        ca-certificates                           \\\n        gdb-multiarch                             \\\n        libpixman-1-dev                           \\\n        libglib2.0-dev                            \\\n        libusb-1.0.0-dev                          \\\n        locales                                   \\\n        python3                                   \\\n        ruby                                      \\\n        ruby-dev                                  \\\n    ;                                             \\\n    # GCC AArch64 tools\n    if [ \"$(uname -m)\" = \"aarch64\" ]; then wget ${GCC_AARCH64}; else wget ${GCC_X86_64}; fi; \\\n    tar -xf gcc-arm-10*;                                                                     \\\n    cp                                                                                       \\\n        gcc-arm-10*/bin/aarch64-none-elf-objdump                                             \\\n        gcc-arm-10*/bin/aarch64-none-elf-readelf                                             \\\n        gcc-arm-10*/bin/aarch64-none-elf-nm                                                  \\\n        /usr/local/bin/;                                                                     \\\n    rm -rf gcc-arm-10*;                                                                      \\\n    # Ruby dependencies\n    gem install bundler;                             \\\n    bundle config set --local without 'development'; \\\n    bundle install --retry 3;                        \\\n    # QEMU\n    git clone https://gitlab.com/qemu-project/qemu.git;        \\\n    cd qemu;                                                   \\\n    git checkout tags/v6.1.0;                                  \\\n    ./configure --target-list=aarch64-softmmu --enable-modules \\\n        --enable-tcg-interpreter --enable-debug-tcg            \\\n        --python=/usr/bin/python3;                             \\\n    make -j10;                                                 \\\n    make install;                                              \\\n    cd ..;                                                     \\\n    rm -rf qemu;                                               \\\n    # Openocd\n    git clone --depth 1 https://git.code.sf.net/p/openocd/code openocd; \\\n    cd openocd;                                                         \\\n    ./bootstrap;                                                        \\\n    ./configure --enable-ftdi;                                          \\\n    make -j10;                                                          \\\n    make install;                                                       \\\n    # GDB\n    wget -P ~ git.io/.gdbinit; \\\n    # Cleanup\n    apt-get purge -y --auto-remove $tempPkgs; \\\n    apt-get autoremove -q -y;                 \\\n    apt-get clean -q -y;                      \\\n    rm -rf /var/lib/apt/lists/*\n\n# Locales\nRUN locale-gen en_US.UTF-8\n\nENV LANG=en_US.UTF-8   \\\n    LANGUAGE=en_US:en  \\\n    LC_ALL=en_US.UTF-8 \\\n    RUBYOPT=-W0\n\n# Openocd\nCOPY rpi3.cfg /openocd/\nCOPY rpi4.cfg /openocd/\n\n# GDB\nCOPY auto /root/.gdbinit.d/auto\n"
  },
  {
    "path": "docker/rustembedded-osdev-utils/Makefile",
    "content": "## SPDX-License-Identifier: MIT OR Apache-2.0\n##\n## Copyright (c) 2019-2023 Andre Richter <andre.o.richter@gmail.com>\n\n# Reference followed: https://www.docker.com/blog/getting-started-with-docker-for-arm-on-linux\n\nTAG := 2021.12\n\ndefault: build_local\n\nbuild_local:\n\tcp ../../Gemfile .\n\tdocker build                                           \\\n\t    --tag rustembedded/osdev-utils:$(TAG)              \\\n\t    --build-arg VCS_REF=`git rev-parse --short HEAD` .\n\trm Gemfile\n\nbuildx_push:\n\tcp ../../Gemfile .\n\tdocker buildx build                                    \\\n\t    --push                                             \\\n\t    --platform linux/arm64/v8,linux/amd64              \\\n\t    --tag rustembedded/osdev-utils:$(TAG)              \\\n\t    --build-arg VCS_REF=`git rev-parse --short HEAD` .\n\trm Gemfile\n"
  },
  {
    "path": "docker/rustembedded-osdev-utils/auto",
    "content": "dashboard -layout assembly expressions history memory registers source stack"
  },
  {
    "path": "docker/rustembedded-osdev-utils/rpi3.cfg",
    "content": "# Script from\n# https://www.suse.com/c/debugging-raspberry-pi-3-with-jtag/\n# with minor adaptions.\n\ntransport select jtag\n\n# we need to enable srst even though we don't connect it\nreset_config trst_and_srst\n\nadapter_khz 1000\njtag_ntrst_delay 500\n\nif { [info exists CHIPNAME] } {\n  set _CHIPNAME $CHIPNAME\n} else {\n  set _CHIPNAME rpi3\n}\n\n#\n# Main DAP\n#\nif { [info exists DAP_TAPID] } {\n   set _DAP_TAPID $DAP_TAPID\n} else {\n   set _DAP_TAPID 0x4ba00477\n}\n\njtag newtap $_CHIPNAME tap -irlen 4 -ircapture 0x1 -irmask 0xf -expected-id $_DAP_TAPID -enable\ndap create $_CHIPNAME.dap -chain-position $_CHIPNAME.tap\n\nset _TARGETNAME $_CHIPNAME.core\nset _CTINAME $_CHIPNAME.cti\n\nset DBGBASE {0x80010000 0x80012000 0x80014000 0x80016000}\nset CTIBASE {0x80018000 0x80019000 0x8001a000 0x8001b000}\nset _cores 4\n\nfor { set _core 0 } { $_core < $_cores } { incr _core } {\n\n    cti create $_CTINAME.$_core -dap $_CHIPNAME.dap -ap-num 0 \\\n        -ctibase [lindex $CTIBASE $_core]\n\n    target create $_TARGETNAME$_core aarch64 \\\n        -dap $_CHIPNAME.dap -coreid $_core \\\n        -dbgbase [lindex $DBGBASE $_core] -cti $_CTINAME.$_core\n\n    $_TARGETNAME$_core configure -event reset-assert-post \"aarch64 dbginit\"\n    $_TARGETNAME$_core configure -event gdb-attach { halt }\n}\n"
  },
  {
    "path": "docker/rustembedded-osdev-utils/rpi4.cfg",
    "content": "# Script from\n# https://www.suse.com/c/debugging-raspberry-pi-3-with-jtag/\n# with minor adaptions.\n\ntransport select jtag\n\n# we need to enable srst even though we don't connect it\nreset_config trst_and_srst\n\nadapter_khz 1000\njtag_ntrst_delay 500\n\nif { [info exists CHIPNAME] } {\n  set _CHIPNAME $CHIPNAME\n} else {\n  set _CHIPNAME rpi4\n}\n\n#\n# Main DAP\n#\nif { [info exists DAP_TAPID] } {\n   set _DAP_TAPID $DAP_TAPID\n} else {\n   set _DAP_TAPID 0x4ba00477\n}\n\njtag newtap $_CHIPNAME tap -irlen 4 -ircapture 0x1 -irmask 0xf -expected-id $_DAP_TAPID -enable\ndap create $_CHIPNAME.dap -chain-position $_CHIPNAME.tap\n\nset _TARGETNAME $_CHIPNAME.core\nset _CTINAME $_CHIPNAME.cti\n\nset DBGBASE {0x80410000 0x80510000 0x80610000 0x80710000}\nset CTIBASE {0x80420000 0x80520000 0x80620000 0x80720000}\nset _cores 4\n\nfor { set _core 0 } { $_core < $_cores } { incr _core } {\n\n    cti create $_CTINAME.$_core -dap $_CHIPNAME.dap -ap-num 0 \\\n        -ctibase [lindex $CTIBASE $_core]\n\n    target create $_TARGETNAME$_core aarch64 \\\n        -dap $_CHIPNAME.dap -coreid $_core \\\n        -dbgbase [lindex $DBGBASE $_core] -cti $_CTINAME.$_core\n\n    $_TARGETNAME$_core configure -event reset-assert-post \"aarch64 dbginit\"\n    $_TARGETNAME$_core configure -event gdb-attach { halt }\n}\n"
  },
  {
    "path": "rust-toolchain.toml",
    "content": "[toolchain]\nchannel = \"nightly-2022-10-13\"\ncomponents = [\"rust-src\", \"llvm-tools-preview\", \"rustfmt\"]\ntargets = [\"aarch64-unknown-none-softfloat\"]\n"
  },
  {
    "path": "utils/devtool/copyright.rb",
    "content": "# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'rubygems'\nrequire 'bundler/setup'\nrequire 'colorize'\n\ndef copyright_check_files(source_files)\n    source_files.sort.each do |f|\n        puts 'Checking for copyright: '.light_blue + f.to_s\n\n        years = copyright_years(f)\n        unless years.include?(Time.now.year)\n            puts \"\\tOnly found years: #{years}\".red\n            return false\n        end\n    end\n\n    true\nend\n\ndef copyright_years(file)\n    years = []\n    File.readlines(file).grep(/.*Copyright.*/).each do |x|\n        years << x.scan(/\\d\\d\\d\\d/).map!(&:to_i)\n    end\n\n    years.flatten\nend\n"
  },
  {
    "path": "utils/devtool.rb",
    "content": "#!/usr/bin/env ruby\n# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2020-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'rubygems'\nrequire 'bundler/setup'\nrequire 'colorize'\nrequire 'fileutils'\nrequire_relative 'devtool/copyright'\n\n# Actions for tutorial folders.\nclass TutorialCrate\n    attr_reader :folder\n\n    def initialize(folder)\n        @folder = folder\n    end\n\n    def tutorial?\n        /[0-9]/.match?(@folder[0])\n    end\n\n    def clean\n        puts 'Cleaning '.light_blue + @folder\n\n        # No output needed.\n        Dir.chdir(@folder) { `make clean` }\n    end\n\n    def update\n        puts \"\\n\\n\"\n        puts 'Updating '.light_blue + @folder\n\n        Dir.chdir(@folder) { system('cargo update') }\n    end\n\n    def clippy(bsp)\n        puts \"\\n\\n\"\n        puts \"Clippy #{@folder} - BSP: #{bsp}\".light_blue\n\n        Dir.chdir(@folder) { exit(1) unless system(\"BSP=#{bsp} make clippy\") }\n    end\n\n    def fmt_cargo_rust(args)\n        Dir.chdir(@folder) { exit(1) unless system(\"cargo fmt #{args}\") }\n    end\n\n    def make(bsp)\n        puts \"\\n\\n\"\n        puts \"Make #{@folder} - BSP: #{bsp}\".light_blue\n\n        Dir.chdir(@folder) { exit(1) unless system(\"BSP=#{bsp} make\") }\n    end\n\n    def test(bsp)\n        return unless boot_test?\n\n        puts \"\\n\\n\"\n        puts \"Test #{@folder} - BSP: #{bsp}\".light_blue\n\n        Dir.chdir(@folder) { exit(1) unless system(\"BSP=#{bsp} make test\") }\n    end\n\n    def test_boot(bsp)\n        return unless boot_test?\n\n        puts \"\\n\\n\"\n        puts \"Test Boot #{@folder} - BSP: #{bsp}\".light_blue\n\n        Dir.chdir(@folder) { exit(1) unless system(\"BSP=#{bsp} make test_boot\") }\n    end\n\n    def test_unit(bsp)\n        return unless unit_integration_tests?\n\n        puts \"\\n\\n\"\n        puts \"Test Unit #{@folder} - BSP: #{bsp}\".light_blue\n\n        Dir.chdir(@folder) { exit(1) unless system(\"BSP=#{bsp} make test_unit\") }\n    end\n\n    def test_integration(bsp)\n        return unless unit_integration_tests?\n\n        puts \"\\n\\n\"\n        puts \"Test Integration #{@folder} - BSP: #{bsp}\".light_blue\n\n        Dir.chdir(@folder) { exit(1) unless system(\"BSP=#{bsp} make test_integration\") }\n    end\n\n    private\n\n    def boot_test?\n        Dir.exist?(\"#{@folder}/tests\") || Dir.exist?(\"#{@folder}/kernel/tests\")\n    end\n\n    def unit_integration_tests?\n        !Dir.glob(\"#{@folder}/kernel/tests/00_*.rs\").empty?\n    end\nend\n\n# Forks commands to all applicable receivers.\nclass DevTool\n    def initialize\n        @user_has_supplied_crates = false\n        @bsp = bsp_from_env || SUPPORTED_BSPS.first\n\n        cl = user_supplied_crate_list || Dir['*/Cargo.toml']\n        @crates = cl.map { |c| TutorialCrate.new(c.delete_suffix('/Cargo.toml')) }\n    end\n\n    def clean\n        @crates.each(&:clean)\n    end\n\n    def update\n        @crates.each(&:update)\n    end\n\n    def clippy(bsp = nil)\n        bsp ||= @bsp\n\n        @crates.each { |c| c.clippy(bsp) }\n    end\n\n    def diff\n        tuts = tutorials.map(&:folder)\n        padding = tuts.map(&:length).max\n\n        tuts[0..-2].each_with_index do |original, i|\n            update = tuts[i + 1]\n            diff_pair(original, update, padding)\n        end\n    end\n\n    def fmt\n        fmt_cargo_rust(check: false)\n        puts\n        fmt_prettier(check: false)\n    end\n\n    def fmt_check\n        fmt_cargo_rust(check: true)\n        puts\n        fmt_prettier(check: true)\n    end\n\n    def make(bsp = nil)\n        bsp ||= @bsp\n\n        @crates.each { |c| c.make(bsp) }\n    end\n\n    def make_xtra\n        return if @user_has_supplied_crates\n\n        puts \"\\n\\n\"\n        puts 'Make Xtra stuff'.light_blue\n        system('cd *_uart_chainloader && bash update.sh')\n        system('cd X1_JTAG_boot && bash update.sh')\n    end\n\n    def test(bsp = nil)\n        bsp ||= @bsp\n\n        @crates.each { |c| c.test(bsp) }\n    end\n\n    def test_boot(bsp = nil)\n        bsp ||= @bsp\n\n        @crates.each { |c| c.test_boot(bsp) }\n    end\n\n    def test_unit(bsp = nil)\n        bsp ||= @bsp\n\n        @crates.each { |c| c.test_unit(bsp) }\n    end\n\n    def test_integration(bsp = nil)\n        bsp ||= @bsp\n\n        @crates.each { |c| c.test_integration(bsp) }\n    end\n\n    def copyright\n        exit(1) unless copyright_check_files(copyright_source_files)\n    end\n\n    def misspell\n        puts 'Misspell'.light_blue\n\n        translations = ['README.CN.md', 'README.ES.md']\n        files = tracked_files.reject { |f| translations.include?(File.basename(f)) }\n        files = files.join(' ')\n\n        exit(1) unless system(\".vendor/misspell -error #{files}\")\n    end\n\n    def rubocop\n        puts 'Rubocop'.light_blue\n        exit(1) unless system('bundle exec rubocop')\n    end\n\n    def ready_for_publish_no_rust\n        clean\n        fmt\n        rubocop\n        copyright\n        diff\n        misspell\n        clean\n    end\n\n    def ready_for_publish\n        ready_for_publish_no_rust\n\n        make_xtra\n        clippy('rpi4')\n        clippy('rpi3')\n        test_boot('rpi3')\n        test_unit('rpi3')\n        test_integration('rpi3')\n        clean\n    end\n\n    private\n\n    SUPPORTED_BSPS = %w[rpi3 rpi4].freeze\n\n    def bsp_from_env\n        bsp = ENV.fetch('BSP', nil)\n\n        return bsp if SUPPORTED_BSPS.include?(bsp)\n\n        nil\n    end\n\n    def fmt_cargo_rust(check: false)\n        args = '-- --check' if check\n\n        @crates.each do |c|\n            print 'Rust cargo fmt '.light_blue\n            print \"#{args} \".light_blue unless args.nil?\n            puts c.folder\n\n            Process.fork { c.fmt_cargo_rust(args) }\n        end\n        Process.waitall\n    end\n\n    def fmt_prettier(check: false)\n        args = if check\n                   '--check'\n               else\n                   '--write'\n               end\n\n        args += if @user_has_supplied_crates\n                    \" #{@crates.map(&:folder).join(' ')}\"\n                else\n                    ' .'\n                end\n\n        puts 'Prettier:'.light_blue\n        exit(1) unless system(\"./node_modules/.bin/prettier #{args}\")\n    end\n\n    def user_supplied_crate_list\n        folders = ARGV.drop(1)\n\n        return nil if folders.empty?\n\n        crates = folders.map { |d| \"#{d}/Cargo.toml\" }.sort\n        crates.each do |c|\n            unless File.exist?(c)\n                puts \"Crate not found: #{c}\"\n                exit(1)\n            end\n        end\n\n        @user_has_supplied_crates = true\n        crates\n    end\n\n    def tutorials\n        @crates.select(&:tutorial?)\n    end\n\n    def tracked_files\n        crate_list = @crates.map(&:folder).join(' ') if @user_has_supplied_crates\n\n        `git ls-files #{crate_list}`.split(\"\\n\") # crates_list == nil means all files\n    end\n\n    def diff_pair(original, update, padding)\n        # Only diff adjacent tutorials. This checks the numbers of the tutorial folders.\n        return unless original[0..1].to_i + 1 == update[0..1].to_i\n\n        # Skip for tutorial 11. Due to the change to virtual manifest, the diff is rather\n        # unreadable.\n        if original[0..1].to_i == 11\n            puts 'Skipping '.light_yellow +\n                 \"#{original}: Too noisy due to change to virtual manifest\"\n            return\n        end\n\n        puts 'Diffing  '.light_blue + original.ljust(padding) + \" -> #{update}\"\n        system(\"bash utils/diff_tut_folders.bash #{original} #{update}\")\n    end\n\n    def copyright_source_files\n        extensions = ['.S', '.rs', '.rb']\n\n        # NOTE: The selection result is the return value of the function.\n        tracked_files.select do |f|\n            next unless File.exist?(f)\n            next if f.include?('build.rs')\n            next if f.include?('boot_test_string.rb')\n\n            f.include?('Makefile') ||\n                f.include?('Dockerfile') ||\n                extensions.include?(File.extname(f))\n        end\n    end\nend\n\n## -------------------------------------------------------------------------------------------------\n## Execution starts here\n## -------------------------------------------------------------------------------------------------\ntool = DevTool.new\ncmd = ARGV[0]\ncommands = tool.public_methods(false).sort\n\nif commands.include?(cmd&.to_sym)\n    tool.public_send(cmd)\nelse\n    puts \"Usage: ./#{__FILE__.split('/').last} COMMAND [optional list of folders]\"\n    puts\n    puts 'Commands:'\n    commands.each { |m| puts \"  #{m}\" }\n    exit(1)\nend\n"
  },
  {
    "path": "utils/diff_tut_folders.bash",
    "content": "#!/usr/bin/env bash\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2018-2023 Andre Richter <andre.o.richter@gmail.com>\n\nDIFF=$(\n    diff -uNr \\\n\t -x README.md \\\n\t -x README.CN.md \\\n\t -x README.ES.md \\\n\t -x kernel8.img \\\n\t -x Cargo.lock \\\n\t -x target \\\n\t $1 $2 \\\n\t| sed -r \"s/[12][90][127][0-9]-[0-9][0-9]-[0-9][0-9] .*//g\" \\\n\t| sed -r \"s/[[:space:]]*$//g\" \\\n\t| sed -r \"s/%/modulo/g\" \\\n        | sed -r \"s/diff -uNr -x README.md -x README.CN.md -x README.ES.md -x kernel8.img -x Cargo.lock -x target/\\ndiff -uNr/g\"\n     )\n\nHEADER=\"## Diff to previous\"\nORIGINAL=$(\n    cat $2/README.md \\\n\t| sed -rn \"/$HEADER/q;p\"\n\t)\n\necho \"$ORIGINAL\" > \"$2/README.md\"\nprintf \"\\n$HEADER\\n\" >> \"$2/README.md\"\nprintf \"\\`\\`\\`diff\\n\" >> \"$2/README.md\"\necho \"$DIFF\" >> \"$2/README.md\"\nprintf \"\\n\\`\\`\\`\\n\" >> \"$2/README.md\"\n"
  },
  {
    "path": "utils/update_copyright.rb",
    "content": "#!/usr/bin/env ruby\n# frozen_string_literal: true\n\n# SPDX-License-Identifier: MIT OR Apache-2.0\n#\n# Copyright (c) 2021-2023 Andre Richter <andre.o.richter@gmail.com>\n\nrequire 'date'\n\nfiles = `git ls-files`.split(\"\\n\")\nfiles = files.delete_if { |f| File.symlink?(f) }\nfiles = files.join(' ')\n\nyear = Date.today.year\n\n`sed -i -- 's,\\\\(Copyright .* 20..\\\\)-20..,\\\\1-#{year},g' #{files}`\n`sed -i -- 's,\\\\(Copyright .* #{year - 1}\\\\) ,\\\\1-#{year} ,g' #{files}`\n"
  }
]