[ { "path": ".dockerignore", "content": "dist\npublic\ndata\nscripts/rarity/target\nnode_modules" }, { "path": ".github/workflows/deploy.yml", "content": "# Simple workflow for deploying static content to GitHub Pages\nname: Deploy static content to Pages\n\non:\n # Runs on pushes targeting the default branch\n push:\n branches: [\"master\"]\n\n # Allows you to run this workflow manually from the Actions tab\n workflow_dispatch:\n\n# Sets the GITHUB_TOKEN permissions to allow deployment to GitHub Pages\npermissions:\n contents: read\n pages: write\n id-token: write\n\n# Allow one concurrent deployment\nconcurrency:\n group: \"pages\"\n cancel-in-progress: true\n\njobs:\n # Single deploy job since we're just deploying\n deploy:\n environment:\n name: github-pages\n url: ${{ steps.deployment.outputs.page_url }}\n runs-on: ubuntu-latest\n steps:\n - name: Checkout\n uses: actions/checkout@v4\n - name: Install pnpm\n uses: pnpm/action-setup@v4\n - name: Set up Node\n uses: actions/setup-node@v4\n with:\n node-version: 20\n cache: \"pnpm\"\n - name: Install dependencies\n run: pnpm install\n - name: Build\n run: pnpm run build\n env:\n PUBLIC_BASE_PATH: /isbn-visualization\n - name: Setup Pages\n uses: actions/configure-pages@v4\n - name: Upload artifact\n uses: actions/upload-pages-artifact@v3\n with:\n # Upload dist folder\n path: \"./dist\"\n - name: Deploy to GitHub Pages\n id: deployment\n uses: actions/deploy-pages@v4\n" }, { "path": ".gitignore", "content": "node_modules\ndist\ndata\nnotes.md\n/public/prefix-data\n/public/images\n/public" }, { "path": ".vscode/launch.json", "content": "{\n // Use IntelliSense to learn about possible attributes.\n // Hover to view descriptions of existing attributes.\n // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387\n \"version\": \"0.2.0\",\n \"configurations\": [\n {\n \"name\": \"tsx\",\n \"type\": \"node\",\n \"request\": \"launch\",\n\n // Debug current file in VSCode\n \"program\": \"${file}\",\n\n /*\n * Path to tsx binary\n * Assuming locally installed\n */\n \"runtimeExecutable\": \"tsx\",\n\n /*\n * Open terminal when debugging starts (Optional)\n * Useful to see console.logs\n */\n \"console\": \"integratedTerminal\",\n \"internalConsoleOptions\": \"neverOpen\",\n\n // Files to exclude from debugger (e.g. call stack)\n \"skipFiles\": [\n // Node.js internal core modules\n \"/**\",\n\n // Ignore all dependencies (optional)\n \"${workspaceFolder}/node_modules/**\"\n ],\n \"args\": [\"publication_date\", \"3\"]\n }\n ]\n}\n" }, { "path": ".vscode/settings.json", "content": "{\n \"editor.formatOnSave\": true,\n \"editor.codeActionsOnSave\": {\n \"source.organizeImports\": \"explicit\"\n },\n \"files.exclude\": {\n \"**/.git\": true,\n \"**/.svn\": true,\n \"**/.hg\": true,\n \"**/CVS\": true,\n \"**/.DS_Store\": true,\n \"**/Thumbs.db\": true,\n \"data\": true\n }\n}\n" }, { "path": "Dockerfile", "content": "# Build rust\nFROM rust:1.85 AS rust-builder\nRUN apt-get update && apt-get install -y cmake\nADD scripts/rarity /app/scripts/rarity\nWORKDIR /app/scripts/rarity\nRUN cargo build --release\n\nFROM rust:1.85 AS oxipng\nRUN cargo install oxipng\n\n# pnpm base\nFROM node:22-slim AS base\nENV PNPM_HOME=\"/pnpm\"\nENV PATH=\"$PNPM_HOME:$PATH\"\nRUN corepack enable\n\nFROM base AS prod-deps\nCOPY . /app\nWORKDIR /app\nRUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile\n\n# clean result\nFROM base\nRUN apt-get update && apt-get install -y pngquant zopfli pv zstd\nCOPY --from=prod-deps /app/node_modules /app/node_modules\nCOPY --from=rust-builder /app/scripts/rarity/target/release/rarity /app/scripts/rarity/target/release/rarity\nCOPY --from=oxipng /usr/local/cargo/bin/oxipng /usr/bin/oxipng\nCOPY . /app\nWORKDIR /app\nRUN pnpm -v # ensure pnpm is downloaded by corepack\nCMD [\"/app/scripts/process-all.sh\"]" }, { "path": "LICENSE.md", "content": "I like the concept of giving back, so I settled on the AGPL as the\ndefault license for all my personal projects.\n\nThis isn't set in stone, so feel free to write me at\n`phireskyde+git@gmail.com` if you need something else.\n\n---\n\n### GNU AFFERO GENERAL PUBLIC LICENSE\n\nVersion 3, 19 November 2007\n\nCopyright © 2007 Free Software Foundation, Inc.\n<>\n\nEveryone is permitted to copy and distribute verbatim copies of this\nlicense document, but changing it is not allowed.\n\n### Preamble\n\nThe GNU Affero General Public License is a free, copyleft license for\nsoftware and other kinds of works, specifically designed to ensure\ncooperation with the community in the case of network server software.\n\nThe licenses for most software and other practical works are designed to\ntake away your freedom to share and change the works. By contrast, our\nGeneral Public Licenses are intended to guarantee your freedom to share\nand change all versions of a program--to make sure it remains free\nsoftware for all its users.\n\nWhen we speak of free software, we are referring to freedom, not price.\nOur General Public Licenses are designed to make sure that you have the\nfreedom to distribute copies of free software (and charge for them if\nyou wish), that you receive source code or can get it if you want it,\nthat you can change the software or use pieces of it in new free\nprograms, and that you know you can do these things.\n\nDevelopers that use our General Public Licenses protect your rights with\ntwo steps: (1) assert copyright on the software, and (2) offer you this\nLicense which gives you legal permission to copy, distribute and/or\nmodify the software.\n\nA secondary benefit of defending all users' freedom is that improvements\nmade in alternate versions of the program, if they receive widespread\nuse, become available for other developers to incorporate. Many\ndevelopers of free software are heartened and encouraged by the\nresulting cooperation. However, in the case of software used on network\nservers, this result may fail to come about. The GNU General Public\nLicense permits making a modified version and letting the public access\nit on a server without ever releasing its source code to the public.\n\nThe GNU Affero General Public License is designed specifically to ensure\nthat, in such cases, the modified source code becomes available to the\ncommunity. It requires the operator of a network server to provide the\nsource code of the modified version running there to the users of that\nserver. Therefore, public use of a modified version, on a publicly\naccessible server, gives the public access to the source code of the\nmodified version.\n\nAn older license, called the Affero General Public License and published\nby Affero, was designed to accomplish similar goals. This is a different\nlicense, not a version of the Affero GPL, but Affero has released a new\nversion of the Affero GPL which permits relicensing under this license.\n\nThe precise terms and conditions for copying, distribution and\nmodification follow.\n\n### TERMS AND CONDITIONS\n\n#### 0. Definitions.\n\n\"This License\" refers to version 3 of the GNU Affero General Public\nLicense.\n\n\"Copyright\" also means copyright-like laws that apply to other kinds of\nworks, such as semiconductor masks.\n\n\"The Program\" refers to any copyrightable work licensed under this\nLicense. Each licensee is addressed as \"you\". \"Licensees\" and\n\"recipients\" may be individuals or organizations.\n\nTo \"modify\" a work means to copy from or adapt all or part of the work\nin a fashion requiring copyright permission, other than the making of an\nexact copy. The resulting work is called a \"modified version\" of the\nearlier work or a work \"based on\" the earlier work.\n\nA \"covered work\" means either the unmodified Program or a work based on\nthe Program.\n\nTo \"propagate\" a work means to do anything with it that, without\npermission, would make you directly or secondarily liable for\ninfringement under applicable copyright law, except executing it on a\ncomputer or modifying a private copy. Propagation includes copying,\ndistribution (with or without modification), making available to the\npublic, and in some countries other activities as well.\n\nTo \"convey\" a work means any kind of propagation that enables other\nparties to make or receive copies. Mere interaction with a user through\na computer network, with no transfer of a copy, is not conveying.\n\nAn interactive user interface displays \"Appropriate Legal Notices\" to\nthe extent that it includes a convenient and prominently visible feature\nthat (1) displays an appropriate copyright notice, and (2) tells the\nuser that there is no warranty for the work (except to the extent that\nwarranties are provided), that licensees may convey the work under this\nLicense, and how to view a copy of this License. If the interface\npresents a list of user commands or options, such as a menu, a prominent\nitem in the list meets this criterion.\n\n#### 1. Source Code.\n\nThe \"source code\" for a work means the preferred form of the work for\nmaking modifications to it. \"Object code\" means any non-source form of a\nwork.\n\nA \"Standard Interface\" means an interface that either is an official\nstandard defined by a recognized standards body, or, in the case of\ninterfaces specified for a particular programming language, one that is\nwidely used among developers working in that language.\n\nThe \"System Libraries\" of an executable work include anything, other\nthan the work as a whole, that (a) is included in the normal form of\npackaging a Major Component, but which is not part of that Major\nComponent, and (b) serves only to enable use of the work with that Major\nComponent, or to implement a Standard Interface for which an\nimplementation is available to the public in source code form. A \"Major\nComponent\", in this context, means a major essential component (kernel,\nwindow system, and so on) of the specific operating system (if any) on\nwhich the executable work runs, or a compiler used to produce the work,\nor an object code interpreter used to run it.\n\nThe \"Corresponding Source\" for a work in object code form means all the\nsource code needed to generate, install, and (for an executable work)\nrun the object code and to modify the work, including scripts to control\nthose activities. However, it does not include the work's System\nLibraries, or general-purpose tools or generally available free programs\nwhich are used unmodified in performing those activities but which are\nnot part of the work. For example, Corresponding Source includes\ninterface definition files associated with source files for the work,\nand the source code for shared libraries and dynamically linked\nsubprograms that the work is specifically designed to require, such as\nby intimate data communication or control flow between those subprograms\nand other parts of the work.\n\nThe Corresponding Source need not include anything that users can\nregenerate automatically from other parts of the Corresponding Source.\n\nThe Corresponding Source for a work in source code form is that same\nwork.\n\n#### 2. Basic Permissions.\n\nAll rights granted under this License are granted for the term of\ncopyright on the Program, and are irrevocable provided the stated\nconditions are met. This License explicitly affirms your unlimited\npermission to run the unmodified Program. The output from running a\ncovered work is covered by this License only if the output, given its\ncontent, constitutes a covered work. This License acknowledges your\nrights of fair use or other equivalent, as provided by copyright law.\n\nYou may make, run and propagate covered works that you do not convey,\nwithout conditions so long as your license otherwise remains in force.\nYou may convey covered works to others for the sole purpose of having\nthem make modifications exclusively for you, or provide you with\nfacilities for running those works, provided that you comply with the\nterms of this License in conveying all material for which you do not\ncontrol copyright. Those thus making or running the covered works for\nyou must do so exclusively on your behalf, under your direction and\ncontrol, on terms that prohibit them from making any copies of your\ncopyrighted material outside their relationship with you.\n\nConveying under any other circumstances is permitted solely under the\nconditions stated below. Sublicensing is not allowed; section 10 makes\nit unnecessary.\n\n#### 3. Protecting Users' Legal Rights From Anti-Circumvention Law.\n\nNo covered work shall be deemed part of an effective technological\nmeasure under any applicable law fulfilling obligations under article 11\nof the WIPO copyright treaty adopted on 20 December 1996, or similar\nlaws prohibiting or restricting circumvention of such measures.\n\nWhen you convey a covered work, you waive any legal power to forbid\ncircumvention of technological measures to the extent such circumvention\nis effected by exercising rights under this License with respect to the\ncovered work, and you disclaim any intention to limit operation or\nmodification of the work as a means of enforcing, against the work's\nusers, your or third parties' legal rights to forbid circumvention of\ntechnological measures.\n\n#### 4. Conveying Verbatim Copies.\n\nYou may convey verbatim copies of the Program's source code as you\nreceive it, in any medium, provided that you conspicuously and\nappropriately publish on each copy an appropriate copyright notice; keep\nintact all notices stating that this License and any non-permissive\nterms added in accord with section 7 apply to the code; keep intact all\nnotices of the absence of any warranty; and give all recipients a copy\nof this License along with the Program.\n\nYou may charge any price or no price for each copy that you convey, and\nyou may offer support or warranty protection for a fee.\n\n#### 5. Conveying Modified Source Versions.\n\nYou may convey a work based on the Program, or the modifications to\nproduce it from the Program, in the form of source code under the terms\nof section 4, provided that you also meet all of these conditions:\n\n- a\\) The work must carry prominent notices stating that you modified it,\n and giving a relevant date.\n- b\\) The work must carry prominent notices stating that it is released\n under this License and any conditions added under section 7. This\n requirement modifies the requirement in section 4 to \"keep intact\n all notices\".\n- c\\) You must license the entire work, as a whole, under this License to\n anyone who comes into possession of a copy. This License will therefore\n apply, along with any applicable section 7 additional terms, to the\n whole of the work, and all its parts, regardless of how they\n are packaged. This License gives no permission to license the work in\n any other way, but it does not invalidate such permission if you have\n separately received it.\n- d\\) If the work has interactive user interfaces, each must display\n Appropriate Legal Notices; however, if the Program has interactive\n interfaces that do not display Appropriate Legal Notices, your work need\n not make them do so.\n\nA compilation of a covered work with other separate and independent\nworks, which are not by their nature extensions of the covered work, and\nwhich are not combined with it such as to form a larger program, in or\non a volume of a storage or distribution medium, is called an\n\"aggregate\" if the compilation and its resulting copyright are not used\nto limit the access or legal rights of the compilation's users beyond\nwhat the individual works permit. Inclusion of a covered work in an\naggregate does not cause this License to apply to the other parts of the\naggregate.\n\n#### 6. Conveying Non-Source Forms.\n\nYou may convey a covered work in object code form under the terms of\nsections 4 and 5, provided that you also convey the machine-readable\nCorresponding Source under the terms of this License, in one of these\nways:\n\n- a\\) Convey the object code in, or embodied in, a physical product\n (including a physical distribution medium), accompanied by the\n Corresponding Source fixed on a durable physical medium customarily used\n for software interchange.\n- b\\) Convey the object code in, or embodied in, a physical product\n (including a physical distribution medium), accompanied by a written\n offer, valid for at least three years and valid for as long as you offer\n spare parts or customer support for that product model, to give anyone\n who possesses the object code either (1) a copy of the Corresponding\n Source for all the software in the product that is covered by this\n License, on a durable physical medium customarily used for software\n interchange, for a price no more than your reasonable cost of physically\n performing this conveying of source, or (2) access to copy the\n Corresponding Source from a network server at no charge.\n- c\\) Convey individual copies of the object code with a copy of the\n written offer to provide the Corresponding Source. This alternative is\n allowed only occasionally and noncommercially, and only if you received\n the object code with such an offer, in accord with subsection 6b.\n- d\\) Convey the object code by offering access from a designated place\n (gratis or for a charge), and offer equivalent access to the\n Corresponding Source in the same way through the same place at no\n further charge. You need not require recipients to copy the\n Corresponding Source along with the object code. If the place to copy\n the object code is a network server, the Corresponding Source may be on\n a different server (operated by you or a third party) that supports\n equivalent copying facilities, provided you maintain clear directions\n next to the object code saying where to find the Corresponding Source.\n Regardless of what server hosts the Corresponding Source, you remain\n obligated to ensure that it is available for as long as needed to\n satisfy these requirements.\n- e\\) Convey the object code using peer-to-peer transmission, provided you\n inform other peers where the object code and Corresponding Source of the\n work are being offered to the general public at no charge under\n subsection 6d.\n\nA separable portion of the object code, whose source code is excluded\nfrom the Corresponding Source as a System Library, need not be included\nin conveying the object code work.\n\nA \"User Product\" is either (1) a \"consumer product\", which means any\ntangible personal property which is normally used for personal, family,\nor household purposes, or (2) anything designed or sold for\nincorporation into a dwelling. In determining whether a product is a\nconsumer product, doubtful cases shall be resolved in favor of coverage.\nFor a particular product received by a particular user, \"normally used\"\nrefers to a typical or common use of that class of product, regardless\nof the status of the particular user or of the way in which the\nparticular user actually uses, or expects or is expected to use, the\nproduct. A product is a consumer product regardless of whether the\nproduct has substantial commercial, industrial or non-consumer uses,\nunless such uses represent the only significant mode of use of the\nproduct.\n\n\"Installation Information\" for a User Product means any methods,\nprocedures, authorization keys, or other information required to install\nand execute modified versions of a covered work in that User Product\nfrom a modified version of its Corresponding Source. The information\nmust suffice to ensure that the continued functioning of the modified\nobject code is in no case prevented or interfered with solely because\nmodification has been made.\n\nIf you convey an object code work under this section in, or with, or\nspecifically for use in, a User Product, and the conveying occurs as\npart of a transaction in which the right of possession and use of the\nUser Product is transferred to the recipient in perpetuity or for a\nfixed term (regardless of how the transaction is characterized), the\nCorresponding Source conveyed under this section must be accompanied by\nthe Installation Information. But this requirement does not apply if\nneither you nor any third party retains the ability to install modified\nobject code on the User Product (for example, the work has been\ninstalled in ROM).\n\nThe requirement to provide Installation Information does not include a\nrequirement to continue to provide support service, warranty, or updates\nfor a work that has been modified or installed by the recipient, or for\nthe User Product in which it has been modified or installed. Access to a\nnetwork may be denied when the modification itself materially and\nadversely affects the operation of the network or violates the rules and\nprotocols for communication across the network.\n\nCorresponding Source conveyed, and Installation Information provided, in\naccord with this section must be in a format that is publicly documented\n(and with an implementation available to the public in source code\nform), and must require no special password or key for unpacking,\nreading or copying.\n\n#### 7. Additional Terms.\n\n\"Additional permissions\" are terms that supplement the terms of this\nLicense by making exceptions from one or more of its conditions.\nAdditional permissions that are applicable to the entire Program shall\nbe treated as though they were included in this License, to the extent\nthat they are valid under applicable law. If additional permissions\napply only to part of the Program, that part may be used separately\nunder those permissions, but the entire Program remains governed by this\nLicense without regard to the additional permissions.\n\nWhen you convey a copy of a covered work, you may at your option remove\nany additional permissions from that copy, or from any part of it.\n(Additional permissions may be written to require their own removal in\ncertain cases when you modify the work.) You may place additional\npermissions on material, added by you to a covered work, for which you\nhave or can give appropriate copyright permission.\n\nNotwithstanding any other provision of this License, for material you\nadd to a covered work, you may (if authorized by the copyright holders\nof that material) supplement the terms of this License with terms:\n\n- a\\) Disclaiming warranty or limiting liability differently from the terms\n of sections 15 and 16 of this License; or\n- b\\) Requiring preservation of specified reasonable legal notices or\n author attributions in that material or in the Appropriate Legal Notices\n displayed by works containing it; or\n- c\\) Prohibiting misrepresentation of the origin of that material, or\n requiring that modified versions of such material be marked in\n reasonable ways as different from the original version; or\n- d\\) Limiting the use for publicity purposes of names of licensors or\n authors of the material; or\n- e\\) Declining to grant rights under trademark law for use of some trade\n names, trademarks, or service marks; or\n- f\\) Requiring indemnification of licensors and authors of that material\n by anyone who conveys the material (or modified versions of it) with\n contractual assumptions of liability to the recipient, for any liability\n that these contractual assumptions directly impose on those licensors\n and authors.\n\nAll other non-permissive additional terms are considered \"further\nrestrictions\" within the meaning of section 10. If the Program as you\nreceived it, or any part of it, contains a notice stating that it is\ngoverned by this License along with a term that is a further\nrestriction, you may remove that term. If a license document contains a\nfurther restriction but permits relicensing or conveying under this\nLicense, you may add to a covered work material governed by the terms of\nthat license document, provided that the further restriction does not\nsurvive such relicensing or conveying.\n\nIf you add terms to a covered work in accord with this section, you must\nplace, in the relevant source files, a statement of the additional terms\nthat apply to those files, or a notice indicating where to find the\napplicable terms.\n\nAdditional terms, permissive or non-permissive, may be stated in the\nform of a separately written license, or stated as exceptions; the above\nrequirements apply either way.\n\n#### 8. Termination.\n\nYou may not propagate or modify a covered work except as expressly\nprovided under this License. Any attempt otherwise to propagate or\nmodify it is void, and will automatically terminate your rights under\nthis License (including any patent licenses granted under the third\nparagraph of section 11).\n\nHowever, if you cease all violation of this License, then your license\nfrom a particular copyright holder is reinstated (a) provisionally,\nunless and until the copyright holder explicitly and finally terminates\nyour license, and (b) permanently, if the copyright holder fails to\nnotify you of the violation by some reasonable means prior to 60 days\nafter the cessation.\n\nMoreover, your license from a particular copyright holder is reinstated\npermanently if the copyright holder notifies you of the violation by\nsome reasonable means, this is the first time you have received notice\nof violation of this License (for any work) from that copyright holder,\nand you cure the violation prior to 30 days after your receipt of the\nnotice.\n\nTermination of your rights under this section does not terminate the\nlicenses of parties who have received copies or rights from you under\nthis License. If your rights have been terminated and not permanently\nreinstated, you do not qualify to receive new licenses for the same\nmaterial under section 10.\n\n#### 9. Acceptance Not Required for Having Copies.\n\nYou are not required to accept this License in order to receive or run a\ncopy of the Program. Ancillary propagation of a covered work occurring\nsolely as a consequence of using peer-to-peer transmission to receive a\ncopy likewise does not require acceptance. However, nothing other than\nthis License grants you permission to propagate or modify any covered\nwork. These actions infringe copyright if you do not accept this\nLicense. Therefore, by modifying or propagating a covered work, you\nindicate your acceptance of this License to do so.\n\n#### 10. Automatic Licensing of Downstream Recipients.\n\nEach time you convey a covered work, the recipient automatically\nreceives a license from the original licensors, to run, modify and\npropagate that work, subject to this License. You are not responsible\nfor enforcing compliance by third parties with this License.\n\nAn \"entity transaction\" is a transaction transferring control of an\norganization, or substantially all assets of one, or subdividing an\norganization, or merging organizations. If propagation of a covered work\nresults from an entity transaction, each party to that transaction who\nreceives a copy of the work also receives whatever licenses to the work\nthe party's predecessor in interest had or could give under the previous\nparagraph, plus a right to possession of the Corresponding Source of the\nwork from the predecessor in interest, if the predecessor has it or can\nget it with reasonable efforts.\n\nYou may not impose any further restrictions on the exercise of the\nrights granted or affirmed under this License. For example, you may not\nimpose a license fee, royalty, or other charge for exercise of rights\ngranted under this License, and you may not initiate litigation\n(including a cross-claim or counterclaim in a lawsuit) alleging that any\npatent claim is infringed by making, using, selling, offering for sale,\nor importing the Program or any portion of it.\n\n#### 11. Patents.\n\nA \"contributor\" is a copyright holder who authorizes use under this\nLicense of the Program or a work on which the Program is based. The work\nthus licensed is called the contributor's \"contributor version\".\n\nA contributor's \"essential patent claims\" are all patent claims owned or\ncontrolled by the contributor, whether already acquired or hereafter\nacquired, that would be infringed by some manner, permitted by this\nLicense, of making, using, or selling its contributor version, but do\nnot include claims that would be infringed only as a consequence of\nfurther modification of the contributor version. For purposes of this\ndefinition, \"control\" includes the right to grant patent sublicenses in\na manner consistent with the requirements of this License.\n\nEach contributor grants you a non-exclusive, worldwide, royalty-free\npatent license under the contributor's essential patent claims, to make,\nuse, sell, offer for sale, import and otherwise run, modify and\npropagate the contents of its contributor version.\n\nIn the following three paragraphs, a \"patent license\" is any express\nagreement or commitment, however denominated, not to enforce a patent\n(such as an express permission to practice a patent or covenant not to\nsue for patent infringement). To \"grant\" such a patent license to a\nparty means to make such an agreement or commitment not to enforce a\npatent against the party.\n\nIf you convey a covered work, knowingly relying on a patent license, and\nthe Corresponding Source of the work is not available for anyone to\ncopy, free of charge and under the terms of this License, through a\npublicly available network server or other readily accessible means,\nthen you must either (1) cause the Corresponding Source to be so\navailable, or (2) arrange to deprive yourself of the benefit of the\npatent license for this particular work, or (3) arrange, in a manner\nconsistent with the requirements of this License, to extend the patent\nlicense to downstream recipients. \"Knowingly relying\" means you have\nactual knowledge that, but for the patent license, your conveying the\ncovered work in a country, or your recipient's use of the covered work\nin a country, would infringe one or more identifiable patents in that\ncountry that you have reason to believe are valid.\n\nIf, pursuant to or in connection with a single transaction or\narrangement, you convey, or propagate by procuring conveyance of, a\ncovered work, and grant a patent license to some of the parties\nreceiving the covered work authorizing them to use, propagate, modify or\nconvey a specific copy of the covered work, then the patent license you\ngrant is automatically extended to all recipients of the covered work\nand works based on it.\n\nA patent license is \"discriminatory\" if it does not include within the\nscope of its coverage, prohibits the exercise of, or is conditioned on\nthe non-exercise of one or more of the rights that are specifically\ngranted under this License. You may not convey a covered work if you are\na party to an arrangement with a third party that is in the business of\ndistributing software, under which you make payment to the third party\nbased on the extent of your activity of conveying the work, and under\nwhich the third party grants, to any of the parties who would receive\nthe covered work from you, a discriminatory patent license (a) in\nconnection with copies of the covered work conveyed by you (or copies\nmade from those copies), or (b) primarily for and in connection with\nspecific products or compilations that contain the covered work, unless\nyou entered into that arrangement, or that patent license was granted,\nprior to 28 March 2007.\n\nNothing in this License shall be construed as excluding or limiting any\nimplied license or other defenses to infringement that may otherwise be\navailable to you under applicable patent law.\n\n#### 12. No Surrender of Others' Freedom.\n\nIf conditions are imposed on you (whether by court order, agreement or\notherwise) that contradict the conditions of this License, they do not\nexcuse you from the conditions of this License. If you cannot convey a\ncovered work so as to satisfy simultaneously your obligations under this\nLicense and any other pertinent obligations, then as a consequence you\nmay not convey it at all. For example, if you agree to terms that\nobligate you to collect a royalty for further conveying from those to\nwhom you convey the Program, the only way you could satisfy both those\nterms and this License would be to refrain entirely from conveying the\nProgram.\n\n#### 13. Remote Network Interaction; Use with the GNU General Public License.\n\nNotwithstanding any other provision of this License, if you modify the\nProgram, your modified version must prominently offer all users\ninteracting with it remotely through a computer network (if your version\nsupports such interaction) an opportunity to receive the Corresponding\nSource of your version by providing access to the Corresponding Source\nfrom a network server at no charge, through some standard or customary\nmeans of facilitating copying of software. This Corresponding Source\nshall include the Corresponding Source for any work covered by version 3\nof the GNU General Public License that is incorporated pursuant to the\nfollowing paragraph.\n\nNotwithstanding any other provision of this License, you have permission\nto link or combine any covered work with a work licensed under version 3\nof the GNU General Public License into a single combined work, and to\nconvey the resulting work. The terms of this License will continue to\napply to the part which is the covered work, but the work with which it\nis combined will remain governed by version 3 of the GNU General Public\nLicense.\n\n#### 14. Revised Versions of this License.\n\nThe Free Software Foundation may publish revised and/or new versions of\nthe GNU Affero General Public License from time to time. Such new\nversions will be similar in spirit to the present version, but may\ndiffer in detail to address new problems or concerns.\n\nEach version is given a distinguishing version number. If the Program\nspecifies that a certain numbered version of the GNU Affero General\nPublic License \"or any later version\" applies to it, you have the option\nof following the terms and conditions either of that numbered version or\nof any later version published by the Free Software Foundation. If the\nProgram does not specify a version number of the GNU Affero General\nPublic License, you may choose any version ever published by the Free\nSoftware Foundation.\n\nIf the Program specifies that a proxy can decide which future versions\nof the GNU Affero General Public License can be used, that proxy's\npublic statement of acceptance of a version permanently authorizes you\nto choose that version for the Program.\n\nLater license versions may give you additional or different permissions.\nHowever, no additional obligations are imposed on any author or\ncopyright holder as a result of your choosing to follow a later version.\n\n#### 15. Disclaimer of Warranty.\n\nTHERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY\nAPPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT\nHOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM \"AS IS\" WITHOUT\nWARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT\nLIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A\nPARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF\nTHE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME\nTHE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.\n\n#### 16. Limitation of Liability.\n\nIN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING\nWILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR\nCONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,\nINCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES\nARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT\nNOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES\nSUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE\nWITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGES.\n\n#### 17. Interpretation of Sections 15 and 16.\n\nIf the disclaimer of warranty and limitation of liability provided above\ncannot be given local legal effect according to their terms, reviewing\ncourts shall apply local law that most closely approximates an absolute\nwaiver of all civil liability in connection with the Program, unless a\nwarranty or assumption of liability accompanies a copy of the Program in\nreturn for a fee.\n\nEND OF TERMS AND CONDITIONS\n\n### How to Apply These Terms to Your New Programs\n\nIf you develop a new program, and you want it to be of the greatest\npossible use to the public, the best way to achieve this is to make it\nfree software which everyone can redistribute and change under these\nterms.\n\nTo do so, attach the following notices to the program. It is safest to\nattach them to the start of each source file to most effectively state\nthe exclusion of warranty; and each file should have at least the\n\"copyright\" line and a pointer to where the full notice is found.\n\n \n Copyright (C) \n\n This program is free software: you can redistribute it and/or modify\n it under the terms of the GNU Affero General Public License as\n published by the Free Software Foundation, either version 3 of the\n License, or (at your option) any later version.\n\n This program is distributed in the hope that it will be useful,\n but WITHOUT ANY WARRANTY; without even the implied warranty of\n MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n GNU Affero General Public License for more details.\n\n You should have received a copy of the GNU Affero General Public License\n along with this program. If not, see .\n\nAlso add information on how to contact you by electronic and paper mail.\n\nIf your software can interact with users remotely through a computer\nnetwork, you should also make sure that it provides a way for users to\nget its source. For example, if your program is a web application, its\ninterface could display a \"Source\" link that leads users to an archive\nof the code. There are many ways you could offer source, and different\nsolutions will be better for different programs; see section 13 for the\nspecific requirements.\n\nYou should also get your employer (if you work as a programmer) or\nschool, if any, to sign a \"copyright disclaimer\" for the program, if\nnecessary. For more information on this, and how to apply and follow the\nGNU AGPL, see <>.\n" }, { "path": "README.md", "content": "# ISBN Visualization\n\n**Please read https://phiresky.github.io/blog/2025/visualizing-all-books-in-isbn-space/ for the live version and description of this project**\n\nScreenshots:\n\n![Screenshot](src/assets/screenshot.png)\n![Screenshot 2](src/assets/screenshot2.png)\n\n## Setup\n\nFetch the main repo and (if you want) the precomputed data.\n\n```bash\n# code\ngit clone git@github.com:phiresky/isbn-visualization.git\n# precomputed prefix data\ngit clone git@github.com:phiresky/isbn-visualization-json.git\n# precomputed png datasets\ngit clone git@github.com:phiresky/isbn-visualization-images.git\ncd isbn-visualization\nmkdir public\nln -s $PWD/../isbn-visualization-images public/images\nln -s $PWD/../isbn-visualization-json/prefix-data public/prefix-data\n```\n\nThen install the JS dependencies. You'll need [pnpm](https://pnpm.io/). The easiest way is corepack, which is bundled with nodejs:\n\n```bash\ncorepack enable\npnpm install\n# run in dev mode (WARNING: perf in dev mode is worse than prod mode!)\npnpm run dev\n# build in prod mode\npnpm run build\n# serve from any static http server (example)\ncd dist && python3 -m http.server\n```\n\n## Preprocessing scripts\n\nThis repo contains a few scripts to generate the relevant data for the web viewer.\n\n### Running in docker\n\nYou can build a docker container containing all relevant code using\n\n```bash\ndocker build -t phiresky/isbn-visualization .\n```\n\n### `scripts/process-all.sh`\n\nA convenience script to run the JS build and all processing steps that have not been run yet:\n\nInputs:\n\n- PUBLIC_BASE_PATH: the url prefix you will host the project under (e.g. /isbn-visualization)\n- DATA_DIR: the directory the input data files are in and intermediary products will be stored\n- OUTPUT_DIR_PUBLIC: the output dir that you will host on your webhost (under PUBLIC_BASE_PATH)\n\nRun in docker:\n\n```bash\ndocker run --rm -it \\\n -e PUBLIC_BASE_PATH=/isbn-visualization \\\n -e DATA_DIR=/data \\\n -e OUTPUT_DIR_PUBLIC=/public \\\n -v ./data:/data \\\n -v ./public:/public \\\n phiresky/isbn-visualization\n```\n\nDirectly:\n\n```\nPUBLIC_BASE_PATH=/ OUTPUT_DIR_PUBLIC=./public DATA_DIR=./data ./scripts/process-all.sh\n```\n\n### `scripts/gen-prefixes.ts`\n\nThis script generates the json files representing the groups/publisher ranges.\n\n- Input: `isbngrp_records.jsonl.seekable.zst`\n- Output: `public/prefix-data/*.json` (split by size), `data/prefix-data.json` (the full data)\n\n```bash\npnpm tsx scripts/gen-prefixes.ts .../aa_meta__aacid__isbngrp_records__20240920T194930Z--20240920T194930Z.jsonl.seekable.zst\n# compress them with zopfli (if you don't want to install zopfli, use `gzip -9 public/prefix-data/*.json`)\nscripts/minify-prefix-data.sh\n```\n\n### `scripts/rarity`\n\nThis one written in Rust for performance. You'll need the [Rust compiler](https://www.rust-lang.org/).\n\n- Input: aa_meta**aacid**worldcat\\_\\_20241230T203056Z--20241230T203056Z.jsonl.seekable.zst\n- Output: `data/library_holding_data.sqlite3`\n\n```bash\ncd scripts/rarity\nexport RUSTFLAGS=\"-C target-cpu=native\"\ncargo run --release -- ~/Downloads/annas_archive_meta__aacid__worldcat__20241230T203056Z--20241230T203056Z.jsonl.seekable.zst\n```\n\nIt takes 20min-1h to process the 250GByte source file.\n\n### `scripts/write-images`\n\nThis script generates the png datasets.\n\nUse `pnpm tsx scripts/write-images list` to list datasets:\n\n```\nSpecial datasets: [ 'publishers', 'all', 'rarity' ]\nNormal datasets: [...depends on\n]\n```\n\nThe syntax is `pnpm tsx scripts/write-images [dataset] [zoom-level|all]`\n\nUse all to generate all zoom levels from 1-4.\n\nInput:\n\n- for the `all` and normal datasets: `data/aa_isbn13_codes_20241204T185335Z.benc.zst` (or set env var `INPUT_BENC=path`)\n- for the `publisher` dataset: `data/prefix-data.json` (generated by `scripts/gen-prefixes.ts`, or set env var `INPUT_PREFIX_DATA=path`)\n- for the `rarity` dataset: `data/library_holding_data.sqlite3` (generated by `scripts/rarity` or set env var `INPUT_HOLDING_SQLITE=path`)\n\nOutput:\n\n- `public/images/tiled/[dataset]/zoom-{1,2,3,4}/*.png`\n- `public/images/tiled/[dataset]/written.json` with the list of images (only if zoom level=all)\n- `public/images/tiled/[dataset]/stats.json`\n\n```bash\n# you might want to run some these in parallel, each takes a 1-10 minutes.\n\nfor dataset in all publishers rarity publication_date cadal_ssno cerlalc duxiu_ssid edsebk gbooks goodreads ia isbndb isbngrp libby md5 nexusstc nexusstc_download oclc ol rgb trantor; do\n pnpm tsx scripts/write-images $dataset all\ndone\n```\n\nSpecial datasets:\n\n#### Dataset `all`\n\nAggregates all datasets, sets white pixels for every book in any of the datasets, black pixels otherwise.\nZoomed out views contain the average, so a pixel with 50% existing books will be brightness 50%.\n\n#### Dataset `publication_date`\n\nThe red in each pixel is the average publication year (minus 1800, clamped to 0-255). The green pixel is the same. The blue pixel is the ratio of books present in the dataset (255 = 100%).\n\n#### Dataset `publishers`\n\nPublishers are assigned an incrementing integer ID by unique `registrant_name`. This integer is stored in the PNG RGB: `publisherId = red * 65536 + green * 256 + blue`.\n\nZoomed out views contain non-aggregated data (publisher ranges smaller than a pixel will not appear).\n\n#### Dataset `rarity`\n\nThe variables holdingCount, editionCount, bookCount are set in the r,g,b colors respectively.\n\nZoomed out views contain the sum of each of the values. If one of the values is ≥ 255, all values are scaled down accordingly. For example:\n\n`r=4,g=2,b=1` means that there is exactly one book in this pixel with 4 holdings and 2 editions\n`r=10,g=3,b=3` means there's three books with a total of 10 holdings and 3 editions\n`r=10,g=3,b=255` means thre's more than 254 books, with on average `10/255` holdings per book, and `3/255` editions per book.\n`r=255,g=10,b=30` means there's more than 254 holdings, with on average `255/10` holdings per edition and `255/30` books per edition\n\n#### Other datasets\n\nThe other datasets contain the data directly from the benc file (white=exists, black=does not exist)\n\n### `scripts/merge-stats.ts`\n\nMerges the statistics from the different datasets into a single file.\n\n- Input: `public/images/tiled/*/stats.json`\n- Output: `public/prefix-data/stats.json`\n\n```bash\npnpm tsx scripts/merge-stats.ts\n```\n\n### `scripts/minify-images.sh` (optional)\n\nMinify the images using [oxipng](https://github.com/shssoichiro/oxipng) and [pngquant](https://pngquant.org/) (for lossy datasets).\n\nThis reduces image size by 5-50%!\n\n```bash\nscripts/minify-images.sh public/images/tiled/\n# or\nscripts/minify-images.sh public/images/tiled/[dataset]\n```\n\n## Running the main web viewer\n\nURLs and paths are configured in `src/config.ts`. The default \"advanced config\", stored in the URL, is configured in `src/lib/RuntimeConfiguration.ts`.\n\nDevelopment: `pnpm run dev`\n\nRuns the app in the development mode.
\nOpen [http://localhost:5173](http://localhost:5173) to view it in the browser.\n\nThe page will reload if you make edits.
\n\nYou can use the following debug objects exposed in the dev console:\n\n- `store`: the main state store which can be manipulated, e.g. `store.showGrid = false`\n- `threejsRoot`: the main threejs objects, e.g. `console.log(threejsRoot.camera.zoom)`\n- `isbnlib`: the `isbn3` library for parsing ISBNs\n\n### `pnpm run build`\n\nBuilds the app for production to the `dist` folder.
\nIt bundles the project in production mode and optimizes the build for the best performance.\nIf the app should not be hosted in the root path of a domain, set the env var e.g. `PUBLIC_BASE_PATH=/isbn-visualization`.\n\n### Deployment\n\nYou can deploy the `dist` folder to any static host provider (netlify, surge, now, etc.)\n" }, { "path": "eslint.config.mjs", "content": "// @ts-check\n\nimport eslint from \"@eslint/js\";\nimport eslintPluginPrettierRecommended from \"eslint-plugin-prettier/recommended\";\nimport tseslint from \"typescript-eslint\";\n\nexport default tseslint.config(\n {\n ignores: [\"public/\", \"data/\", \"dist/\", \"node_modules/\", \"scripts/rarity/\"],\n },\n eslint.configs.recommended,\n tseslint.configs.strictTypeChecked,\n tseslint.configs.stylisticTypeChecked,\n eslintPluginPrettierRecommended,\n {\n languageOptions: {\n parserOptions: {\n projectService: true,\n tsconfigRootDir: import.meta.dirname,\n },\n },\n rules: {\n \"@typescript-eslint/restrict-template-expressions\": \"off\",\n \"@typescript-eslint/no-unused-vars\": [\n \"error\",\n { argsIgnorePattern: \"^_\", varsIgnorePattern: \"^_\" },\n ],\n \"@typescript-eslint/no-unnecessary-condition\": [\n \"error\",\n { allowConstantLoopConditions: true },\n ],\n },\n },\n);\n" }, { "path": "flake.nix", "content": "{\n inputs = {\n nixpkgs.url = \"github:NixOS/nixpkgs/nixos-unstable\";\n #rust-overlay = {\n # url = \"github:oxalica/rust-overlay\";\n # inputs = {\n # nixpkgs.follows = \"nixpkgs\";\n # };\n #};\n };\n outputs = { self, nixpkgs, flake-utils, /*rust-overlay*/ }:\n flake-utils.lib.eachDefaultSystem\n (system:\n let\n #overlays = [ (import rust-overlay) ];\n pkgs = import nixpkgs {\n inherit system /*overlays*/;\n };\n #rustToolchain = pkgs.pkgsBuildHost.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml;\n #nativeBuildInputs = with pkgs; [ rustToolchain pkg-config wasm-bindgen-cli ];\n buildInputs = with pkgs; [ pnpm openssl nodejs ];\n in\n with pkgs;\n {\n devShells.default = mkShell {\n # 👇 and now we can just inherit them\n inherit buildInputs /*nativeBuildInputs*/;\n # shellHook = ''\n # # For rust-analyzer 'hover' tooltips to work.\n # export RUST_SRC_PATH=${pkgs.rustPlatform.rustLibSrc}\n # '';\n };\n }\n );\n}\n" }, { "path": "index.html", "content": "\n\n \n \n \n \n \n ISBN Visualization\n \n \n \n \n
\n\n \n \n\n" }, { "path": "package.json", "content": "{\n \"name\": \"@phiresky/isbn-visualization\",\n \"version\": \"0.0.0\",\n \"description\": \"\",\n \"type\": \"module\",\n \"scripts\": {\n \"lint\": \"tsc && eslint .\",\n \"start\": \"vite\",\n \"dev\": \"vite\",\n \"build\": \"pnpm lint && vite build\",\n \"serve\": \"vite preview\"\n },\n \"license\": \"MIT\",\n \"devDependencies\": {\n \"@eslint/js\": \"^9.21.0\",\n \"eslint\": \"^9.21.0\",\n \"eslint-config-prettier\": \"^10.0.1\",\n \"eslint-plugin-prettier\": \"^5.2.3\",\n \"prettier\": \"^3.5.2\",\n \"typescript\": \"^5.7.3\",\n \"typescript-eslint\": \"^8.24.1\",\n \"vite\": \"^6.1.1\"\n },\n \"dependencies\": {\n \"@react-three/drei\": \"^10.0.1\",\n \"@react-three/fiber\": \"9.0.4\",\n \"@types/bencode\": \"^2.0.4\",\n \"@types/better-sqlite3\": \"^7.6.12\",\n \"@types/node\": \"^22.13.5\",\n \"@types/react\": \"^19.0.10\",\n \"@types/react-dom\": \"^19.0.4\",\n \"@types/three\": \"^0.173.0\",\n \"@vitejs/plugin-react-swc\": \"^3.8.0\",\n \"bencode\": \"^4.0.0\",\n \"better-sqlite3\": \"^11.8.1\",\n \"isbn3\": \"^1.2.7\",\n \"jsbarcode\": \"^3.11.6\",\n \"lru-cache\": \"^11.0.2\",\n \"mobx\": \"^6.13.6\",\n \"mobx-react-lite\": \"^4.1.0\",\n \"mobx-utils\": \"^6.1.0\",\n \"prando\": \"^6.0.1\",\n \"react\": \"^19.0.0\",\n \"react-dom\": \"^19.0.0\",\n \"react-select\": \"^5.10.0\",\n \"sharp\": \"^0.33.5\",\n \"simple-zstd\": \"^1.4.2\",\n \"three\": \"^0.173.0\",\n \"tsx\": \"^4.19.3\",\n \"zlib\": \"^1.0.5\"\n },\n \"packageManager\": \"pnpm@10.5.0+sha512.11106a5916c7406fe4b8cb8e3067974b8728f47308a4f5ac5e850304afa6f57e2847d7950dfe78877d8d36bfb401d381c4215db3a4c3547ffa63c14333a6fa51\",\n \"pnpm\": {\n \"onlyBuiltDependencies\": [\n \"@swc/core\",\n \"better-sqlite3\",\n \"esbuild\",\n \"sharp\"\n ]\n },\n \"prettier\": {}\n}\n" }, { "path": "scripts/gen-book-titles-sqlite.ts", "content": "import sqlite from \"better-sqlite3\";\nimport { createReadStream } from \"fs\";\nimport fs from \"fs/promises\";\nimport readline from \"readline\";\nimport zlib from \"zlib\";\ninterface Record {\n _index: \"aarecords__9\";\n _id: string;\n _source: {\n id: \"string\";\n file_unified_data: {\n title_best: string;\n author_best: string;\n publisher_best: string;\n identifiers_unified: {\n aarecord_id: string[];\n\n md5?: string[];\n sha1?: string[];\n isbn10?: string[];\n isbn13?: string[];\n };\n };\n };\n}\n\nfunction connect(dbName: string) {\n const db = sqlite(dbName);\n // enable wal mode\n db.prepare(\"PRAGMA journal_mode = WAL\").run();\n // disable synchronous\n db.prepare(\"PRAGMA synchronous = OFF\").run();\n // create table isbns (isbn13, book_id), books (book_id, publisher, author, title)\n db.prepare(\n \"CREATE TABLE IF NOT EXISTS books (book_id INTEGER PRIMARY KEY, publisher TEXT, author TEXT, title TEXT)\",\n ).run();\n db.prepare(\n \"CREATE UNIQUE INDEX IF NOT EXISTS idx_books_publisher_author_title ON books (publisher, author, title)\",\n ).run();\n db.prepare(\n \"CREATE TABLE IF NOT EXISTS isbns (isbn13 INTEGER, book_id INTEGER REFERENCES books(book_id), primary key (isbn13, book_id))\",\n ).run();\n return db;\n}\n\nasync function load(dbName: string, dataDir: string) {\n const db = connect(dbName);\n // readdir, find all dataDir/aarecords__*.json.gz\n const files = (await fs.readdir(dataDir)).filter((f) =>\n /^aarecords__[^.]+\\.json\\.gz$/.exec(f),\n );\n for (const file of files) {\n console.log(`Loading ${file}`);\n // stream read gzipped jsonl file\n const stream = createReadStream(`${dataDir}/${file}`);\n const gunzip = zlib.createGunzip();\n const rl = readline.createInterface({\n input: stream.pipe(gunzip),\n crlfDelay: Infinity,\n });\n // insert or return id\n const book = db.prepare<[string, string, string], { book_id: number }>(\n \"INSERT INTO books (publisher, author, title) VALUES (?, ?, ?) ON CONFLICT (publisher, author, title) DO UPDATE SET publisher = excluded.publisher RETURNING book_id\",\n );\n const isbns = db.prepare(\n \"INSERT OR IGNORE INTO isbns (isbn13, book_id) VALUES (?, ?)\",\n );\n db.exec(\"BEGIN TRANSACTION\");\n for await (const line of rl) {\n // parse json\n const record = JSON.parse(line) as Record;\n // insert into books\n const { title_best, author_best, publisher_best } =\n record._source.file_unified_data;\n const { isbn13 = [], isbn10 } =\n record._source.file_unified_data.identifiers_unified;\n if (!title_best) {\n // console.log(`No title for ${aarecord_id[0]}`);\n continue;\n }\n const rop = book.get(publisher_best, author_best, title_best);\n if (!rop) throw new Error(\"book.get failed\");\n const book_id = rop.book_id;\n if (isbn13.length === 0) {\n // console.log(`No ISBN for ${aarecord_id[0]} ${title_best}`);\n if (isbn10?.length) console.log(`no isbn13, but has isbn10: ${isbn10}`);\n }\n\n // insert into isbns\n for (const isbn of isbn13) {\n isbns.run(isbn, book_id);\n }\n }\n db.exec(\"END TRANSACTION\");\n }\n}\n\n// cmdline args\nconst dbName = process.argv[2];\nconst dataDir = process.argv[3];\nif (!dbName || !dataDir) {\n console.error(\"Usage: gen-sqlite \");\n process.exit(1);\n}\nvoid load(dbName, dataDir);\n" }, { "path": "scripts/gen-prefixes.ts", "content": "import { createReadStream } from \"node:fs\";\nimport { mkdir, writeFile } from \"node:fs/promises\";\nimport { createInterface } from \"node:readline\";\nimport { ZSTDDecompress } from \"simple-zstd\";\nimport {\n addRecord,\n Digit,\n InfoMap,\n LazyInfoMap,\n PrefixInfo,\n} from \"../src/lib/info-map\";\nimport { addIsbnGroups } from \"../src/lib/prefix-data\";\nimport { IsbnPrefixWithDashes } from \"../src/lib/util\";\n\ninterface JsonRecord {\n aacid: string;\n metadata: {\n id: string;\n record: {\n registrant_name: \"foo\";\n agency_name: \"New Zealand\";\n country_name: \"New Zealand\";\n isbns: [\n { isbn: IsbnPrefixWithDashes; isbn_type: \"prefix\" },\n { isbn: \"...\"; isbn_type: \"isbn13\" },\n ];\n };\n };\n}\n\nasync function go() {\n const fname = process.argv[2];\n if (!fname) throw new Error(\"no input filename provided\");\n const map: InfoMap = {};\n let recordCount = 0;\n for await (const line of createInterface(\n createReadStream(fname).pipe(ZSTDDecompress()),\n )) {\n const obj = JSON.parse(line) as JsonRecord;\n if (recordCount % 100000 === 0)\n console.log(`${recordCount}/2700000 records...`);\n recordCount++;\n for (const isbn of obj.metadata.record.isbns) {\n if (isbn.isbn_type === \"prefix\") {\n // console.log(isbn.isbn);\n // if (isbn.isbn.length > 9) continue;\n const r = obj.metadata.record;\n addRecord(map, isbn.isbn, {\n // id: obj.metadata.id,\n registrant_name: r.registrant_name,\n agency_name: r.agency_name,\n country_name: r.country_name,\n source: \"isbngrp\",\n prefix: isbn.isbn,\n });\n }\n }\n }\n addIsbnGroups(map, {\n testMode: false,\n addUnassigned: true,\n });\n const maxDepth = 7;\n const maxInlineDeepChildren = 10;\n const outDir = (process.env.OUTPUT_DIR_PUBLIC ?? \"public\") + \"/prefix-data\";\n const outFileFull = (process.env.DATA_DIR ?? \"data\") + \"/prefix-data.json\";\n\n let nextPublisherId = 1;\n let nextGroupId = 1;\n const publishersIdCache = new Map();\n function countUniquePublishers(map: InfoMap): Set {\n const out = new Set();\n for (const [_digit, info] of Object.entries(map) as [Digit, PrefixInfo][]) {\n if (info.children) {\n const children = countUniquePublishers(info.children);\n info.totalChildren = children.size;\n for (const child of children) {\n out.add(child);\n }\n }\n if (info.info) {\n for (const record of info.info) {\n if (record.source === \"isbngrp\") {\n out.add(record.registrant_name);\n }\n }\n }\n }\n return out;\n }\n countUniquePublishers(map);\n function recurseAssignNumericIds(map: InfoMap) {\n for (const [_digit, info] of Object.entries(map) as [Digit, PrefixInfo][]) {\n if (info.info) {\n const record = info.info[0];\n if (record.source === \"isbngrp\") {\n const cached = publishersIdCache.get(record.registrant_name);\n if (cached) {\n record.numericId = cached;\n } else {\n record.numericId = nextPublisherId++;\n publishersIdCache.set(record.registrant_name, record.numericId);\n }\n } else {\n if (record.name !== \"Unassigned\") {\n record.numericId = nextGroupId++;\n }\n }\n }\n if (info.children) {\n recurseAssignNumericIds(info.children);\n }\n }\n }\n recurseAssignNumericIds(map);\n console.log(\n `assigned ${nextPublisherId} publisher ids, ${nextGroupId} group ids`,\n );\n\n async function recurseOrRemoveAndWrite(\n layer: InfoMap,\n depth: number,\n prefix: string,\n ): Promise {\n await mkdir(outDir, { recursive: true });\n if (depth >= maxDepth && Object.keys(layer).length) {\n const fname = `${prefix}.json`;\n await writeFile(`${outDir}/${fname}`, JSON.stringify(layer));\n return { lazy: fname };\n } else {\n const out: LazyInfoMap = {};\n for (const [digit, info] of Object.entries(layer) as [\n Digit,\n PrefixInfo,\n ][]) {\n out[digit] = {\n ...info,\n children:\n info.totalChildren <= maxInlineDeepChildren\n ? info.children\n : await recurseOrRemoveAndWrite(\n info.children ?? {},\n depth + 1,\n `${prefix}${digit}`,\n ),\n };\n }\n return out;\n }\n }\n await writeFile(outFileFull, JSON.stringify(map));\n console.log(`wrote ${recordCount} records to ${outFileFull}`);\n const lazyMap = await recurseOrRemoveAndWrite(map, 0, \"\");\n await writeFile(`${outDir}/root.json`, JSON.stringify(lazyMap));\n console.log(`wrote lazy map to ${outDir}/root.json`);\n}\n\nvoid go();\n" }, { "path": "scripts/merge-stats.ts", "content": "import { readFileSync, writeFileSync } from \"fs\";\nimport { mergeStats, StatsMap } from \"../src/lib/stats\";\nimport { IsbnPrefixWithoutDashes } from \"../src/lib/util\";\n\nconst dir = process.env.OUTPUT_DIR_PUBLIC ?? \"public\";\nconst out: StatsMap = {};\nfor (const dataset of [\"all\", \"publication_date\", \"rarity\", \"publishers\"]) {\n const f = JSON.parse(\n readFileSync(`${dir}/images/tiled/${dataset}/stats.json`, \"utf-8\"),\n ) as StatsMap;\n for (const k of Object.keys(f) as IsbnPrefixWithoutDashes[]) {\n if (out[k]) {\n const v = f[k];\n if (v === undefined) continue;\n mergeStats(out[k], v);\n } else out[k] = f[k];\n }\n}\n\nconst outFile = `${dir}/prefix-data/stats.json`;\nconsole.log(`Writing to ${outFile}`);\nwriteFileSync(outFile, JSON.stringify(out));\n" }, { "path": "scripts/minify-images.sh", "content": "#!/bin/bash\nset -euo pipefail\n\nlines=\"$(find \"$1\" -name '*.png' | wc -l)\"\n\nfind \"$1\" -name '*.png' | sort | pv -l --size=$lines | while read f; do \n if [[ ! -f \"$f.timestamp\" ]] || [[ \"$f\" -nt \"$f.timestamp\" ]] ; then\n echo -n \"Re-compressing $f \"\n cp \"$f\" \"$f.orig\" --preserve=all\n # if in rarity or publishers dir, don't quantize (lossy)\n if [[ \"$f\" == *\"/rarity/\"* ]] || [[ \"$f\" == *\"/publishers/\"* ]] || [[ \"$f\" == *\"/publication_date/zoom-4\"* ]]; then\n echo losslessly...\n true\n else\n echo lossily...\n pngquant \"$f\" --ext .png --skip-if-larger --force || true\n fi\n oxipng \"$f\" -r -o max --strip all\n touch \"$f.timestamp\"\n fi\ndone" }, { "path": "scripts/minify-prefix-data.sh", "content": "#!/bin/bash\nset -euo pipefail\n\nJOBS=\"${JOBS:-$(nproc)}\"\n\nOUTPUT_DIR_PUBLIC=\"${OUTPUT_DIR_PUBLIC:-public}\"\n\necho compressing files in $OUTPUT_DIR_PUBLIC/prefix-data with zopfli using $JOBS threads\nfor f in $OUTPUT_DIR_PUBLIC/prefix-data/*.json; do\n (\n # .. do your stuff here\n echo \"zopfli $f..\"\n zopfli \"$f\" && rm \"$f\"\n ) &\n\n # allow to execute up to $N jobs in parallel\n while [[ $(jobs -r -p | wc -l) -ge $JOBS ]]; do\n # now there are $N jobs already running, so wait here for any job\n # to be finished so there is a place to start next one.\n wait -n\n done\n\ndone\n\n# no more jobs to be started but wait for pending jobs\n# (all need to be finished)\nwait\n\necho \"all done\"" }, { "path": "scripts/process-all.sh", "content": "#!/bin/bash\nset -euo pipefail\n\n# for each env var, check if file exists and make path absolute\n\n# default INPUT_ISBNGRP_DUMP to DATA_DIR/aa_meta__aacid__isbngrp_records__20240920T194930Z--20240920T194930Z.jsonl.seekable.zst\nINPUT_ISBNGRP_DUMP=\"${INPUT_ISBNGRP_DUMP:-\"$DATA_DIR/annas_archive_meta__aacid__isbngrp_records__20240920T194930Z--20240920T194930Z.jsonl.seekable.zst\"}\"\nINPUT_WORLDCAT_DUMP=\"${INPUT_WORLDCAT_DUMP:-\"$DATA_DIR/annas_archive_meta__aacid__worldcat__20241230T203056Z--20241230T203056Z.jsonl.seekable.zst\"}\"\nINPUT_BENC=\"${INPUT_BENC:-\"$DATA_DIR/aa_isbn13_codes_20241204T185335Z.benc.zst\"}\"\n# annas_archive_meta__aacid__worldcat__20241230T203056Z--20241230T203056Z.jsonl.seekable.zst\nfor var in INPUT_ISBNGRP_DUMP INPUT_WORLDCAT_DUMP INPUT_BENC OUTPUT_DIR_PUBLIC DATA_DIR; do\n if [ -z \"${!var-}\" ]; then\n echo \"Required env variable not set: $var\"\n exit 1\n fi\n if [ ! -f \"${!var}\" ] && [ ! -d \"${!var}\" ]; then\n echo \"File not found: ${!var} (from $var)\"\n exit 1\n fi\n export $var=\"$(realpath \"${!var}\")\"\ndone\n\n# go to repo root\ncd \"$(dirname \"$0\")/..\"\n\n\n# build web components to out dir\nif [ ! -f \"$OUTPUT_DIR_PUBLIC/index.html\" ]; then\n echo \"Running pnpm build\"\n rm -rf \"$OUTPUT_DIR_PUBLIC/assets\" # ensure we don't have old assets\n pnpm build\n cp -r dist/* \"$OUTPUT_DIR_PUBLIC/\"\nelse\n echo \"Skipping pnpm build as $OUTPUT_DIR_PUBLIC/index.html already exists\"\nfi\n\n# run only if DATA_DIR/prefix-data.json does not exist\nif [ ! -f \"$DATA_DIR/prefix-data.json\" ]; then\n echo \"Running gen-prefixes.ts\"\n pnpm tsx scripts/gen-prefixes.ts \"$INPUT_ISBNGRP_DUMP\"\nelse\n echo \"Skipping gen-prefixes.ts as $DATA_DIR/prefix-data.json already exists\"\nfi\n\nif [ ! -f \"$OUTPUT_DIR_PUBLIC/prefix-data/root.json.gz\" ]; then\n echo \"Running scripts/minify-prefix-data.sh\"\n scripts/minify-prefix-data.sh\nelse\n echo \"Skipping scripts/minify-prefix-data.sh as $OUTPUT_DIR_PUBLIC/prefix-data/root.json.gz already exists\"\nfi\n\n\n# run only if DATA_DIR/library_holding_data.sqlite3 does not exist\nif [ ! -f \"$DATA_DIR/library_holding_data.sqlite3\" ]; then\n echo \"Running scripts/rarity\"\n scripts/rarity/target/release/rarity \"$INPUT_WORLDCAT_DUMP\"\nelse\n echo \"Skipping scripts/rarity as $DATA_DIR/library_holding_data.sqlite3 already exists\"\nfi\n\nJOBS=\"${JOBS:-$(nproc)}\"\n\nfor dataset in all publishers rarity publication_date cadal_ssno cerlalc duxiu_ssid edsebk gbooks goodreads ia isbndb isbngrp libby md5 nexusstc nexusstc_download oclc ol rgb trantor; do\n if [ ! -f \"$OUTPUT_DIR_PUBLIC/images/tiled/$dataset/written.json\" ]; then\n echo \"Running scripts/write-images $dataset all\"\n pnpm tsx scripts/write-images $dataset all &\n else\n echo \"Skipping scripts/write-images $dataset all as $OUTPUT_DIR_PUBLIC/images/tiled/$dataset/written.json already exists\"\n fi\n\n # allow to execute up to $N jobs in parallel\n while [[ $(jobs -r -p | wc -l) -ge $JOBS ]]; do\n # now there are $N jobs already running, so wait here for any job\n # to be finished so there is a place to start next one.\n wait -n\n done\ndone\nwait\n\n# merge-stats\nif [ ! -f \"$OUTPUT_DIR_PUBLIC/prefix-data/stats.json\" ] && [ ! -f \"$OUTPUT_DIR_PUBLIC/prefix-data/stats.json.gz\" ] ; then\n echo \"Running scripts/merge-stats.ts\"\n pnpm tsx scripts/merge-stats.ts\nelse\n echo \"Skipping scripts/merge-stats.ts as $OUTPUT_DIR_PUBLIC/prefix-data/stats.json already exists\"\nfi\n\n# minify-images\n\nfor dataset in \"$OUTPUT_DIR_PUBLIC/images/tiled/\"*; do\n echo \"Running scripts/minify-images.sh $dataset &\"\n scripts/minify-images.sh \"$dataset\" &\n # allow to execute up to $N jobs in parallel\n while [[ $(jobs -r -p | wc -l) -ge $JOBS ]]; do\n # now there are $N jobs already running, so wait here for any job\n # to be finished so there is a place to start next one.\n wait -n\n done\ndone\nwait\n\nif [ ! -d \"$OUTPUT_DIR_PUBLIC/title-data\" ]; then\n echo \"Running scripts/write-titles.ts\"\n pnpm tsx scripts/write-titles.ts\nelse\n echo \"Skipping scripts/write-titles.ts as $OUTPUT_DIR_PUBLIC/title-data already exists\"\nfi" }, { "path": "scripts/rarity/.gitignore", "content": "/target\n" }, { "path": "scripts/rarity/Cargo.toml", "content": "[package]\nname = \"rarity\"\nversion = \"0.1.0\"\nedition = \"2021\"\n\n[dependencies]\nsimd-json = { version = \"*\", default-features = false, features = [\"serde_impl\", \"known-key\"] }\nrusqlite = { version = \"0.30\", features = [\"bundled\"] }\nzstd = \"0.13.2\"\nhumansize = \"*\"\nserde = { version = \"1.0\", features = [\"derive\"] }\nparking_lot = \"0.12.3\"\ncrossbeam-channel = \"0.5.14\"\nnum_cpus = \"1.16.0\"\nsnmalloc-rs = { version = \"0.3.7\", features = [\"lto\", \"native-cpu\"] }\nmemory-stats = \"1.2.0\"\nregex = \"1.11.1\"\n\n[profile.release]\ncodegen-units = 1\nlto = \"fat\"\n" }, { "path": "scripts/rarity/src/main.rs", "content": "#[global_allocator]\n// better performance than the default malloc\nstatic ALLOC: snmalloc_rs::SnMalloc = snmalloc_rs::SnMalloc;\nuse crossbeam_channel::{bounded, Sender};\nuse humansize::{format_size, BINARY};\nuse parking_lot::Mutex as PLMutex;\nuse rusqlite::{params, Connection};\nuse serde::Deserialize;\nuse std::fs::File;\nuse std::io::{self, BufRead, BufReader};\nuse std::sync::{Arc, LazyLock};\nuse std::time::{Duration, Instant};\nuse zstd::Decoder;\n\nconst CHANNEL_BATCH_SIZE: usize = 10000;\n\n// Type aliases\ntype OclcIdNumeric = u64;\ntype Isbn = String;\n\n// Enum to represent the different metadata types\n#[derive(Deserialize, Debug)]\n#[serde(tag = \"type\")]\nenum RawRecord {\n #[serde(rename = \"title_json\")]\n TitleJson { record: TitleRecord },\n #[serde(rename = \"search_holdings_summary_all_editions\")]\n SearchHoldings {\n // oclc_number: String,\n // from_filenames: Vec,\n record: HoldingsRecord,\n },\n\n #[serde(untagged)]\n Other {},\n}\n\n#[derive(Deserialize, Debug)]\nstruct TitleRecord {\n #[serde(rename = \"oclcNumber\")]\n oclc_number: String,\n title: Option,\n creator: Option,\n //#[serde(rename = \"totalEditions\")]\n //total_editions: u32,\n // isbn13: Option,\n isbns: Vec,\n #[serde(rename = \"machineReadableDate\")]\n machine_readable_date: Option,\n date: Option,\n #[serde(rename = \"publicationDate\")]\n publication_date: Option,\n}\n\n#[derive(Deserialize, Debug)]\nstruct HoldingsRecord {\n oclc_number: OclcIdNumeric,\n total_holding_count: u32,\n total_editions: u32,\n}\n\n#[derive(Deserialize, Debug)]\nstruct JsonRecord {\n metadata: RawRecord,\n}\n\n// Result type for parsed records\n#[derive(Debug)]\nenum ParsedRecord {\n Title {\n oclc_num: OclcIdNumeric,\n title: Option,\n creator: Option,\n isbn: Vec,\n publication_date: Option,\n },\n Holdings {\n oclc_num: OclcIdNumeric,\n holdings: (u32, u32),\n },\n}\n\nfn format_si_number(num: u64) -> String {\n format_size(num, BINARY)\n}\n\nstruct ZstdStreamWithProgress {\n reader: R,\n bytes_read: u64,\n bytes_read_last: u64,\n total_size: u64,\n last_update: Instant,\n}\n\nimpl ZstdStreamWithProgress {\n fn new(reader: R, total_size: u64) -> Self {\n Self {\n reader,\n bytes_read: 0,\n bytes_read_last: 0,\n total_size,\n last_update: Instant::now(),\n }\n }\n}\n\nimpl io::Read for ZstdStreamWithProgress {\n fn read(&mut self, buf: &mut [u8]) -> io::Result {\n let bytes = self.reader.read(buf)?;\n self.bytes_read += bytes as u64;\n\n if self.last_update.elapsed() >= Duration::from_secs(1) {\n eprintln!(\n \"read {} / {} ({:.2}%, {}/s)\",\n format_si_number(self.bytes_read),\n format_si_number(self.total_size),\n (self.bytes_read as f64 / self.total_size as f64) * 100.0,\n format_si_number(\n (self.bytes_read - self.bytes_read_last) / self.last_update.elapsed().as_secs()\n )\n );\n self.last_update = Instant::now();\n self.bytes_read_last = self.bytes_read;\n }\n\n Ok(bytes)\n }\n}\n\nfn process_batch(lines: Vec, record_count: u64) -> Vec {\n lines\n .into_iter()\n .enumerate()\n .flat_map(|(i, line)| {\n let mut json_buffer = line.into_bytes();\n let record: JsonRecord = match simd_json::serde::from_slice(&mut json_buffer) {\n Ok(v) => v,\n Err(e) => {\n eprintln!(\n \"Error parsing JSON at record {}: {}\",\n record_count + i as u64,\n e\n );\n return vec![];\n }\n };\n\n match record.metadata {\n RawRecord::TitleJson { record } => {\n if let Ok(oclc_num) = record.oclc_number.parse() {\n return vec![ParsedRecord::Title {\n oclc_num,\n isbn: record\n .isbns\n .iter()\n .filter_map(|isbn| {\n let int: i64 = isbn.parse().ok()?;\n if int < 978_000_000_000_0 || int >= 980_000_000_000_0 {\n return None;\n }\n Some(int)\n })\n .collect(),\n publication_date: parse_publication_date(&record),\n title: record.title,\n creator: record.creator,\n }];\n }\n }\n RawRecord::SearchHoldings { record, .. } => {\n return vec![ParsedRecord::Holdings {\n oclc_num: record.oclc_number,\n holdings: (record.total_holding_count, record.total_editions),\n }];\n }\n _ => {}\n }\n vec![]\n })\n .collect()\n}\n\n// try each of the three date fields in order (machineReadableDate, publicationDate, date), parse them with the regex \".*\\b([12]\\d\\d\\d)\\b.*\", fall back to next if regex fails\nfn parse_single_date(date: &str) -> Option {\n static RE: LazyLock =\n LazyLock::new(|| regex::Regex::new(r\".*\\b([12]\\d\\d\\d)\\b.*\").unwrap());\n\n RE.captures(date)\n .and_then(|cap| cap.get(1))\n .and_then(|m| m.as_str().parse().ok())\n}\nfn parse_publication_date(record: &TitleRecord) -> Option {\n record\n .machine_readable_date\n .as_ref()\n .and_then(|date| parse_single_date(date))\n .or_else(|| {\n record\n .publication_date\n .as_ref()\n .and_then(|date| parse_single_date(date))\n })\n .or_else(|| {\n record\n .date\n .as_ref()\n .and_then(|date| parse_single_date(date))\n })\n}\n\nfn reader_thread(reader: impl BufRead, sender: Sender>) -> io::Result<()> {\n let mut batch = Vec::with_capacity(CHANNEL_BATCH_SIZE);\n for line in reader.lines() {\n batch.push(line?);\n\n if batch.len() >= CHANNEL_BATCH_SIZE {\n let mut new_batch = Vec::with_capacity(CHANNEL_BATCH_SIZE);\n std::mem::swap(&mut batch, &mut new_batch);\n sender\n .send(new_batch)\n .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?;\n }\n }\n\n // Send the final batch if it's not empty\n if !batch.is_empty() {\n let _ = sender.send(batch);\n }\n\n Ok(())\n}\n\nfn setup_database(conn: &Connection) -> rusqlite::Result<()> {\n // performance pragmas\n conn.execute_batch(\"PRAGMA synchronous = OFF\")?;\n conn.execute_batch(\"PRAGMA journal_mode = WAL\")?;\n conn.execute_batch(\"PRAGMA cache_size = 100000\")?;\n conn.execute_batch(\"PRAGMA temp_store = MEMORY\")?;\n conn.execute_batch(\"PRAGMA mmap_size = 30000000000\")?;\n conn.execute_batch(\n \"CREATE TABLE IF NOT EXISTS isbn_data (\n oclc_number INTEGER NOT NULL,\n isbn13 INTEGER NOT NULL,\n publication_date INTEGER,\n title TEXT,\n creator TEXT,\n PRIMARY KEY (oclc_number, isbn13)\n );\n CREATE INDEX IF NOT EXISTS isbn_oclc_number ON isbn_data (isbn13);\n \",\n )?;\n\n conn.execute(\n \"CREATE TABLE IF NOT EXISTS holdings_data (\n oclc_number INTEGER PRIMARY KEY,\n holding_count INTEGER NOT NULL,\n edition_count INTEGER NOT NULL\n )\",\n [],\n )?;\n\n Ok(())\n}\n\nfn main() -> io::Result<()> {\n let args: Vec = std::env::args().collect();\n let fname = args.get(1).expect(\"no input filename provided\");\n // output env var DATA_DIR\n let out_dir = std::env::var(\"DATA_DIR\").unwrap_or_else(|_| \"../../data\".to_string());\n // Initialize SQLite database\n let conn = Connection::open(format!(\"{}/library_holding_data.sqlite3\", out_dir))\n .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?;\n setup_database(&conn).map_err(|e| io::Error::new(io::ErrorKind::Other, e))?;\n\n let file = File::open(fname)?;\n let file_size = file.metadata()?.len();\n\n let progress_reader = ZstdStreamWithProgress::new(file, file_size);\n let decoder = Decoder::new(progress_reader)?;\n let reader = BufReader::new(decoder);\n\n // Shared database connection\n let db = Arc::new(PLMutex::new(conn));\n let record_count = Arc::new(PLMutex::new(0u64));\n\n let parser_threads: usize = num_cpus::get();\n // Channel for passing batches of lines\n let (sender, receiver) = bounded(parser_threads * 4);\n\n // Spawn reader thread\n let reader_handle = std::thread::spawn(move || reader_thread(reader, sender));\n\n // Process batches in parallel\n let processing_threads: Vec<_> = (0..parser_threads)\n .map(|_| {\n let receiver = receiver.clone();\n let db = Arc::clone(&db);\n let record_count = Arc::clone(&record_count);\n\n std::thread::spawn(move || {\n while let Ok(batch) = receiver.recv() {\n let current_count = {\n let mut count = record_count.lock();\n *count += batch.len() as u64;\n *count\n };\n\n if current_count % 1000000 < CHANNEL_BATCH_SIZE as u64 {\n println!(\n \"{} records... {{ memory: {} }}\",\n current_count,\n format_si_number(get_memory_usage())\n );\n }\n\n let parsed_records = process_batch(batch, current_count);\n store_to_db(&db, parsed_records).unwrap();\n }\n })\n })\n .collect();\n\n // Wait for reader to finish\n reader_handle.join().expect(\"Reader thread panicked\")?;\n\n // Wait for all processing threads to finish\n for handle in processing_threads {\n handle.join().expect(\"Processing thread panicked\");\n }\n\n Ok(())\n}\n\nfn store_to_db(\n db: &Arc>,\n records: Vec,\n) -> Result<(), rusqlite::Error> {\n let mut db = db.lock();\n let tx = db.transaction().unwrap();\n\n for record in records {\n match record {\n ParsedRecord::Title {\n oclc_num,\n isbn,\n publication_date,\n title,\n creator,\n } => {\n for isbn in isbn {\n tx.prepare_cached(\n \"INSERT OR IGNORE INTO isbn_data (oclc_number, isbn13, publication_date, title, creator) VALUES (?1, ?2, ?3, ?4, ?5)\",\n )?\n .execute(params![oclc_num, isbn, publication_date, title, creator])?;\n }\n }\n ParsedRecord::Holdings { oclc_num, holdings } => {\n tx.prepare_cached(\n \"INSERT OR IGNORE INTO holdings_data (oclc_number, holding_count, edition_count) VALUES (?1, ?2, ?3)\")?.execute(\n params![oclc_num, holdings.0 as i64, holdings.1 as i64],\n )?;\n }\n }\n }\n tx.commit().unwrap();\n\n Ok(())\n}\n\nfn get_memory_usage() -> u64 {\n memory_stats::memory_stats()\n .map(|e| e.physical_mem as u64)\n .unwrap_or(0)\n}\n" }, { "path": "scripts/write-images/ImageTiler.ts", "content": "import { mkdir } from \"fs/promises\";\nimport sharp from \"sharp\";\nimport { ImageTile, channelMax } from \".\";\nimport {\n IMG_WIDTH,\n IsbnPrefixWithoutDashes,\n IsbnRelative,\n ProjectionConfig,\n relativeToIsbnPrefix,\n statsConfig,\n totalIsbns,\n} from \"../../src/lib/util\";\nimport { bookshelfConfig } from \"../../src/projections/bookshelf\";\n\nexport class StatsAggregator {\n statistics = new Map>();\n\n addStatistic(isbn: IsbnRelative, obj: Record) {\n const isbnFull = relativeToIsbnPrefix(isbn);\n for (\n let i = statsConfig.minPrefixLength;\n i <= statsConfig.maxPrefixLength;\n i++\n ) {\n const prefix = isbnFull.slice(0, i) as IsbnPrefixWithoutDashes;\n let stats = this.statistics.get(prefix);\n if (!stats) {\n stats = {};\n this.statistics.set(prefix, stats);\n }\n for (const [key, value] of Object.entries(obj)) {\n stats[key] = (stats[key] || 0) + value;\n }\n }\n }\n}\nexport class ImageTiler {\n images = new Map();\n written = new Set();\n config: ProjectionConfig;\n totalBooksPerPixel: number;\n // only set for first zoom level\n stats?: StatsAggregator;\n postprocessPixels?: (\n img: ImageTile,\n totalBooksPerPixel: number,\n ) => void | Promise;\n constructor(\n private prefixLength: number,\n private tiledDir: string,\n ) {\n const { width, height } =\n prefixLength === 4\n ? { width: 100000, height: 20000 }\n : { width: IMG_WIDTH * Math.sqrt(10 ** (prefixLength - 1)) };\n this.config =\n /* linearConfig({\n scale: Math.sqrt(scale),\n aspectRatio: 5 / 4,\n });*/\n bookshelfConfig({ width, height });\n\n this.totalBooksPerPixel =\n totalIsbns / this.config.pixelWidth / this.config.pixelHeight;\n console.log(`total books per pixel: ${this.totalBooksPerPixel}`);\n }\n logProgress(progress: number) {\n console.log(\n `Progress for ${this.tiledDir}: ${(progress * 100).toFixed(2)}%...`,\n );\n }\n async init() {\n console.log(`Generating ${this.tiledDir}...`);\n await mkdir(this.tiledDir, { recursive: true });\n }\n #getImage(relativeIsbn: number): ImageTile {\n const prefix = Math.floor(relativeIsbn / 10 ** (10 - this.prefixLength));\n const startIsbn = prefix * 10 ** (10 - this.prefixLength);\n const endIsbn = startIsbn + 10 ** (10 - this.prefixLength) - 1;\n const start = this.config.relativeIsbnToCoords(startIsbn as IsbnRelative);\n const end = this.config.relativeIsbnToCoords(endIsbn as IsbnRelative);\n let image = this.images.get(prefix);\n if (this.written.has(prefix))\n throw Error(`tile ${prefix} already finalized`);\n if (!image) {\n const width = Math.ceil(end.x + end.width - start.x);\n const height = Math.ceil(end.y + end.height - start.y);\n image = {\n x: start.x,\n y: start.y,\n width,\n height,\n img: new Float32Array(width * height * 3),\n };\n this.images.set(prefix, image);\n }\n return image;\n }\n colorIsbn(\n relativeIsbn: IsbnRelative,\n color: [number, number, number],\n options: {\n addToPixel: boolean;\n scaleColors: boolean;\n scaleColorByTileScale: boolean;\n } = { addToPixel: true, scaleColorByTileScale: true, scaleColors: true },\n ) {\n const channels = 3;\n const image = this.#getImage(relativeIsbn);\n // const x = Math.floor((position / scale) % dimensions.width);\n // const y = Math.floor(position / scale / dimensions.width);\n // eslint-disable-next-line prefer-const\n let { x, y, width, height } =\n this.config.relativeIsbnToCoords(relativeIsbn);\n x -= image.x;\n y -= image.y;\n // if we are scaling by tile scale, we want to consider pixels that are < 50% filled. If not,\n // we want to only include those >= 50% filled. Since the center of a pixel is at (0.5, 0.5), this means rounding gives us the bound (lower bound inclusive, upper bound exclusive)\n const minX = options.scaleColorByTileScale ? Math.floor(x) : Math.round(x);\n let maxX = options.scaleColorByTileScale\n ? Math.ceil(x + width)\n : Math.round(x + width);\n const minY = options.scaleColorByTileScale ? Math.floor(y) : Math.round(y);\n let maxY = options.scaleColorByTileScale\n ? Math.ceil(y + height)\n : Math.round(y + height);\n // but, if no pixel would be put, put a pixel\n if (minX === maxX) maxX++;\n if (minY === maxY) maxY++;\n for (let xo = minX; xo < maxX; xo++) {\n for (let yo = minY; yo < maxY; yo++) {\n const pixelIndex = (yo * image.width + xo) * channels;\n // we may have some pixels that we only want to fractionally fill\n let scaleColor = options.scaleColors ? channelMax : 1;\n if (options.scaleColorByTileScale) {\n const filWidth = Math.min(x + width, xo + 1) - Math.max(x, xo);\n const filHeight = Math.min(y + height, yo + 1) - Math.max(y, yo);\n scaleColor *= filWidth * filHeight;\n }\n if (options.addToPixel) {\n image.img[pixelIndex] += color[0] * scaleColor;\n image.img[pixelIndex + 1] += color[1] * scaleColor;\n image.img[pixelIndex + 2] += color[2] * scaleColor;\n } else {\n image.img[pixelIndex] = color[0] * scaleColor;\n image.img[pixelIndex + 1] = color[1] * scaleColor;\n image.img[pixelIndex + 2] = color[2] * scaleColor;\n }\n }\n }\n }\n async #writeAndPurgeImage(prefix: number) {\n await this.writeImage(prefix);\n this.images.delete(prefix);\n this.written.add(prefix);\n }\n async writeImage(prefix: number) {\n if (this.written.has(prefix)) throw Error(\"image already written\");\n const image = this.images.get(prefix);\n if (!image) throw Error(\"no image\");\n if (this.postprocessPixels)\n await this.postprocessPixels(image, this.totalBooksPerPixel);\n const img = sharp(image.img, {\n raw: {\n width: image.width,\n height: image.height,\n channels: 3,\n premultiplied: false,\n },\n });\n const paddedPrefix = String(prefix).padStart(this.prefixLength, \"0\");\n /*const withSubdirs = paddedPrefix\n .replace(/(.{4})/g, \"$1/\")\n .replace(/\\/$/, \"\");\n if (withSubdirs.includes(\"/\")) {\n await mkdir(dirname(withSubdirs), { recursive: true });\n }*/\n const fname = `${this.tiledDir}/${paddedPrefix}.png`;\n console.log(`writing tile ${fname}`);\n await img.toFile(fname);\n // await new Promise((resolve) => setTimeout(resolve, 1000));\n img.destroy();\n }\n async writeAll() {\n await this.purgeToLength(0);\n }\n async purgeToLength(len: number) {\n while (this.images.size > len) {\n const image = this.images.keys().next();\n if (image.value === undefined) throw Error(\"impossibor\");\n await this.#writeAndPurgeImage(image.value);\n }\n }\n\n async finish() {\n console.log(`writing ${this.images.size} remaining tiles`);\n await this.writeAll();\n console.log(`wrote ${this.written.size} tiles`);\n\n console.log(\"Done.\");\n }\n}\n" }, { "path": "scripts/write-images/index.ts", "content": "import { writeFile } from \"fs/promises\";\nimport { ImageTiler, StatsAggregator } from \"./ImageTiler\";\nimport * as modules from \"./modules\";\nimport { loadSparseDataToMemory } from \"./modules/single-sparse\";\n\nexport type IsbnData = Partial>;\n\n/** sharp / vips uses a channel max of 1e16 for float32 images for some reason */\nexport const channelMax = 65535;\n\n/** info of one tile of a tiled image */\nexport interface ImageTile {\n x: number;\n y: number;\n width: number;\n height: number;\n img: Float32Array;\n}\n\nexport type ProcessSingleZoom = (tiler: ImageTiler) => Promise;\nasync function processAllZoomLevels(\n dataset: string,\n minLevel = 1,\n maxLevel = 4,\n): Promise {\n const stats = new StatsAggregator();\n const processIsbnData = await loadData(dataset, stats);\n const written = [];\n const dir = `${process.env.OUTPUT_DIR_PUBLIC ?? \"public\"}/images/tiled/${dataset}`;\n for (let level = minLevel; level <= maxLevel; level++) {\n const tiledDir = `${dir}/zoom-${level}`;\n const tiler = new ImageTiler(level, tiledDir);\n if (level === minLevel) tiler.stats = stats;\n await tiler.init();\n await processIsbnData(tiler);\n await tiler.finish();\n const w = tiler.written;\n for (const prefix of w) {\n written.push(prefix.toString().padStart(level, \"0\"));\n }\n if (level === minLevel) {\n await writeFile(\n `${dir}/stats.json`,\n JSON.stringify(Object.fromEntries(stats.statistics)),\n );\n }\n }\n if (minLevel === 1 && maxLevel === 4) {\n await writeFile(`${dir}/written.json`, JSON.stringify(written));\n }\n}\n\nconst specialDatasets = [\"publishers\", \"all\", \"rarity\", \"publication_date\"];\nasync function loadData(\n dataset: string,\n stats: StatsAggregator,\n): Promise {\n if (dataset === \"publishers\") {\n return await modules.publishers();\n } else if (dataset === \"rarity\") {\n return modules.rarity(stats);\n } else if (dataset === \"all\") {\n return await modules.all(stats);\n } else if (dataset === \"publication_date\") {\n return modules.publication_date(stats);\n } else {\n return await modules.single(dataset);\n }\n}\nasync function main() {\n // Main execution\n const dataset = process.argv[2];\n if (!dataset) throw Error(\"dataset arg required, use list to list\");\n if (dataset === \"list\") {\n console.log(specialDatasets, Object.keys(await loadSparseDataToMemory()));\n return;\n }\n const level = process.argv[3];\n if (!level) throw Error(\"level arg required (1,2,3,4 or all)\");\n if (level === \"all\") {\n await processAllZoomLevels(dataset);\n } else {\n await processAllZoomLevels(dataset, +level, +level);\n }\n}\n\nvoid main();\n" }, { "path": "scripts/write-images/modules/aggregate-dense.ts", "content": "import { IsbnData, ProcessSingleZoom } from \"..\";\nimport { IsbnRelative, totalIsbns } from \"../../../src/lib/util\";\nimport { ImageTiler, StatsAggregator } from \"../ImageTiler\";\nimport { loadSparseDataToMemory } from \"./single-sparse\";\n\nexport async function colorImageWithDenseIsbns(\n tiler: ImageTiler,\n isbnsBinaryUint8: Uint8Array,\n): Promise {\n if (isbnsBinaryUint8.length !== totalIsbns) throw Error(\"wrong length\");\n const addcolor = [1, 1, 1] as [number, number, number];\n for (let i = 0; i < isbnsBinaryUint8.length; i++) {\n const relativeIsbn = i as IsbnRelative;\n if (relativeIsbn % 2e6 === 0) {\n tiler.logProgress(relativeIsbn / totalIsbns);\n await tiler.purgeToLength(1);\n }\n if (isbnsBinaryUint8[i]) {\n tiler.colorIsbn(relativeIsbn, addcolor);\n tiler.stats?.addStatistic(relativeIsbn, { dataset_all: 1 });\n }\n }\n}\nexport function aggregateDatasets(\n datasets: IsbnData,\n stats: StatsAggregator,\n): Uint8Array {\n const out = new Uint8Array(totalIsbns);\n for (const dataset in datasets) {\n console.log(\"adding data for dataset\", dataset);\n const data = datasets[dataset];\n\n let position = 0;\n let isbnStreak = true;\n if (!data) throw Error(\"no data\");\n for (const value of data) {\n if (isbnStreak) {\n for (let j = 0; j < value; j++) {\n out[position as IsbnRelative] = 1;\n stats.addStatistic(position as IsbnRelative, {\n [`dataset_${dataset}`]: 1,\n });\n position++;\n }\n } else {\n position += value;\n }\n\n isbnStreak = !isbnStreak;\n }\n }\n return out;\n}\n\nexport default async function aggregateDense(\n stats: StatsAggregator,\n): Promise {\n const dataSet = await loadSparseDataToMemory();\n const data = aggregateDatasets(dataSet, stats);\n return (tiler) => colorImageWithDenseIsbns(tiler, data);\n}\n" }, { "path": "scripts/write-images/modules/index.ts", "content": "export { default as all } from \"./aggregate-dense\";\nexport { default as publication_date } from \"./publication_date\";\nexport { default as publishers } from \"./publishers\";\nexport { default as rarity } from \"./rarity\";\nexport { default as single } from \"./single-sparse\";\n" }, { "path": "scripts/write-images/modules/publication_date.ts", "content": "import sqlite3 from \"better-sqlite3\";\nimport { channelMax, ImageTile, ProcessSingleZoom } from \"..\";\nimport {\n fullIsbnToRelative,\n Isbn13Number,\n IsbnRelative,\n IsbnStrWithChecksum,\n totalIsbns,\n} from \"../../../src/lib/util\";\nimport { ImageTiler, StatsAggregator } from \"../ImageTiler\";\n\nexport function loadPublicationDateData(\n dbName: string,\n stats: StatsAggregator,\n) {\n const db = sqlite3(dbName);\n let i = 0;\n const maxOclcNumber = db\n .prepare(\"select max(oclc_number) from isbn_data\")\n .pluck()\n .get() as number;\n\n const isbns = new Uint8Array(totalIsbns);\n for (const row of db\n .prepare<\n [],\n {\n oclc_number: number;\n isbn13: Isbn13Number;\n publication_date: number | null;\n }\n >(\"select * from isbn_data where publication_date is not null\")\n .iterate()) {\n if (++i % 1000000 === 0)\n console.log(\n \"loading publication date data\",\n ((row.oclc_number / maxOclcNumber) * 100).toFixed(1) + \"%\",\n i,\n row,\n );\n // isbns.set(+row.isbn as Isbn13Number, row.oclc_number);\n const isbnRel = fullIsbnToRelative(\n String(row.isbn13) as IsbnStrWithChecksum,\n );\n if (isbnRel < 0 || isbnRel >= totalIsbns) {\n throw new Error(`invalid isbn: ${row.isbn13} ${isbnRel}`);\n }\n if (row.publication_date !== null) {\n // range 1800 - 2055\n isbns[isbnRel] = Math.min(255, Math.max(1, row.publication_date - 1800));\n stats.addStatistic(isbnRel, {\n publication_date: row.publication_date,\n publication_date_count: 1,\n });\n }\n }\n return isbns;\n}\n\nexport default function rarityModule(\n stats: StatsAggregator,\n): ProcessSingleZoom {\n const dataset = loadPublicationDateData(\n process.env.INPUT_HOLDING_SQLITE ?? \"data/library_holding_data.sqlite3\",\n stats,\n );\n return (tiler) => processPublicationData(tiler, dataset);\n}\nasync function processPublicationData(\n tiler: ImageTiler,\n dataset: Uint8Array,\n): Promise {\n tiler.postprocessPixels = postprocessPixels;\n for (let i = 0; i < totalIsbns; i++) {\n const relativeIsbn = i as IsbnRelative;\n if (relativeIsbn % 2e6 === 0) {\n tiler.logProgress(relativeIsbn / totalIsbns);\n await tiler.purgeToLength(1);\n }\n const publicationDate = dataset[i]; // - 1800\n if (publicationDate)\n tiler.colorIsbn(relativeIsbn, [publicationDate, 1, 1], {\n addToPixel: true,\n scaleColors: false,\n scaleColorByTileScale: false,\n });\n }\n}\n\nfunction postprocessPixels(image: ImageTile, totalBooksPerPixel: number) {\n for (let i = 0; i < image.img.length; i += 3) {\n let publicationDate = image.img[i];\n const bookCount = image.img[i + 1];\n // verify all are ints\n if (!Number.isInteger(publicationDate)) {\n throw new Error(\"non-integer value\");\n }\n // compute average date\n if (bookCount > 0) {\n publicationDate /= bookCount;\n }\n if (bookCount === 0 && publicationDate !== 0) {\n console.log({ i, publicationDate, bookCount });\n throw new Error(\"invalid publication date\");\n }\n if (bookCount > 0 && (publicationDate < 0 || publicationDate > 255)) {\n console.log({ i, publicationDate, bookCount });\n throw new Error(\"invalid publication date\");\n }\n // scale to channelMax\n publicationDate *= channelMax / 255;\n image.img[i] = publicationDate;\n image.img[i + 1] = publicationDate;\n image.img[i + 2] = (bookCount / totalBooksPerPixel) * channelMax;\n }\n}\n" }, { "path": "scripts/write-images/modules/publishers.ts", "content": "import { readFile } from \"fs/promises\";\nimport { ProcessSingleZoom } from \"..\";\nimport { InfoMap, LazyPrefixInfo } from \"../../../src/lib/info-map\";\nimport { getGroupHierarchy } from \"../../../src/lib/prefix-data\";\nimport {\n IsbnRelative,\n lastIsbnInPrefix,\n relativeToIsbnPrefix,\n removeDashes,\n totalIsbns,\n} from \"../../../src/lib/util\";\nimport { ImageTiler } from \"../ImageTiler\";\n\nexport async function processPublishersData(\n tiler: ImageTiler,\n publishersData: LazyPrefixInfo,\n): Promise {\n let color: [number, number, number] | null = null;\n let curPrefixEnd = -1;\n for (\n let relativeIsbn = 0 as IsbnRelative;\n relativeIsbn < totalIsbns;\n relativeIsbn++\n ) {\n if (relativeIsbn % 2e6 === 0) {\n tiler.logProgress(relativeIsbn / totalIsbns);\n await tiler.purgeToLength(1);\n }\n if (relativeIsbn > curPrefixEnd) {\n const isbn = relativeToIsbnPrefix(relativeIsbn);\n const data = getGroupHierarchy(publishersData, isbn);\n if (typeof data === \"function\") {\n throw Error(\n \"found lazy data in full data dump from /data, this is impossible\",\n );\n }\n if (data.outers.length >= 2) {\n const pr = data.outers[1]?.info?.[0].prefix;\n if (!pr) throw Error(\"not handled\");\n curPrefixEnd = lastIsbnInPrefix(removeDashes(pr));\n } else {\n curPrefixEnd = relativeIsbn + 9;\n }\n if (data.outers.length === 0) {\n // throw Error(`no data for ${isbn}, previous ended at ${curPrefixEnd}`);\n color = null;\n continue;\n }\n color = null;\n const publisherId = data.outers[1]?.info?.[0].numericId;\n // publisherId to RGB\n if (publisherId) {\n color = [0, 0, 0];\n color[0] = ((publisherId & 0xff0000) >> 16) / 255;\n color[1] = ((publisherId & 0x00ff00) >> 8) / 255;\n color[2] = (publisherId & 0x0000ff) / 255;\n tiler.stats?.addStatistic(relativeIsbn, {\n publisher_blocks: 1,\n });\n }\n\n /* console.log(\n `color from ${isbn} to ${curPrefixEnd + isbnEANStart}: ${color}`\n );*/\n }\n if (color) {\n tiler.colorIsbn(relativeIsbn, color, {\n addToPixel: false,\n scaleColors: true,\n scaleColorByTileScale: false,\n });\n }\n }\n}\n\nexport async function loadPublishersData() {\n const publishersData = {\n children: JSON.parse(\n await readFile(\n process.env.INPUT_PREFIX_DATA ?? `data/prefix-data.json`,\n \"utf8\",\n ),\n ) as InfoMap,\n totalChildren: 0,\n };\n return publishersData;\n}\n\nexport default async function publishersModule(): Promise {\n const publishersData = await loadPublishersData();\n return (tiler) => processPublishersData(tiler, publishersData);\n}\n" }, { "path": "scripts/write-images/modules/rarity.ts", "content": "import sqlite3 from \"better-sqlite3\";\nimport { channelMax, ImageTile, ProcessSingleZoom } from \"..\";\nimport {\n fullIsbnToRelative,\n Isbn13Number,\n IsbnRelative,\n IsbnStrWithChecksum,\n totalIsbns,\n} from \"../../../src/lib/util\";\nimport { ImageTiler, StatsAggregator } from \"../ImageTiler\";\n\nexport function loadRarityData(dbName: string, stats: StatsAggregator) {\n const db = sqlite3(dbName);\n let i = 0;\n const maxOclcNumber = db\n .prepare(\"select max(oclc_number) from isbn_data\")\n .pluck()\n .get() as number;\n\n const isbns = new Uint8Array(totalIsbns * 2);\n for (const row of db\n .prepare<\n [],\n {\n oclc_number: number;\n isbn13: Isbn13Number;\n publication_date: number;\n holding_count: number;\n edition_count: number;\n }\n >(\n \"select * from isbn_data join holdings_data on isbn_data.oclc_number = holdings_data.oclc_number\",\n )\n .iterate()) {\n if (++i % 1000000 === 0)\n console.log(\n \"loading rarity data\",\n ((row.oclc_number / maxOclcNumber) * 100).toFixed(1) + \"%\",\n i,\n row,\n );\n // isbns.set(+row.isbn as Isbn13Number, row.oclc_number);\n const isbnRel = fullIsbnToRelative(\n String(row.isbn13) as IsbnStrWithChecksum,\n );\n if (isbnRel < 0 || isbnRel >= totalIsbns) {\n throw new Error(`invalid isbn: ${row.isbn13} ${isbnRel}`);\n }\n const existingHolding = isbns[2 * isbnRel];\n const existingEdition = isbns[2 * isbnRel + 1];\n isbns[2 * isbnRel] = Math.min(row.holding_count + existingHolding, 255);\n // add 1 to edition count as a \"exists\" marker\n isbns[2 * isbnRel + 1] = Math.min(\n (existingEdition || 1) + row.edition_count,\n 255,\n );\n\n stats.addStatistic(isbnRel, {\n rarity_holdingCount: row.holding_count,\n rarity_editionCount: row.edition_count,\n rarity_exists: 1,\n });\n /*if (existingHolding || existingEdition) {\n console.log(\"multiple entries for \", row, {\n existingHolding,\n existingEdition,\n });\n }*/\n }\n return isbns;\n}\n\n/*if (require.main === module) {\n const dbName = process.argv[2];\n if (!dbName) throw new Error(\"no db name provided\");\n loadRarityData(dbName);\n}*/\n\nexport default function rarityModule(\n stats: StatsAggregator,\n): ProcessSingleZoom {\n const dataset = loadRarityData(\n process.env.INPUT_HOLDING_SQLITE ?? \"data/library_holding_data.sqlite3\",\n stats,\n );\n return (tiler) => processRarityData(tiler, dataset);\n}\nasync function processRarityData(\n tiler: ImageTiler,\n dataset: Uint8Array,\n): Promise {\n tiler.postprocessPixels = postprocessPixels;\n for (let i = 0; i < totalIsbns; i++) {\n const relativeIsbn = i as IsbnRelative;\n if (relativeIsbn % 2e6 === 0) {\n tiler.logProgress(relativeIsbn / totalIsbns);\n await tiler.purgeToLength(1);\n }\n const holdingCount = dataset[2 * i];\n let editionCount = dataset[2 * i + 1];\n const exists = editionCount > 0; // we added 1 to editionCount as an \"exists\" marker\n if (exists) editionCount -= 1;\n if (holdingCount || editionCount || exists) {\n tiler.colorIsbn(relativeIsbn, [holdingCount, editionCount, 1], {\n addToPixel: true,\n scaleColors: false,\n scaleColorByTileScale: false,\n });\n }\n }\n}\n\nfunction postprocessPixels(image: ImageTile) {\n for (let i = 0; i < image.img.length; i += 3) {\n let holdingsCount = image.img[i];\n let editionCount = image.img[i + 1];\n let bookCount = image.img[i + 2];\n // verify all are ints\n if (\n !Number.isInteger(holdingsCount) ||\n !Number.isInteger(editionCount) ||\n !Number.isInteger(bookCount)\n ) {\n throw new Error(\"non-integer value\");\n }\n // verify all are positive\n if (holdingsCount < 0 || editionCount < 0 || bookCount < 0) {\n throw new Error(\"negative value\");\n }\n // verify all are 0 if bookCount is 0\n if (bookCount === 0 && (holdingsCount || editionCount)) {\n throw new Error(\"non-zero value with zero book count\");\n }\n\n // scale the colors\n const maxValue = Math.max(holdingsCount, editionCount, bookCount);\n const needScaleDown = maxValue >= 255;\n if (needScaleDown) {\n const scale = 255 / maxValue;\n holdingsCount *= scale;\n editionCount *= scale;\n bookCount *= scale;\n }\n // scale to channelMax\n holdingsCount *= channelMax / 255;\n editionCount *= channelMax / 255;\n bookCount *= channelMax / 255;\n /*console.log({\n holdingsCount,\n editionCount,\n bookCount,\n maxValue,\n foo: image.img.slice(i, i + 3),\n });*/\n image.img[i] = holdingsCount;\n image.img[i + 1] = editionCount;\n image.img[i + 2] = bookCount;\n }\n}\n" }, { "path": "scripts/write-images/modules/single-sparse.ts", "content": "import bencode from \"bencode\";\nimport { createReadStream } from \"node:fs\";\nimport { ZSTDDecompress } from \"simple-zstd\";\nimport { IsbnData, ProcessSingleZoom } from \"..\";\nimport { IsbnRelative } from \"../../../src/lib/util\";\nimport { ImageTiler } from \"../ImageTiler\";\nexport const INPUT_FILENAME =\n process.env.INPUT_BENC ??\n `${process.env.DATA_DIR ?? \"data\"}/aa_isbn13_codes_20241204T185335Z.benc.zst`;\n\nexport async function colorImageWithSparseIsbns(\n tiler: ImageTiler,\n packedIsbnsBinary: Uint32Array,\n): Promise {\n const addcolor = [1, 1, 1] as [number, number, number];\n\n let position = 0;\n let isbnStreak = true;\n\n for (const value of packedIsbnsBinary) {\n if (isbnStreak) {\n for (let j = 0; j < value; j++) {\n const isbn = position as IsbnRelative;\n tiler.colorIsbn(isbn, addcolor);\n // tiler.stats?.addStatistic(isbn, { count: 1 });\n\n position++;\n }\n } else {\n position += value;\n await tiler.purgeToLength(1);\n }\n\n isbnStreak = !isbnStreak;\n }\n}\n\nexport async function loadSparseDataToMemory(): Promise {\n // Read and decompress the input file\n const fileStream = createReadStream(INPUT_FILENAME);\n return new Promise((resolve) => {\n const chunks: Buffer[] = [];\n fileStream\n .pipe(ZSTDDecompress())\n .on(\"data\", (chunk: Buffer) => chunks.push(chunk))\n .on(\"end\", () => {\n const data = Buffer.concat(chunks);\n const isbnData = bencode.decode(data) as Record;\n // Convert Uint8Array to Uint32Array\n const isbnData2: IsbnData = {};\n for (const [k, v] of Object.entries(isbnData)) {\n if (v.byteOffset !== 0) {\n throw new Error(\n `packedIsbnsBinaryUint8 must be aligned to 0, is ${v.byteOffset}`,\n );\n }\n const packedIsbnsBinary = new Uint32Array(v.buffer);\n isbnData2[k] = packedIsbnsBinary;\n }\n resolve(isbnData2);\n });\n });\n}\n\nexport default async function singleSparse(\n dataset: string,\n): Promise {\n const data = await loadSparseDataToMemory();\n const dataa = data[dataset];\n if (!dataa) {\n throw new Error(`dataset ${dataset} not found`);\n }\n return (tiler) => colorImageWithSparseIsbns(tiler, dataa);\n}\n" }, { "path": "scripts/write-titles.ts", "content": "import sqlite3 from \"better-sqlite3\";\nimport { mkdirSync, writeFileSync } from \"fs\";\nimport path from \"path\";\nimport {\n Isbn13Number,\n IsbnRelative,\n relativeToFullIsbn,\n splitNameJson,\n totalIsbns,\n} from \"../src/lib/util\";\n\nexport function loadPublicationDateData(dbName: string) {\n const db = sqlite3(dbName);\n // perf options\n db.pragma(\"cache_size = 100000\");\n //mmap\n db.pragma(\"journal_mode = WAL\");\n db.pragma(\"synchronous = OFF\");\n db.pragma(\"temp_store = MEMORY\");\n db.pragma(\"mmap_size = 300000000000\");\n\n const blockSize = 10000;\n const prefixLength = 12 - Math.log10(blockSize);\n const dirSegmentLength = 3;\n for (let isbn = 0; isbn < totalIsbns; isbn += blockSize) {\n const first = relativeToFullIsbn(isbn as IsbnRelative);\n const next = relativeToFullIsbn((isbn + blockSize) as IsbnRelative);\n const rows = db\n .prepare<\n [Isbn13Number, Isbn13Number],\n {\n isbn13: Isbn13Number;\n title: string | null;\n creator: string | null;\n }\n >(\n \"select isbn13,title as title, creator as creator from isbn_data where isbn13 >= ? and isbn13 < ? group by isbn13 order by isbn13\",\n )\n .all(+first as Isbn13Number, +next as Isbn13Number);\n for (const row of rows) {\n const maxL = 70;\n if (row.title && row.title.length > maxL)\n row.title = row.title.slice(0, maxL) + \"...\";\n if (row.creator && row.creator.length > maxL)\n row.creator = row.creator.slice(0, maxL) + \"...\";\n }\n if (isbn % 1000000 === 0)\n console.log(\n `loading range ${first}, done: ${((isbn / totalIsbns) * 100).toFixed(\n 1,\n )}%`,\n );\n if (rows.length === 0) continue;\n const prefixStr = first.slice(0, prefixLength);\n const fname =\n `${process.env.OUTPUT_DIR_PUBLIC ?? \"public\"}/title-data/` +\n splitNameJson(prefixStr, dirSegmentLength);\n mkdirSync(path.dirname(fname), { recursive: true });\n writeFileSync(fname, JSON.stringify(rows));\n }\n}\n\nloadPublicationDateData(\n `${process.env.DATA_DIR ?? \"data\"}/library_holding_data.sqlite3`,\n);\n" }, { "path": "src/App.tsx", "content": "import { useMemo, type FC } from \"react\";\n\nimport { IsbnMap } from \"./components/IsbnMap\";\nimport { bookshelfConfig } from \"./projections/bookshelf\";\n\nconst App: FC = () => {\n const config = useMemo(\n () =>\n bookshelfConfig({\n width: Math.min(\n 1500,\n document.body.clientWidth,\n (document.body.clientHeight / 2) * Math.sqrt(10),\n ),\n }),\n [],\n );\n return ;\n};\n\nexport default App;\n" }, { "path": "src/components/Controls.tsx", "content": "import isbnlib from \"isbn3\";\nimport { observer, useLocalObservable } from \"mobx-react-lite\";\nimport { fromPromise } from \"mobx-utils\";\nimport React, { useMemo, useRef } from \"react\";\nimport { OptionProps, components } from \"react-select\";\nimport AsyncSelect from \"react-select/async\";\nimport Select from \"react-select/base\";\nimport { default as config, default as staticConfig } from \"../config\";\nimport { GoogleBooksItem, googleBooksQuery } from \"../lib/google-books\";\nimport { Store } from \"../lib/Store\";\nimport { IsbnPrefixWithoutDashes, IsbnStrWithChecksum } from \"../lib/util\";\nimport { Legend } from \"./Legend\";\n\nexport const Controls: React.FC<{ store: Store }> = observer(function Controls({\n store,\n}) {\n const state = useLocalObservable(() => ({\n showSettings: false,\n showDatasetChooser: false,\n }));\n const stats = useMemo(\n () =>\n fromPromise(\n store.statsCalculator.getStats(\n \"978\" as IsbnPrefixWithoutDashes,\n \"979\" as IsbnPrefixWithoutDashes,\n ),\n ),\n [],\n );\n return (\n
\n
\n ISBN Visualization{\" \"}\n {stats.case({\n fulfilled(stats) {\n return (\n \n Showing{\" \"}\n {(\n stats[`dataset_${store.runtimeConfig.dataset}`] ??\n stats.dataset_all ??\n 0\n ).toLocaleString()}{\" \"}\n books\n \n );\n },\n })}\n {state.showSettings && (\n <>\n \n\n {\n store.switchDataset(store.runtimeConfig.dataset, true);\n }}\n >\n Reset Settings\n \n \n )}\n {!state.showSettings && (\n (state.showDatasetChooser = true)}\n >\n Preset:{\" \"}\n {(() => {\n const ds = staticConfig.datasetOptions.find(\n (e) => e.id === store.runtimeConfig.dataset,\n );\n if (!ds) return null;\n return (\n <>\n {ds.name}\n
\n {ds.description}\n \n );\n })()}\n \n )}\n
\n \n {state.showSettings ? (\n \n ) : (\n \n )}\n {state.showDatasetChooser && (\n
\n
\n \n
\n Choose a Preset{\" \"}\n {\n state.showSettings = !state.showSettings;\n state.showDatasetChooser = false;\n }}\n >\n ⚙ {state.showSettings ? \"Done\" : \"Advanced\"}\n \n
\n\n {\n state.showDatasetChooser = false;\n }}\n >\n Close\n \n \n {staticConfig.datasetOptions.map((d) => (\n \n {\n state.showDatasetChooser = false;\n store.switchDataset(d.id, true);\n }}\n >\n {d.name} [{d.id}]
\n {d.description && {d.description}}\n \n \n ))}\n
\n
\n )}\n
\n );\n});\n\nconst BookOption: React.FC> = (p) => {\n return (\n \n {p.data.volumeInfo.title}\n
\n {p.data.volumeInfo.authors?.join(\", \")}\n \n );\n};\n\nexport interface MinimalGoogleBooksItem {\n id: string;\n volumeInfo: {\n title?: string;\n authors?: string[];\n industryIdentifiers?: GoogleBooksItem[\"volumeInfo\"][\"industryIdentifiers\"];\n };\n}\nconst MainStuff: React.FC<{ store: Store }> = observer(function MainStuff({\n store,\n}) {\n const selectRef = useRef>(null);\n return (\n <>\n

\n Drag/Zoom like a map. Tap to show details of an ISBN! Right-click-drag\n to show stats.\n

\n \n

\n \n \n );\n});\n\nconst Settings: React.FC<{ store: Store }> = observer(function Settings({\n store,\n}) {\n const config = store.runtimeConfig;\n return (\n <>\n

\n \n \n \n \n \n (to make it easier to see sparse data)\n
\n
\n Publisher settings\n \n \n \n \n (each publisher's range is highlighted with a random color)\n \n\n \n
\n
\n Zoom Settings\n \n \n \n \n
\n
\n Data Filters\n \n \n
\n \n {\n store.shaderError = \"\";\n config.customShader = e.currentTarget.value;\n }}\n />\n {store.shaderError && (\n
\n Shader Error:{\" \"}\n \n {store.shaderError}\n \n
\n )}\n \n );\n});\n\nconst LoadProgress = observer(function LoadProgress({\n store,\n}: {\n store: Store;\n}) {\n if (store.inProgress.size === 0) return /* green checkmark emoji */ \"✅\";\n const errors = [...store.inProgress].filter((e) => e[1]);\n /* red cross emoji */\n if (errors.length > 0)\n return errors.map((e, i) => (\n
\n ❌ {e[0]}: {String(e[1])}\n
\n ));\n return
;\n});\n" }, { "path": "src/components/EanBarcode.tsx", "content": "export function EanBarcode(props: { ean: string }) {\n return {props.ean};\n}\n" }, { "path": "src/components/Highlight.tsx", "content": "import { Html } from \"@react-three/drei\";\nimport { hyphenate } from \"isbn3\";\nimport { observer } from \"mobx-react-lite\";\nimport React from \"react\";\nimport { LazyPrefixInfo } from \"../lib/info-map\";\nimport { Store } from \"../lib/Store\";\nimport { relativeToFullIsbn, removeDashes, siNumber } from \"../lib/util\";\nimport { getPlanePosition } from \"../lib/view-utils\";\nimport { AbbrevStats, maxZoomForStats } from \"./StatsShow\";\n\nexport const PublisherHighlightShow: React.FC<{ store: Store }> = observer(\n function PublisherHighlightShow({ store }) {\n if (store.highlightedIsbn.type === \"done\") return null;\n if (!store.highlightedPublisher) return null;\n const isbn = store.highlightedPublisher.relative;\n const isbnFull = relativeToFullIsbn(isbn);\n const loc = getPlanePosition(store.projection, isbn, isbn);\n const publisher = store.highlightedPublisher.data?.[1]?.info?.[0];\n return (\n <>\n \n \n {/**/}\n \n
\n ISBN {hyphenate(isbnFull) || isbnFull}\n
\n {(store.highlightedPublisher.data &&\n store.highlightedPublisher.obj && (\n \n )) ??
Unassigned or unknown range
}\n
\n {publisher && (\n \n )}\n Click to show book details\n
\n {store.floatZoomFactor < maxZoomForStats && (\n Right-click-drag to show region stats\n )}\n
\n \n \n \n \n );\n },\n);\nexport const HighlightShow: React.FC<{ store: Store }> = observer(\n function HighlightShow({ store }) {\n if (store.highlightedIsbn.type === \"todo\") return null;\n const isbn = store.highlightedIsbn.relative;\n const loc = getPlanePosition(store.projection, isbn, isbn);\n return (\n <>\n \n {/* */}\n \n \n \n \n {/**/}\n \n \n \n \n \n \n );\n },\n);\n\nconst HighlightCircle: React.FC<{ store: Store }> = observer(\n function HighlightCircle(props: { store: Store }) {\n const store = props.store;\n const circleRadius = 30;\n const circleStroke = 4;\n if (store.floatZoomFactor > 5000) return null;\n const svg = (s: React.CSSProperties) => (\n \n {\" \"}\n \n \n );\n return (\n
\n {svg({ filter: \"drop-shadow(0 0 4px black)\", zIndex: 0 })}\n {/*svg({ zIndex: 30 })*/}\n
\n );\n },\n);\nconst IsbnInfo = observer(function IsbnInfo(props: { store: Store }) {\n const o = props.store.highlightedIsbn;\n if (o.type === \"todo\") return \"Hover to see ISBN info\";\n let groupInfo;\n if (o.obj) {\n const i = o.obj;\n if (!i.prefix) return
imposs: no prefix?
;\n const prefixLen = i.prefix.length + i.group.length;\n const totalDigits = 13 - 1; // 13 minus check digit\n const numBooksInGroup = 10 ** (totalDigits - prefixLen);\n const numBooksInPublisher =\n 10 ** (totalDigits - prefixLen - i.publisher.length);\n\n groupInfo = (\n
\n \n
\n {/*Article: {i.article}*/}\n {o.rarity &&\n (o.rarity.bookCount === 0 ? (\n <>(no holding data)\n ) : (\n <>\n {o.rarity.holdingCount === 255 ? \">250\" : o.rarity.holdingCount}{\" \"}\n known libraries hold copies of{\" \"}\n {o.rarity.editionCount >= 254 ? \">250\" : o.rarity.editionCount}{\" \"}\n editions of this book\n \n ))}\n
\n
\n \n Details\n \n Num possible ISBNs in group: {siNumber(numBooksInGroup)}
\n Num possible ISBNs in publisher: {siNumber(numBooksInPublisher)}{\" \"}\n
\n {o.groupInfo.flatMap((g) =>\n (g.info ?? []).map((info, i) => (\n
  • \n {info.source === \"publisher-ranges\" ? (\n <>\n Group {info.prefix}: {info.name}\n \n ) : (\n <>\n Publisher {info.prefix}: {info.registrant_name} (\n {info.country_name})\n \n )}\n
    • \n )),\n )}\n
    \n Look up book on\n
      \n {props.store.externalSearchEngines.map((d) => (\n
    • \n \n {d.name}\n \n
      • \n ))}\n
    \n
    \n );\n } else {\n groupInfo =
    Unassigned or unknown range
    ;\n }\n const isbn = o.obj?.isbn13h ?? relativeToFullIsbn(o.relative);\n\n return (\n <>\n {\n e.stopPropagation();\n }}\n >\n {\n e.preventDefault();\n e.stopPropagation();\n props.store.highlightedIsbn = { type: \"todo\" };\n }}\n >\n Close\n \n {\n props.store.zoomAnimateToHighlight();\n }}\n >\n Fly to book\n \n Book:{\" \"}\n {o.googleBookDetails === \"todo\" ? (\n <>{isbn}...\n ) : o.googleBookDetails === null ? (\n <>{isbn} (not found on Google Books)\n ) : (\n <>\n \n {o.googleBookDetails.volumeInfo.title}\n
    \n by {o.googleBookDetails.volumeInfo.authors?.join(\", \")}\n
    \n ISBN: {isbn}\n \n )}\n
    \n {groupInfo}\n
    \n \n );\n});\n\nfunction GroupInfo({\n obj,\n groupInfo,\n}: {\n obj: ISBN;\n groupInfo: LazyPrefixInfo[];\n}) {\n const publisherPrefix = `${obj.prefix}${obj.group}${obj.publisher}`;\n const computedPublisherInfo = groupInfo.find((g) => {\n const myPrefix = g.info?.[0]?.prefix;\n return myPrefix && removeDashes(myPrefix) === publisherPrefix;\n });\n\n return (\n <>\n Group {obj.prefix}-{obj.group}: {obj.groupname}
    \n Publisher {obj.publisher}:{\" \"}\n \n {computedPublisherInfo?.info &&\n computedPublisherInfo.info[0].source === \"isbngrp\"\n ? computedPublisherInfo.info[0].registrant_name\n : \"unknown\"}\n {\" \"}\n {(computedPublisherInfo?.info?.length ?? 0) > 1\n ? `(+${(computedPublisherInfo?.info?.length ?? 0) - 1} more)`\n : \"\"}\n \n );\n}\n" }, { "path": "src/components/ImageTree.tsx", "content": "import { Plane } from \"@react-three/drei\";\nimport { observer, useLocalObservable } from \"mobx-react-lite\";\nimport { fromPromise } from \"mobx-utils\";\nimport { useEffect, useState } from \"react\";\nimport { Blending } from \"three\";\nimport { DetailLevelObservable } from \"../lib/DetailLevelObservable\";\nimport { Store } from \"../lib/Store\";\nimport {\n digits,\n isbnPrefixAppend,\n isbnPrefixToRelative,\n IsbnPrefixWithDashes,\n IsbnPrefixWithoutDashes,\n ProjectionConfig,\n removeDashes,\n} from \"../lib/util\";\n\nexport const ImageTree: React.FC<{\n config: ProjectionConfig;\n store: Store;\n prefix: IsbnPrefixWithDashes;\n blending: Blending;\n}> = observer(function _ImageTree(props) {\n const groupPrefix = removeDashes(props.prefix);\n\n const view = props.store.getDetailLevel(groupPrefix);\n if (!view.container) return null;\n return ;\n});\nconst GroupShowInner: React.FC<{\n config: ProjectionConfig;\n store: Store;\n prefix: IsbnPrefixWithDashes;\n view: DetailLevelObservable;\n blending: Blending;\n}> = observer(function _GroupShowInner({ view, ...props }) {\n const { position, width, height } = view.planePosition;\n const groupPrefix = removeDashes(props.prefix);\n const [hasChildren, setHasChildren] = useState(false);\n useEffect(() => {\n void (async () => {\n setHasChildren(\n (\n await Promise.all(\n props.store.shaderUtil.shaderProgram.requiredTextures.map(\n (dataset) =>\n props.store\n .imageLoader(dataset)\n .getHasChildren(isbnPrefixToRelative(groupPrefix)),\n ),\n )\n ).some((e) => e),\n );\n })();\n }, [groupPrefix, props.store.shaderUtil.shaderProgram.requiredTextures]);\n\n return (\n <>\n {view.image && (\n \n )}\n {view.imageChildren &&\n hasChildren &&\n digits.map((i) => {\n return (\n \n );\n })}\n \n );\n});\n\nconst PrefixImage: React.FC<{\n store: Store;\n prefix: IsbnPrefixWithoutDashes;\n position: [number, number, number];\n args: [number, number];\n blending: Blending;\n}> = observer((props) => {\n const prefix = isbnPrefixToRelative(props.prefix);\n const { material } = useLocalObservable(() => ({\n get _material() {\n return fromPromise(props.store.shaderUtil.getIsbnShaderMaterial(prefix));\n },\n get material() {\n return this._material.case({\n fulfilled: (m) => {\n if (m) {\n m.refreshUniforms();\n return m.material;\n }\n },\n });\n },\n }));\n if (!material) return null;\n return (\n \n );\n});\n" }, { "path": "src/components/IsbnGrid.tsx", "content": "import { Grid } from \"@react-three/drei\";\nimport { computed } from \"mobx\";\nimport { observer, useLocalObservable } from \"mobx-react-lite\";\nimport config from \"../config\";\nimport { Store } from \"../lib/Store\";\nimport { totalIsbns } from \"../lib/util\";\n\nexport const IsbnGrid: React.FC<{ store: Store }> = observer(function IsbnGrid({\n store,\n}) {\n const zoomLevel = computed(() => {\n const zoom = store.floatZoomFactor;\n\n const zoomLevel = Math.round(Math.log10(zoom) * 2) - 1;\n if (zoomLevel < 0) return 0;\n const maxZoom = 6;\n if (zoomLevel > maxZoom) return maxZoom;\n return zoomLevel;\n }).get();\n const maxShowZoom = store.runtimeConfig.doBookshelfEffect ? 6 : 8;\n const color =\n zoomLevel > 4 ? config.bookshelfColorHex : store.runtimeConfig.gridColor;\n return (\n <>\n \n {store.runtimeConfig.gridLevels >= 2 && zoomLevel + 1 <= maxShowZoom && (\n \n )}\n {store.runtimeConfig.gridLevels >= 3 && zoomLevel + 2 <= maxShowZoom && (\n \n )}\n \n );\n});\n\nconst IsbnGridLevel: React.FC<{\n store: Store;\n zoomLevel: number;\n thickness: number;\n z: number;\n color: string;\n}> = observer(function IsbnGridLevel(props) {\n const { store } = props;\n\n const thickness = props.zoomLevel > 4 ? props.thickness + 1 : props.thickness;\n const pwidth = store.projection.pixelWidth;\n const pheight = store.projection.pixelHeight;\n\n const outerGridWidth = 10;\n const outerGridHeight = totalIsbns / 1e9;\n const width = outerGridWidth * 10 ** Math.floor(props.zoomLevel / 2);\n const height = outerGridHeight * 10 ** Math.floor((props.zoomLevel + 1) / 2);\n\n let innerOnly = 1;\n if (props.zoomLevel > 5) {\n innerOnly = 100;\n }\n const { position } = useLocalObservable(\n () => ({\n get position() {\n let position: [number, number, number] = [\n store.projection.pixelWidth / 2,\n -store.projection.pixelHeight / 2,\n props.z,\n ];\n if (props.zoomLevel > 5) {\n position = [\n store.view.minX + store.view.width / 2,\n -(store.view.minY + store.view.height / 2),\n props.z,\n ];\n position[0] -= position[0] % (pwidth / width);\n position[1] -= position[1] % (pheight / height);\n }\n return position;\n },\n }),\n { position: computed.struct },\n );\n return (\n \n );\n});\n" }, { "path": "src/components/IsbnMap.tsx", "content": "import { OrbitControls, OrthographicCamera, Plane } from \"@react-three/drei\";\nimport { Canvas, ThreeEvent } from \"@react-three/fiber\";\nimport * as isbnlib from \"isbn3\";\nimport { observer } from \"mobx-react-lite\";\nimport React, { useEffect, useMemo, useState } from \"react\";\nimport * as THREE from \"three\";\nimport { MeshStandardMaterial, NoToneMapping } from \"three\";\nimport { shaderErrorToString } from \"../lib/shader-error\";\nimport { Store } from \"../lib/Store\";\nimport { IsbnPrefixWithDashes, ProjectionConfig } from \"../lib/util\";\nimport { Controls } from \"./Controls\";\nimport { HighlightShow, PublisherHighlightShow } from \"./Highlight\";\nimport { ImageTree } from \"./ImageTree\";\nimport { IsbnGrid } from \"./IsbnGrid\";\nimport { MiniMap } from \"./MiniMap\";\nimport { StatsShow } from \"./StatsShow\";\nimport { TextTree } from \"./TextTree\";\nObject.assign(window, { isbnlib });\n\nlet pointerMoved = 0;\nlet isPointerDown = -1;\nlet cancelHighlight = false;\n\nexport const IsbnMap: React.FC<{ config: ProjectionConfig }> = observer(\n function IsbnView(props: { config: ProjectionConfig }) {\n const [store] = useState(() => new Store(props.config));\n Object.assign(window, { store });\n\n useEffect(() => {\n function cancelHighlightListener() {\n if (cancelHighlight) store.highlightedPublisher = null;\n else cancelHighlight = true;\n }\n function cancelZoom() {\n // cancel flight on scroll\n cancelAnimationFrame(store.animationRequestId);\n }\n\n window.addEventListener(\"wheel\", cancelZoom);\n window.addEventListener(\"pointermove\", cancelHighlightListener);\n return () => {\n window.removeEventListener(\"wheel\", cancelZoom);\n window.removeEventListener(\"pointermove\", cancelHighlightListener);\n };\n }, []);\n\n const transparent = useMemo(\n () =>\n new MeshStandardMaterial({\n color: \"green\",\n transparent: true,\n opacity: 0,\n }),\n [],\n );\n return (\n <>\n {\n Object.assign(window, { threejsRoot });\n // threejsRoot.gl.debug.onShaderError = e => store.shaderError = e;\n store.camera = threejsRoot.camera as THREE.OrthographicCamera;\n threejsRoot.gl.debug.onShaderError = (...args) => {\n const err = shaderErrorToString(...args);\n console.warn(err);\n store.shaderError = err;\n };\n }}\n scene={{ background: new THREE.Color(\"#1d2636\") }}\n gl={{ toneMapping: NoToneMapping }}\n >\n \n {\n store.orbitControls = e;\n }}\n enableDamping={false}\n makeDefault\n enableRotate={false}\n enablePan={true}\n mouseButtons={{ LEFT: THREE.MOUSE.PAN }}\n zoomToCursor={true}\n minZoom={0.5}\n maxZoom={20000}\n touches={{ ONE: THREE.TOUCH.PAN, TWO: THREE.TOUCH.DOLLY_PAN }}\n onChange={(e) => {\n store.updateView(e);\n }}\n />\n {/**/}\n ) => {\n cancelAnimationFrame(store.animationRequestId);\n pointerMoved = 0;\n isPointerDown = e.button;\n if (e.button === 2) {\n store.highlightedPublisher = null;\n const x = e.point.x + props.config.pixelWidth / 2;\n const y = props.config.pixelHeight / 2 - e.point.y;\n store.updateHighlightedStats(x, y, \"start\");\n }\n }}\n onPointerUp={(e: ThreeEvent) => {\n isPointerDown = -1;\n if (\n (e.nativeEvent.target as Element | null)?.tagName !== \"CANVAS\"\n )\n return;\n if (pointerMoved < 4) {\n e.stopPropagation();\n const x = e.point.x + props.config.pixelWidth / 2;\n const y = props.config.pixelHeight / 2 - e.point.y;\n if (e.button === 2) {\n // store.updateStats(x, y, \"end\");\n store.highlightedStats = null;\n } else {\n store.updateHighlight(x, y, false);\n }\n }\n }}\n onPointerMove={(e: ThreeEvent) => {\n cancelHighlight = false;\n pointerMoved++;\n const x = e.point.x + props.config.pixelWidth / 2;\n const y = props.config.pixelHeight / 2 - e.point.y;\n if (isPointerDown === 2) {\n if (\n (e.nativeEvent.target as Element | null)?.tagName !== \"CANVAS\"\n )\n return;\n store.updateHighlightedStats(x, y, \"end\");\n }\n if (isPointerDown === -1 && e.pointerType === \"mouse\") {\n if (\n (e.nativeEvent.target as Element | null)?.tagName !== \"CANVAS\"\n )\n return;\n e.stopPropagation();\n\n store.updateHighlight(x, y, true);\n }\n }}\n />\n \n \n \n \n {store.runtimeConfig.showPublisherNames && (\n <>\n \n \n \n )}\n \n \n {store.runtimeConfig.showGrid && }\n \n \n \n \n \n );\n },\n);\n" }, { "path": "src/components/Legend.tsx", "content": "import { observer } from \"mobx-react-lite\";\nimport Select, {\n components,\n OptionProps,\n SingleValueProps,\n} from \"react-select\";\nimport { defaultColorSchemeMeaning } from \"../config\";\nimport { Store } from \"../lib/Store\";\nexport const gradientsPngUrl = new URL(\n \"../assets/gradients.png\",\n import.meta.url,\n).toString();\nconst w = 230;\nconst h = 20;\nconst totalGradientsInPng = 9;\nconst options = Array.from({ length: totalGradientsInPng })\n .map((_, value) => ({\n value,\n }))\n .filter((e) => e.value !== 1);\n\nexport const Legend: React.FC<{ store: Store }> = observer(\n function Legend(props) {\n const dataset = props.store.currentDataset;\n let meaning = dataset.colorSchemeMeaning;\n if (meaning === null) return;\n meaning ??= defaultColorSchemeMeaning;\n\n return (\n
    \n \n isSearchable={false}\n value={options.find(\n (o) => o.value === props.store.runtimeConfig.colorGradient,\n )}\n getOptionValue={(e) => e.value.toString()}\n onChange={(e) => {\n if (e) props.store.runtimeConfig.colorGradient = e.value;\n }}\n options={options}\n components={{\n Option: ColorGradientOption,\n SingleValue: ColorGradientSingleValue,\n }}\n store={props.store}\n />\n
    \n );\n },\n);\n\nconst ColorGradientOption: React.FC> = (p) => {\n return (\n \n \n \n );\n};\n\nconst ColorGradientSingleValue: React.FC> =\n observer((p) => {\n const meaning =\n p.selectProps.store.currentDataset.colorSchemeMeaning ??\n defaultColorSchemeMeaning;\n return (\n \n {/*
    Legend
    */}\n
    \n \n {meaning.markers.map((m) => (\n \n \n {m.label}\n
    \n ))}\n
    \n
    \n     \n );\n });\n\nconst Gradient: React.FC<{ value: number }> = (props) => (\n \n);\n" }, { "path": "src/components/MiniMap.tsx", "content": "/* eslint-disable @typescript-eslint/unbound-method -- mobx binds observable methods */\nimport { computed } from \"mobx\";\nimport { Observer, observer, useLocalObservable } from \"mobx-react-lite\";\nimport React, { useRef } from \"react\";\nimport { Store } from \"../lib/Store\";\nimport {\n firstIsbnInPrefix,\n isbnPrefixFromRelative,\n IsbnPrefixRelative,\n lastIsbnInPrefix,\n} from \"../lib/util\";\nimport { getPlanePosition } from \"../lib/view-utils\";\n\n// Default blocks configuration\nconst DEFAULT_BLOCKS = [\n { pos: \"00\", text: \"EN\" }, // Row 0, Col 0\n { pos: \"01\", text: \"EN\" }, // Row 0, Col 1\n { pos: \"02\", text: \"FR\" }, // Row 0, Col 2\n { pos: \"03\", text: \"DE\" }, // Row 0, Col 3\n { pos: \"04\", text: \"JP\" }, // Row 0, Col 4\n { pos: \"05\", text: \"SU\" }, // R1ow 0, Col 5\n { pos: \"07\", text: \"CN\" }, // Row 0, Col 7\n { pos: \"18\", text: \"US\" }, // Row 1, Col 8\n { pos: \"065\", text: \"BR\" },\n // XX blocks for column 6 (prefix '0' for row 0)\n ...Array.from({ length: 4 }, (_, i) => ({\n pos: `06${i}`,\n text: `6${i}`,\n })),\n // XX blocks for column 8 (prefix '0' for row 0)\n ...Array.from({ length: 10 }, (_, i) => ({\n pos: `08${i}`,\n text: [\"CS\", \"IN\", \"NO\", \"PL\", \"ES\", \"BR\", \"YU\", \"DK\", \"IT\", \"KR\"][i],\n })),\n // XX blocks for column 9 (prefix '0' for row 0)\n ...Array.from({ length: 10 }, (_, i) => ({\n pos: `09${i}`,\n text: [\"NL\", \"SE\", \"\", \"IN\", \"NL\"][i] ?? `9${i}`,\n })),\n { pos: \"110\", text: \"FR\" },\n { pos: \"111\", text: \"KR\" },\n { pos: \"112\", text: \"IT\" },\n { pos: \"113\", text: \"ES\" },\n] as BlockConfig[];\n\ninterface Overlay {\n x: number;\n y: number;\n width: number;\n height: number;\n}\n\ninterface BlockConfig {\n pos: IsbnPrefixRelative; // Two digits (row/col) for main blocks, four digits (row/col/subdivision) for XX\n text: string;\n color?: string;\n}\n\ninterface MinimapSVGProps {\n blocks?: BlockConfig[];\n store: Store;\n}\n\n// Constants\nconst SQRT10 = Math.sqrt(10);\nconst WIDTH = 100;\nconst HEIGHT = WIDTH * (2 / SQRT10);\nconst ROW_HEIGHT = HEIGHT / 2;\nconst CELL_WIDTH = WIDTH / 10;\nconst XX_HEIGHT = ROW_HEIGHT / 10;\n\nlet dragDistance = 0;\n\nconst MinimapSVG: React.FC = observer(\n ({ blocks = DEFAULT_BLOCKS, store }) => {\n const svgRef = useRef(null);\n\n const scale = store.projection.pixelWidth / WIDTH;\n const state = useLocalObservable(() => ({\n isDragging: false,\n dragStart: { x: 0, y: 0 },\n get overlay() {\n const fakeScale = 0.5;\n const w = store.view.width / store.projection.pixelWidth;\n const overlay = {\n x: store.view.minX / scale + fakeScale,\n y: store.view.minY / scale + fakeScale,\n width: Math.max(0.5, store.view.width / scale - w * fakeScale * 2),\n height: Math.max(0.5, store.view.height / scale - w * fakeScale * 2),\n };\n return overlay;\n },\n setOverlay(o: Overlay) {\n store.setView(\n (o.x + o.width / 2) * scale,\n (o.y + o.height / 2) * scale,\n );\n },\n // Convert screen coordinates to SVG coordinates\n getLocalCoordinates(event: React.MouseEvent | MouseEvent): {\n x: number;\n y: number;\n } {\n if (!svgRef.current) return { x: 0, y: 0 };\n\n const CTM = svgRef.current.getScreenCTM();\n if (!CTM) return { x: 0, y: 0 };\n\n const point = svgRef.current.createSVGPoint();\n point.x = event.clientX;\n point.y = event.clientY;\n const transformed = point.matrixTransform(CTM.inverse());\n\n return {\n x: transformed.x,\n y: transformed.y,\n };\n },\n\n handleMouseDown(event: React.MouseEvent) {\n dragDistance = 0;\n event.preventDefault();\n if (store.floatZoomFactor <= 1) return;\n const coords = this.getLocalCoordinates(event);\n this.isDragging = true;\n this.dragStart = {\n x: coords.x - this.overlay.x,\n y: coords.y - this.overlay.y,\n };\n window.addEventListener(\"pointermove\", this.handleMouseMove);\n window.addEventListener(\"pointerup\", this.handleMouseUp);\n },\n\n handleMouseMove(event: MouseEvent) {\n if (!this.isDragging) return;\n dragDistance++;\n const coords = this.getLocalCoordinates(event);\n const newX = Math.max(\n 0,\n Math.min(WIDTH - this.overlay.width, coords.x - this.dragStart.x),\n );\n const newY = Math.max(\n 0,\n Math.min(HEIGHT - this.overlay.height, coords.y - this.dragStart.y),\n );\n\n this.setOverlay({\n ...this.overlay,\n x: newX,\n y: newY,\n });\n },\n\n handleMouseUp() {\n this.isDragging = false;\n\n window.removeEventListener(\"pointermove\", this.handleMouseMove);\n window.removeEventListener(\"pointerup\", this.handleMouseUp);\n },\n }));\n\n return (\n \n {/* Background */}\n \n\n {/* Blocks */}\n {blocks.map((b) => (\n \n ))}\n\n {/* Grid lines */}\n \n {Array.from({ length: 11 }, (_, i) => (\n \n ))}\n {Array.from({ length: 3 }, (_, i) => (\n \n ))}\n \n\n \n {() => {\n const overlay = state.overlay;\n const widthRatio = overlay.width / WIDTH;\n\n return (\n <>\n \n {/* if very small, draw a white circle around the rect */}\n {widthRatio < 0.02 && (\n \n )}\n \n );\n }}\n \n \n );\n },\n);\n\n// Helper to parse position string\nconst parsePosition = (pos: string) => {\n if (pos.length === 2) {\n // Main block: row/column\n return {\n row: parseInt(pos[0]),\n column: parseInt(pos[1]),\n subdivision: null,\n };\n } else if (pos.length === 3) {\n // XX block: row/column/subdivision\n return {\n row: parseInt(pos[0]),\n column: parseInt(pos[1]),\n subdivision: parseInt(pos.slice(2)),\n };\n }\n throw new Error(\n \"Position must be 2 digits for main blocks or 3 digits for XX blocks\",\n );\n};\n// Color generation\nconst generateColor = (pos: string): string => {\n const { column, subdivision } = parsePosition(pos);\n const isXX = subdivision !== null;\n\n if (isXX) {\n const baseHues: Record = {\n 6: 180, // Cyan-based\n 8: 280, // Purple-based\n 9: 30, // Orange-based\n };\n const hue = (baseHues[column] || 0) + subdivision * 10;\n return `hsl(${hue}, 80%, ${60 + subdivision * 2}%)`;\n } else {\n const baseColors: Record = {\n \"00\": \"#4a90e2\", // EN\n \"01\": \"#4a90e2\", // EN\n \"02\": \"#50c878\", // FR\n \"03\": \"#daa520\", // DE\n \"04\": \"#ff6b6b\", // JP\n \"05\": \"#9370db\", // RU\n \"07\": \"#ff4500\", // CN\n \"18\": \"#4169e1\", // US\n };\n return baseColors[pos] || \"#808080\";\n }\n};\n// Helper to determine if a position represents an XX block\nconst isXXBlock = (pos: string): boolean => pos.length === 3;\n\n// Helper to get block dimensions\nconst getBlockDimensions = (pos: string) => {\n const { row, column, subdivision } = parsePosition(pos);\n\n if (subdivision !== null) {\n return {\n x: column * CELL_WIDTH,\n y: row * ROW_HEIGHT + subdivision * XX_HEIGHT,\n width: CELL_WIDTH,\n height: XX_HEIGHT,\n };\n } else {\n return {\n x: column * CELL_WIDTH,\n y: row * ROW_HEIGHT,\n width: CELL_WIDTH,\n height: ROW_HEIGHT,\n };\n }\n};\nconst RenderBlock: React.FC<{ block: BlockConfig; store: Store }> = observer(\n ({ block, store }) => {\n const { pos, text } = block;\n const dims = getBlockDimensions(pos);\n const isHovered = computed(() => store.minimapHoveredCell === pos).get();\n const isXX = isXXBlock(pos);\n\n function setHovered(pos: IsbnPrefixRelative | null) {\n store.minimapHoveredCell = pos;\n if (pos) {\n const p = isbnPrefixFromRelative(pos);\n store.highlightedStats = { prefixStart: p, prefixEnd: p };\n } else {\n store.highlightedStats = null;\n }\n }\n\n return (\n {\n setHovered(pos);\n }}\n onPointerLeave={() => {\n setHovered(null);\n }}\n onClick={() => {\n if (dragDistance > 4) return;\n const start = firstIsbnInPrefix(isbnPrefixFromRelative(pos));\n const end = lastIsbnInPrefix(isbnPrefixFromRelative(pos));\n const p = getPlanePosition(store.projection, start, end);\n store.zoomAnimateTo(\n p.xStart + p.width / 2,\n p.yStart + p.height / 2,\n { 2: 2, 3: 0.9 * Math.sqrt(10) ** 2 }[pos.length] ?? 1,\n 1,\n );\n }}\n style={{ cursor: \"pointer\" }}\n >\n \n \n {text}\n \n \n );\n },\n);\n\nexport const MiniMap: React.FC<{ store: Store }> = observer(\n function MiniMap(props) {\n return (\n
    \n \n {props.store.resetZoomButton && (\n {\n props.store.resetZoomButton = false;\n props.store.zoomAnimateTo(\n props.store.projection.pixelWidth / 2,\n props.store.projection.pixelHeight / 2,\n 1,\n 1,\n );\n }}\n >\n Reset Zoom\n \n )}\n
    \n );\n },\n);\n" }, { "path": "src/components/SingleBookCover.tsx", "content": "import * as isbnlib from \"isbn3\";\nimport { observer } from \"mobx-react-lite\";\nimport { fromPromise } from \"mobx-utils\";\nimport { useMemo } from \"react\";\nimport { Store } from \"../lib/Store\";\nimport { IsbnStrWithChecksum } from \"../lib/util\";\nimport { EanBarcode } from \"./EanBarcode\";\n\nfunction dot(v1: [number, number], v2: [number, number]) {\n return v1[0] * v2[0] + v1[1] * v2[1];\n}\n\n// Helper function to emulate GLSL's fract function\n// Returns the fractional part of a number\nfunction fract(x: number) {\n return x - Math.floor(x);\n}\n\n// Random function translated from GLSL\n// Takes an array of 2 numbers (representing vec2)\nfunction rande(co: [number, number]) {\n return fract(Math.sin(dot(co, [12.9898, 78.233])) * 2);\n}\n\nexport function bookHeight(bookIndex: [number, number]) {\n const minBookHeight = 0.8;\n const maxBookHeight = 0.95;\n const r = 1.2;\n const re = rande([bookIndex[0] * r, bookIndex[1] * r]);\n return minBookHeight + (maxBookHeight - minBookHeight) * re;\n}\n\nexport const SingleBookCover = observer(function SingleBookCover({\n isbn,\n store,\n}: {\n store: Store;\n isbn: IsbnStrWithChecksum;\n}) {\n const fetchTitleJson = useMemo(\n () => fromPromise(store.titleFetcher.fetchTitle(isbn)),\n [isbn],\n );\n const titleInfo = store.cachedGoogleBooks.get(isbn);\n const [y1, x1, y2, x2, y3, x3, _checksum] = isbn.slice(-7);\n const [x, y] = [+(x1 + x2 + x3), +(y1 + y2 + y3)];\n const bookHeightE = bookHeight([x, 999 - y]);\n // console.log(isbn, x, y);\n const title =\n titleInfo?.volumeInfo.title ??\n fetchTitleJson.case({ fulfilled: (t) => t?.title });\n const author =\n titleInfo?.volumeInfo.authors?.join(\", \") ??\n fetchTitleJson.case({ fulfilled: (t) => t?.creator });\n return (\n \n
    \n
    \n
    ISBN {isbnlib.hyphenate(isbn)}
    \n \n
    \n
    \n 40\n ? \"muchtext\"\n : \"littletext\"\n }`}\n >\n
    {title}
    \n
    \n {author ? `by ${author}` : \"\"}\n
    \n \n \n );\n});\n" }, { "path": "src/components/StatsShow.tsx", "content": "import { Html, Plane } from \"@react-three/drei\";\nimport { observer, useLocalObservable } from \"mobx-react-lite\";\nimport { fromPromise } from \"mobx-utils\";\nimport React, { useMemo } from \"react\";\nimport { MeshBasicMaterial } from \"three\";\nimport { BlockStats } from \"../lib/stats\";\nimport { Store } from \"../lib/Store\";\nimport {\n firstIsbnInPrefix,\n IsbnPrefixWithDashes,\n IsbnPrefixWithoutDashes,\n lastIsbnInPrefix,\n removeDashes,\n} from \"../lib/util\";\nimport { getPlanePosition } from \"../lib/view-utils\";\n\nexport const maxZoomForStats = 40;\nexport const StatsShow: React.FC<{ store: Store }> = observer(\n function StatsShow({ store }) {\n const material = useMemo(\n () =>\n new MeshBasicMaterial({\n color: \"#ccffcc\",\n transparent: true,\n opacity: 0.9,\n }),\n [],\n );\n const state = useLocalObservable(() => ({\n get edge() {\n if (!store.highlightedStats) return null;\n let p1: string = store.highlightedStats.prefixStart;\n let p2: string = store.highlightedStats.prefixEnd;\n p1 = p1.slice(0, p2.length);\n p2 = p2.slice(0, p1.length);\n if (p2 < p1) [p1, p2] = [p2, p1];\n while (p1.slice(0, -1) !== p2.slice(0, -1)) {\n p1 = p1.slice(0, -1);\n p2 = p2.slice(0, -1);\n }\n /*if (p2 > p1) {\n p2 = String(+p2 - 1);\n }*/\n return [p1 as IsbnPrefixWithoutDashes, p2 as IsbnPrefixWithoutDashes];\n },\n get stats() {\n if (!this.edge) return null;\n return fromPromise(\n store.statsCalculator.getStats(this.edge[0], this.edge[1]),\n );\n },\n }));\n if (!state.edge) return null;\n const [p1, p2] = state.edge;\n\n const start = firstIsbnInPrefix(p1);\n const end = lastIsbnInPrefix(p2);\n const plane = getPlanePosition(store.projection, start, end);\n\n return (\n \n \n {store.floatZoomFactor > maxZoomForStats && (\n \n \n Zoom out to view stats\n \n \n )}\n \n
    \n {state.stats?.case({\n pending: () => <>Loading...,\n rejected: () => <>Error,\n fulfilled: (stats) => (\n <>\n e + \"-\") as IsbnPrefixWithDashes\n }\n prefixEnd={\n p2.replace(/^.../, (e) => e + \"-\") as IsbnPrefixWithDashes\n }\n />\n
    \n Details\n \n \n \n \n \n \n \n {Object.entries(stats).map((s) => (\n \n \n \n \n \n ))}\n \n
    {(stats.dataset_all ?? 0).toLocaleString()}\n books total\n
    {s[1]?.toLocaleString()}in {s[0]}\n (\n {(((s[1] ?? 0) / (stats.dataset_all ?? 0)) * 100)\n .toFixed(2)\n .padStart(5, \" \")}\n %)\n
    \n
    \n Right click to close\n \n ),\n })}\n
    \n \n     \n );\n },\n);\nexport const AbbrevStats: React.FC<{\n store: Store;\n prefixStart: IsbnPrefixWithDashes;\n prefixEnd: IsbnPrefixWithDashes;\n}> = observer(function AbbrevStats({ store, prefixStart, prefixEnd }) {\n const [p1, p2] = [removeDashes(prefixStart), removeDashes(prefixEnd)];\n const stats = useMemo(\n () => fromPromise(store.statsCalculator.getStats(p1, p2)),\n [p1, p2],\n );\n return (\n
    \n {stats.case({\n fulfilled: (r) => (\n \n ),\n })}\n
    \n );\n});\n\nfunction StatsSummary(props: {\n stats: BlockStats;\n prefixStart: IsbnPrefixWithDashes;\n prefixEnd: IsbnPrefixWithDashes;\n}) {\n const r = props.stats;\n if (!r.dataset_all) return <>;\n return (\n
    \n Stats for {props.prefixStart}\n {props.prefixStart !== props.prefixEnd ? <> to {props.prefixEnd} : \"\"}:\n
    \n Known books: {r.dataset_all.toLocaleString()}\n
    \n dataset_md5: {r.dataset_md5 ?? 0} (\n {(((r.dataset_md5 ?? 0) / r.dataset_all) * 100).toFixed(2)}\n %)\n
    \n Average publication year:{\" \"}\n {r.publication_date_count && (\n <>\n {((r.publication_date ?? 0) / r.publication_date_count).toFixed(0)} (\n {r.publication_date_count.toFixed(0)} samples)\n
    \n \n )}\n Average holdings:{\" \"}\n {((r.rarity_holdingCount ?? 0) / (r.rarity_exists ?? 0)).toFixed(1)}{\" \"}\n libraries\n
    \n Average editions:{\" \"}\n {((r.rarity_editionCount ?? 0) / (r.rarity_exists ?? 0)).toFixed(1)}\n
    \n );\n}\n" }, { "path": "src/components/TextTree.tsx", "content": "import { Html } from \"@react-three/drei\";\nimport { observer } from \"mobx-react-lite\";\nimport { useEffect, useState } from \"react\";\nimport { useDelay } from \"../lib/delayRender\";\nimport { DetailLevelObservable } from \"../lib/DetailLevelObservable\";\nimport { DIGITS, LazyPrefixInfo } from \"../lib/info-map\";\nimport { getGroup, resolveOnePrefixLevel } from \"../lib/prefix-data\";\nimport { Store } from \"../lib/Store\";\nimport {\n calculateCheckDigit,\n digits,\n isbnPrefixAppend,\n IsbnPrefixWithDashes,\n IsbnStrWithChecksum,\n ProjectionConfig,\n removeDashes,\n} from \"../lib/util\";\nimport { SingleBookCover } from \"./SingleBookCover\";\nexport const TextTree: React.FC<{\n config: ProjectionConfig;\n store: Store;\n prefix: IsbnPrefixWithDashes;\n}> = observer(function _TextTree(props) {\n const groupPrefix = removeDashes(props.prefix);\n\n const view = props.store.getDetailLevel(groupPrefix);\n if (!view.container) return null;\n return ;\n});\nconst GroupShowInner: React.FC<{\n config: ProjectionConfig;\n store: Store;\n prefix: IsbnPrefixWithDashes;\n view: DetailLevelObservable;\n}> = observer(function _GroupShowInner({ view, ...props }) {\n const { position, width, height } = view.planePosition;\n const [groupO, setGroupO] = useState(null);\n\n useEffect(() => {\n void (async () => {\n // resolve group plus one child level\n let g = getGroup(props.store.rootPrefixInfo, props.prefix);\n const jsonRoot = props.store.runtimeConfig.jsonRoot;\n if (typeof g === \"function\")\n g = await props.store.trackAsyncProgress(\n `resolvePublishers(${props.prefix})`,\n g(jsonRoot),\n );\n if (g?.children && \"lazy\" in g.children) {\n await resolveOnePrefixLevel(g, jsonRoot);\n }\n setGroupO(g);\n })();\n }, [props.prefix]);\n return (\n <>\n \n\n {view.textChildren &&\n // groupO?.children &&\n digits.map((i) => {\n return (\n \n );\n })}\n \n );\n});\nconst RenderGroup: React.FC<{\n store: Store;\n group: LazyPrefixInfo | null;\n prefix: IsbnPrefixWithDashes;\n position: [number, number, number];\n width: number;\n height: number;\n view: DetailLevelObservable;\n}> = observer(function _RenderGroup({\n store,\n group,\n prefix,\n position,\n width,\n height,\n view,\n}) {\n const shouldDelay = useDelay();\n if (!shouldDelay) return null;\n const plainPrefix = removeDashes(prefix);\n const isSingleBook = plainPrefix.length === 11;\n // console.log(\"RenderGroup \" + prefix);\n if (!view.textOpacity || (!group?.children && !isSingleBook)) return null;\n const smSize = Math.min(width, height);\n const vertical = height > width;\n const showVertical = store.runtimeConfig.groupTextVertical || isSingleBook;\n if (group || isSingleBook) {\n return (\n \n \n {isSingleBook ? (\n \n ) : group ? (\n \n ) : (\n \"impossible\"\n )}\n \n \n );\n }\n return null;\n});\n\nfunction ChildBooks(props: {\n prefix: IsbnPrefixWithDashes;\n vertical: boolean;\n showVertical: boolean;\n store: Store;\n}) {\n const plainPrefix = removeDashes(props.prefix);\n return (\n
    \n {DIGITS.map((digit) => (\n
    \n \n
    \n ))}\n
    \n );\n}\n\nfunction ChildGroupNames(props: {\n prefix: IsbnPrefixWithDashes;\n group: LazyPrefixInfo;\n vertical: boolean;\n showVertical: boolean;\n store: Store;\n}) {\n if (!props.group.children) return null;\n if (\"lazy\" in props.group.children) {\n console.warn(\"lazy group, should be impossible\", props.prefix);\n return null;\n }\n const children = props.group.children;\n const prefixWithAppendedDash = props.group.info?.[0].prefix\n ? props.group.info[0].prefix + \"-\"\n : props.prefix;\n return (\n
    \n {DIGITS.map((digit) => {\n const child = children[digit];\n return (\n \n {child && (\n \n )}\n
    \n );\n })}\n \n );\n}\nconst GroupNameTxt = function GroupName(props: {\n prefix: string;\n group: LazyPrefixInfo;\n}) {\n const firstInfo = props.group.info?.[0];\n const infoCount = props.group.info?.length ?? 0;\n if (!firstInfo) {\n return (\n \n {`${props.group.totalChildren} publisher${\n props.group.totalChildren > 1 ? \"s\" : \"\"\n }`}\n
    \n {`(${props.prefix})`}\n     \n );\n }\n if (firstInfo.source === \"isbngrp\") {\n return (\n <>\n {firstInfo.registrant_name}\n
    \n \n {`(${firstInfo.prefix}) ${\n infoCount > 1 ? `(+${infoCount - 1} more)` : \"\"\n }`}\n \n \n );\n }\n return (\n <>\n {firstInfo.name}\n
    \n {`(${firstInfo.prefix}-)`}\n \n );\n};\n" }, { "path": "src/config.ts", "content": "import { MinimalGoogleBooksItem } from \"./components/Controls\";\nimport { RuntimeConfiguration } from \"./lib/RuntimeConfiguration\";\nimport { IsbnStrWithChecksum } from \"./lib/util\";\n\nexport interface DatasetOption {\n id: string;\n name: string;\n description?: string;\n runtimeConfig?: Partial;\n colorSchemeMeaning?: ColorSchemeMeaning | null;\n}\nexport interface ColorSchemeMeaning {\n title: string;\n markers: { value: number; label: string }[];\n}\nexport const defaultColorSchemeMeaning = {\n title: \"Books\",\n markers: [\n { value: 0, label: \"0%\" },\n { value: 0.5, label: \"50% allocated\" },\n { value: 1, label: \"100%\" },\n ],\n};\nexport default {\n bookshelfColor: [0.5, 0.1, 0.1, 1.0],\n bookshelfColorHex: \"#7f1a1a\",\n datasetOptions: [\n {\n id: \"all\",\n name: \"All Known Books\",\n description: \"Books in various sources\",\n },\n {\n id: \"publication_date\",\n name: \"Publication Date\",\n description: \"Shows the publication year of books\",\n runtimeConfig: {\n shaderGlow: 4,\n colorGradient: 2,\n },\n colorSchemeMeaning: {\n title: \"Publication year\",\n markers: [\n { value: 0, label: \"≤1985\" },\n { value: 0.25, label: \"\" },\n // { value: (2000 - 1985) / (2025 - 1985), label: \"2000\" },\n { value: 0.5, label: String((2025 + 1985) / 2) },\n { value: 0.75, label: \"\" },\n { value: 1, label: \"2025\" },\n ],\n },\n },\n {\n id: \"all-md5\",\n name: \"All ISBNs (red), md5s (green)\",\n description:\n \"Shows which proportion of books have at least one file in AA.\",\n runtimeConfig: {\n colorGradient: 4,\n },\n colorSchemeMeaning: {\n title: \"File Availability\",\n markers: [\n { value: 0, label: \"Missing\" },\n { value: 0.5, label: \"50% present\" },\n { value: 1, label: \"100%\" },\n ],\n },\n },\n {\n id: \"rarity\",\n name: \"Rarity data\",\n description:\n \"Shows which books are rare, based on how many libraries they are in.\",\n colorSchemeMeaning: {\n title: \"Rarity\",\n markers: Array.from({ length: 21 }).map((_, i) => ({\n value: (i / 20.0) ** 2,\n label: { 0: \"0 libraries\", 10: \"10\", 20: \"20+\" }[i] ?? \"\",\n })),\n },\n runtimeConfig: {\n colorGradient: 4,\n },\n },\n {\n id: \"publishers\",\n name: \"Publisher Ranges\",\n description:\n \"Assigns a random color to each unique publisher so the prefixes of each one are visible.\",\n runtimeConfig: {\n publishersColorSchema: \"hsl\",\n },\n colorSchemeMeaning: null,\n },\n {\n id: \"gbooks\",\n name: \"Google Books\",\n description: \"Books that are or were present in Google Books are white.\",\n },\n { id: \"md5\", name: \"Files in AA\" },\n { id: \"cadal_ssno\", name: \"CADAL SSNOs\" },\n { id: \"cerlalc\", name: \"CERLALC data leak\" },\n { id: \"duxiu_ssid\", name: \"DuXiu SSIDs\" },\n { id: \"edsebk\", name: \"EBSCOhost’s eBook Index\" },\n { id: \"goodreads\", name: \"Goodreads\" },\n { id: \"ia\", name: \"Internet Archive\" },\n { id: \"isbndb\", name: \"ISBNdb\" },\n { id: \"isbngrp\", name: \"ISBN Global Register of Publishers\" },\n { id: \"libby\", name: \"Libby\" },\n { id: \"nexusstc\", name: \"Nexus/STC\" },\n { id: \"oclc\", name: \"OCLC/Worldcat\" },\n { id: \"ol\", name: \"OpenLibrary\" },\n { id: \"rgb\", name: \"Russian State Library\" },\n { id: \"trantor\", name: \"Imperial Library of Trantor\" },\n ] as DatasetOption[],\n exampleBooks: [\n {\n id: \"gatsby\",\n volumeInfo: {\n title: \"The Great Gatsby\",\n authors: [\"F. Scott Fitzgerald\"],\n industryIdentifiers: [\n {\n type: \"ISBN_13\",\n identifier: \"9780743273565\" as IsbnStrWithChecksum,\n },\n ],\n },\n },\n {\n id: \"ctacher\",\n volumeInfo: {\n title: \"The Catcher in the Rye\",\n authors: [\"J.D. Salinger\"],\n industryIdentifiers: [\n {\n type: \"ISBN_13\",\n identifier: \"9780316769488\" as IsbnStrWithChecksum,\n },\n ],\n },\n },\n {\n id: \"got\",\n volumeInfo: {\n title: \"A Game of Thrones\",\n authors: [\"George R. R. Martin\"],\n industryIdentifiers: [\n {\n type: \"ISBN_13\",\n identifier: \"9780553381689\" as IsbnStrWithChecksum,\n },\n ],\n },\n },\n {\n id: \"hp1\",\n volumeInfo: {\n title: \"Harry Potter and the Philosopher's Stone\",\n authors: [\"J.K. Rowling\"],\n industryIdentifiers: [\n {\n type: \"ISBN_13\",\n identifier: \"9780590353427\" as IsbnStrWithChecksum,\n },\n ],\n },\n },\n ] as MinimalGoogleBooksItem[],\n externalSearchEngines: [\n {\n name: \"Google Books\",\n url: \"https://books.google.com/books?vid=ISBN%s\", //\"https://www.google.com/search?udm=36&q=isbn%3A%s\",\n },\n { name: \"Worldcat\", url: \"https://worldcat.org/isbn/%s\" },\n ],\n jsonCompression: \"gzip\",\n};\n" }, { "path": "src/index.css", "content": "html,\nbody {\n width: 100%;\n height: 100%;\n box-sizing: border-box;\n margin: 0;\n font-family: system-ui, sans-serif;\n}\n* {\n box-sizing: border-box;\n}\n#root {\n width: 100%;\n height: 100%;\n --group-width: 252px;\n --group-height: calc(var(--group-width) / sqrt(10));\n}\n\ncode {\n font-family: source-code-pro, Menlo, Monaco, Consolas, \"Courier New\",\n monospace;\n}\n\n.no-cursor {\n pointer-events: none;\n}\n\n.controls {\n position: absolute;\n top: 1rem;\n left: 1rem;\n background: #ffffff;\n padding: 1.5rem;\n z-index: 100;\n width: 400px;\n max-width: calc(100vw - 2rem);\n border-radius: 12px;\n box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.1),\n 0 2px 4px -1px rgba(0, 0, 0, 0.06);\n transition: all 0.3s ease;\n max-height: calc(100vh - 2rem);\n border: 1px solid rgba(229, 231, 235, 0.5);\n}\n\n.controls:hover {\n box-shadow: 0 10px 15px -3px rgba(0, 0, 0, 0.1),\n 0 4px 6px -2px rgba(0, 0, 0, 0.05);\n}\n\n.controls .head {\n display: flex;\n flex-direction: row;\n align-items: flex-start;\n justify-content: space-between;\n gap: 0.5rem;\n margin-bottom: 0.75rem;\n flex-wrap: wrap;\n}\n\n.controls .head b {\n font-weight: 600;\n color: #111827;\n line-height: 1.1;\n margin-right: 0.5rem;\n}\n\nbutton {\n padding: 0.375rem 0.75rem;\n border-radius: 6px;\n border: 1px solid #e5e7eb;\n background: #f9fafb;\n color: #374151;\n font-size: 0.875rem;\n transition: all 0.2s ease;\n cursor: pointer;\n}\n.controls button.preset {\n display: block;\n width: 100%;\n}\n\n.controls button:hover {\n background: #f3f4f6;\n border-color: #d1d5db;\n}\n\n.controls button:active {\n background: #e5e7eb;\n transform: translateY(1px);\n}\n\n.controls.advanced {\n background: #f8fafc;\n}\n\n@media (max-width: 640px) {\n .controls {\n top: 0;\n left: 0;\n right: 0;\n padding: 0.75rem;\n max-width: 100vw;\n border-radius: 0;\n max-height: 70vh;\n border-bottom-left-radius: 12px;\n border-bottom-right-radius: 12px;\n }\n\n .controls .head {\n padding-bottom: 0.5rem;\n margin-bottom: 0.5rem;\n gap: 0.375rem;\n }\n\n .controls .head b {\n margin-right: 0.375rem;\n }\n\n .controls button {\n padding: 0.25rem 0.5rem;\n font-size: 0.813rem;\n }\n\n .controls p {\n margin: 0.5rem 0;\n font-size: 0.875rem;\n line-height: 1.25;\n }\n\n .controls label {\n font-size: 0.875rem;\n }\n}\n\n.controls.advanced {\n max-height: 100vh;\n overflow-y: auto;\n}\n\n.controls .form-row {\n display: flex;\n align-items: center;\n gap: 1rem;\n}\n.controls .form-row > div:first-child {\n flex: 1;\n}\n.stats-table {\n td:first-child {\n text-align: right;\n }\n}\n.group-name {\n color: white;\n /*text-shadow: rgb(0, 0, 0) 2px 0px 0px, rgb(0, 0, 0) 1.75517px 0.958851px 0px,\n rgb(0, 0, 0) 1.0806px 1.68294px 0px, rgb(0, 0, 0) 0.141474px 1.99499px 0px,\n rgb(0, 0, 0) -0.832294px 1.81859px 0px,\n rgb(0, 0, 0) -1.60229px 1.19694px 0px, rgb(0, 0, 0) -1.97999px 0.28224px 0px,\n rgb(0, 0, 0) -1.87291px -0.701566px 0px,\n rgb(0, 0, 0) -1.30729px -1.51361px 0px,\n rgb(0, 0, 0) -0.421592px -1.95506px 0px,\n rgb(0, 0, 0) 0.567324px -1.91785px 0px,\n rgb(0, 0, 0) 1.41734px -1.41108px 0px,\n rgb(0, 0, 0) 1.92034px -0.558831px 0px, rgb(0, 0, 0) 0 0 8px,\n rgb(0, 0, 0) 0 0 8px, rgb(0, 0, 0) 0 0 8px, rgb(0, 0, 0) 0 0 8px,\n rgb(0, 0, 0) 0 0 8px, rgb(0, 0, 0) 0 0 8px;*/\n /* border: 1px solid black; */\n text-align: center;\n align-content: center;\n overflow: hidden;\n width: var(--group-width);\n font-size: 1.2rem;\n height: var(--group-height);\n -webkit-text-stroke: 7px black;\n text-stroke: 7px black;\n paint-order: stroke fill;\n}\n.group-name.vertical {\n float: left;\n width: var(--group-height);\n height: var(--group-width);\n font-size: 0.9rem;\n}\n.group-name-wrap.vertical {\n width: calc(var(--group-width) * sqrt(10));\n}\n.group-name span {\n /* background: black; */\n}\n.group-name small {\n font-size: 75%;\n color: #ddd;\n}\n\n.lds-dual-ring {\n /* change color here */\n color: #1c4c5b;\n}\n.lds-dual-ring,\n.lds-dual-ring:after {\n box-sizing: border-box;\n}\n.lds-dual-ring {\n display: inline-block;\n width: 20px;\n height: 20px;\n}\n.lds-dual-ring:after {\n content: \" \";\n display: block;\n width: 16px;\n height: 16px;\n margin: 2px;\n border-radius: 50%;\n border: 6.4px solid currentColor;\n border-color: currentColor transparent currentColor transparent;\n animation: lds-dual-ring 1.2s linear infinite;\n}\n@keyframes lds-dual-ring {\n 0% {\n transform: rotate(0deg);\n }\n 100% {\n transform: rotate(360deg);\n }\n}\n\n@font-face {\n font-family: \"Libre Barcode EAN13 Text\";\n src: url(./LibreBarcodeEAN13Text-Regular.ttf) format(\"truetype\");\n}\n.ean13 {\n font-family: \"Libre Barcode EAN13 Text\", \"Adobe NotDef\";\n /* Setting this explicitly was necessary for IOS, version 13.7, Safari and Chrome.*/\n font-size: 3rem;\n font-feature-settings: \"calt\" 1;\n overflow: auto;\n}\n\n.single-book-wrap {\n width: var(--group-width);\n height: var(--group-height);\n overflow: hidden;\n}\n.single-book {\n color: black;\n width: 100%;\n height: 100%;\n /* border-right: 5px solid green; */\n display: flex;\n padding-left: 3%; /** shader bookshelf height 0.03 */\n flex-direction: row;\n align-items: center;\n}\n.single-book .titleinfo {\n flex-grow: 1;\n text-align: center;\n}\n.single-book .titleinfo .author {\n font-size: 0.8rem;\n}\n.single-book .titleinfo.muchtext {\n font-size: 0.7rem;\n}\n.single-book .titleinfo.muchtext .author {\n font-size: 0.6rem;\n}\n.single-book .unknown {\n opacity: 0.5;\n}\n.single-book .isbn {\n font-size: 0.3rem;\n}\n.single-book .isbn-and-barcode > div {\n display: block;\n transform: rotate(90deg);\n text-align: center;\n}\n.single-book .ean13 {\n display: block;\n font-size: 2.8rem;\n}\n\n.dataset-chooser-wrap {\n background: rgba(0, 0, 0, 0.6);\n position: fixed;\n top: 0;\n left: 0;\n width: 100%;\n height: 100%;\n}\n.dataset-chooser {\n position: fixed;\n top: 50%;\n left: 50%;\n transform: translate(-50%, -50%);\n background: white;\n min-width: 300px;\n max-width: 400px;\n max-height: 70vh;\n overflow: auto;\n padding: 1em;\n box-shadow: 0 0 4px 4px black;\n border-radius: 12px;\n}\n.dataset-chooser button {\n display: inline-block;\n}\n\n.dataset-chooser button.choose-dataset {\n margin-bottom: 1em;\n width: 100%;\n}\n\n.isbn-highlight,\n.stats-highlight {\n background: white;\n box-shadow: rgba(0, 0, 0, 0.2) 0px 8px 24px;\n border-radius: 8px;\n padding: 1rem;\n min-width: 350px;\n max-width: 450px;\n font-size: 0.9375rem;\n line-height: 1.5;\n border: 1px solid rgba(0, 0, 0, 0.1);\n max-height: 400px;\n overflow-y: auto;\n}\n.isbn-highlight {\n transform: translateX(-50%) translateY(10px);\n}\n\n.isbn-highlight h2 {\n font-size: 1.25rem;\n font-weight: 600;\n margin: 0 0 0.5rem 0;\n color: #111827;\n}\n\n.isbn-highlight .isbn-title {\n font-size: 1rem;\n font-weight: 600;\n margin-bottom: 0.75rem;\n}\n\n.isbn-highlight .group-info {\n margin: 0.75rem 0;\n padding: 0.75rem 0;\n border-top: 1px solid #e5e7eb;\n border-bottom: 1px solid #e5e7eb;\n}\n\n.isbn-highlight .stats-section {\n display: grid;\n grid-template-columns: auto 1fr;\n gap: 0.5rem 1rem;\n margin: 0.75rem 0;\n}\n\n.isbn-highlight .stats-label {\n font-weight: 500;\n color: #4b5563;\n}\n\n.isbn-highlight .stats-value {\n color: #111827;\n}\n\n.isbn-highlight .instructions {\n margin-top: 0.75rem;\n font-weight: 500;\n color: #4b5563;\n}\n\n.isbn-highlight small {\n display: block;\n margin-top: 0.25rem;\n color: #6b7280;\n}\n\n.isbn-highlight button {\n padding: 0.375rem 0.75rem;\n border-radius: 6px;\n border: 1px solid #e5e7eb;\n background: #f9fafb;\n color: #374151;\n font-size: 0.875rem;\n transition: all 0.2s ease;\n cursor: pointer;\n}\n\n.isbn-highlight button:hover {\n background: #f3f4f6;\n border-color: #d1d5db;\n}\n\n.isbn-highlight img {\n margin-right: 0.5rem;\n max-height: 85px;\n border: 1px solid #e5e7eb;\n border-radius: 4px;\n}\n\n.isbn-highlight details {\n margin-top: 0.75rem;\n}\n\n.isbn-highlight summary {\n color: #4b5563;\n font-weight: 500;\n margin-bottom: 0.5rem;\n}\n\n.isbn-highlight ul {\n margin: 0.5rem 0;\n padding-left: 1.5rem;\n}\n\n.isbn-highlight li {\n margin: 0.25rem 0;\n}\n\n.isbn-highlight a {\n color: #2563eb;\n text-decoration: none;\n}\n\n.isbn-highlight a:hover {\n text-decoration: underline;\n}\n\n.float-button {\n float: right;\n margin-left: 1ex;\n}\n\n.minimap {\n position: absolute;\n right: 0;\n background: white;\n top: 0;\n margin: 1rem;\n width: 300px;\n height: calc(300px * 2 / sqrt(10));\n z-index: 100;\n border-radius: 1em;\n overflow: hidden;\n box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.1),\n 0 2px 4px -1px rgba(0, 0, 0, 0.06);\n border: 2px solid black;\n background: black;\n}\n\n@media (max-width: 640px) {\n .minimap {\n top: auto;\n bottom: 0;\n margin: 0;\n width: 180px;\n height: calc(180px * 2 / sqrt(10));\n }\n .minimap button {\n padding: 0.1rem;\n }\n}\n" }, { "path": "src/index.tsx", "content": "import { isWebGL2Available } from \"@react-three/drei\";\nimport { configure } from \"mobx\";\nimport { StrictMode } from \"react\";\nimport { createRoot } from \"react-dom/client\";\nimport App from \"./App\";\nimport \"./index.css\";\nconfigure({ enforceActions: \"never\", computedRequiresReaction: true });\n\nconst root = document.getElementById(\"root\");\nif (!root) throw new Error(\"No root element found with id 'root'\");\ncreateRoot(root).render(\n \n \n ,\n);\n\nfunction testCompat() {\n const available = isWebGL2Available();\n if (!available) alert(\"WebGL2 not available, please upgrade your browser!\");\n}\n\ntestCompat();\n" }, { "path": "src/lib/DetailLevelObservable.ts", "content": "import { computed, makeObservable } from \"mobx\";\nimport { Store } from \"./Store\";\nimport {\n IMG_WIDTH,\n IsbnPrefixWithoutDashes,\n IsbnRelative,\n firstIsbnInPrefix,\n lastIsbnInPrefix,\n} from \"./util\";\nimport { getPlanePosition, simplifyView } from \"./view-utils\";\n\nconst minPrefixLength = 3; // 978-, 979\nconst maxPrefixLength = 11;\n\nexport class DetailLevelObservable {\n planePosition: ReturnType;\n rect: { xStart: number; xEnd: number; yStart: number; yEnd: number };\n isbnStart: IsbnRelative;\n isbnEnd: IsbnRelative;\n parent: DetailLevelObservable | null = null;\n constructor(\n private store: Store,\n private prefix: IsbnPrefixWithoutDashes,\n ) {\n makeObservable(this, {\n viewVisible: computed,\n container: computed,\n textOpacity: computed,\n textChildren: computed,\n imageChildren: computed,\n image: computed,\n });\n this.parent =\n prefix.length > 2\n ? store.getDetailLevel(prefix.slice(0, -1) as IsbnPrefixWithoutDashes)\n : null;\n this.isbnStart = firstIsbnInPrefix(prefix);\n this.isbnEnd = lastIsbnInPrefix(prefix);\n\n this.planePosition = getPlanePosition(\n store.projection,\n this.isbnStart,\n this.isbnEnd,\n );\n this.rect = this.planePosition;\n }\n get viewVisible() {\n if (this.parent?.viewVisible === \"invisible\") return \"invisible\";\n if (this.parent?.viewVisible === \"visible\") return \"visible\";\n const v = simplifyView(this.store.view, this.rect);\n return v;\n }\n get container() {\n // return this.prefix.length < 8;\n // console.log(\"update container\", this.prefix);\n if (this.viewVisible === \"invisible\") return false;\n return true;\n }\n get textOpacity() {\n const innermost = this.prefix.length === maxPrefixLength;\n const outermost = this.prefix.length === minPrefixLength;\n const textSwitchLevel = this.store.runtimeConfig.textMinZoomLevel;\n const textSwitchLevelFull = textSwitchLevel * 1.5 - textSwitchLevel;\n const opa1 = outermost\n ? 1\n : Math.max(\n 0,\n Math.min(\n 1,\n (getScale(this.rect, this.store, 0) - textSwitchLevel) /\n textSwitchLevelFull,\n ),\n );\n // show 2 levels at the same time\n const showLevels = this.store.runtimeConfig.textLevelCount;\n const opa2 = innermost\n ? 1\n : Math.max(\n 0,\n Math.min(\n 1,\n 1 -\n (getScale(this.rect, this.store, -showLevels) - textSwitchLevel) /\n textSwitchLevelFull,\n ),\n );\n return Math.min(opa1, opa2);\n }\n get textChildren() {\n const outermost = this.prefix.length === minPrefixLength;\n return (\n this.prefix.length <= maxPrefixLength &&\n (outermost ||\n getScale(this.rect, this.store, -1) >=\n this.store.runtimeConfig.textMinZoomLevel)\n );\n }\n\n #imageRelativeLevel(relativeLevel: number) {\n if (this.viewVisible === \"invisible\") return false;\n if (this.prefix.length === minPrefixLength) return true;\n const nextLargerImgScale = getScale(\n this.rect,\n this.store,\n relativeLevel + 1,\n );\n return (\n this.prefix.length <= 6 &&\n nextLargerImgScale >= this.store.runtimeConfig.imgMinZoomLevel\n );\n }\n\n get image() {\n return this.#imageRelativeLevel(0);\n }\n get imageChildren() {\n return this.#imageRelativeLevel(-1);\n }\n}\nexport interface DetailLevel {\n container: boolean;\n text: boolean;\n children: boolean;\n}\nexport function getScale(\n rect: { xEnd: number; xStart: number; yEnd: number; yStart: number },\n store: Store,\n relativeLevel: number,\n) {\n const imgWidthInPixels = (rect.xEnd - rect.xStart) * store.floatZoomFactor;\n const isVertical = rect.xEnd - rect.xStart < rect.yEnd - rect.yStart;\n const imgScale =\n imgWidthInPixels / (isVertical ? IMG_WIDTH / Math.sqrt(10) : IMG_WIDTH);\n const nextImgScale = imgScale * 10 ** (relativeLevel / 2);\n return nextImgScale;\n}\n" }, { "path": "src/lib/ImageLoader.ts", "content": "import {\n LinearFilter,\n LinearMipMapLinearFilter,\n MagnificationTextureFilter,\n MinificationTextureFilter,\n NearestFilter,\n Texture,\n TextureLoader,\n} from \"three\";\nimport { Store } from \"./Store\";\nimport { IsbnPrefixRelative } from \"./util\";\nexport class ImageLoader {\n path: string;\n loader: TextureLoader = new TextureLoader();\n textures = new Map();\n existing: Promise>;\n hasChildren: Promise>;\n static maxZoomPrefixLength = 4; // images with nearest zoom have prefix length 4\n minFilter: MinificationTextureFilter = LinearMipMapLinearFilter;\n magFilter: MagnificationTextureFilter = LinearFilter;\n constructor(\n root: string,\n private dataset: string,\n private store: Store,\n ) {\n this.path = `${root}/${dataset}`;\n this.minFilter = dataset === \"publishers\" ? NearestFilter : NearestFilter;\n\n this.magFilter = dataset === \"publishers\" ? NearestFilter : NearestFilter;\n this.existing = store.trackAsyncProgress(\n `${this.path}/written.json`,\n this.loadExisting(),\n );\n this.hasChildren = this.loadHasChildren();\n }\n private async loadExisting() {\n try {\n const res = await fetch(`${this.path}/written.json`);\n const json = (await res.json()) as IsbnPrefixRelative[];\n return new Set(json);\n } catch (cause) {\n throw Error(`Could not load written.json for ${this.dataset}`, { cause });\n }\n }\n private async loadHasChildren() {\n const existing = await this.existing;\n const out = new Set();\n for (const prefix of existing) {\n for (let i = 1; i < prefix.length; i++) {\n out.add(prefix.slice(0, i) as IsbnPrefixRelative);\n }\n }\n return out;\n }\n async getHasChildren(prefix: IsbnPrefixRelative): Promise {\n const hasChildren = await this.hasChildren;\n return hasChildren.has(prefix);\n }\n async getTexture(prefix: IsbnPrefixRelative): Promise {\n const gotten = this.textures.get(prefix);\n if (gotten) {\n return gotten;\n }\n if (!(await this.existing).has(prefix)) {\n return null;\n }\n try {\n const path = `${this.path}/zoom-${prefix.length}/${prefix}.png`;\n const t = await this.store.trackAsyncProgress(\n `loadTexture(${path})`,\n this.loader.loadAsync(path),\n );\n if (prefix.length === ImageLoader.maxZoomPrefixLength)\n t.magFilter = NearestFilter;\n else t.magFilter = this.magFilter;\n // t.colorSpace = THREE.SRGBColorSpace;\n t.minFilter = this.minFilter;\n this.textures.set(prefix, t);\n return t;\n } catch (e) {\n console.error(e);\n return null;\n }\n }\n}\n" }, { "path": "src/lib/RuntimeConfiguration.ts", "content": "import config from \"../config\";\n\n/** Any value not set is set according to the dataset-specific defaults. */\nexport interface RuntimeConfiguration {\n /**\n * The identifier of the dataset to display. Also affects the defaults of all the other options.\n * Datasets are defined in config.ts (datasetOptions).\n */\n dataset: string;\n /** If true, when zoomed in, each pixel will have a book-like style applied to it in the shader */\n doBookshelfEffect: boolean;\n /** If true the group/publisher names will be vertical (for zoom levels where the bounding box is vertical) */\n groupTextVertical: boolean;\n /** Each publisher range gets a random unique color. This number, between 0.0-1.0, determines how bright that color is.\n * If 0, the publishers images will not be loaded.\n */\n publishersBrightness: number;\n /**\n * Number from 0.0-10, determines how much the brightness is increased when zoomed out (so sparse data is easier to see).\n * The exact effect depends on the shader (passed as uniform float GLOW)\n */\n shaderGlow: number;\n /** If true, the publisher names are overlaid over their bounding box. */\n showPublisherNames: boolean;\n /** Show a grid to visualize the boundaries of each ISBN digit */\n showGrid: boolean;\n /** How many grid levels to display (1 to 3) */\n gridLevels: number;\n /** Determines how the color is assigned to publishers */\n publishersColorSchema: \"dark\" | \"hsl\";\n /** Determines the minimum size at which text of each zoom level is displayed (performance-critical, good values are around 0.04-0.2) */\n textMinZoomLevel: number;\n /** Determines how many text levels are shown simultaneously. Their size depends on textMinZoomLevel. Can be floating number (e.g. 1.5) */\n textLevelCount: number;\n /** Determines the zoom level at which to load the next image level. 1.0 means that when an image tile is magnified 1.0x,\n * the next level is loaded (so in that case images would always be displayed down-scaled or at 1:1 size)\n */\n imgMinZoomLevel: number;\n /** the GLSL fragment shader template snippet to replace the default one with. See shaders.ts for defaults */\n customShader: string;\n /** The color of the grid as a hex string (\"#555544\") */\n gridColor: string;\n /** If set, filter out books that were published before this year */\n filterMinimumPublicationYear: number;\n /** If set, filter out books that were published after this year */\n filterMaximumPublicationYear: number;\n /** The index of the color gradient to choose from gradients.png. The meaning of the color scale depends on the dataset. */\n colorGradient: number;\n\n /** The URL prefix for image tiles (used like `${imagesRoot}/${dataset}/zoom-1/1.png`) */\n imagesRoot: string;\n /** The URL prefix of json files (publishers and stats) */\n jsonRoot: string;\n /** The URL prefix of the json files containing book titles (if any) */\n titlesRoot: string;\n}\n\nlet baseUrl = import.meta.env.BASE_URL;\nif (baseUrl === \"/\") baseUrl = \"\";\n\nconst defaultDataset = \"publication_date\";\nfunction isMobile() {\n const minWidth = 768; // Minimum width for desktop devices\n return window.innerWidth < minWidth || screen.width < minWidth;\n}\nexport function defaultRuntimeConfig(dataset: string): RuntimeConfiguration {\n const ds = config.datasetOptions.find((d) => d.id === dataset);\n // on mobile, defaults for performance\n const mobile = {\n textMinZoomLevel: 0.12,\n textLevelCount: 1.66,\n imgMinZoomLevel: 1.8,\n };\n return {\n dataset,\n doBookshelfEffect: true,\n groupTextVertical: true,\n publishersBrightness: 0.5,\n shaderGlow: 5,\n showPublisherNames: true,\n showGrid: true,\n gridLevels: 2,\n publishersColorSchema: \"hsl\",\n textMinZoomLevel: 0.09,\n textLevelCount: 2,\n imgMinZoomLevel: 1.2,\n customShader: \"\",\n gridColor: \"#555544\",\n filterMinimumPublicationYear: -1,\n filterMaximumPublicationYear: -1,\n colorGradient: 6,\n imagesRoot:\n window.origin === \"https://phiresky.github.io\"\n ? \"/isbn-visualization-images/tiled\"\n : baseUrl + \"/images/tiled\",\n jsonRoot:\n window.origin === \"https://phiresky.github.io\"\n ? \"/isbn-visualization-json/prefix-data\"\n : baseUrl + \"/prefix-data\",\n titlesRoot:\n window.origin === \"https://phiresky.github.io\" ||\n window.location.hostname === \"localhost\"\n ? \"https://isbn-titles.phiresky.xyz\"\n : baseUrl + \"/title-data\",\n\n ...(isMobile() ? mobile : {}),\n ...ds?.runtimeConfig,\n };\n}\n\nexport function loadRuntimeConfigFromURL(): RuntimeConfiguration {\n const url = new URLSearchParams(window.location.search);\n const base = defaultRuntimeConfig(\n url.get(\"dataset\") ?? defaultDataset,\n ) as unknown as Record;\n for (const key in base) {\n const value = url.get(key);\n if (value !== null) {\n if (typeof base[key] === \"number\") {\n base[key] = parseFloat(value);\n } else if (typeof base[key] === \"boolean\") {\n base[key] = value === \"true\";\n } else if (typeof base[key] === \"string\") {\n base[key] = value;\n } else {\n throw new Error(`Unknown type for ${key}`);\n }\n }\n }\n return base as unknown as RuntimeConfiguration;\n}\n\n// set, to url, only values not same as base, taking care of nulls as well\nexport function saveRuntimeConfigToURL(_config: RuntimeConfiguration) {\n const config = _config as unknown as Record;\n const base = defaultRuntimeConfig(_config.dataset) as unknown as Record<\n string,\n unknown\n >;\n const url = new URLSearchParams();\n if (_config.dataset !== defaultDataset) {\n url.set(\"dataset\", _config.dataset);\n }\n for (const key in config) {\n if (config[key] !== base[key]) {\n url.set(key, String(config[key]));\n }\n }\n\n debounceSetUrl(url);\n}\n\nfunction debounce(\n func: (...args: A[]) => void,\n wait: number,\n): (...args: A[]) => void {\n let timeout: number;\n return function (this: unknown, ...args: A[]) {\n clearTimeout(timeout);\n timeout = window.setTimeout(() => {\n func.apply(this, args);\n }, wait);\n };\n}\n\nconst debounceSetUrl = debounce((url: URLSearchParams) => {\n window.history.replaceState({}, \"\", `?${url.toString()}`);\n}, 500);\n" }, { "path": "src/lib/Store.ts", "content": "import { OrbitControlsChangeEvent } from \"@react-three/drei\";\nimport * as isbnlib from \"isbn3\";\nimport { autorun, makeAutoObservable, observable, runInAction } from \"mobx\";\nimport { createTransformer } from \"mobx-utils\";\nimport { Camera, OrthographicCamera, Vector3, Vector3Like } from \"three\";\nimport config from \"../config\";\nimport { DetailLevelObservable } from \"./DetailLevelObservable\";\nimport { plotSmartTrajectory, Point3D } from \"./flight\";\nimport { GoogleBooksItem, googleBooksQueryIsbn } from \"./google-books\";\nimport { ImageLoader } from \"./ImageLoader\";\nimport { LazyPrefixInfo } from \"./info-map\";\nimport { getGroupHierarchy, LazyPrefixInfoWithParents } from \"./prefix-data\";\nimport {\n defaultRuntimeConfig,\n loadRuntimeConfigFromURL,\n RuntimeConfiguration,\n saveRuntimeConfigToURL,\n} from \"./RuntimeConfiguration\";\nimport { ShaderUtil } from \"./shaders\";\nimport { StatsCalculator } from \"./stats\";\nimport { TitleFetcher } from \"./TitleFetcher\";\nimport {\n firstIsbnInPrefix,\n fullIsbnToRelative,\n isbnPrefixToRelative,\n IsbnPrefixWithoutDashes,\n IsbnRelative,\n IsbnStrWithChecksum,\n lastIsbnInPrefix,\n ProjectionConfig,\n relativeToFullIsbn,\n relativeToIsbnPrefix,\n statsConfig,\n} from \"./util\";\nimport { ViewParams } from \"./view-utils\";\n\ninterface RarityInfo {\n holdingCount: number;\n editionCount: number;\n bookCount: number;\n}\nexport class Store {\n view: ViewParams;\n camera?: OrthographicCamera;\n orbitControls?: OrbitControlsChangeEvent[\"target\"] | null = null;\n statsCalculator = new StatsCalculator(this);\n minimapHoveredCell: string | null = null;\n\n highlightedIsbn:\n | { type: \"todo\" }\n | {\n type: \"done\";\n isbn: IsbnStrWithChecksum;\n obj: ISBN | null;\n relative: IsbnRelative;\n groupInfo: LazyPrefixInfo[];\n googleBookDetails: GoogleBooksItem | null | \"todo\";\n rarity: RarityInfo | null;\n } = {\n type: \"todo\",\n };\n #imageLoader = new Map();\n rootPrefixInfo: LazyPrefixInfo = {\n children: { lazy: \"root.json\" },\n totalChildren: 0,\n };\n\n bookDetails = new Map();\n projection: ProjectionConfig;\n externalSearchEngines: { name: string; url: string }[] = [];\n shaderUtil = new ShaderUtil(this);\n animationRequestId = 0;\n runtimeConfig: RuntimeConfiguration;\n\n /** numeric id of publisher to highlight */\n highlightedPublisher: {\n relative: IsbnRelative;\n obj: ISBN | null;\n data: LazyPrefixInfo[] | null;\n } | null = null;\n highlightedStats: {\n prefixStart: IsbnPrefixWithoutDashes;\n prefixEnd: IsbnPrefixWithoutDashes;\n } | null = null;\n resetZoomButton = false;\n shaderError = \"\";\n titleFetcher = new TitleFetcher(this);\n constructor(projectionConfig: ProjectionConfig) {\n this.projection = projectionConfig;\n this.runtimeConfig = loadRuntimeConfigFromURL();\n makeAutoObservable(this, {\n view: observable.deep,\n rootPrefixInfo: observable.shallow,\n highlightedIsbn: observable.shallow,\n orbitControls: false,\n animationRequestId: false,\n highlightedPublisher: observable.shallow,\n });\n this.view = {\n minX: 0,\n minY: 0,\n maxX: this.projection.pixelWidth,\n maxY: this.projection.pixelHeight,\n width: this.projection.pixelWidth,\n height: this.projection.pixelHeight,\n };\n const params = new URLSearchParams(window.location.search);\n this.addExternalSearchEngines(params);\n autorun(() => {\n saveRuntimeConfigToURL(this.runtimeConfig);\n });\n }\n get floatZoomFactor() {\n return this.projection.pixelWidth / this.view.width;\n }\n addExternalSearchEngines(params: URLSearchParams) {\n this.externalSearchEngines.push(...config.externalSearchEngines);\n const searchEngines = params.getAll(\"external\");\n for (const engine of searchEngines) {\n const [name, url] = [\n engine.slice(0, engine.indexOf(\":\")),\n engine.slice(engine.indexOf(\":\") + 1),\n ];\n\n this.externalSearchEngines.push({ name, url });\n }\n }\n getDetailLevel = createTransformer(\n (prefix: IsbnPrefixWithoutDashes) =>\n new DetailLevelObservable(this, prefix),\n );\n imageLoader(dataset: string) {\n let l = this.#imageLoader.get(dataset);\n if (!l) {\n l = new ImageLoader(this.runtimeConfig.imagesRoot, dataset, this);\n this.#imageLoader.set(dataset, l);\n }\n return l;\n }\n async getBookDetail(isbn: IsbnStrWithChecksum) {\n const b = this.bookDetails.get(isbn);\n if (b) return b;\n const r = await this.googleBooksQueryIsbn(isbn);\n this.bookDetails.set(isbn, r);\n return r;\n }\n\n cachedGoogleBooks = new Map();\n async googleBooksQueryIsbn(isbn: IsbnStrWithChecksum) {\n const cached = this.cachedGoogleBooks.get(isbn);\n if (cached) return Promise.resolve(cached);\n const result = await this.trackAsyncProgress(\n `googleBooksQuery(${isbn})`,\n googleBooksQueryIsbn(isbn),\n );\n this.cachedGoogleBooks.set(isbn, result);\n return result;\n }\n\n updateHighlight(x: number, y: number, isHover: boolean) {\n const relativeIsbn = this.projection.coordsToRelativeIsbn(x, y);\n if (isHover) {\n this.updateHighlightedPublisher(relativeIsbn);\n } else {\n this.updateHighlightedIsbn(\n relativeToFullIsbn(relativeIsbn),\n relativeIsbn,\n );\n }\n }\n updateHighlightedPublisher(relativeIsbn: IsbnRelative) {\n const isbnStr = relativeToIsbnPrefix(relativeIsbn);\n const groupInfo = getGroupHierarchy(this.rootPrefixInfo, isbnStr);\n const isbnInst = isbnlib.parse(relativeToFullIsbn(relativeIsbn));\n const oldOne = this.highlightedPublisher?.relative;\n this.highlightedPublisher = {\n relative: relativeIsbn,\n obj: isbnInst,\n data: null,\n };\n if (typeof groupInfo === \"function\") {\n this.highlightedPublisher.data = (\n getGroupHierarchy(\n this.rootPrefixInfo,\n isbnStr,\n false,\n ) as LazyPrefixInfoWithParents\n ).outers;\n\n //groupInfo().then((info) => (this.highlightedGroupInfo = info.outers));\n if (oldOne !== relativeIsbn)\n this.debounceFetchGroupData(() =>\n groupInfo(this.runtimeConfig.jsonRoot).then((info) => {\n if (this.highlightedPublisher)\n this.highlightedPublisher.data = info.outers;\n }),\n );\n } else {\n this.highlightedPublisher.data = groupInfo.outers;\n }\n }\n\n updateHighlightedStats(x: number, y: number, mode: \"start\" | \"end\") {\n const relativeIsbn = this.projection.coordsToRelativeIsbn(x, y);\n const prefix = relativeToIsbnPrefix(relativeIsbn).slice(\n 0,\n statsConfig.maxPrefixLength,\n ) as IsbnPrefixWithoutDashes;\n if (!this.highlightedStats || mode === \"start\")\n this.highlightedStats = { prefixStart: prefix, prefixEnd: prefix };\n else this.highlightedStats.prefixEnd = prefix;\n }\n inProgress = new Map();\n trackAsyncProgress(_id: string, p: Promise) {\n let id = _id;\n let copy = 1;\n while (this.inProgress.has(id)) id = _id + \" \" + String(++copy);\n runInAction(() => this.inProgress.set(id, null));\n //console.time(id);\n void p.then(() => {\n this.inProgress.delete(id);\n //console.timeEnd(id);\n });\n p.catch((e: unknown) => {\n this.inProgress.set(id, String(e));\n console.timeEnd(id);\n console.warn(id, \"ERROR\", e);\n });\n return p;\n }\n updateHighlightedIsbn(\n fullIsbn: IsbnStrWithChecksum,\n relativeIsbn?: IsbnRelative,\n ) {\n if (!relativeIsbn) relativeIsbn = fullIsbnToRelative(fullIsbn);\n const isbnStr = relativeToIsbnPrefix(relativeIsbn);\n // getGroup(store, prefix)\n const isbnInst = isbnlib.parse(fullIsbn);\n if (\n this.highlightedIsbn.type !== \"todo\" &&\n this.highlightedIsbn.relative &&\n this.highlightedIsbn.relative === relativeIsbn\n )\n return;\n this.highlightedIsbn = {\n type: \"done\",\n relative: relativeIsbn,\n isbn: fullIsbn,\n obj: isbnInst,\n groupInfo: [],\n googleBookDetails: \"todo\",\n rarity: null,\n };\n const groupInfo = getGroupHierarchy(this.rootPrefixInfo, isbnStr);\n if (typeof groupInfo === \"function\") {\n this.highlightedIsbn.groupInfo = (\n getGroupHierarchy(\n this.rootPrefixInfo,\n isbnStr,\n false,\n ) as LazyPrefixInfoWithParents\n ).outers;\n //groupInfo().then((info) => (this.highlightedGroupInfo = info.outers));\n this.debounceFetchGroupData(() =>\n groupInfo(this.runtimeConfig.jsonRoot).then((info) => {\n if (this.highlightedIsbn.type === \"done\")\n this.highlightedIsbn.groupInfo = info.outers;\n }),\n );\n } else {\n this.highlightedIsbn.groupInfo = groupInfo.outers;\n }\n if (this.highlightedIsbn.groupInfo.length > 0) {\n void (async () => {\n const detail = await this.getBookDetail(fullIsbn);\n if (\n this.highlightedIsbn.type === \"done\" &&\n this.highlightedIsbn.relative === relativeIsbn\n )\n this.highlightedIsbn.googleBookDetails = detail;\n })();\n void (async () => {\n const rarity = await this.getRarityOfIsbn(relativeIsbn);\n if (\n this.highlightedIsbn.type === \"done\" &&\n this.highlightedIsbn.relative === relativeIsbn\n )\n this.highlightedIsbn.rarity = rarity;\n })();\n } else {\n this.highlightedIsbn.googleBookDetails = null;\n }\n }\n debounceFetchGroupDataTimeout: ReturnType | null = null;\n // call newFunction only after 1s of inactivity\n debounceFetchGroupData(newFunction: () => Promise) {\n if (this.debounceFetchGroupDataTimeout) {\n clearTimeout(this.debounceFetchGroupDataTimeout);\n }\n this.debounceFetchGroupDataTimeout = setTimeout(() => {\n void this.trackAsyncProgress(\"highlightGroupInfo\", newFunction());\n }, 500);\n }\n updateView(e?: OrbitControlsChangeEvent) {\n if (!e) return;\n const camera = (e as { target: { object: Camera } }).target.object;\n const topLeft = new Vector3(-1, -1, 0).unproject(camera);\n const bottomRight = new Vector3(1, 1, 0).unproject(camera);\n const minX = topLeft.x + this.projection.pixelWidth / 2;\n const maxX = bottomRight.x + this.projection.pixelWidth / 2;\n const minY = -(bottomRight.y - this.projection.pixelHeight / 2);\n const maxY = -(topLeft.y - this.projection.pixelHeight / 2);\n const view = {\n minX,\n maxX,\n minY,\n maxY,\n width: +(maxX - minX).toFixed(8),\n height: +(maxY - minY).toFixed(8),\n };\n this.resetZoomButton = true;\n Object.assign(this.view, view);\n }\n zoomAnimateToHighlight() {\n if (this.highlightedIsbn.type !== \"done\") return;\n const { x, y, width, height } = this.projection.relativeIsbnToCoords(\n this.highlightedIsbn.relative,\n );\n const targetX = x + width / 2;\n const targetY = y + (height * 3) / 4;\n this.zoomAnimateTo(targetX, targetY, 14000, 7);\n }\n setView(targetX: number, targetY: number, zoom?: number) {\n targetX -= this.projection.pixelWidth / 2;\n targetY = this.projection.pixelHeight / 2 - targetY;\n const camera = this.camera;\n if (!camera) return;\n camera.position.x = targetX;\n camera.position.y = targetY;\n if (zoom) camera.zoom = zoom;\n if (!this.orbitControls) return;\n this.orbitControls.target.x = camera.position.x;\n this.orbitControls.target.y = camera.position.y;\n camera.updateProjectionMatrix();\n }\n zoomAnimateTo(\n targetX: number,\n targetY: number,\n targetZoom: number,\n timeScale: number,\n ) {\n targetX -= this.projection.pixelWidth / 2;\n targetY = this.projection.pixelHeight / 2 - targetY;\n const camera = this.camera;\n if (!camera) return;\n const orbitControls = this.orbitControls;\n if (!orbitControls) return;\n const maxZoom = 1; // maxZoom = distance 1. 1/2 * maxZoom = distance 2 => maxZoom/n = distance n;\n const from = new Point3D(\n camera.position.x,\n camera.position.y,\n maxZoom / camera.zoom,\n );\n const to = new Point3D(targetX, targetY, maxZoom / targetZoom);\n console.log(\"xyz space\", {\n from,\n to,\n });\n const setPosition = (position: Vector3Like) => {\n camera.position.x = position.x;\n camera.position.y = position.y;\n camera.zoom = maxZoom / position.z;\n if (!this.orbitControls) return;\n this.orbitControls.target.x = camera.position.x;\n this.orbitControls.target.y = camera.position.y;\n camera.updateProjectionMatrix();\n };\n //const trajectory = getTrajectoryReal2(from, to);\n const trajectory = plotSmartTrajectory(from, to);\n console.log(\"trajectory xyz\", trajectory);\n // lerp each segment in trajectory using it's given length\n const start = performance.now() / timeScale;\n const animate = () => {\n const now = performance.now() / timeScale;\n const time = now - start;\n setPosition(trajectory.position(time));\n if (time < trajectory.duration) {\n this.animationRequestId = requestAnimationFrame(animate);\n }\n };\n animate();\n }\n\n async getRarityOfIsbn(isbn: IsbnRelative): Promise {\n const imgPrefix = relativeToIsbnPrefix(isbn).slice(\n 0,\n 2 + 4,\n ) as IsbnPrefixWithoutDashes;\n\n const img = await this.imageLoader(\"rarity\").getTexture(\n isbnPrefixToRelative(imgPrefix),\n );\n if (!img) return null;\n const imgElement = img.image as HTMLImageElement | null;\n if (!imgElement) throw Error(\"no image element\");\n const imgPos = this.projection.relativeIsbnToCoords(\n firstIsbnInPrefix(imgPrefix),\n );\n const imgPosEnd = this.projection.relativeIsbnToCoords(\n lastIsbnInPrefix(imgPrefix),\n );\n const pos = this.projection.relativeIsbnToCoords(isbn);\n const canvas = new OffscreenCanvas(1, 1);\n const ctx = canvas.getContext(\"2d\");\n if (!ctx) throw Error(\"no canvas context\");\n const xInImg = Math.round(\n ((pos.x - imgPos.x) / (imgPosEnd.x + imgPosEnd.width - imgPos.x)) *\n imgElement.width,\n );\n const yInImg = Math.round(\n ((pos.y - imgPos.y) / (imgPosEnd.y + imgPosEnd.height - imgPos.y)) *\n imgElement.height,\n );\n ctx.drawImage(imgElement, xInImg, yInImg, 1, 1, 0, 0, 1, 1);\n const imgData = ctx.getImageData(0, 0, 1, 1);\n console.log({ ctx, imgElement, imgData, pos, imgPos, xInImg, yInImg });\n return {\n holdingCount: imgData.data[0],\n editionCount: imgData.data[1],\n bookCount: imgData.data[2],\n };\n }\n get currentDataset() {\n const d = config.datasetOptions.find(\n (g) => g.id === this.runtimeConfig.dataset,\n );\n if (!d) throw Error(\"dataset not found\");\n return d;\n }\n\n switchDataset(dataset: string, resetSettings: boolean) {\n const d = config.datasetOptions.find((d) => d.id === dataset);\n if (!d) throw Error(\"dataset not found\");\n this.inProgress.clear();\n if (resetSettings) {\n this.runtimeConfig = defaultRuntimeConfig(d.id);\n } else {\n this.runtimeConfig.dataset = d.id;\n }\n }\n}\n" }, { "path": "src/lib/TitleFetcher.ts", "content": "import { fetchJson } from \"./json-fetch\";\nimport { Store } from \"./Store\";\nimport {\n Isbn13Number,\n IsbnPrefixWithoutDashes,\n IsbnStrWithChecksum,\n splitNameJson,\n} from \"./util\";\n\nexport interface TitleFetchedInfo {\n isbn13: Isbn13Number;\n title: string;\n creator: string;\n}\n\nexport class TitleFetcher {\n cache = new Map<\n IsbnPrefixWithoutDashes,\n Promise>\n >();\n constructor(private store: Store) {}\n async fetchTitle(\n title: IsbnStrWithChecksum,\n ): Promise {\n const prefixStr = title.slice(0, 8) as IsbnPrefixWithoutDashes;\n const fname = splitNameJson(prefixStr, 3);\n\n let gotten = this.cache.get(prefixStr);\n if (!gotten) {\n gotten = fetchJson(\n this.store.runtimeConfig.titlesRoot + \"/\" + fname,\n ).then(\n (data) =>\n new Map(\n data.map((info) => [\n String(info.isbn13) as IsbnStrWithChecksum,\n info,\n ]),\n ),\n );\n this.cache.set(prefixStr, gotten);\n }\n const data = await gotten;\n return data.get(title);\n }\n}\n" }, { "path": "src/lib/delayRender.ts", "content": "import { useEffect, useState } from \"react\";\n\nconst maxPerFrame = 1;\nconst pending: (() => void)[] = [];\nfunction clearT() {\n let remaining = pending.length > 10 ? maxPerFrame * 2 : maxPerFrame;\n while (pending.length > 0 && remaining > 0) {\n pending.pop()?.();\n remaining--;\n }\n requestAnimationFrame(clearT);\n}\nclearT();\n\nexport function useDelay() {\n const [yes, setYes] = useState(false);\n useEffect(() => {\n const fn = () => {\n setYes(true);\n };\n pending.push(fn);\n return () => {\n const inx = pending.indexOf(fn);\n if (inx >= 0) pending.splice(inx, 1);\n };\n }, []);\n return yes;\n}\n" }, { "path": "src/lib/flight.ts", "content": "import { Vector3Like } from \"three/src/math/Vector3\";\n\nclass Point2D {\n constructor(\n public x: number,\n public y: number,\n ) {}\n plus(p: Point2D) {\n return new Point2D(this.x + p.x, this.y + p.y);\n }\n minus(p: Point2D) {\n return new Point2D(this.x - p.x, this.y - p.y);\n }\n mul(s: number) {\n return new Point2D(this.x * s, this.y * s);\n }\n div(s: number) {\n return new Point2D(this.x / s, this.y / s);\n }\n length() {\n return Math.sqrt(this.x ** 2 + this.y ** 2);\n }\n toString() {\n return `(${this.x}, ${this.y})`;\n }\n}\n\nexport class Point3D implements Vector3Like {\n constructor(\n public x: number,\n public y: number,\n public z: number,\n ) {}\n\n plus(p: Point3D) {\n return new Point3D(this.x + p.x, this.y + p.y, this.z + p.z);\n }\n minus(p: Point3D) {\n return new Point3D(this.x - p.x, this.y - p.y, this.z - p.z);\n }\n mul(s: number) {\n return new Point3D(this.x * s, this.y * s, this.z * s);\n }\n div(s: number) {\n return new Point3D(this.x / s, this.y / s, this.z / s);\n }\n neg() {\n return new Point3D(-this.x, -this.y, -this.z);\n }\n length() {\n return Math.sqrt(this.x ** 2 + this.y ** 2 + this.z ** 2);\n }\n distance(p: Point3D) {\n return Math.sqrt(\n (this.x - p.x) ** 2 + (this.y - p.y) ** 2 + (this.z - p.z) ** 2,\n );\n }\n normalize() {\n const l = this.length();\n return new Point3D(this.x / l, this.y / l, this.z / l);\n }\n toString() {\n return `(${this.x}, ${this.y}, ${this.z})`;\n }\n}\n\n// eslint-disable-next-line @typescript-eslint/no-wrapper-object-types\nexport interface Trajectory extends Object {\n position(t: number): Point3D;\n duration: number;\n}\n\nexport function getTrajectoryReal2(\n before: Point3D,\n after: Point3D,\n): Trajectory {\n // add flight duration to each point based on the distance to prev point\n const traj = getTrajectoryReal(before, after) as (Point3D & { t: number })[];\n /*let totalDistance = 0;\n for (let i = 1; i < traj.length; i++) {\n totalDistance += Math.sqrt(\n (traj[i].x - traj[i - 1].x) ** 2 +\n (traj[i].y - traj[i - 1].y) ** 2 +\n (traj[i].z - traj[i - 1].z) ** 2\n );\n }*/\n let totalT = 0;\n // add calculated distance-based flight duration\n for (let i = 0; i < traj.length; i++) {\n const distance =\n i === 0\n ? 0\n : Math.sqrt(\n (traj[i].x - traj[i - 1].x) ** 2 +\n (traj[i].y - traj[i - 1].y) ** 2 +\n (traj[i].z - traj[i - 1].z) ** 2,\n );\n const t =\n distance *\n Math.min(\n 3000,\n i === 0 ? 1 / traj[i].z : (1 / traj[i].z + 1 / traj[i - 1].z) / 2,\n );\n totalT += t;\n traj[i].t = t;\n }\n // limit max flight duration\n if (totalT > 2000) {\n for (const p of traj) {\n p.t *= 2000 / totalT;\n }\n totalT = 2000;\n }\n\n return new TimeInterpolatingTrajectory({\n position(time: number): Point3D {\n let rest = time;\n let curSegment = 1;\n while (true) {\n if (curSegment >= traj.length) {\n return after;\n }\n\n if (rest <= traj[curSegment].t) {\n break;\n }\n\n rest -= traj[curSegment].t;\n curSegment++;\n }\n\n const segment = traj[curSegment];\n const lastSegment = traj[curSegment - 1];\n const progress = rest / segment.t;\n //console.log(\"return \", {curSegment, progress, rest, time, segment, lastSegment});\n return new Point3D(\n lastSegment.x + (segment.x - lastSegment.x) * progress,\n lastSegment.y + (segment.y - lastSegment.y) * progress,\n lastSegment.z + (segment.z - lastSegment.z) * progress,\n );\n },\n duration: totalT,\n toString() {\n return traj.map((p) => p.toString()).join(\" -> \");\n },\n });\n}\n\nfunction getTrajectoryReal(before: Point3D, after: Point3D) {\n const speed = 0.0002;\n const x1 = 0;\n const xScale = 1000;\n const x2 =\n Math.sqrt((after.x - before.x) ** 2 + (after.y - before.y) ** 2) / xScale;\n const y1 = before.z;\n const y2 = after.z;\n const points = getTrajectoryPoints(speed, x1, y1, x2, y2);\n console.log(\"xy space\", {\n from: { x: x1, y: y1 },\n to: { x: x2, y: y2 },\n });\n console.log(\"trajectory xy\", points);\n return points.map((p) => ({\n x: (p.x / x2) * (after.x - before.x) + before.x,\n y: (p.x / x2) * (after.y - before.y) + before.y,\n z: p.y,\n }));\n}\n\nfunction getTrajectoryPoints(\n speed: number,\n x1: number,\n y1: number,\n x2: number,\n y2: number,\n) {\n const { a, b, c } = calculateTrajectory(x1, y1, x2, y2);\n const points = [];\n for (let x = x1; x <= x2; x += speed) {\n const y = a * x ** 2 + b * x + c;\n points.push({ x, y });\n }\n points.push({ x: x2, y: y2 });\n return points;\n}\n\n/** solve ax^2+bx+c based on two x,y pairs plus a set based on the distance between the points */\nfunction calculateTrajectory(x1: number, y1: number, x2: number, y2: number) {\n // Use x-distance for steepness\n const xDistance = Math.abs(x2 - x1);\n // Always make 'a' negative for downward-facing parabola\n const flonk = Math.max(y1, y2);\n // we want aFlonk = -100 when ymax = 1, -1000 when ymax = 0.001\n const a = -xDistance * (1 / flonk) ** 0.3 * 10; // Divided by 5 to make the steepness more manageable\n const b = (y2 - y1 - a * (x2 ** 2 - x1 ** 2)) / (x2 - x1);\n const c = y1 - a * x1 ** 2 - b * x1;\n console.log({ a, b, c });\n return { a, b, c };\n}\n\n/**\n * Takes another trajectory and interplolates the time spend on the curve segments depending on the zoom level\n * It also calculates the total duration of the trajectory according to the distance of the whole trajectory\n */\nclass TimeInterpolatingTrajectory implements Trajectory {\n private points: Point3D[];\n private durations: number[];\n private offsetIndex = 0;\n private offsetTime = 0;\n public duration: number;\n\n constructor(inner: Trajectory) {\n const points: Point3D[] = [];\n let totalDistance = 0;\n\n function dist(p1: Point3D, p2: Point3D) {\n const dx = p1.x - p2.x;\n const dy = p1.y - p2.y;\n const zScale = 8;\n const dz =\n (Math.log10(1 + (1 / p1.z) * zScale) -\n Math.log10(1 + (1 / p2.z) * zScale)) *\n zScale;\n return Math.sqrt(dx * dx + dy * dy + dz * dz);\n }\n\n function duration(p: Point3D) {\n const dur = Math.log10(1 + 1 / p.z / 8);\n console.assert(dur >= 0, \"duration should be positive\");\n return dur;\n }\n\n function add(\n from: number,\n fromP: Point3D,\n to: number,\n toP: Point3D,\n depth: number,\n ) {\n const distance = dist(fromP, toP);\n\n const middle = from + (to - from) / 2;\n const middleP = inner.position(middle);\n const eps = 0.0000001;\n const isLineOnZ =\n Math.abs(fromP.z - toP.z) < eps && Math.abs(fromP.z - middleP.z) < eps;\n\n if (distance <= 2 || isLineOnZ || depth > 400) {\n totalDistance += distance;\n return;\n }\n\n add(from, fromP, middle, middleP, depth + 1);\n points.push(middleP);\n add(middle, middleP, to, toP, depth + 1);\n }\n\n const start = 0;\n const startP = inner.position(start);\n const end = inner.duration;\n const endP = inner.position(end);\n\n //console.log(\"create TimeInterpolatingTrajectory\", { startP, endP, start, end });\n\n points.push(startP);\n\n add(start, startP, end, endP, 0);\n\n points.push(endP);\n\n //console.log(\"Points\", { length: points.length, totalDistance })\n\n const maxDuration = 1000;\n const minDuration = 100;\n const targetDuration = totalDistance * 20;\n //console.log(\"dist-and-dist\", { totalDistance, targetDuration });\n this.duration = Math.max(\n minDuration,\n Math.min(targetDuration, maxDuration),\n );\n\n // create durations\n const durations: number[] = [];\n for (let i = 0; i < points.length - 1; i++) {\n const p1 = points[i];\n const p2 = points[i + 1];\n const dur = (duration(p1) + duration(p2)) / 2;\n durations.push(dur * dist(p1, p2));\n }\n\n // normalize durations\n const totalDuration = durations.reduce((sum, d) => sum + d, 0);\n for (let i = 0; i < durations.length; i++) {\n durations[i] = (durations[i] * this.duration) / totalDuration;\n }\n\n this.points = points;\n this.durations = durations;\n }\n\n position(t: number): Point3D {\n while (true) {\n const rest = t - this.offsetTime;\n const p1 = this.points[this.offsetIndex];\n const p2 = this.points[this.offsetIndex + 1];\n const dur = this.durations[this.offsetIndex];\n\n if (rest < dur) {\n const progress = rest / dur;\n return p1.plus(p2.minus(p1).mul(progress));\n }\n\n if (this.offsetIndex >= this.points.length - 2) {\n return this.points[this.points.length - 1];\n }\n\n //console.log(\"switch to next trajectory\", this.offsetIndex);\n this.offsetTime += dur;\n this.offsetIndex++;\n }\n }\n}\n\n/*\n A trajectory that moves linearly in blub space and makes something between a parable and a circle in real space\n*/\nclass DirectBlubSpaceTrajectory implements Trajectory {\n constructor(\n public start: Point2D,\n public end: Point2D,\n public origin: Point3D,\n public target: Point3D,\n public xyDirection: Point3D,\n public duration: number,\n ) {}\n\n position(t: number): Point3D {\n if (t >= this.duration) {\n return this.target;\n }\n const progress = t / this.duration;\n const pointInBlub = this.start.plus(\n this.end.minus(this.start).mul(progress),\n );\n\n const { dist, zoom } = fromBlubSpace(pointInBlub, segmentSize);\n const pointInReal = this.origin.plus(this.xyDirection.mul(dist));\n pointInReal.z = zoom;\n return pointInReal;\n }\n\n reverse(): Trajectory & { origin: Point3D } {\n // eslint-disable-next-line @typescript-eslint/no-this-alias\n const self = this;\n return {\n duration: self.duration,\n origin: self.target,\n position(t: number): Point3D {\n if (t >= self.duration) {\n return self.origin;\n }\n return self.position(self.duration - t);\n },\n };\n }\n}\n\n// A trajectory that moves on a line from origin to target in real space\nclass RealSpaceTrajectory implements Trajectory {\n direct: Point3D;\n constructor(\n private origin: Point3D,\n private target: Point3D,\n public duration: number,\n ) {\n this.direct = target.minus(origin);\n }\n\n position(t: number): Point3D {\n if (t >= this.duration) {\n return this.target;\n }\n const progress = t / this.duration;\n return this.origin.plus(this.direct.mul(progress));\n }\n}\n\n/**\n * Moves along multiple trajectories\n */\nclass CompositeTrajectory implements Trajectory {\n constructor(private trajectories: Trajectory[]) {\n this.duration = trajectories.reduce((sum, t) => sum + t.duration, 0);\n }\n duration: number;\n\n // private offsetTime: number = 0;\n // private offsetIndex: number = 0;\n // position(t: number): Point3D {\n // let rest = t - this.offsetTime;\n // while (this.offsetIndex < this.trajectories.length - 1 && rest >= this.trajectories[this.offsetIndex].duration) {\n // rest -= this.trajectories[this.offsetIndex].duration;\n // this.offsetTime += this.trajectories[this.offsetIndex].duration;\n // this.offsetIndex++;\n // console.log(\"switch to next trajectory\", this.offsetIndex);\n // }\n // return this.trajectories[this.offsetIndex].position(rest);\n // }\n\n position(t: number): Point3D {\n let rest = t;\n for (const trajectory of this.trajectories) {\n if (rest < trajectory.duration) {\n return trajectory.position(rest);\n }\n rest -= trajectory.duration;\n }\n return this.trajectories[this.trajectories.length - 1].position(\n this.trajectories[this.trajectories.length - 1].duration,\n );\n }\n}\n\nconst segmentSize = 500;\nconst minZoom = 2;\nconst targetMinZoom = 1;\n\nexport function plotSmartTrajectory(\n origin: Point3D,\n target: Point3D,\n): Trajectory {\n return new TimeInterpolatingTrajectory(\n plotSmartTrajectoryInner(origin, target),\n );\n}\n\nfunction plotSmartTrajectoryInner(\n origin: Point3D,\n target: Point3D,\n): Trajectory {\n const direct = target.minus(origin);\n const xyDistance = Math.sqrt(direct.x ** 2 + direct.y ** 2);\n const xyDirection = new Point3D(\n direct.x / xyDistance,\n direct.y / xyDistance,\n 0,\n );\n\n // console.log(\"plotSmartTrajectory\", {\n // origin,\n // target,\n // xyDistance,\n // xyDirection,\n // });\n if (xyDistance <= 0) {\n // if xyDistance is 0 we just have to zoom in\n return new RealSpaceTrajectory(origin, target, 1);\n }\n\n function makeIndirectTrajectory(): Trajectory {\n //console.assert(origin.z <= targetMinZoom)\n //console.assert(target.z <= targetMinZoom);\n const zoomOutTrajectory = makeFullZoomOutTrajectory(\n origin,\n xyDirection,\n 1 / 3,\n targetMinZoom,\n );\n const zoomInTrajectory = makeFullZoomOutTrajectory(\n target,\n xyDirection.neg(),\n 1 / 3,\n targetMinZoom,\n ).reverse();\n const topLevelZoomTrajectory = new RealSpaceTrajectory(\n zoomOutTrajectory.target,\n zoomInTrajectory.origin,\n 1 / 10,\n );\n //console.log(\"makeIndirectTrajectory\", { zoomOutTrajectory, topLevelZoomTrajectory, zoomInTrajectory });\n return new CompositeTrajectory([\n zoomOutTrajectory,\n topLevelZoomTrajectory,\n zoomInTrajectory,\n ]);\n }\n\n // first check if we can project the flight path into a blub space segment (aka a half circle)\n if (xyDistance < segmentSize) {\n const start = toBlubSpace(0, origin.z, segmentSize);\n const end = toBlubSpace(xyDistance, target.z, segmentSize);\n\n if (origin.z <= targetMinZoom && target.z <= targetMinZoom) {\n // if we are not already zoomed out more than targetMinZoom\n // check whether the line intersects the targetMinZoom circle\n // otherwise we would zoom out more than targetMinZoom\n if (\n lineIntersectsCircle(\n start,\n end,\n new Point2D(0, 0),\n zoomToBlubRadius(targetMinZoom),\n )\n ) {\n // if the line intersects the targetMinZoom circle, we need to zoom out to targetMinZoom and then do a full zoom in\n return makeIndirectTrajectory();\n }\n }\n\n //console.log(\"plotSmartTrajectory\", { start, end });\n return new DirectBlubSpaceTrajectory(\n start,\n end,\n origin,\n target,\n xyDirection,\n 1,\n );\n } else if (origin.z > targetMinZoom) {\n // if we are zoomed out more than the targetMinZoom,\n // don't zoom out at all, just move at the current zoom level and then zoom in\n const zoomInTrajectory = makeFullZoomOutTrajectory(\n target,\n xyDirection.neg(),\n 1 / 2,\n origin.z,\n ).reverse();\n const topLevelZoomTrajectory = new RealSpaceTrajectory(\n origin,\n zoomInTrajectory.origin,\n 1 / 2,\n );\n //console.log(\"onlyZoomIn\", { topLevelZoomTrajectory, zoomInTrajectory });\n return new CompositeTrajectory([topLevelZoomTrajectory, zoomInTrajectory]);\n } else {\n // the targets are further away than a segment, so we need to zoom out fully to the target zoom level and zoom in again\n return makeIndirectTrajectory();\n }\n}\n/**\n * Checks whether the line between p1 and p2 intersects a circle with center circle and radius radius\n */\nfunction lineIntersectsCircle(\n p1: Point2D,\n p2: Point2D,\n circle: Point2D,\n radius: number,\n) {\n // no idea whats going on here... it's adapted from the INTERNET!\n const v1 = {\n x: p2.x - p1.x,\n y: p2.y - p1.y,\n };\n const v2 = {\n x: p1.x - circle.x,\n y: p1.y - circle.y,\n };\n const b = -2 * (v1.x * v2.x + v1.y * v2.y);\n const c = 2 * (v1.x * v1.x + v1.y * v1.y);\n const d = Math.sqrt(\n b * b - 2 * c * (v2.x * v2.x + v2.y * v2.y - radius * radius),\n );\n if (isNaN(d)) {\n // no intercept\n return false;\n }\n const u1 = (b - d) / c; // these represent the unit distance of point one and two on the line\n const u2 = (b + d) / c;\n return (u1 <= 1 && u1 >= 0) || (u2 <= 1 && u2 >= 0);\n}\n\n/**\n * Creates a trajectory that zooms out from origin to the targetZoom level in the xy direction\n * Because it moves to the tangent of the targetZoom circle in blub space, it makes a nice approaching curve in real space\n */\nfunction makeFullZoomOutTrajectory(\n origin: Point3D,\n xyDirection: Point3D,\n duration: number,\n targetZoom: number,\n): DirectBlubSpaceTrajectory {\n const zoomOutStart = toBlubSpace(0, origin.z, segmentSize);\n const zoomOutEnd = circleTangentPointFrom0Origin(\n zoomOutStart,\n zoomToBlubRadius(targetZoom),\n );\n const { dist: zoomOutEndDist, zoom: zoomOutEndZoom } = fromBlubSpace(\n zoomOutEnd,\n segmentSize,\n );\n const zoomOutEndInReal = origin.plus(xyDirection.mul(zoomOutEndDist));\n zoomOutEndInReal.z = zoomOutEndZoom;\n\n //console.log(\"makeFullZoomOutTrajectory\", { origin, zoomOutStart, zoomOutEnd, zoomOutEndInReal });\n\n return new DirectBlubSpaceTrajectory(\n zoomOutStart,\n zoomOutEnd,\n origin,\n zoomOutEndInReal,\n xyDirection,\n duration,\n );\n}\n\n/**\n * Calculates the point on the circle with radius radius so that the line from p to the calculated point is tangent to the circle\n */\nfunction circleTangentPointFrom0Origin(p: Point2D, radius: number): Point2D {\n const startRadius = p.length();\n const zoomOutAngle = Math.acos(radius / startRadius);\n return new Point2D(\n Math.cos(zoomOutAngle) * radius,\n Math.sin(zoomOutAngle) * radius,\n );\n}\n\nconst zoomRadiusOffset = minZoom + 0.1;\nfunction zoomToBlubRadius(zoom: number): number {\n return zoomRadiusOffset - zoom;\n}\n\nfunction blubRadiusToZoom(radius: number): number {\n return zoomRadiusOffset - radius;\n}\n\nfunction toBlubSpace(dist: number, zoom: number, segmentSize: number): Point2D {\n const radius = zoomToBlubRadius(zoom);\n const angle = (dist / segmentSize) * Math.PI;\n return new Point2D(Math.cos(angle) * radius, Math.sin(angle) * radius);\n}\n\nfunction fromBlubSpace(\n p: Point2D,\n segmentSize: number,\n): { dist: number; zoom: number } {\n const radius = Math.sqrt(p.x ** 2 + p.y ** 2);\n const angle = Math.atan2(p.y, p.x);\n return {\n dist: (angle / Math.PI) * segmentSize,\n zoom: blubRadiusToZoom(radius),\n };\n}\n" }, { "path": "src/lib/google-books.ts", "content": "import { IsbnStrWithChecksum } from \"./util\";\n\ninterface GoogleBooksResponse {\n items?: GoogleBooksItem[];\n totalItems: number;\n}\nexport interface GoogleBooksItem {\n kind: string;\n id: string;\n etag: string;\n selfLink: string;\n volumeInfo: Partial<{\n title: string;\n authors: string[];\n industryIdentifiers: (\n | {\n type: \"ISBN_10\";\n identifier: string;\n }\n | {\n type: \"ISBN_13\";\n identifier: IsbnStrWithChecksum;\n }\n )[];\n publisher: string;\n publishedDate: string;\n description: string;\n readingModes: {\n text: boolean;\n image: boolean;\n };\n pageCount: number;\n printType: string;\n categories: string[];\n averageRating: number;\n ratingsCount: number;\n maturityRating: string;\n allowAnonLogging: boolean;\n contentVersion: string;\n panelizationSummary: {\n containsEpubBubbles: boolean;\n containsImageBubbles: boolean;\n };\n imageLinks: {\n smallThumbnail: string;\n thumbnail: string;\n };\n language: string;\n previewLink: string;\n infoLink: string;\n canonicalVolumeLink: string;\n }>;\n saleInfo: {\n country: string;\n saleability: string;\n isEbook: boolean;\n };\n accessInfo: {\n country: string;\n viewability: string;\n embeddable: boolean;\n publicDomain: boolean;\n textToSpeechPermission: string;\n epub: {\n isAvailable: boolean;\n };\n pdf: {\n isAvailable: boolean;\n };\n webReaderLink: string;\n accessViewStatus: string;\n quoteSharingAllowed: boolean;\n };\n searchInfo: {\n textSnippet: string;\n };\n}\n\nexport async function googleBooksQuery(query: string) {\n const r = await fetch(\n `https://www.googleapis.com/books/v1/volumes?q=${encodeURIComponent(query)}`,\n );\n const r_1 = (await r.json()) as GoogleBooksResponse;\n return r_1.items ?? [];\n}\n\nexport async function googleBooksQueryIsbn(\n isbn: IsbnStrWithChecksum,\n): Promise {\n const r = await googleBooksQuery(`isbn:${isbn}`);\n if (r.length === 0) return null;\n if (r.length > 1) console.warn(\"multiple results for isbn\", isbn, r);\n return r[0];\n}\n" }, { "path": "src/lib/info-map.ts", "content": "import { IsbnPrefixWithDashes } from \"./util\";\n\nexport type Digit = \"0\" | \"1\" | \"2\" | \"3\" | \"4\" | \"5\" | \"6\" | \"7\" | \"8\" | \"9\";\nexport const DIGITS: Digit[] = [\n \"0\",\n \"1\",\n \"2\",\n \"3\",\n \"4\",\n \"5\",\n \"6\",\n \"7\",\n \"8\",\n \"9\",\n];\nexport type PrefixInfoData =\n | {\n // id: string;\n numericId?: number;\n registrant_name: string;\n agency_name: string;\n country_name: string;\n source: \"isbngrp\";\n prefix: IsbnPrefixWithDashes;\n }\n | {\n source: \"publisher-ranges\";\n numericId?: number;\n name: string;\n prefix: IsbnPrefixWithDashes;\n color?: string;\n };\nexport interface PrefixInfo {\n children?: InfoMap;\n info?: PrefixInfoData[];\n totalChildren: number;\n}\nexport type InfoMap = Partial>;\n\nexport interface NeedLazyLoad {\n lazy: string | Promise;\n}\nexport interface LazyPrefixInfo {\n children?: LazyInfoMap;\n info?: PrefixInfoData[];\n totalChildren: number;\n}\nexport type LazyInfoMap = NeedLazyLoad | Partial>;\n\nexport function addRecord(\n map: InfoMap,\n prefix: IsbnPrefixWithDashes,\n record: PrefixInfoData,\n) {\n let layer = map;\n // eslint-disable-next-line @typescript-eslint/no-misused-spread\n for (const [i, _digit] of [...prefix].entries()) {\n if (_digit === \"-\") continue;\n const digit = _digit as Digit;\n layer[digit] ??= { totalChildren: 0 };\n const target = layer[digit];\n const isLast = i === prefix.length - 1;\n if (isLast) {\n target.info ??= [];\n target.info.push(record);\n } else {\n target.totalChildren++;\n target.children ??= {};\n layer = target.children;\n }\n }\n}\n" }, { "path": "src/lib/json-fetch.ts", "content": "export async function fetchJson(fname: string) {\n const config = (await import(\"../config\")).default;\n const gzip = config.jsonCompression === \"gzip\";\n const res = await fetch(`${fname}${gzip ? \".gz\" : \"\"}`);\n if (!res.ok)\n throw Error(String(res.status) + \" \" + res.statusText, { cause: res });\n let stream = res.body;\n if (!stream) throw Error(\"No body\");\n if (\n gzip &&\n /* vite dev server has a bug where it sends gzip files as content-encoding gzip\n */\n (import.meta.env.MODE !== \"development\" || fname.startsWith(\"https://\"))\n )\n stream = stream.pipeThrough(new DecompressionStream(\"gzip\"));\n const map = (await new Response(stream).json()) as T;\n return map;\n}\n" }, { "path": "src/lib/prefix-data.ts", "content": "import isbnlib from \"isbn3\";\nimport {\n addRecord,\n Digit,\n InfoMap,\n LazyInfoMap,\n LazyPrefixInfo,\n PrefixInfo,\n PrefixInfoData,\n} from \"./info-map\";\nimport { fetchJson } from \"./json-fetch\";\nimport {\n digits,\n isbnEANStart,\n IsbnPrefixWithDashes,\n IsbnPrefixWithoutDashes,\n totalIsbns,\n} from \"./util\";\n\nconst testGroups: PrefixInfoData[] = [];\nfor (let x = 1; x <= 10; x++) {\n testGroups.push(\n ...digits.map((i) => ({\n prefix: `978-${String(i).padStart(x, \"0\")}` as IsbnPrefixWithDashes,\n source: \"publisher-ranges\" as const,\n name: \"test\",\n })),\n );\n}\n\nexport async function resolveOnePrefixLevel(\n prefix: LazyPrefixInfo,\n fetchRoot: string,\n): Promise<{ children?: InfoMap; info?: PrefixInfoData[] }> {\n if (prefix.children && \"lazy\" in prefix.children) {\n if (typeof prefix.children.lazy === \"string\") {\n const fname = prefix.children.lazy;\n prefix.children.lazy = (async () => {\n const map = await fetchJson(`${fetchRoot}/${fname}`);\n prefix.children = map;\n })();\n }\n await prefix.children.lazy;\n return prefix as PrefixInfo;\n }\n return prefix as PrefixInfo;\n}\nexport function addIsbnGroups(\n prefixData: InfoMap,\n {\n testMode = false,\n addUnassigned,\n }: { testMode: boolean; addUnassigned: boolean },\n) {\n if (testMode) {\n // empty\n prefixData[9] = { totalChildren: 0 };\n for (const group of testGroups) addRecord(prefixData, group.prefix, group);\n return;\n }\n for (const [prefix, group] of Object.entries(isbnlib.groups) as [\n IsbnPrefixWithDashes,\n (typeof isbnlib.groups)[string],\n ][]) {\n addRecord(prefixData, prefix, {\n ...group,\n prefix,\n source: \"publisher-ranges\",\n });\n }\n const musicPrefix = \"979-0\" as IsbnPrefixWithDashes;\n addRecord(prefixData, musicPrefix, {\n prefix: musicPrefix,\n name: \"Sheet Music (ISMNs)\",\n source: \"publisher-ranges\",\n });\n if (addUnassigned) {\n const rootPrefixInfo = {\n children: prefixData,\n totalChildren: 0,\n };\n for (let i = 0; i < totalIsbns / 1e8; i++) {\n const range = String(isbnEANStart + i * 1e8);\n const prefix = (range[0] +\n range[1] +\n range[2] +\n \"-\" +\n range[3]) as IsbnPrefixWithDashes;\n if (\n !getGroup(rootPrefixInfo, prefix) &&\n !digits.some((e) =>\n getGroup(rootPrefixInfo, (prefix + e) as IsbnPrefixWithDashes),\n )\n ) {\n addRecord(prefixData, prefix, {\n prefix,\n name: \"Unassigned\",\n color: \"black\",\n source: \"publisher-ranges\",\n });\n }\n }\n }\n}\nexport interface LazyPrefixInfoWithParents {\n outers: LazyPrefixInfo[];\n inner: LazyPrefixInfo | null;\n}\nexport function getGroupHierarchy(\n rootPrefixInfo: LazyPrefixInfo,\n prefix: IsbnPrefixWithDashes | IsbnPrefixWithoutDashes,\n allowFetch = true,\n):\n | LazyPrefixInfoWithParents\n | ((prefixRoot: string) => Promise) {\n const infos: LazyPrefixInfo[] = [];\n let cur: LazyPrefixInfo = rootPrefixInfo;\n for (const c of prefix as Iterable) {\n if (c === \"-\") continue;\n if (cur.info) infos.push(cur);\n if (cur.children) {\n if (\"lazy\" in cur.children) {\n if (allowFetch) {\n return async (fetchRoot: string) => {\n await resolveOnePrefixLevel(cur, fetchRoot);\n const res = getGroupHierarchy(rootPrefixInfo, prefix);\n // flatten\n if (typeof res === \"function\") return await res(fetchRoot);\n return res;\n };\n } else {\n return { outers: infos, inner: null };\n }\n }\n }\n if (!cur.children?.[c]) return { outers: infos, inner: null };\n cur = cur.children[c];\n }\n return { outers: infos, inner: cur };\n}\nexport function getGroup(\n rootPrefixInfo: LazyPrefixInfo,\n prefix: IsbnPrefixWithDashes,\n):\n | LazyPrefixInfo\n | null\n | ((fetchRoot: string) => Promise) {\n const h = getGroupHierarchy(rootPrefixInfo, prefix);\n if (typeof h === \"function\")\n return (fetchRoot: string) => h(fetchRoot).then((h) => h.inner);\n return h.inner;\n}\n" }, { "path": "src/lib/shader-error.ts", "content": "// https://github.com/Mugen87/three.js/blob/a94de2aa8b2ef1830adeea3be5d1d1eca5b6e1f4/src/renderers/webgl/WebGLProgram.js\nfunction handleSource(string: string, errorLine: number) {\n const lines = string.split(\"\\n\");\n const lines2 = [];\n\n const from = Math.max(errorLine - 6, 0);\n const to = Math.min(errorLine + 6, lines.length);\n\n for (let i = from; i < to; i++) {\n const line = i + 1;\n lines2.push(`${line === errorLine ? \">\" : \" \"} ${line}: ${lines[i]}`);\n }\n\n return lines2.join(\"\\n\");\n}\n\nfunction getShaderErrors(\n gl: WebGLRenderingContext,\n shader: WebGLShader,\n type: string,\n) {\n const status = gl.getShaderParameter(shader, gl.COMPILE_STATUS) as string;\n const log = gl.getShaderInfoLog(shader);\n if (!log) return \"\";\n const errors = log.trim();\n\n if (status && errors === \"\") return \"\";\n\n const errorMatches = /ERROR: 0:(\\d+)/.exec(errors);\n if (errorMatches) {\n // --enable-privileged-webgl-extension\n // console.log( '**' + type + '**', gl.getExtension( 'WEBGL_debug_shaders' ).getTranslatedShaderSource( shader ) );\n const source = gl.getShaderSource(shader);\n if (!source) return errors;\n const errorLine = parseInt(errorMatches[1]);\n return (\n type.toUpperCase() +\n \"\\n\\n\" +\n errors +\n \"\\n\\n\" +\n handleSource(source, errorLine)\n );\n } else {\n return errors;\n }\n}\n\nexport const shaderErrorToString = (\n gl: WebGLRenderingContext,\n program: WebGLProgram,\n glVertexShader: WebGLShader,\n glFragmentShader: WebGLShader,\n) => {\n const programLog = gl.getProgramInfoLog(program)?.trim();\n const vertexErrors = getShaderErrors(gl, glVertexShader, \"vertex\");\n const fragmentErrors = getShaderErrors(gl, glFragmentShader, \"fragment\");\n\n const err =\n \"THREE.WebGLProgram: Shader Error \" +\n String(gl.getError()) +\n \" - \" +\n \"VALIDATE_STATUS \" +\n String(gl.getProgramParameter(program, gl.VALIDATE_STATUS)) +\n \"\\n\\n\" +\n \"Program Info Log: \" +\n String(programLog) +\n \"\\n\" +\n vertexErrors +\n \"\\n\" +\n fragmentErrors;\n\n return err;\n};\n" }, { "path": "src/lib/shaders.ts", "content": "import { makeAutoObservable } from \"mobx\";\nimport * as THREE from \"three\";\nimport { ShaderMaterial } from \"three\";\nimport { gradientsPngUrl } from \"../components/Legend\";\nimport config from \"../config\";\nimport { ImageLoader } from \"./ImageLoader\";\nimport { Store } from \"./Store\";\nimport { IsbnPrefixRelative, removeDashes } from \"./util\";\n\nconst uniforms = {\n GLOW: \"int\",\n IS_IMAGE_MAX_ZOOM: \"bool\",\n DO_BOOKSHELF_EFFECT: \"bool\",\n CURRENT_ZOOM: \"float\",\n PUBLISHERS_BRIGHTNESS: \"float\",\n HIGHLIGHTED_PUBLISHER_ID: \"int\",\n HIGHLIGHTED_PUBLISHER_PREFIX_LENGTH: \"int\",\n PUBLISHERS_COLOR_SCHEME: \"int\",\n MIN_PUBLICATION_YEAR: \"int\",\n MAX_PUBLICATION_YEAR: \"int\",\n CHOSEN_COLOR_GRADIENT: \"int\",\n\n gradients: \"sampler2D\",\n};\ntype UniformNames = keyof typeof uniforms;\nconst makeFragmentShader = (colorFn: string) => `\nin vec2 vUv;\n// allow up to 6 datasets\nuniform sampler2D col1;\nuniform sampler2D col2;\nuniform sampler2D col3;\nuniform sampler2D col4;\nuniform sampler2D col5;\nuniform sampler2D col6;\nuniform sampler2D col7; // dummy data\n\n${Object.entries(uniforms)\n .map(([name, type]) => `uniform ${type} ${name};`)\n .join(\"\\n\")}\n\nout vec4 fragColor;\n\nivec4 getOrigRGB(vec4 c) {\n return ivec4(c * 255.0);\n}\n// less random but reproducible in JS\nfloat rand(vec2 co){\n // return fract(sin(dot(co, vec2(12.9898, 78.233)))); // fract(length(co) / 1000.0));\n return fract(sin(dot(co, vec2(12.9898, 78.233))) * 2.);\n}\n// more random\nhighp float rand2(vec2 co) {\n highp float a = 12.9898;\n highp float b = 78.233;\n highp float c = 43758.5453;\n highp float dt= dot(co.xy, vec2(a,b));\n highp float sn= mod(dt,3.14);\n return fract(sin(sn) * c);\n}\n\nfloat getBookDecoration(vec2 positionWithinPixel, vec2 bookIndex) {\n float pattern = rand2(bookIndex * 2.3);\n float patternScale = 3.0 + rand2(bookIndex * 3.7) * 6.0; // Random scale between 4 and 12\n float pattern2 = rand2(bookIndex * 5.1); // Second pattern selector\n float decoration = 0.0;\n\n if (positionWithinPixel.y < 0.23) return 0.0;\n \n vec2 patternUV = positionWithinPixel;\n \n if (pattern < 0.25) {\n // Vertical stripes with varying frequency\n decoration = step(0.5, fract(patternUV.y * patternScale));\n } else if (pattern < 0.5) {\n // Diagonal stripes\n float diagonalPos = (patternUV.x + patternUV.y) * patternScale / 2.0;\n decoration = step(0.5, fract(diagonalPos));\n } else if (pattern < 0.75) {\n // Dots with varying size and spacing\n vec2 dotUV = fract(patternUV * patternScale) - 0.5;\n float dotSize = 0.15 + rand2(bookIndex * 7.3) * 0.2;\n decoration = 1.0 - smoothstep(dotSize, dotSize + 0.05, length(dotUV));\n } else {\n // Mixed pattern based on second random value\n if (pattern2 < 0.33) {\n // Checkerboard\n vec2 checkUV = floor(patternUV * patternScale);\n decoration = mod(checkUV.x + checkUV.y, 2.0);\n } else if (pattern2 < 0.66) {\n // Diamond pattern with varying size\n vec2 diamondUV = fract(patternUV * patternScale) - 0.5;\n float diamondSize = 0.2 + rand2(bookIndex * 9.1) * 0.3;\n decoration = 1.0 - smoothstep(diamondSize, diamondSize + 0.05, abs(diamondUV.x) + abs(diamondUV.y));\n } else {\n // Crosshatch\n float hatch1 = step(0.5, fract((patternUV.x - patternUV.y) * patternScale));\n float hatch2 = step(0.5, fract((patternUV.x + patternUV.y) * patternScale));\n decoration = min(hatch1 + hatch2, 1.0);\n }\n }\n \n return decoration;\n}\n\nvec4 bookshelfOverlay(vec4 bookColor) {\n if (!DO_BOOKSHELF_EFFECT) return bookColor;\n \n vec2 textureSize = vec2(textureSize($first_dataset, 0));\n vec2 bookIndex = floor(vUv.xy * textureSize);\n vec2 positionWithinPixel = mod(vUv.xy * textureSize, 1.0);\n float bookshelfHeight = 0.03;\n vec4 bookshelfColor = vec4(${config.bookshelfColor});\n \n positionWithinPixel.y -= bookshelfHeight;\n vec4 bgColor = (positionWithinPixel.y < 0.0 ? bookshelfColor : vec4(0.0));\n \n vec2 distanceToEdge = (0.5 - abs(positionWithinPixel - 0.5));\n float minBookWidth = 0.9;\n float maxBookWidth = 0.99;\n float bookWidth = minBookWidth + (maxBookWidth - minBookWidth) * rand(bookIndex);\n \n if (distanceToEdge.x < (1.0 - bookWidth)) return bgColor;\n \n float minBookHeight = 0.8;\n float maxBookHeight = 0.95;\n float bookHeight = minBookHeight + (maxBookHeight - minBookHeight) * rand(bookIndex * 1.2);\n vec2 bookCenter = vec2(0.5, bookHeight / 2.0);\n float stretchy = 1.5;\n \n if (length((positionWithinPixel - bookCenter) * vec2(1.0, stretchy * sqrt(10.0))) > stretchy * 1.6 * bookHeight) \n return bgColor;\n \n float decoration = getBookDecoration(positionWithinPixel, bookIndex);\n \n // Mix original color with generated color and decoration\n // vec4 cla = min(bookColor, vec4(1.0));\n vec4 decoratedColor = mix(bookColor, bookColor * 0.7, decoration * 0.5);\n \n // Apply edge shading\n return decoratedColor - decoratedColor * (vec4(1.0) * 3.0 * pow(0.5-distanceToEdge.x, 2.0));\n}\n\nvec4 bookshelfOverlayDependingOnZoom(vec4 bookColor) {\n if (!IS_IMAGE_MAX_ZOOM) return bookColor;\n float minZoom = 90.0;\n if (CURRENT_ZOOM < 90.0) return bookColor;\n vec4 c1 = bookshelfOverlay(bookColor);\n float maxZoom = minZoom * sqrt(10.0);\n float fadeIn = clamp((CURRENT_ZOOM - minZoom) / (maxZoom - minZoom), 0.0, 1.0);\n return mix(bookColor, c1, fadeIn);\n}\n\nfloat brightnessWithGlow(float brightness) {\n return clamp(brightness * float(2 * GLOW + 1), 0., 1.);\n}\n\nvec4 texture2DWithGlow(sampler2D col1, vec2 vUv) {\n vec2 textureSize = vec2(textureSize(col1, 0));\n vec4 books = texture2D(col1, vUv);\n if (IS_IMAGE_MAX_ZOOM && CURRENT_ZOOM > 50.0) return books;\n if (GLOW >= 1) {\n books *= float(GLOW);\n books += texture2D(col1, vUv + vec2(1, 0) / textureSize);\n books += texture2D(col1, vUv + vec2(0, 1) / textureSize);\n books += texture2D(col1, vUv + vec2(-1, 0) / textureSize);\n books += texture2D(col1, vUv + vec2(0, -1) / textureSize);\n }\n if (GLOW >= 2) {\n books *= float(GLOW);\n books += texture2D(col1, vUv + vec2(1, 1) / textureSize);\n books += texture2D(col1, vUv + vec2(-1, 1) / textureSize);\n books += texture2D(col1, vUv + vec2(1, -1) / textureSize);\n books += texture2D(col1, vUv + vec2(-1, -1) / textureSize);\n }\n if (GLOW >= 3) {\n books *= float(GLOW);\n books += texture2D(col1, vUv + vec2(2, 0) / textureSize);\n books += texture2D(col1, vUv + vec2(0, 2) / textureSize);\n books += texture2D(col1, vUv + vec2(-2, 0) / textureSize);\n books += texture2D(col1, vUv + vec2(0, -2) / textureSize);\n }\n return books;\n}\n\nvec4 publisherColorDark(int publisherId) {\n if (publisherId == 0) return vec4(0.0);\n float random = rand2(vec2(float((publisherId%256)) * 54.1, float(publisherId) / 15260.1));\n vec4 color1 = vec4(0.396, 0.263, 0.229, 1.);\n vec4 color2 = vec4(0.129, 0.263, 0.396, 1.);\n vec4 pubColor = mix(color1, color2, random);\n return pubColor;\n}\nvec3 hsv2rgb(vec3 c) {\n vec4 K = vec4(1.0, 2.0 / 3.0, 1.0 / 3.0, 3.0);\n vec3 p = abs(fract(c.xxx + K.xyz) * 6.0 - K.www);\n return c.z * mix(K.xxx, clamp(p - K.xxx, 0.0, 1.0), c.y);\n}\n\n#define PI 3.1415926538\nvec4 heatmapColor(float x) {\n // each color is 20 pixels high, so take the middle of that\n float ySize = float(textureSize(gradients, 0).y);\n return texture2D(gradients, vec2(x, (ySize - 10. - float(CHOSEN_COLOR_GRADIENT) * 20.) / ySize));\n // https://stackoverflow.com/questions/28495390/thermal-imaging-palette-table\n // return vec4(sqrt(x), pow(x,3.), (sin(2. * PI * x) >= 0. ? sin(2. * PI * x) : 0.), 1.0);\n}\n\nvec4 publisherColorHSL(int publisherId) {\n if (publisherId == 0) return vec4(0.0);\n float random = rand2(vec2(float((publisherId%258)) * 54.1, float(publisherId) / 15260.1));\n return vec4(hsv2rgb(vec3(random, 0.5, 0.5)), 1.0);\n}\nvec4 withPublisherColor(vec4 bookColor, float publishersBrightness) {\n vec4 pubColorRaw = texture2D($dataset_publishers, vUv);\n ivec4 data = getOrigRGB(pubColorRaw);\n int publisherId = data.r * 65536 + data.g * 256 + data.b;\n vec4 publisherColor = PUBLISHERS_COLOR_SCHEME == 1 ? publisherColorHSL(publisherId) : publisherColorDark(publisherId);\n vec4 color = bookColor + publishersBrightness * publisherColor;\n if (HIGHLIGHTED_PUBLISHER_ID != 0) {\n if (HIGHLIGHTED_PUBLISHER_ID == publisherId) {\n float minZoom = 0.05 * pow(10., float(HIGHLIGHTED_PUBLISHER_PREFIX_LENGTH) / 2.0);\n float maxZoom = minZoom * sqrt(10.0);\n float fadeOut = clamp((CURRENT_ZOOM - minZoom) / (maxZoom - minZoom), 0.4, 1.0);\n return mix(vec4(137.0, 196.0, 244.0, 255.0) / 255., color, fadeOut);\n }\n }\n return color;\n}\n\nvec4 filterPublicationRange(vec4 bookColor) {\n if (MIN_PUBLICATION_YEAR == -1 && MAX_PUBLICATION_YEAR == -1) return bookColor;\n ivec4 data = getOrigRGB(texture2D($dataset_publication_date, vUv));\n // zero means no data, 1 means 1801 or before, 255 means 2055 or later\n if (data.r == 0) return vec4(0.);\n int publicationYear = (data.r + 1800);\n if (MIN_PUBLICATION_YEAR != -1 && publicationYear < MIN_PUBLICATION_YEAR) return vec4(0.);\n if (MAX_PUBLICATION_YEAR != -1 && publicationYear > MAX_PUBLICATION_YEAR) return vec4(0.);\n return bookColor;\n}\n\nvec4 postprocessColor(vec4 bookColor, float publishersBrightness) {\n return bookshelfOverlayDependingOnZoom(withPublisherColor(filterPublicationRange(bookColor), publishersBrightness));\n}\n\nvec4 postprocessColor(vec4 bookColor) {\n return postprocessColor(bookColor, PUBLISHERS_BRIGHTNESS);\n}\n\n\n${colorFn}\n\nvoid main() {\n fragColor = colorOfPixel(vUv);\n}`;\n\ntype TypedUniforms = Record<\n UniformNames,\n { value: number | boolean | THREE.Texture }\n>;\nexport class ShaderUtil {\n constructor(private store: Store) {\n makeAutoObservable(this);\n }\n gradientsTexture = new THREE.TextureLoader()\n .loadAsync(gradientsPngUrl)\n .then((t) => {\n t.colorSpace = THREE.NoColorSpace;\n t.magFilter = THREE.LinearFilter;\n t.minFilter = THREE.LinearFilter;\n return t;\n });\n\n get shaderColorFn() {\n if (this.store.runtimeConfig.customShader) {\n return this.store.runtimeConfig.customShader;\n }\n if (this.store.runtimeConfig.dataset === \"all-md5\") {\n return `vec4 colorOfPixel(vec2 uv) {\n vec4 present_all = texture2D($dataset_all, uv);\n vec4 present_md5 = texture2D($dataset_md5, uv);\n\n // vec4 present_gradient = vec4(present_all.x - present_md5.x, present_md5.x + present_all.x * 0.1, present_all.x * 0.1, 1);\n vec4 present_gradient = heatmapColor(present_md5.x / present_all.x) * brightnessWithGlow(present_all.x);\n // add publishers only in background (when brightness of gradient > 0.1);\n float publisherStrength = length(present_gradient) > 1.1 ? 0.0 : PUBLISHERS_BRIGHTNESS;\n return postprocessColor(present_gradient, publisherStrength);\n}\n`;\n } else if (this.store.runtimeConfig.dataset === \"rarity\") {\n return `vec4 colorOfPixel(vec2 uv) {\n vec4 data = texture2D($dataset_rarity, uv);\n // create linear gradient between red and green\n vec4 colorRare = vec4(1.0,0.0,0.0,1);\n vec4 colorCommon = vec4(0.0,1.0,0,1);\n // png range 0-255, shader range 0-1\n ivec4 dataOrig = getOrigRGB(data);\n int holdingCount = dataOrig.r;\n int editionCount = dataOrig.g;\n int bookCount = dataOrig.b;\n if (bookCount != 0) {\n float averageHoldingPerBook = float(holdingCount) / float(bookCount);\n // make gradient between 0 (rare) and 1 (common)\n float rarity = clamp(pow(averageHoldingPerBook / 20.0, 2.0), 0., 1.);\n // float presence = max(data.r, max(data.g, data.b)); // since we scale down one of the values when max > 255\n return postprocessColor(heatmapColor(rarity)); // * brightnessWithGlow(presence);\n }\n return postprocessColor(vec4(0.));\n}`;\n } else if (this.store.runtimeConfig.dataset === \"publication_date\") {\n return `vec4 colorOfPixel(vec2 uv) {\n vec4 bookColor = texture2D($dataset_publication_date, uv);\n if (bookColor.r != 0.) {\n float publicationYear = (bookColor.r * 255.) + 1800.; // average publication year in this pixel\n float fillRate = bookColor.b; // 0-100% number of books present\n float minYear = 1985.;\n float maxYear = 2025.;\n float brightness = brightnessWithGlow(fillRate);\n bookColor = heatmapColor(clamp((publicationYear - minYear) / (maxYear - minYear), 0., 1.)) * brightness;\n }\n return postprocessColor(bookColor);\n}`;\n } else if (this.store.runtimeConfig.dataset === \"publishers\") {\n return `\nvec4 colorOfPixel(vec2 uv) {\n // 1.0 stands for brightness of publishers = 100%\n return postprocessColor(vec4(0.), 1.0);\n}`;\n } else {\n return `vec4 colorOfPixel(vec2 uv) {\n vec4 bookColor = texture2D($dataset_${this.store.runtimeConfig.dataset}, uv);\n return postprocessColor(heatmapColor(bookColor.r));\n}`;\n }\n }\n\n get shaderProgram() {\n const requiredTextures: string[] = [];\n const cfg = this.store.runtimeConfig;\n const fragmentShader = makeFragmentShader(this.shaderColorFn).replace(\n /\\$([a-z0-9_]+)/g,\n (_, dataset: string) => {\n if (\n dataset === \"dataset_publishers\" &&\n cfg.publishersBrightness === 0 &&\n cfg.dataset !== \"publishers\"\n ) {\n return `col7`; // fake / empty texture\n } else if (\n dataset === \"dataset_publication_date\" &&\n cfg.dataset !== \"publication_date\" &&\n cfg.filterMinimumPublicationYear === -1 &&\n cfg.filterMaximumPublicationYear === -1\n ) {\n return `col7`; // fake / empty texture\n } else if (dataset === \"first_dataset\") {\n return cfg.dataset === \"dataset_publishers\" ? `col1` : `col2`;\n } else {\n requiredTextures.push(dataset.replace(\"dataset_\", \"\"));\n return `col${requiredTextures.length}`;\n }\n },\n );\n return { fragmentShader, requiredTextures };\n }\n\n async getIsbnShaderMaterial(\n prefix: IsbnPrefixRelative,\n ): Promise<{ material: ShaderMaterial; refreshUniforms: () => void } | null> {\n const { requiredTextures, fragmentShader } = this.shaderProgram;\n const textures = await Promise.all(\n requiredTextures.map((d) => this.store.imageLoader(d).getTexture(prefix)),\n );\n const gradientsTexture = await this.gradientsTexture;\n const isMaxZoom = prefix.length >= ImageLoader.maxZoomPrefixLength;\n\n const material = new ShaderMaterial({\n lights: false,\n uniforms: {\n DO_BOOKSHELF_EFFECT: { value: true },\n CURRENT_ZOOM: { value: 1 },\n IS_IMAGE_MAX_ZOOM: { value: isMaxZoom },\n GLOW: { value: 0 },\n PUBLISHERS_BRIGHTNESS: { value: 0 },\n HIGHLIGHTED_PUBLISHER_ID: { value: 0 },\n HIGHLIGHTED_PUBLISHER_PREFIX_LENGTH: { value: 0 },\n PUBLISHERS_COLOR_SCHEME: { value: 0 },\n gradients: { value: gradientsTexture },\n ...Object.fromEntries(\n textures.map((_, i) => [`col${i + 1}`, { value: textures[i] }]),\n ),\n MIN_PUBLICATION_YEAR: { value: -1 },\n MAX_PUBLICATION_YEAR: { value: -1 },\n CHOSEN_COLOR_GRADIENT: { value: 0 },\n } satisfies TypedUniforms,\n glslVersion: THREE.GLSL3,\n // blending,\n vertexShader: `\nvarying vec2 vUv;\nvoid main() {\n vUv = uv;\n gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0);\n}\n`,\n fragmentShader,\n });\n return {\n material,\n refreshUniforms: () => {\n const config = this.store.runtimeConfig;\n const unis = material.uniforms as TypedUniforms;\n // material.uniforms.col1.value = texture1;\n // material.uniforms.col2.value = texture2;\n unis.DO_BOOKSHELF_EFFECT.value = config.doBookshelfEffect;\n unis.CURRENT_ZOOM.value = this.store.floatZoomFactor;\n unis.GLOW.value = config.shaderGlow;\n unis.PUBLISHERS_BRIGHTNESS.value = config.publishersBrightness;\n const publisherInfo =\n this.store.highlightedPublisher?.data?.[1]?.info?.[0];\n unis.HIGHLIGHTED_PUBLISHER_ID.value = publisherInfo?.numericId ?? 0;\n unis.HIGHLIGHTED_PUBLISHER_PREFIX_LENGTH.value = publisherInfo\n ? removeDashes(publisherInfo.prefix).length - 2\n : 0;\n unis.PUBLISHERS_COLOR_SCHEME.value = [undefined, \"hsl\", \"dark\"].indexOf(\n config.publishersColorSchema,\n );\n unis.MIN_PUBLICATION_YEAR.value = config.filterMinimumPublicationYear;\n unis.MAX_PUBLICATION_YEAR.value = config.filterMaximumPublicationYear;\n unis.CHOSEN_COLOR_GRADIENT.value = config.colorGradient;\n },\n };\n }\n}\n" }, { "path": "src/lib/stats.ts", "content": "import { fetchJson } from \"./json-fetch\";\nimport { Store } from \"./Store\";\nimport { IsbnPrefixWithoutDashes } from \"./util\";\n\nexport type StatsMap = Partial>;\nexport type BlockStats = Partial>;\nexport class StatsCalculator {\n #data: StatsMap | null = null;\n constructor(private store: Store) {}\n #getRanges(\n startPrefix: IsbnPrefixWithoutDashes,\n endPrefix: IsbnPrefixWithoutDashes,\n ) {\n const components: string[] = [];\n function recurse(prefix: string, left: string, right: string) {\n if (left.length === 0 && right.length === 0) {\n components.push(prefix);\n return;\n }\n const leftDigit = left[0];\n const rightDigit = right[0];\n if (leftDigit === rightDigit) {\n recurse(prefix + left[0], left.slice(1), right.slice(1));\n return;\n }\n if (leftDigit > rightDigit) {\n throw Error(\"leftDigit > rightDigit\");\n }\n if (left.length === 1) components.push(prefix + leftDigit);\n else if (left.length > 1) recurse(prefix + leftDigit, left.slice(1), \"9\");\n for (let i = +leftDigit + 1; i < +rightDigit; i++) {\n components.push(prefix + String(i));\n }\n if (right.length === 1) components.push(prefix + rightDigit);\n else if (right.length > 1)\n recurse(prefix + rightDigit, \"0\", right.slice(1));\n }\n\n recurse(\"\", startPrefix, endPrefix);\n return components as IsbnPrefixWithoutDashes[];\n }\n async #fetchStats(): Promise {\n if (!this.#data) {\n this.#data = await fetchJson(\n `${this.store.runtimeConfig.jsonRoot}/stats.json`,\n );\n }\n return this.#data;\n }\n async getStats(\n startPrefix: IsbnPrefixWithoutDashes,\n endPrefix: IsbnPrefixWithoutDashes,\n ) {\n const ranges = this.#getRanges(startPrefix, endPrefix);\n const stats = await this.#fetchStats();\n const output: BlockStats = {};\n for (const range of ranges) {\n const cur = stats[range];\n if (cur) mergeStats(output, cur);\n }\n return output;\n }\n}\n\nexport function mergeStats(target: BlockStats, source: BlockStats) {\n for (const key in source) {\n target[key] = (target[key] ?? 0) + (source[key] ?? 0);\n }\n}\n" }, { "path": "src/lib/types-select.d.ts", "content": "import type {} from \"react-select/base\";\nimport { Store } from \"./Store\";\n// This import is necessary for module augmentation.\n// It allows us to extend the 'Props' interface in the 'react-select/base' module\n// and add our custom property 'myCustomProp' to it.\n\ndeclare module \"react-select/base\" {\n export interface Props<\n Option,\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n IsMulti extends boolean,\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n Group extends GroupBase