Full Code of g-plane/tiny-package-manager for AI

Repository: g-plane/tiny-package-manager
Branch: master
Commit: 643fe89cf76e
Files: 19
Total size: 21.0 KB

Directory structure:
gitextract_g2jw__t_/

├── .editorconfig
├── .eslintrc.yml
├── .github/
│   └── FUNDING.yml
├── .gitignore
├── .vscode/
│   └── settings.json
├── LICENSE
├── README.md
├── azure-pipelines.yml
├── dprint.json
├── package.json
├── src/
│   ├── cli.ts
│   ├── index.ts
│   ├── install.ts
│   ├── list.ts
│   ├── lock.ts
│   ├── log.ts
│   ├── resolve.ts
│   └── utils.ts
└── tsconfig.json

================================================
FILE CONTENTS
================================================

================================================
FILE: .editorconfig
================================================
root = true

[*]
indent_style = space
indent_size = 2
end_of_line = lf
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true

================================================
FILE: .eslintrc.yml
================================================
extends:
  - gplane
  - gplane/node
  - gplane/typescript


================================================
FILE: .github/FUNDING.yml
================================================
github: g-plane


================================================
FILE: .gitignore
================================================
dist/
node_modules/
yarn-error.log


================================================
FILE: .vscode/settings.json
================================================
{
  "eslint.validate": [
    "javascript",
    "javascriptreact",
    {
      "language": "typescript",
      "autoFix": true
    }
  ]
}


================================================
FILE: LICENSE
================================================
MIT License

Copyright (c) 2018-present Pig Fang

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.


================================================
FILE: README.md
================================================
# Tiny Package Manager

> A very very simple demo and guide for explaining package manager.

## Introduction

As a JavaScript developer, you may use package manager like [npm](https://www.npmjs.com/) or [Yarn](https://yarnpkg.com/)
frequently.

However, do you know how a package manager works? Or, you may be curious about how to build a package manager.

Well, the purpose of this guide is not to let you re-invent a new wheel.
There is no need to do that because both npm and Yarn are mature and stable enough.
The purpose is just to explain how a package manager works under the hood.
You can read the code, and the comments will explain how it works.

Note: To simplify the guide and make it as simple as possible,
this demo doesn't handle some edge cases and catch errors and exceptions.
If you are really curious about that,
it's recommended to read the source code of [npm](https://github.com/npm/npm) or [Yarn](https://github.com/yarnpkg/yarn).

## Features

- [x] Download packages to `node_modules` directory.
- [x] Simple CLI.
- [x] Simply resolve dependency conflicts.
- [x] Flatten dependencies tree.
- [x] Support lock file. (Like `yarn.lock` or `package-lock.json`)
- [x] Add a new package through CLI. (Like `yarn add` or `npm i <package>` command)
- [ ] Run lifecycle scripts. (`preinstall` and `postinstall`)
- [ ] Symlink the `bin` files.

## How to start?

Read the source code in the `src` directory.
You can read the `src/index.ts` file in the beginning.

If you would like to try this simple package manager,
just install it globally:

Via Yarn:

```
$ yarn global add tiny-package-manager
```

Via npm:

```
$ npm i -g tiny-package-manager
```

Then just go to a directory which contains valid `package.json` and run:

```
$ tiny-pm
```

## License

MIT License (c) 2018-present [Pig Fang](https://gplane.win/)


================================================
FILE: azure-pipelines.yml
================================================
pool:
  vmImage: 'Ubuntu-16.04'

steps:
- task: NodeTool@0
  inputs:
    versionSpec: '10.x'
  displayName: 'Install Node.js'

- script: |
    yarn
    yarn test
  displayName: 'Run tests'


================================================
FILE: dprint.json
================================================
{
  "extends": "https://dprint.gplane.win/2024-01.json"
}


================================================
FILE: package.json
================================================
{
  "name": "tiny-package-manager",
  "version": "1.1.0",
  "description": "The tiny package manager.",
  "main": "dist/index.js",
  "repository": "g-plane/tiny-package-manager",
  "author": "Pig Fang <g-plane@hotmail.com>",
  "license": "MIT",
  "private": false,
  "bin": {
    "tiny-pm": "dist/cli.js"
  },
  "files": [
    "dist"
  ],
  "scripts": {
    "dev": "tsc -p . -w",
    "build": "tsc -p .",
    "prepublishOnly": "npm run build",
    "lint": "eslint . -f=beauty --ext=.ts",
    "test": "npm run lint",
    "fmt": "dprint fmt"
  },
  "engines": {
    "node": ">8"
  },
  "dependencies": {
    "find-up": "^5.0.0",
    "fs-extra": "^11.1.0",
    "js-yaml": "^4.1.0",
    "log-update": "^4.0.0",
    "progress": "^2.0.3",
    "semver": "^7.3.8",
    "tar": "^6.1.13",
    "undici": "^5.26.4",
    "yargs": "^17.6.2"
  },
  "devDependencies": {
    "@gplane/tsconfig": "^5.0.0",
    "@types/fs-extra": "^11.0.1",
    "@types/js-yaml": "^4.0.5",
    "@types/progress": "^2.0.5",
    "@types/semver": "^7.3.13",
    "@types/tar": "^6.1.3",
    "@types/yargs": "^17.0.20",
    "@typescript-eslint/eslint-plugin": "^5.49.0",
    "@typescript-eslint/parser": "^5.49.0",
    "dprint": "^0.45.0",
    "eslint": "^8.33.0",
    "eslint-config-gplane": "^6.2.2",
    "eslint-formatter-beauty": "^3.0.0",
    "typescript": "^4.9.4"
  }
}


================================================
FILE: src/cli.ts
================================================
import yargs from 'yargs'
import pm from '.'

/*
 * This file is for CLI usage.
 * There isn't too much logic about package manager here.
 * For details please consult the documentation of `yargs` module.
 */

yargs
  .usage('tiny-pm <command> [args]')
  .version()
  .alias('v', 'version')
  .help()
  .alias('h', 'help')
  .command(
    'install',
    'Install the dependencies.',
    (argv) => {
      argv.option('production', {
        type: 'boolean',
        description: 'Install production dependencies only.',
      })

      argv.boolean('save-dev')
      argv.boolean('dev')
      argv.alias('D', 'dev')

      return argv
    },
    pm
  )
  .command(
    '*',
    'Install the dependencies.',
    (argv) =>
      argv.option('production', {
        type: 'boolean',
        description: 'Install production dependencies only.',
      }),
    pm
  )
  .parse()


================================================
FILE: src/index.ts
================================================
import findUp from 'find-up'
import * as fs from 'fs-extra'
import type yargs from 'yargs'
import install from './install'
import list, { PackageJson } from './list'
import * as lock from './lock'
import * as log from './log'
import * as utils from './utils'

/**
 * Welcome to learning about how to build a package manager.
 * In this guide I will tell you how to build a
 * very very simple package manager like npm or Yarn.
 *
 * I will use ES2017 syntax in this guide,
 * so please make sure you know about it.
 *
 * Also this guide is written in TypeScript.
 * Don't worry if you don't know TypeScript,
 * just treat it as JavaScript with some type annotations.
 * If you have learned Flow, that's great,
 * because they are similar.
 *
 * To make this guide as simple as possible,
 * I haven't handled some edge cases.
 *
 * Good luck and let's start!   :)
 *
 * This is just the main file of the whole tiny package manager,
 * but not all of the logic,
 * because I split them into different modules and files for better management.
 */

export default async function(args: yargs.Arguments) {
  // Find and read the `package.json`.
  const jsonPath = (await findUp('package.json'))!
  const root = await fs.readJson(jsonPath)

  /*
   * If we are adding new packages by running `tiny-pm install <packageName>`,
   * collect them through CLI arguments.
   * This purpose is to behaves like `npm i <packageName>` or `yarn add`.
   */
  const additionalPackages = args._.slice(1)
  if (additionalPackages.length) {
    if (args['save-dev'] || args.dev) {
      root.devDependencies = root.devDependencies || {}
      /*
       * At this time we don't specific version now, so set it empty.
       * And we will fill it later after fetched the information.
       */
      additionalPackages.forEach((pkg) => (root.devDependencies[pkg] = ''))
    } else {
      root.dependencies = root.dependencies || {}
      /*
       * At this time we don't specific version now, so set it empty.
       * And we will fill it later after fetched the information.
       */
      additionalPackages.forEach((pkg) => (root.dependencies[pkg] = ''))
    }
  }

  /*
   * In production mode,
   * we just need to resolve production dependencies.
   */
  if (args.production) {
    delete root.devDependencies
  }

  // Read the lock file
  await lock.readLock()

  // Generate the dependencies information.
  const info = await list(root)

  // Save the lock file asynchronously.
  lock.writeLock()

  /*
   * Prepare for the progress bar.
   * Note that we re-compute the number of packages.
   * Because of the duplication,
   * number of resolved packages is not equivalent to
   * the number of packages to be installed.
   */
  log.prepareInstall(
    Object.keys(info.topLevel).length + info.unsatisfied.length
  )

  // Install top level packages.
  await Promise.all(
    Object.entries(info.topLevel).map(([name, { url }]) => install(name, url))
  )

  // Install packages which have conflicts.
  await Promise.all(
    info.unsatisfied.map((item) => install(item.name, item.url, `/node_modules/${item.parent}`))
  )

  beautifyPackageJson(root)

  // Save the `package.json` file.
  fs.writeJson(jsonPath, root, { spaces: 2 })

  // That's all! Everything should be finished if no errors occurred.
}

/**
 * Beautify the `dependencies` field and `devDependencies` field.
 */
function beautifyPackageJson(packageJson: PackageJson) {
  if (packageJson.dependencies) {
    packageJson.dependencies = utils.sortKeys(packageJson.dependencies)
  }

  if (packageJson.devDependencies) {
    packageJson.devDependencies = utils.sortKeys(packageJson.devDependencies)
  }
}


================================================
FILE: src/install.ts
================================================
import * as fs from 'fs-extra'
import * as tar from 'tar'
import { request } from 'undici'
import * as log from './log'

export default async function(name: string, url: string, location = '') {
  // Prepare for the directory which is for installation
  const path = `${process.cwd()}${location}/node_modules/${name}`

  // Create directories recursively.
  await fs.mkdirp(path)

  const response = await request(url)
  /*
   * The response body is a readable stream
   * and the `tar.extract` accepts a readable stream,
   * so we don't need to create a file to disk,
   * and just extract the stuff directly.
   */
  response.body
    ?.pipe(tar.extract({ cwd: path, strip: 1 }))
    .on('close', log.tickInstalling) // Update the progress bar
}


================================================
FILE: src/list.ts
================================================
import * as semver from 'semver'
import * as lock from './lock'
import * as log from './log'
import resolve from './resolve'

interface DependenciesMap {
  [dependency: string]: string
}
// eslint-disable-next-line @typescript-eslint/no-type-alias
type DependencyStack = Array<{
  name: string,
  version: string,
  dependencies: { [dep: string]: string },
}>
export interface PackageJson {
  dependencies?: DependenciesMap
  devDependencies?: DependenciesMap
}

/*
 * The `topLevel` variable is to flatten packages tree
 * to avoid duplication.
 */
const topLevel: {
  [name: string]: { url: string, version: string },
} = Object.create(null)

/*
 * However, there may be dependencies conflicts,
 * so this variable is for that.
 */
const unsatisfied: Array<{ name: string, parent: string, url: string }> = []

async function collectDeps(
  name: string,
  constraint: string,
  stack: DependencyStack = [],
) {
  // Retrieve a single manifest by name from the lock.
  const fromLock = lock.getItem(name, constraint)

  /*
   * Fetch the manifest information.
   * If that manifest is not existed in the lock,
   * fetch it from network.
   */
  const manifest = fromLock || (await resolve(name))

  // Add currently resolving module to CLI
  log.logResolving(name)

  /*
   * Use the latest version of a package
   * while it will conform the semantic version.
   * However, if no semantic version is specified,
   * use the latest version.
   */
  const versions = Object.keys(manifest)
  const matched = constraint
    ? semver.maxSatisfying(versions, constraint)
    : versions[versions.length - 1] // The last one is the latest.
  if (!matched) {
    throw new Error('Cannot resolve suitable package.')
  }

  const matchedManifest = manifest[matched]!

  if (!topLevel[name]) {
    /*
     * If this package is not existed in the `topLevel` map,
     * just put it.
     */
    topLevel[name] = { url: matchedManifest.dist.tarball, version: matched }
  } else if (semver.satisfies(topLevel[name]!.version, constraint)) {
    const conflictIndex = checkStackDependencies(name, matched, stack)
    if (conflictIndex === -1) {
      /*
       * Remember to return this function to skip the dependencies checking.
       * This may avoid dependencies circulation.
       */
      return
    }
    /*
     * Because of the module resolution algorithm of Node.js,
     * there may be some conflicts in the dependencies of dependency.
     * How to check it? See the `checkStackDependencies` function below.
     * ----------------------------
     * We just need information of the previous **two** dependencies
     * of the dependency which has conflicts.
     * :(  Not sure if it's right.
     */
    unsatisfied.push({
      name,
      parent: stack
        .map(({ name }) => name) // eslint-disable-line no-shadow
        .slice(conflictIndex - 2)
        .join('/node_modules/'),
      url: matchedManifest.dist.tarball,
    })
  } else {
    /*
     * Yep, the package is already existed in that map,
     * but it has conflicts because of the semantic version.
     * So we should add a record.
     */
    unsatisfied.push({
      name,
      parent: stack.at(-1)!.name,
      url: matchedManifest.dist.tarball,
    })
  }

  // Don't forget to collect the dependencies of our dependencies.
  const dependencies = matchedManifest.dependencies ?? {}

  // Save the manifest to the new lock.
  lock.updateOrCreate(`${name}@${constraint}`, {
    version: matched,
    url: matchedManifest.dist.tarball,
    shasum: matchedManifest.dist.shasum,
    dependencies,
  })

  /*
   * Collect the dependencies of dependency,
   * so it's time to be deeper.
   */
  if (dependencies) {
    stack.push({
      name,
      version: matched,
      dependencies,
    })
    await Promise.all(
      Object.entries(dependencies)
        // The filter below is to prevent dependency circulation
        .filter(([dep, range]) => !hasCirculation(dep, range, stack))
        .map(([dep, range]) => collectDeps(dep, range, stack.slice()))
    )
    stack.pop()
  }

  /*
   * Return the semantic version range to
   * add missing semantic version range in `package.json`.
   */
  if (!constraint) {
    return { name, version: `^${matched}` }
  }
}

/**
 * This function is to check if there are conflicts in the
 * dependencies of dependency, not the top level dependencies.
 */
function checkStackDependencies(
  name: string,
  version: string,
  stack: DependencyStack,
) {
  return stack.findIndex(({ dependencies }) => {
    const semverRange = dependencies[name]
    /*
     * If this package is not as a dependency of another package,
     * this is safe and we just return `true`.
     */
    if (!semverRange) {
      return true
    }

    // Semantic version checking.
    return semver.satisfies(version, semverRange)
  })
}

/**
 * This function is to check if there is dependency circulation.
 *
 * If a package is existed in the stack and it satisfy the semantic version,
 * it turns out that there is dependency circulation.
 */
function hasCirculation(name: string, range: string, stack: DependencyStack) {
  return stack.some(
    (item) => item.name === name && semver.satisfies(item.version, range)
  )
}

/**
 * To simplify this guide,
 * We intend to support `dependencies` and `devDependencies` fields only.
 */
export default async function(rootManifest: PackageJson) {
  /*
   * For both production dependencies and development dependencies,
   * if the package name and the semantic version are returned,
   * we should add them to the `package.json` file.
   * This is necessary when adding new packages.
   */

  // Process production dependencies
  if (rootManifest.dependencies) {
    ;(
      await Promise.all(
        Object.entries(rootManifest.dependencies).map((pair) => collectDeps(...pair))
      )
    )
      .filter(Boolean)
      .forEach(
        (item) => (rootManifest.dependencies![item!.name] = item!.version)
      )
  }

  // Process development dependencies
  if (rootManifest.devDependencies) {
    ;(
      await Promise.all(
        Object.entries(rootManifest.devDependencies).map((pair) => collectDeps(...pair))
      )
    )
      .filter(Boolean)
      .forEach(
        (item) => (rootManifest.devDependencies![item!.name] = item!.version)
      )
  }

  return { topLevel, unsatisfied }
}


================================================
FILE: src/lock.ts
================================================
import * as fs from 'fs-extra'
import * as yaml from 'js-yaml'
import type { Manifest } from './resolve'
import * as utils from './utils'

// Define the type of the lock tree.
interface Lock {
  [index: string]: {
    version: string,
    url: string,
    shasum: string,
    dependencies: { [dependency: string]: string },
  }
}

// ------------ The LOCK is here. ---------------------

/*
 * Why we use two separated locks?
 * This is useful when removing packages.
 * When adding or removing packages,
 * the lock file can be updated automatically without any manual operations.
 */

/*
 * This is the old lock.
 * The old lock is only for reading from the lock file,
 * so the old lock should be read only except reading the lock file.
 */
const oldLock: Lock = Object.create(null)

/*
 * This is the new lock.
 * The new lock is only for writing to the lock file,
 * so the new lock should be written only except saving the lock file.
 */
const newLock: Lock = Object.create(null)

// ----------------------------------------------------

/**
 * Save the information of a package to the lock.
 * If that information is not existed in the lock, create it.
 * Otherwise, just update it.
 */
export function updateOrCreate(name: string, info: Lock[string]) {
  // Create it if that information is not existed in the lock.
  if (!newLock[name]) {
    newLock[name] = Object.create(null)
  }

  // Then update it.
  Object.assign(newLock[name]!, info)
}

/**
 * Retrieve the information of a package by name and it's semantic
 * version range.
 *
 * Note that we don't return the data directly.
 * That is, we just do format the data,
 * which make the data structure similar to npm registry.
 *
 * This can let us avoid changing the logic of the `collectDeps`
 * function in the `list` module.
 */
export function getItem(name: string, constraint: string): Manifest | null {
  /*
   * Retrieve an item by a key from the lock.
   * The format of the key is similar and inspired by Yarn's `yarn.lock` file.
   */
  const item = oldLock[`${name}@${constraint}`]

  // Return `null` instead of `undefined` if we cannot find that.
  if (!item) {
    return null
  }

  // Convert the data structure as the comment above.
  return {
    [item.version]: {
      dependencies: item.dependencies,
      dist: { shasum: item.shasum, tarball: item.url },
    },
  }
}

/**
 * Simply save the lock file.
 */
export async function writeLock() {
  /*
   * Sort the keys of the lock before save it.
   * This is necessary because each time you use the package manager,
   * the order will not be same.
   * Sort it makes useful for git diff.
   */
  await fs.writeFile(
    './tiny-pm.yml',
    yaml.dump(utils.sortKeys(newLock), { noRefs: true })
  )
}

/**
 * Simply read the lock file.
 * Skip it if we cannot find the lock file.
 */
export async function readLock() {
  if (await fs.pathExists('./tiny-pm.yml')) {
    Object.assign(
      oldLock,
      yaml.load(await fs.readFile('./tiny-pm.yml', 'utf-8'))
    )
  }
}


================================================
FILE: src/log.ts
================================================
import logUpdate from 'log-update'
import ProgressBar from 'progress'

let progress: ProgressBar

/**
 * Update currently resolved module.
 * This is similar to Yarn.
 */
export function logResolving(name: string) {
  logUpdate(`[1/2] Resolving: ${name}`)
}

/**
 * Use a friendly progress bar.
 */
export function prepareInstall(count: number) {
  logUpdate('[1/2] Finished resolving.')
  progress = new ProgressBar('[2/2] Installing [:bar]', {
    complete: '#',
    total: count,
  })
}

/**
 * This is for update the progress bar
 * once tarball extraction is finished.
 */
export function tickInstalling() {
  progress.tick()
}


================================================
FILE: src/resolve.ts
================================================
import { request } from 'undici'

// Just type definition and this can be ignored.
export interface Manifest {
  [version: string]: {
    dependencies?: { [dep: string]: string },
    dist: { shasum: string, tarball: string },
  }
}

// This allows us use a custom npm registry.
const REGISTRY = process.env.REGISTRY || 'https://registry.npmjs.org/'

/*
 * Use cache to prevent duplicated network request,
 * when asking the same package.
 */
const cache: { [dep: string]: Manifest } = Object.create(null)

export default async function(name: string): Promise<Manifest> {
  /*
   * If the requested package manifest is existed in cache,
   * just return it directly.
   */
  const cached = cache[name]
  if (cached) {
    return cached
  }

  const response = await request(`${REGISTRY}${name}`)

  const json = (await response.body.json()) as
    | { error: string }
    | { versions: Manifest }
  if ('error' in json) {
    throw new ReferenceError(`No such package: ${name}`)
  }

  // Add the manifest info to cache and return it.
  return (cache[name] = json.versions)
}


================================================
FILE: src/utils.ts
================================================
export function sortKeys<T extends { [key: string]: any }>(obj: T) {
  return Object.fromEntries(
    Object.entries(obj).sort(([a], [b]) => a.localeCompare(b))
  )
}


================================================
FILE: tsconfig.json
================================================
{
  "extends": "@gplane/tsconfig",
  "compilerOptions": {
    "target": "es2020",
    "module": "commonjs",
    "outDir": "dist"
  }
}