Repository: dtinth/todo-actions Branch: master Commit: a466c8c81cd2 Files: 29 Total size: 40.6 KB Directory structure: gitextract_9vuzh9_k/ ├── .github/ │ └── workflows/ │ ├── push-process-todo-comments.yml │ └── push-test.yml ├── .gitignore ├── .prettierrc ├── Dockerfile ├── LICENSE ├── README.md ├── entrypoint.sh ├── jest.config.js ├── package.json ├── src/ │ ├── CLIEntrypoint.ts │ ├── CodeRepository.ts │ ├── DataStore.ts │ ├── File.ts │ ├── MongoDB.ts │ ├── ProcessId.ts │ ├── TaskInformationGenerator.ts │ ├── TaskManagementSystem.ts │ ├── TaskUpdater.ts │ ├── TodoActionsMain.test.ts │ ├── TodoActionsMain.ts │ ├── TodoParser.test.ts │ ├── TodoParser.ts │ ├── __mocks__/ │ │ ├── CodeRepository.ts │ │ ├── DataStore.ts │ │ ├── TaskManagementSystem.ts │ │ └── World.ts │ └── types.ts └── tsconfig.json ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/workflows/push-process-todo-comments.yml ================================================ on: push: branches: - master name: Process TODO comments jobs: collectTODO: name: Collect TODO runs-on: ubuntu-latest steps: - uses: actions/checkout@master - name: Collect TODO uses: ./ env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} TODO_ACTIONS_MONGO_URL: ${{ secrets.TODO_ACTIONS_MONGO_URL }} ================================================ FILE: .github/workflows/push-test.yml ================================================ on: push name: Test jobs: yarnInstall: name: yarn install runs-on: ubuntu-latest steps: - uses: actions/checkout@master - name: yarn install uses: Borales/actions-yarn@7f2a9167277e57a749fc97441aec0056d2b13948 with: args: install - name: typecheck uses: Borales/actions-yarn@7f2a9167277e57a749fc97441aec0056d2b13948 with: args: tsc - name: test uses: Borales/actions-yarn@7f2a9167277e57a749fc97441aec0056d2b13948 with: args: test ================================================ FILE: .gitignore ================================================ # Logs logs *.log npm-debug.log* yarn-debug.log* yarn-error.log* # Runtime data pids *.pid *.seed *.pid.lock # Directory for instrumented libs generated by jscoverage/JSCover lib-cov # Coverage directory used by tools like istanbul coverage # nyc test coverage .nyc_output # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) .grunt # Bower dependency directory (https://bower.io/) bower_components # node-waf configuration .lock-wscript # Compiled binary addons (https://nodejs.org/api/addons.html) build/Release # Dependency directories node_modules/ jspm_packages/ # TypeScript v1 declaration files typings/ # Optional npm cache directory .npm # Optional eslint cache .eslintcache # Optional REPL history .node_repl_history # Output of 'npm pack' *.tgz # Yarn Integrity file .yarn-integrity # dotenv environment variables file .env # next.js build output .next /lib/ ================================================ FILE: .prettierrc ================================================ { "singleQuote": true, "semi": false, "trailingComma": "all" } ================================================ FILE: Dockerfile ================================================ FROM node:12.6.0 LABEL "com.github.actions.name"="todo-actions" LABEL "com.github.actions.description"="Convert TODO comments into issues" LABEL "com.github.actions.icon"="alert-circle" LABEL "com.github.actions.color"="gray-dark" LABEL "repository"="http://github.com/dtinth/todo-actions" LABEL "homepage"="http://github.com/dtinth/todo-actions" LABEL "maintainer"="dtinth " ENV GIT_COMMITTER_NAME=TODO ENV GIT_AUTHOR_NAME=TODO ENV EMAIL=todo-actions[bot]@users.noreply.github.com RUN mkdir -p /app ADD entrypoint.sh package.json tsconfig.json yarn.lock /app/ RUN cd /app && yarn --frozen-lockfile ADD src /app/src RUN cd /app && yarn build ENTRYPOINT ["/app/entrypoint.sh"] ================================================ FILE: LICENSE ================================================ MIT License Copyright (c) 2019 Thai Pangsakulyanont Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: README.md ================================================ # todo-actions Turn TODO comments inside source code into GitHub issues and closes them when they are gone. Runs on GitHub Actions. This project is hugely inspired by [0pdd](https://www.yegor256.com/2017/04/05/pdd-in-action.html). ## Features - Turns TODO comments into GitHub issues. A TODO comment looks like this: ```js // TODO: Add integration test for TodoActionsMain. // // Code that interface with external data have been separated into their own modules. // These includes: // // - `DataStore` // - `CodeRepository` // - `TaskManagementSystem` // // They can be mocked by creating a mock version using `__mocks__` folder. // https://jestjs.io/docs/en/manual-mocks ``` …and it gets turned into an issue like this: > [Screenshot](https://github.com/dtinth/todo-actions/issues/35) The first line is the title. The rest becomes the issue body. - The GitHub issue is updated whenever the text inside the TODO comment changes. This allows elaboration and collaboration on TODO comments. - Once the TODO comment is removed, the corresponding issue is automatically closed. This allows fine-grained task management, and also allows new contributors to easily contribute to the code base. > Screenshot As a case study, when we [used](https://wonderful.software/elect-live/pdd/) the [0pdd](./docs/images/elect-live-example.png) tool on [codeforthailand/election-live](https://github.com/codeforthailand/election-live) project, it helped us attract 20+ contributors and visualized the work that got done in just 7 days: > Screenshot ## Usage ### Before You Start **Before you begin, you'll need a running MongoDB instance** This action uses MongoDB to keep track of TODO comments and their associated issues. You can get a free instance on [MongoDB Atlas](https://www.mongodb.com/cloud/atlas). The same MongoDB database can be used with multiple repositories. 1. Once you have a MongoDB instance running, you need to get a URL (known as a “connection string on MongoDB’s Cloud service) to connect to your database. Follow [MongoDB’s instructions](https://docs.atlas.mongodb.com/connect-to-cluster/) for how to connect to a cluster. 2. Once you have the connection string, copy it and go to your repository’s “Settings” tab, then to “Secrets” > ![Screenshot of a repository’s “Secrets” page, inside the Settings tab.](./docs/images/github_secrets_screenshot.png) 3. Click “Add a new secret”, give it the name TODO_ACTIONS_MONGO_URL, and paste in the MongoDB connection sctring. ### Setting up 1. In the repository where you want to set up this action, click the “Actions” tab > Screenshot of a repository's navigation tabs, with “Actions” highlighted 2. On the Actions page, click “Set up a workflow yourself” (If you already have actions set up, click “New workflow” in the left sidebar first.) > The “Actions” page for a repository, with an outline drawn around the “Set up a workflow yourself” button 3. This will bring you to the GitHub workflow editor. Copy the below code into the editor: ```yml name: Create issues from todos on: push: branches: - master jobs: todos: runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 - name: todo-actions uses: dtinth/todo-actions@master env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} TODO_ACTIONS_MONGO_URL: ${{ secrets.TODO_ACTIONS_MONGO_URL }} ``` _Recommended: Rename `main.yml` to something else, such as `todos.yml`_ 4. Complete the workflow creation by clicking “Start commit” and committing the new `yml` file to your repo. 5. Commit your changes. You should see the workflow running on GitHub under **Actions** tab. ## Development ### Glossary This tool is designed to be task management system-agnostic. That is, in the future it may be used with tools other than GitHub issues. Therefore, inside the code base, instead of “issues,” `todo-actions` calls them tasks. - **TODO comment:** A TODO comment inside the source code. It begins with a _TODO marker_, and followed by a block of text whose first line is the title and the rest is the body. ``` // TODO: Title here // Body here ``` A TODO comment may be in one of 3 stages: - **new:** This TODO comment is newly added. To ensure that we can reliably track the TODO comment, even when its title or body changes, we need to assign a unique identifier to it. - **identified:** This TODO comment has been identified. However a _Task_ has not been created for this TODO comment yet. - **associated:** A _Task_ has been created for this TODO comment. - **TODO marker:** The text that denotes a TODO comment. It begins with the word `TODO`, may contain a _reference_ inside square brackets, and ends with a colon. In order for the marker to be recognized, it must follow a whitespace, and no alphanumeric character may precede it. | Stage | Example marker | | ---------- | ----------------------------------- | | new | `TODO:` | | identified | `TODO [$5d20dc8e6a26d44c2afd08c6]:` | | associated | `TODO [#1]:` | - **Repository:** A GitHub repository. Don't use the word “project” when you mean “repository.” - **Task Management System:** e.g. GitHub Issues, GitHub Projects, Trello, Taskworld, JIRA, etc. - **Task:** A work item inside a _Task Management System_ that can be created and completed by `todo-actions`. e.g. an issue, a card, a ticket, or a task. - **To complete a task** means “to close an issue,” “to move a card to done,” or “to mark as completed/resolved,” depending on the task management system you use. ### Implementation overview 1. A `push` event causes the action to run in GitHub Actions. If the current branch is master, it continues. Otherwise, it is aborted. 2. The action scans for `TODO` comments. ``` // TODO: implement this thing ``` 3. Each new TODO marker is then replaced with a unique ID. ``` // TODO [$5d20dc8e6a26d44c2afd08c6]: implement this thing ``` 4. The change is committed and pushed to the repository. If the push is successful, then we have successfully uniquely identified each to-do comment. Otherwise, someone else has made another commit to the repository, and the action is aborted. 5. For each `TODO` marker, create a GitHub issue. Then replace the marker with the issue number. ``` // TODO [#1]: implement this thing ``` 6. The change is committed and pushed to the repository. If the push is successful, then it is done. Otherwise, someone else has made another commit to the repository, the action on that commit will take care of committing. ================================================ FILE: entrypoint.sh ================================================ #!/bin/sh sh -c "node /app/lib/CLIEntrypoint.js $*" ================================================ FILE: jest.config.js ================================================ module.exports = { moduleFileExtensions: ['ts', 'tsx', 'js'], transform: { '^.+\\.tsx?$': 'ts-jest', }, testMatch: ['**/src/**/*.test.+(ts|tsx|js)'], } ================================================ FILE: package.json ================================================ { "devDependencies": { "@types/jest": "^24.0.15", "@types/lodash.sortby": "^4.7.6", "@types/mongodb": "^3.1.28", "@types/node": "^12.0.12", "jest": "^24.8.0", "madge": "^3.4.4", "ts-jest": "^24.0.2", "ts-node": "^8.3.0", "typescript": "^3.5.2" }, "scripts": { "test": "jest", "build": "rm -rf lib && tsc", "dev": "ts-node -r dotenv/config src/CLIEntrypoint.ts" }, "dependencies": { "@octokit/graphql": "^3.0.1", "@octokit/rest": "^16.28.4", "dotenv": "^8.0.0", "lodash.sortby": "^4.7.0", "mongodb": "^3.2.7", "tkt": "1.1.0" } } ================================================ FILE: src/CLIEntrypoint.ts ================================================ import { cli } from 'tkt' import { runMain } from './TodoActionsMain' import * as MongoDB from './MongoDB' cli() .command('$0', 'Collect TODOs and create issues', {}, async args => { await runMain() await MongoDB.close() }) .parse() ================================================ FILE: src/CodeRepository.ts ================================================ import { existsSync, readFileSync } from 'fs' import { logger, invariant } from 'tkt' import { execSync, execFileSync } from 'child_process' import { IFile } from './types' import { File } from './File' const log = logger('CodeRepository') const event = process.env.GITHUB_EVENT_PATH && existsSync(process.env.GITHUB_EVENT_PATH) ? (log.debug('Found GitHub Action event file'), JSON.parse(readFileSync(process.env.GITHUB_EVENT_PATH, 'utf8'))) : (log.debug('No GitHub Action event file found'), null) export const repoContext = { repositoryNodeId: process.env.GITHUB_REPO_NODE_ID || (event && event.repository && event.repository.node_id) || invariant( false, 'GitHub Repo Node ID not found, either in GitHub Action event payload and GITHUB_REPO_NODE_ID environment variable.', ), repositoryOwner: process.env.GITHUB_REPO_OWNER || (event && event.repository && event.repository.full_name.split('/')[0]) || invariant( false, 'GitHub Repo Owner not found, either in GitHub Action event payload and GITHUB_REPO_OWNER environment variable.', ), repositoryName: process.env.GITHUB_REPO_NAME || (event && event.repository && event.repository.full_name.split('/')[1]) || invariant( false, 'GitHub Repo Name not found, either in GitHub Action event payload and GITHUB_REPO_NAME environment variable.', ), defaultBranch: process.env.GITHUB_REPO_DEFAULT_BRANCH || (event && event.repository && event.repository.default_branch) || invariant( false, 'GitHub Repo Default Branch not found, either in GitHub Action event payload and GITHUB_REPO_DEFAULT_BRANCH environment variable.', ), } type CodeRepositoryState = { files: IFile[] saveChanges(commitMessage: string): Promise } export async function scanCodeRepository(): Promise { log.info('Search for files with TODO tags...') const filesWithTodoMarker = execSync('git grep -Il TODO', { encoding: 'utf8', }) .split('\n') .filter(name => name) const files: IFile[] = [] log.info('Parsing TODO tags...') for (const filePath of filesWithTodoMarker) { const file = new File(filePath) files.push(file) } return { files, async saveChanges(commitMessage) { const changedFiles = files.filter(file => file.contents.changed) log.info('Files changed: %s', changedFiles.length) if (changedFiles.length === 0) { return } for (const file of changedFiles) { file.save() } execFileSync('git', ['add', ...changedFiles.map(file => file.fileName)]) execFileSync('git', ['commit', '-m', commitMessage], { stdio: 'inherit', }) if (!process.env.GITHUB_TOKEN) { throw `Maybe you forgot to enable the GITHUB_TOKEN secret?` } execSync( 'git push "https://x-access-token:$GITHUB_TOKEN@github.com/$GITHUB_REPOSITORY.git" HEAD:"$GITHUB_REF"', { stdio: 'inherit' }, ) }, } } ================================================ FILE: src/DataStore.ts ================================================ import { logger, invariant } from 'tkt' import { ITodo, ITaskState } from './types' import { ObjectId } from 'mongodb' import { getMongoDb } from './MongoDB' import { currentProcessId } from './ProcessId' const log = logger('DataStore') type TaskResolutionProcedure = | { existingTaskReference: string } | { acquireTaskCreationLock(): Promise } type TaskCreationLock = { finish(taskReference: string, state: ITaskState): Promise } type Task = { taskReference: string state: ITaskState markAsCompleted(): Promise updateState(newState: ITaskState): Promise } export async function beginTaskResolution( todoUniqueKey: string, repositoryId: string, todo: ITodo, ): Promise { const db = await getMongoDb() const _id = ObjectId.createFromHexString(todoUniqueKey) // Ensure a task exists in the database. const task = await db.tasks.findOneAndUpdate( { _id: _id }, { $setOnInsert: { _id: _id, repositoryId: repositoryId, taskReference: null, createdAt: new Date(), ownerProcessId: null, ownerProcessTimestamp: null, }, }, { upsert: true, returnOriginal: false }, ) if (!task.value) { throw new Error('Failed to upsert a task.') } if (task.value.taskReference) { log.debug( 'Found already-existing identifier %s for TODO %s.', task.value.taskReference, todoUniqueKey, ) return { existingTaskReference: task.value.taskReference } } return { async acquireTaskCreationLock() { // Acquire a lock... log.debug( 'Acquiring lock for TODO %s (currentProcessId=%s).', todoUniqueKey, currentProcessId, ) const lockedTask = await db.tasks.findOneAndUpdate( { _id: _id, $or: [ { ownerProcessTimestamp: null }, { ownerProcessTimestamp: { $lt: new Date(Date.now() - 60e3) } }, ], }, { $set: { ownerProcessId: currentProcessId, ownerProcessTimestamp: new Date(), }, }, { returnOriginal: false }, ) if (!lockedTask.value) { throw new Error('Failed to acquire a lock for this task.') } return { async finish(taskReference, state) { // Associate log.debug( 'Created task %s for TODO %s. Saving changes.', taskReference, todoUniqueKey, ) await db.tasks.findOneAndUpdate( { _id: _id }, { $set: { taskReference: taskReference, hash: state.hash } }, ) }, } }, } } export async function findAllUncompletedTasks( repositoryId: string, ): Promise { const db = await getMongoDb() const result = await db.tasks .find({ repositoryId: repositoryId, completed: { $ne: true }, taskReference: { $ne: null }, }) .toArray() return result.map(taskData => { return { taskReference: taskData.taskReference || invariant(false, 'Unexpected unassociated task.'), state: { hash: taskData.hash || '', }, async markAsCompleted() { await db.tasks.findOneAndUpdate( { _id: taskData._id }, { $set: { completed: true } }, ) }, async updateState(newState) { await db.tasks.findOneAndUpdate( { _id: taskData._id }, { $set: { hash: newState.hash } }, ) }, } as Task }) } ================================================ FILE: src/File.ts ================================================ import { IFile, IFileContents } from './types' export class File implements IFile { fileName: string contents: FileContents constructor(fileName: string) { this.fileName = fileName this.contents = new FileContents( require('fs').readFileSync(fileName, 'utf8'), ) } save() { if (this.contents.changed) { require('fs').writeFileSync( this.fileName, this.contents.toString(), 'utf8', ) this.contents.changed = false } } } /** * A mock file. */ export class MockFile implements IFile { fileName: string contents: FileContents constructor(fileName: string, contents: string) { this.fileName = fileName this.contents = new FileContents(contents) } save() { this.contents.changed = false } } export class FileContents implements IFileContents { lines: string[] changed: boolean constructor(contents: string) { this.lines = contents.split('\n') this.changed = false } changeLine(lineIndex: number, newLineContents: string) { this.lines[lineIndex] = newLineContents this.changed = true } toString() { return this.lines.join('\n') } } ================================================ FILE: src/MongoDB.ts ================================================ import { Collection, ObjectId, MongoClient } from 'mongodb' import { invariant, logger } from 'tkt' type TaskSchema = { /** * Globally-unique ID for the task. */ _id: ObjectId /** * String identifying the repository. * This should be stable, i.e. does not change even though project is renamed. */ repositoryId: string /** * The identifier of the associated task. */ taskReference: string | null /** * `true` if issue is completed. */ completed?: boolean /** * When the task is created. */ createdAt: Date /** * ID of the process creating it. */ ownerProcessId: string | null /** * Timestamp at which the lock was acquired. */ ownerProcessTimestamp: Date | null /** * Hash of the task body contents */ hash?: string } let mongoPromise: Promise<{ client: MongoClient tasks: Collection }> const log = logger('mongo') export async function getMongoDb() { if (mongoPromise) return mongoPromise return (mongoPromise = (async () => { const { MongoClient } = await import('mongodb') log.info('Connecting...') const client = new MongoClient( process.env.TODO_ACTIONS_MONGO_URL || invariant( false, 'Missing environment variable: TODO_ACTIONS_MONGO_URL', ), ) await client.connect() log.info('Connected!') const db = client.db() const tasks = db.collection('tasks') tasks.createIndex({ repositoryId: 1 }) return { client, tasks: tasks, } })()) } export async function close() { if (!mongoPromise) return const mongo = await mongoPromise mongo.client.close() } ================================================ FILE: src/ProcessId.ts ================================================ import { ObjectId } from 'bson' export const currentProcessId = new ObjectId().toHexString() ================================================ FILE: src/TaskInformationGenerator.ts ================================================ import { ITodo, ITaskState } from './types' import { createHash } from 'crypto' import { repoContext } from './CodeRepository' type TaskInformation = { state: ITaskState title: string body: string } export function generateTaskInformationFromTodo(todo: ITodo): TaskInformation { const title = todo.title const file = todo.file.fileName // TODO [#31]: Also link to end line in addition to just the starting line. // This requires changing `IFile` interface and `File` class to also keep track of where the TODO comment ends. const line = todo.startLine const owner = repoContext.repositoryOwner const repo = repoContext.repositoryName const defaultBranch = repoContext.defaultBranch const url = `https://github.com/${owner}/${repo}/blob/${defaultBranch}/${file}#L${line}` const link = `[${file}:${line}](${url})` const body = [ todo.body, '', '---', `_` + `This issue has been automatically created by [todo-actions](https://github.com/apps/todo-actions) based on a TODO comment found in ${link}. ` + `It will automatically be closed when the TODO comment is removed from the default branch (${defaultBranch}).` + `_`, ].join('\n') return { state: { hash: createHash('md5') .update(title) .update(body) .digest('hex'), }, title, body, } } ================================================ FILE: src/TaskManagementSystem.ts ================================================ import { invariant, logger } from 'tkt' import * as CodeRepository from './CodeRepository' const log = logger('TaskManagementSystem') type TaskInformation = { title: string body: string } export async function createTask( information: TaskInformation, ): Promise { const graphql = require('@octokit/graphql').defaults({ headers: { authorization: `token ${process.env.GITHUB_TOKEN || invariant(false, 'Required GITHUB_TOKEN variable.')}`, }, }) const result = await graphql( ` mutation CreateIssue($input: CreateIssueInput!) { createIssue(input: $input) { issue { number } } } `, { input: { repositoryId: CodeRepository.repoContext.repositoryNodeId, title: information.title, body: information.body, }, }, ) log.debug('Create issue result:', result) return result.createIssue.issue.number ? `#${result.createIssue.issue.number}` : invariant( false, 'Failed to get issue number out of createIssue API call.', ) } export async function completeTask(taskReference: string): Promise { const Octokit = (await import('@octokit/rest')).default const octokit = new Octokit({ auth: `token ${process.env.GITHUB_TOKEN || invariant(false, 'Required GITHUB_TOKEN variable.')}`, }) const result = await octokit.issues.update({ owner: CodeRepository.repoContext.repositoryOwner, repo: CodeRepository.repoContext.repositoryName, issue_number: +taskReference.substr(1), state: 'closed', }) log.debug('Issue close result:', result.data) } export async function updateTask( taskReference: string, information: TaskInformation, ): Promise { const Octokit = (await import('@octokit/rest')).default const octokit = new Octokit({ auth: `token ${process.env.GITHUB_TOKEN || invariant(false, 'Required GITHUB_TOKEN variable.')}`, }) const result = await octokit.issues.update({ owner: CodeRepository.repoContext.repositoryOwner, repo: CodeRepository.repoContext.repositoryName, issue_number: +taskReference.substr(1), title: information.title, body: information.body, }) log.debug('Issue update result:', result.data) } ================================================ FILE: src/TaskUpdater.ts ================================================ import { invariant, logger } from 'tkt' import { ITodo } from './types' import * as CodeRepository from './CodeRepository' import * as TaskManagementSystem from './TaskManagementSystem' import * as DataStore from './DataStore' import * as TaskInformationGenerator from './TaskInformationGenerator' const log = logger('TaskUpdater') export async function ensureAllTodosAreAssociated(todos: ITodo[]) { const references: string[] = [] for (const todo of todos) { const reference = todo.reference || invariant(false, 'Unexpected unidentified TODO marker') const unassociated = reference.startsWith('$') if (unassociated) { // TODO [#37]: Isolate error when creating tasks // Failure to create a task should not prevent the action from progressing forward. // We can simply skip processing this comment for now. // Since this script is designed to be idempotent, it can be retried later. const todoUniqueKey = reference.substr(1) log.debug('Found unresolved TODO %s, resolving task...', todoUniqueKey) const taskReference = await resolveTask(todoUniqueKey, todo) log.debug('Resolved TODO %s => task %s', todoUniqueKey, taskReference) todo.reference = taskReference references.push(taskReference) } } return references } export async function reconcileTasks(todos: ITodo[]) { const uncompletedTasks = await DataStore.findAllUncompletedTasks( CodeRepository.repoContext.repositoryNodeId, ) log.info( 'Number of registered uncompleted tasks: %s', uncompletedTasks.length, ) for (const todo of todos) { const reference = todo.reference || invariant(false, 'Unexpected unidentified TODO marker') invariant( !reference.startsWith('$'), 'Expected all TODO comments to be associated by now.', ) const task = uncompletedTasks.find(t => t.taskReference === reference) if (!task) { log.warn( 'Cannot find a matching task for TODO comment with reference "%s"', reference, ) continue } // TODO [#38]: Isolate error when updating tasks // Failure to update a task should not prevent the action from progressing forward. // We can simply skip processing this task for now. // Since this script is designed to be idempotent, it can be retried later. const { title, body, state, } = TaskInformationGenerator.generateTaskInformationFromTodo(todo) if (task.state.hash !== state.hash) { log.info( 'Hash for "%s" changed: "%s" => "%s" -- must update task.', reference, task.state.hash, state.hash, ) await TaskManagementSystem.updateTask(reference, { title, body }) await task.updateState(state) } else { log.info( 'Hash for "%s" remains unchanged: "%s".', reference, task.state.hash, ) } } for (const task of uncompletedTasks) { if (todos.find(todo => todo.reference === task.taskReference)) continue log.info( 'TODO for task "%s" is gone -- completing task!', task.taskReference, ) // TODO [#39]: Isolate error when completing tasks // Failure to complete a task should not prevent the action from progressing forward. // We can simply skip processing this task for now. // Since this script is designed to be idempotent, it can be retried later. await TaskManagementSystem.completeTask(task.taskReference) await task.markAsCompleted() } } export async function resolveTask( todoUniqueKey: string, todo: ITodo, ): Promise { const resolution = await DataStore.beginTaskResolution( todoUniqueKey, CodeRepository.repoContext.repositoryNodeId, todo, ) if ('existingTaskReference' in resolution) { return resolution.existingTaskReference } const taskCreationLock = await resolution.acquireTaskCreationLock() log.debug('Lock acquired. Now creating task for TODO %s.', todoUniqueKey) const { title, body, state, } = TaskInformationGenerator.generateTaskInformationFromTodo(todo) const taskReference = await TaskManagementSystem.createTask({ title, body }) taskCreationLock.finish(taskReference, state) return taskReference } ================================================ FILE: src/TodoActionsMain.test.ts ================================================ import { runMain } from './TodoActionsMain' import { resetMockWorld } from './__mocks__/World' import sortBy from 'lodash.sortby' jest.mock('./DataStore') jest.mock('./CodeRepository') jest.mock('./TaskManagementSystem') const MARKER = 'TODO' it('works', async () => { const world = resetMockWorld() // Round 1: Arrange world.file( 'main.js', ` // ${MARKER}: Hello world // This is great! `, ) // Round 1: Act await runMain() // Round 1: Assert commits expect(world.commits.length).toEqual(2) expect(world.commits[0].files.get('main.js')).toMatch( new RegExp(`${MARKER} \\[\\$\\w+\\]: Hello world`), ) expect(world.commits[1].files.get('main.js')).toMatch( new RegExp(`${MARKER} \\[#\\d+\\]: Hello world`), ) expect(world.commits[1].message).toMatch(/#1/) expect(world.commits[1].message).toMatch(/#2/) // Round 1: Assert tasks expect(world.tasks.length).toEqual(2) expect(sortBy(world.tasks.map(t => t.title))).toEqual([ 'Hello world', 'Somebody once told me', ]) // Idempotent check await runMain() expect(world.commits.length).toEqual(2) expect(world.tasks.length).toEqual(2) // Round 2: Arrange const task1 = world.tasks.find(t => t.title === 'Hello world')! const task2 = world.tasks.find(t => t.title === 'Somebody once told me')! world.file( 'main.js', ` `, ) // Round 2: Act await runMain() // Round 2: Assert commits // No new commits expected expect(world.commits.length).toEqual(2) // Round 2: Assert tasks expect(task1.completed).toBe(true) expect(task2.completed).toBe(false) expect(task2.title).toBe('Somebody once told me?') }) ================================================ FILE: src/TodoActionsMain.ts ================================================ import { invariant } from 'tkt' import { logger } from 'tkt' import { ObjectId } from 'bson' import { ITodo } from './types' import * as TodoParser from './TodoParser' import * as TaskUpdater from './TaskUpdater' import * as CodeRepository from './CodeRepository' const log = logger('main') export async function runMain() { log.info('Search for files with TODO tags...') const { files, saveChanges } = await CodeRepository.scanCodeRepository() const todoComments: ITodo[] = [] for (const file of files) { // TODO [#22]: Implement ignoring paths if (file.fileName === 'README.md') continue const todos = TodoParser.parseTodos(file) log.info('%s: %s found', file.fileName, todos.length) todoComments.push(...todos) } log.info('Total TODOs found: %s', todoComments.length) const todosWithoutReference = todoComments.filter(todo => !todo.reference) log.info('TODOs without references: %s', todosWithoutReference.length) if (todosWithoutReference.length > 0) { for (const todo of todosWithoutReference) { todo.reference = `$${new ObjectId().toHexString()}` } await saveChanges('Collect TODO comments') } // Every TODO must have a reference by now. for (const todo of todoComments) { invariant( todo.reference, 'TODO "%s" at %s must have a reference by now!', todo.title, todo.file.fileName, ) } // Update all the tasks according to the TODO state. const associated = await TaskUpdater.ensureAllTodosAreAssociated(todoComments) await saveChanges('Update TODO references: ' + associated.join(', ')) // Reconcile all tasks await TaskUpdater.reconcileTasks(todoComments) } ================================================ FILE: src/TodoParser.test.ts ================================================ import { parseTodos } from './TodoParser' import { MockFile } from './File' const MARKER = 'TODO' describe('parseTodos', () => { it('is a function', () => { expect(parseTodos).toBeInstanceOf(Function) }) it('can parse todo', () => { const file = new MockFile( 'main.js', ` // ${MARKER}: Item 1 // ${MARKER}: Item 2 // Body // ${MARKER}: Item 3 // // Extended body // Not part of TODO /* * ${MARKER}: Item 4 * Body * * Extended body */ # ${MARKER}: Item 6 # Body # # Extended body #- # Not part of TODO `, ) const result = parseTodos(file) expect(result).toHaveLength(6) expect(result[0].file).toBe(file) expect(result[0].title).toBe('Item 1') expect(result[1].title).toBe('Item 2') expect(result[2].title).toBe('Item 3') expect(result[3].title).toBe('Item 4') expect(result[4].title).toBe('Item 5') expect(result[5].title).toBe('Item 6') expect(result[0].body).toBe('') expect(result[1].body).toBe('Body') expect(result[2].body).toBe('Extended body') expect(result[3].body).toBe('Body\n\nExtended body') expect(result[4].body).toBe('Body\n\nExtended body') expect(result[5].body).toBe('Body\n\nExtended body') }) it('detects marker with reference', () => { const file = new MockFile( 'main.js', ` // ${MARKER} [#1]: Item 1 // ${MARKER} [$wow]: Item 2 // ${MARKER} [todo-actions#1]: Item 3 // ${MARKER} [https://github.com/dtinth/todo-actions/issues/1]: Item 4 `, ) const result = parseTodos(file) expect(result).toHaveLength(4) expect(result[0].reference).toBe('#1') expect(result[1].reference).toBe('$wow') expect(result[2].reference).toBe('todo-actions#1') expect(result[3].reference).toBe( 'https://github.com/dtinth/todo-actions/issues/1', ) }) it('allows title on the next line', () => { const file = new MockFile( 'main.js', ` // ${MARKER}: // Title // Body `, ) const result = parseTodos(file) expect(result).toHaveLength(1) expect(result[0].file).toBe(file) expect(result[0].title).toBe('Title') expect(result[0].body).toBe('Body') }) }) ================================================ FILE: src/TodoParser.ts ================================================ import { IFile, ITodo } from './types' export function parseTodos(file: IFile): ITodo[] { const out: Todo[] = [] let currentTodo: Todo | undefined for (const [lineIndex, line] of file.contents.lines.entries()) { const match = line.match(/^(\W+\s)TODO(?: \[([^\]\s]+)\])?:(.*)/) if (match) { const todo = new Todo(file, lineIndex, match[1], match[2], match[3]) currentTodo = todo out.push(todo) } else if (currentTodo) { const beforePrefix = line.substr(0, currentTodo.prefix.length) const afterPrefix = line.substr(currentTodo.prefix.length) if ( beforePrefix.trimRight() === currentTodo.prefix.trimRight() && (!afterPrefix || beforePrefix.match(/\s$/)) ) { currentTodo.handleLine(afterPrefix) } else { currentTodo = undefined } } } return out } class Todo implements ITodo { prefix: string line: number suffix: string body: string title: string private currentReference: string | null constructor( public file: IFile, line: number, prefix: string, reference: string | null, suffix: string, ) { this.line = line this.prefix = prefix this.currentReference = reference this.suffix = suffix this.title = suffix.trim() this.body = '' } get reference(): string | null { return this.currentReference } set reference(newRef) { this.currentReference = newRef this.file.contents.changeLine( this.line, `${this.prefix}TODO${newRef ? ` [${newRef}]` : ''}:${this.suffix}`, ) } get startLine(): number { return this.line + 1 } handleLine(line: string) { if (!this.title) { this.title = line } else if (this.body || line) { this.body += (this.body ? '\n' : '') + line } } } ================================================ FILE: src/__mocks__/CodeRepository.ts ================================================ import { mockWorld } from './World' type Real = typeof import('../CodeRepository') export const repoContext: Real['repoContext'] = { repositoryNodeId: '__GITHUB_REPO_NODE_ID__', repositoryOwner: '_dtinth', repositoryName: '_todo-actions', defaultBranch: 'master', } export const scanCodeRepository: Real['scanCodeRepository'] = async () => { const files = [...mockWorld.files.values()] return { files: files, async saveChanges(commitMessage) { if (!files.some(f => f.contents.changed)) return files.forEach(f => f.save()) mockWorld.commits.push({ message: commitMessage, files: new Map(files.map(f => [f.fileName, f.contents.toString()])), }) }, } } ================================================ FILE: src/__mocks__/DataStore.ts ================================================ import { mockWorld } from './World' type Real = typeof import('../DataStore') export const beginTaskResolution: Real['beginTaskResolution'] = async ( todoUniqueKey, repositoryId, ) => { const existing = mockWorld.store.find(entry => entry._id === todoUniqueKey) if (existing) { return { existingTaskReference: existing.reference } } return { async acquireTaskCreationLock() { return { async finish(taskReference, state) { mockWorld.store.push({ _id: todoUniqueKey, reference: taskReference, state: state, completed: false, }) }, } }, } } export const findAllUncompletedTasks: Real['findAllUncompletedTasks'] = async repositoryId => { return mockWorld.store .filter(entry => !entry.completed) .map(entry => { return { taskReference: entry.reference, state: entry.state, async markAsCompleted() { entry.completed = true }, async updateState(newState) { entry.state = newState }, } }) } ================================================ FILE: src/__mocks__/TaskManagementSystem.ts ================================================ import { mockWorld, MockTask } from './World' type Real = typeof import('../TaskManagementSystem') export const createTask: Real['createTask'] = async information => { const number = mockWorld.tasks.length + 1 const task: MockTask = { ...information, number, completed: false } mockWorld.tasks.push(task) return `#${task.number}` } export const completeTask: Real['completeTask'] = async taskReference => { getTask(taskReference).completed = true } export const updateTask: Real['updateTask'] = async ( taskReference, information, ) => { Object.assign(getTask(taskReference), information) } function getTask(taskReference: string) { return mockWorld.tasks.find(t => `#${t.number}` === taskReference)! } ================================================ FILE: src/__mocks__/World.ts ================================================ import { IFile, ITaskState } from '../types' import { MockFile } from '../File' export let mockWorld: MockWorld export type MockTask = { title: string body: string number: number completed: boolean } export type MockDataStoreEntry = { _id: string completed: boolean reference: string state: ITaskState } export type MockCommit = { message: string files: Map } export function resetMockWorld() { mockWorld = new MockWorld() return mockWorld } class MockWorld { files: Map = new Map() branch = 'master' store: MockDataStoreEntry[] = [] tasks: MockTask[] = [] commits: MockCommit[] = [] file(fileName: string, contents: string) { this.files.set(fileName, new MockFile(fileName, contents)) } } ================================================ FILE: src/types.ts ================================================ /** * A representation of a file being processed, * with mutable contents. */ export interface IFile { fileName: string contents: IFileContents /** * Saves the file back into the file system. */ save(): void } export interface IFileContents { changed: boolean /** * File contents as array of lines. * The newline character has been stripped. * May be mutated to change the contents of the file. */ lines: ReadonlyArray /** * Change a line */ changeLine(lineIndex: number, newLineContents: string): void } export interface ITodo { file: IFile startLine: number reference: string | null title: string body: string } export interface ITaskState { hash: string } ================================================ FILE: tsconfig.json ================================================ { "compilerOptions": { "module": "commonjs", "target": "es2016", "strict": true, "skipLibCheck": true, "rootDir": "src", "outDir": "lib", "sourceMap": true, "esModuleInterop": true }, "include": ["src"], "exclude": ["node_modules", "**/node_modules/*"] }