Repository: retorquere/zotero-storage-scanner
Branch: master
Commit: 411b4dec3f78
Files: 14
Total size: 19.7 KB

Directory structure:
gitextract_t_d6xoq3/

├── .eslintignore
├── .eslintrc.json
├── .github/
│   └── workflows/
│       └── release.yml
├── .gitignore
├── README.md
├── chrome.manifest
├── content/
│   ├── zotero-storage-scanner.ts
│   └── zotero-storage-scanner.xul
├── esbuild.js
├── locale/
│   └── en-US/
│       ├── zotero-storage-scanner.dtd
│       └── zotero-storage-scanner.properties
├── package.json
├── skin/
│   └── default/
│       └── overlay.css
└── tsconfig.json

================================================
FILE CONTENTS
================================================

================================================
FILE: .eslintignore
================================================
node_modules
*.d.ts
generator-temp


================================================
FILE: .eslintrc.json
================================================
{
  "root": true,
  "env": {
    "browser": true,
    "es6": true,
    "node": true
  },
  "extends": [
    "eslint:recommended",
    "plugin:@typescript-eslint/recommended",
    "plugin:@typescript-eslint/eslint-recommended",
    "plugin:@typescript-eslint/recommended-requiring-type-checking"
  ],
  "parser": "@typescript-eslint/parser",
  "parserOptions": {
    "project": "tsconfig.json",
    "sourceType": "module"
  },
  "plugins": [
    "eslint-plugin-import",
    "eslint-plugin-prefer-arrow",
    "@typescript-eslint",
    "@typescript-eslint/eslint-plugin"
  ],
  "rules": {
    "@typescript-eslint/adjacent-overload-signatures": "error",
    "@typescript-eslint/array-type": [
      "error",
      {
        "default": "array"
      }
    ],
    "@typescript-eslint/await-thenable": "error",
    "@typescript-eslint/ban-ts-comment": "warn",
    "@typescript-eslint/ban-types": [
      "warn",
      {
        "types": {
          "Object": {
            "message": "Avoid using the `Object` type. Did you mean `object`?"
          },
          "Function": {
            "message": "Avoid using the `Function` type. Prefer a specific function type, like `() => void`."
          },
          "Boolean": {
            "message": "Avoid using the `Boolean` type. Did you mean `boolean`?"
          },
          "Number": {
            "message": "Avoid using the `Number` type. Did you mean `number`?"
          },
          "String": {
            "message": "Avoid using the `String` type. Did you mean `string`?"
          },
          "Symbol": {
            "message": "Avoid using the `Symbol` type.  Did you mean `symbol`?"
          }
        }
      }
    ],
    "@typescript-eslint/consistent-type-assertions": "error",
    "@typescript-eslint/dot-notation": "error",
    "@typescript-eslint/explicit-module-boundary-types": "warn",
    "@typescript-eslint/indent": [
      "error",
      2
    ],
    "@typescript-eslint/member-delimiter-style": [
      "error",
      {
        "multiline": {
          "delimiter": "none",
          "requireLast": false
        },
        "singleline": {
          "delimiter": "comma",
          "requireLast": false
        }
      }
    ],
    "@typescript-eslint/member-ordering": "off",
    "@typescript-eslint/naming-convention": "off",
    "@typescript-eslint/no-array-constructor": "error",
    "@typescript-eslint/no-empty-function": "error",
    "@typescript-eslint/no-empty-interface": "error",
    "@typescript-eslint/no-explicit-any": "off",
    "@typescript-eslint/no-extra-non-null-assertion": "error",
    "@typescript-eslint/no-extra-semi": "error",
    "@typescript-eslint/no-floating-promises": "error",
    "@typescript-eslint/no-for-in-array": "error",
    "@typescript-eslint/no-implied-eval": "off",
    "@typescript-eslint/no-inferrable-types": "error",
    "@typescript-eslint/no-misused-new": "error",
    "@typescript-eslint/no-misused-promises": "error",
    "@typescript-eslint/no-namespace": "error",
    "@typescript-eslint/no-non-null-asserted-optional-chain": "error",
    "@typescript-eslint/no-non-null-assertion": "warn",
    "@typescript-eslint/no-parameter-properties": "off",
    "@typescript-eslint/no-shadow": [
      "error",
      {
        "hoist": "all"
      }
    ],
    "@typescript-eslint/no-this-alias": "error",
    "@typescript-eslint/no-unnecessary-type-assertion": "error",
    "@typescript-eslint/no-unsafe-assignment": "off",
    "@typescript-eslint/no-unsafe-call": "off",
    "@typescript-eslint/no-unsafe-member-access": "off",
    "@typescript-eslint/no-unsafe-return": "error",
    "@typescript-eslint/no-unused-expressions": "error",
    "@typescript-eslint/no-unused-vars": [
      "error",
      {
        "argsIgnorePattern": "^_"
      }
    ],
    "@typescript-eslint/no-use-before-define": "off",
    "@typescript-eslint/no-var-requires": "off",
    "@typescript-eslint/prefer-as-const": "error",
    "@typescript-eslint/prefer-for-of": "error",
    "@typescript-eslint/prefer-function-type": "error",
    "@typescript-eslint/prefer-namespace-keyword": "error",
    "@typescript-eslint/prefer-regexp-exec": "off",
    "@typescript-eslint/quotes": [
      "error",
      "single",
      {
        "avoidEscape": true
      }
    ],
    "@typescript-eslint/require-await": "error",
    "@typescript-eslint/restrict-plus-operands": "error",
    "@typescript-eslint/restrict-template-expressions": "off",
    "@typescript-eslint/semi": [
      "error",
      "never"
    ],
    "@typescript-eslint/triple-slash-reference": [
      "error",
      {
        "path": "always",
        "types": "prefer-import",
        "lib": "always"
      }
    ],
    "@typescript-eslint/unbound-method": "error",
    "@typescript-eslint/unified-signatures": "error",
    "arrow-body-style": "error",
    "arrow-parens": [
      "error",
      "as-needed"
    ],
    "brace-style": [
      "error",
      "stroustrup",
      {
        "allowSingleLine": true
      }
    ],
    "comma-dangle": [
      "error",
      {
        "objects": "always-multiline",
        "arrays": "always-multiline",
        "functions": "never"
      }
    ],
    "complexity": "off",
    "constructor-super": "error",
    "curly": [
      "error",
      "multi-line"
    ],
    "eol-last": "error",
    "eqeqeq": [
      "error",
      "smart"
    ],
    "guard-for-in": "error",
    "id-blacklist": [
      "error",
      "any",
      "Number",
      "number",
      "String",
      "string",
      "Boolean",
      "boolean",
      "Undefined",
      "undefined"
    ],
    "id-match": "error",
    "import/order": "off",
    "linebreak-style": [
      "error",
      "unix"
    ],
    "max-classes-per-file": "off",
    "max-len": [
      "warn",
      {
        "code": 240
      }
    ],
    "new-parens": "off",
    "no-array-constructor": "off",
    "no-bitwise": "error",
    "no-caller": "error",
    "no-cond-assign": "off",
    "no-console": "error",
    "no-debugger": "error",
    "no-empty": [
      "error",
      {
        "allowEmptyCatch": true
      }
    ],
    "no-empty-function": "off",
    "no-eval": "error",
    "no-extra-semi": "off",
    "no-fallthrough": "off",
    "no-implied-eval": "off",
    "no-invalid-this": "off",
    "no-irregular-whitespace": "error",
    "no-magic-numbers": "off",
    "@typescript-eslint/no-magic-numbers": [ "error", {
      "ignore": [ -1, 0, 1, 2 ],
      "ignoreEnums": true
    }],
    "no-new-wrappers": "error",
    "no-redeclare": "error",
    "no-throw-literal": "error",
    "no-trailing-spaces": "error",
    "no-undef-init": "error",
    "no-underscore-dangle": [
      "error",
      {
        "allowAfterThis": true
      }
    ],
    "no-unsafe-finally": "error",
    "no-unused-labels": "error",
    "no-unused-vars": "off",
    "no-var": "error",
    "object-shorthand": "error",
    "one-var": [
      "off",
      "never"
    ],
    "prefer-arrow/prefer-arrow-functions": [
      "error",
      {
        "allowStandaloneDeclarations": true
      }
    ],
    "prefer-const": [
      "error",
      {
        "destructuring": "all"
      }
    ],
    "prefer-object-spread": "error",
    "prefer-template": "error",
    "quote-props": [
      "error",
      "as-needed"
    ],
    "radix": "off",
    "require-await": "off",
    "space-before-function-paren": [
      "error",
      {
        "anonymous": "never",
        "named": "never",
        "asyncArrow": "always"
      }
    ],
    "spaced-comment": [
      "error",
      "always",
      {
        "markers": [
          "/"
        ]
      }
    ],
    "use-isnan": "error",
    "valid-typeof": "off",
    "yoda": "error",
    "@typescript-eslint/consistent-type-definitions": "off",
    "no-new-func": "off"
  },
  "ignorePatterns": [
    "webpack.config.ts",
    "util/*.ts",
    "minitests/*.ts",
    "content/minitests/*.ts"
  ]
}


================================================
FILE: .github/workflows/release.yml
================================================
name: release

on:
  push:

jobs:
  release:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
    - name: install node
      uses: actions/setup-node@v1
      with:
        node-version: 14.x
    - name: Cache node dependencies
      uses: actions/cache@v2
      env:
        cache-name: cache-dependencies
      with:
        path: |
          ~/.npm
        key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('package-lock.json') }}
        restore-keys: |
          ${{ runner.os }}-build-${{ env.cache-name }}-
          ${{ runner.os }}-build-
          ${{ runner.os }}-
    - name: install node dependencies
      run: npm install
    - name: build
      run: npm run build
    - name: release
      run: npm run release
      env:
        GITHUB_TOKEN: ${{ github.token }}


================================================
FILE: .gitignore
================================================
wiki
gen
build
*~
*.swp
*.debug
*.cache
*.status
*.js.map
*.tmp
*.xpi
node_modules
.env
.eslintcache


================================================
FILE: README.md
================================================
# This plugin has been superceeded by Zotero Attachment Scanner

which can be found at https://github.com/SciImage/zotero-attachment-scanner

## Zotero Storage Scanner

A Zotero plugin to remove the broken & duplicate attachment link of the bibliography

Install by downloading the [latest version](https://github.com/retorquere/zotero-storage-scanner/releases/latest).

This plugin scans your attachments of storage to remove the missing & duplicate attachment link. The combination of the duplicated bibliography in Zotero could create a lot of duplicate attachments. Also when you use a plugin/ third-party software to directly remove the PDF of the bibliography, the attachment link becomes broken in Zotero. This plugin is designed to solve the problem.

Install by downloading the latest version and installing it from the Zotero add-ons screen.

## A little background on how the plugin works

There is no UI, this plugin scans your library after being launched from tools->storage scanner in the background.

If you run zotero-storage-scanner you will see a zotero process (name dependand on OS) kick off in your proccess manager (top, activity monitor, Task Manager), however as it works through your library it live updates two tags `#duplicates` and `#broken` as it goes. If those tags have no items attached to them after some time (variable depending on size of library) then you are golden, if there are entries tagged in either then your likely have duplicate articles or a file/DB has issues been identified with the most likely cause being a "missing" PDF sometimes caused by incomplete syncing.


================================================
FILE: chrome.manifest
================================================
content	zotero-storage-scanner				content/
locale	zotero-storage-scanner	en-US		locale/en-US/
skin	zotero-storage-scanner	default		skin/

overlay	chrome://zotero/content/zoteroPane.xul	chrome://zotero-storage-scanner/content/zotero-storage-scanner.xul


================================================
FILE: content/zotero-storage-scanner.ts
================================================
declare const Zotero: any
// declare const Components: any

const monkey_patch_marker = 'StorageScannerMonkeyPatched'

// eslint-disable-next-line @typescript-eslint/no-unused-vars
function patch(object, method, patcher) {
  if (object[method][monkey_patch_marker]) return
  object[method] = patcher(object[method])
  object[method][monkey_patch_marker] = true
}

class StorageScanner {
  private duplicates: string
  private noattachments: string
  private globals: Record<string, any>

  // eslint-disable-next-line @typescript-eslint/require-await
  public async load(globals: Record<string, any>) {
    this.globals = globals
    if (!this.duplicates) {
      // https://groups.google.com/d/msg/zotero-dev/QYNGxqTSpaQ/uvGObVNlCgAJ
      await Zotero.Schema.schemaUpdatePromise

      let attachmentTypeID
      for (const type of await Zotero.DB.queryAsync("select itemTypeID from itemTypes where typeName = 'attachment'")) {
        attachmentTypeID = type.itemTypeID
      }

      this.duplicates = `
        SELECT
          item.itemID,
          attachment.path,
          CASE attachment.linkMode WHEN ${Zotero.Attachments.LINK_MODE_LINKED_URL} THEN 1 ELSE 0 END AS isLinkedURL,
          COALESCE(duplicates.duplicates, 1) as duplicates
        FROM items item
        LEFT JOIN itemAttachments attachment ON attachment.itemID = item.itemID
        LEFT JOIN (
          SELECT parentItemID, contentType, COUNT(*) as duplicates
          FROM itemAttachments
          WHERE linkMode <> ${Zotero.Attachments.LINK_MODE_LINKED_URL} AND itemID NOT IN (select itemID from deletedItems)
          GROUP BY parentItemID, contentType
        ) duplicates ON attachment.parentItemID = duplicates.parentItemID AND attachment.contentType = duplicates.contentType
        WHERE
            item.itemTypeID = ${attachmentTypeID}
          AND
            item.itemID NOT IN (select itemID from deletedItems)
          AND
            (
                attachment.path IS NULL
              OR
                (attachment.linkMode IS NOT NULL AND attachment.linkMode <> ${Zotero.Attachments.LINK_MODE_LINKED_URL})
            )
      `.replace(/\n/g, ' ').trim()
      Zotero.debug(`StorageScanner: duplicates = ${this.duplicates}`)

      this.noattachments = `
	      SELECT item.itemID, COUNT(attachment.itemID) AS attachments
        FROM items item
        JOIN itemTypes ON item.itemTypeID = itemTypes.itemTypeID AND itemTypes.typeName NOT IN ('note', 'attachment')
        LEFT JOIN itemAttachments attachment ON attachment.parentItemID = item.itemID AND attachment.itemID NOT IN (select itemID from deletedItems)
        WHERE item.itemID NOT IN (select itemID from deletedItems)
        GROUP BY item.itemID

        UNION

        SELECT itemID, CASE WHEN parentItemID IS NULL THEN 0 ELSE 1 END as attachments
        FROM itemAttachments attachment
        WHERE attachment.itemID NOT IN (select itemID from deletedItems)
      `.replace(/\n/g, ' ').trim()
    }
  }

  public async scan() {
    if (!this.duplicates) return

    // Zotero.Attachments.LINK_MODE_LINKED_URL // ignore this
    // Zotero.Attachments.LINK_MODE_IMPORTED_URL // snapshot
    // Zotero.Attachments.LINK_MODE_IMPORTED_FILE
    // LINK_MODE_LINKED_FILE

    const attachments = (await Zotero.DB.queryAsync(this.duplicates)) || [] // apparently 'no results' gets me 'null', not an empty list. Sure, ok.

    Zotero.debug(`StorageScanner.attachments: ${attachments.length}`)

    for (const attachment of attachments) {
      Zotero.debug(`StorageScanner.attachment: ${JSON.stringify({itemID: attachment.itemID, path: attachment.path, duplicates: attachment.duplicates})}`)
      const item = await Zotero.Items.getAsync(attachment.itemID)
      /*
        because getAsync isn't "same as get but asynchronously" but "sort
        of same as get but asynchronously, however if the object was not
        already loaded by some user interaction you're out of luck". BTW,
        if you could know at this point that getAsync would get you a loaded
        object, you could just have called "get". Nice.
        https://groups.google.com/d/msg/zotero-dev/QYNGxqTSpaQ/nGKJakGnBAAJ
        https://groups.google.com/d/msg/zotero-dev/naAxXIbpDhU/iSLpXo-UBQAJ
      */
      await item.loadAllData()

      let save = false

      if (this.updateTag(item, '#broken_attachments', !attachment.isLinkedURL && !(await item.getFilePathAsync()))) save = true
      if (this.updateTag(item, '#multiple_attachments_of_same_type', attachment.duplicates > 1)) save = true
      Zotero.debug(`StorageScanner.save: ${save}`)

      if (save) await item.saveTx()
    }

    const items = (await Zotero.DB.queryAsync(this.noattachments)) || []
    Zotero.debug(`StorageScanner.items: ${items.length}`)
    for (const status of items) {
      const item = await Zotero.Items.getAsync(status.itemID)
      await item.loadAllData()

      let save = false
      if (this.updateTag(item, '#nosource ', !status.attachments)) save = true
      if (save) await item.saveTx()
    }
  }

  private updateTag(item, tag, add) {
    Zotero.debug(`StorageScanner.updateTag('${item.id}.${tag}', ist: ${item.hasTag(tag)}, soll: ${add})`)
    if (add) {
      if (item.hasTag(tag)) return false
      item.addTag(tag)
    }
    else {
      if (!item.hasTag(tag)) return false
      item.removeTag(tag)
    }

    return true
  }
}

Zotero.StorageScanner = new StorageScanner


================================================
FILE: content/zotero-storage-scanner.xul
================================================
<?xml version="1.0"?>
<?xml-stylesheet href="chrome://zotero-storage-scanner/skin/overlay.css" type="text/css"?>
<!DOCTYPE window SYSTEM "chrome://zotero-storage-scanner/locale/zotero-storage-scanner.dtd">

<overlay id="zotero-storage-scanner-overlay" xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul">
  <script>
    var globals = Function('return this')();
    var zotero_storage_scanner_zoteropane = null;
  </script>
  <stringbundleset>
    <stringbundle id="zotero-storage-scanner-strings" src="chrome://zotero-storage-scanner/locale/zotero-storage-scanner.properties"/>
  </stringbundleset>

  <menupopup id="menu_ToolsPopup">
    <menuitem label="&zotero-storage-scanner.name;" oncommand="Zotero.StorageScanner.scan()"/>
  </menupopup>
	
	<!-- Include the main extension logic -->
  <script src="chrome://zotero-storage-scanner/content/zotero-storage-scanner.js"/>
  <script>
    window.addEventListener('load', async function() {
      try {
        Zotero.debug('zotero-storage-scanner startup')
        await Zotero.StorageScanner.load(globals)
        Zotero.debug('zotero-storage-scanner started')
      } catch (err) {
        Zotero.debug(`zotero-storage-scanner ZoteroPane overlay error: ${err.message}\n${err.stack ||''}`)
      }
    })
  </script>
</overlay>


================================================
FILE: esbuild.js
================================================
const path = require('path')
const fs = require('fs')
const esbuild = require('esbuild')
const rmrf = require('rimraf')

require('zotero-plugin/copy-assets')
require('zotero-plugin/rdf')
require('zotero-plugin/version')

async function build() {
  rmrf.sync('gen')
  await esbuild.build({
    bundle: true,
    format: 'iife',
    target: ['firefox60'],
    entryPoints: [ 'content/zotero-storage-scanner.ts' ],
    outdir: 'build/content',
    banner: { js: 'if (!Zotero.StorageScanner) {\n' },
    footer: { js: '\n}' },
  })
}

build().catch(err => {
  console.log(err)
  process.exit(1)
})


================================================
FILE: locale/en-US/zotero-storage-scanner.dtd
================================================
<!ENTITY zotero-storage-scanner.name "Storage Scanner">


================================================
FILE: locale/en-US/zotero-storage-scanner.properties
================================================


================================================
FILE: package.json
================================================
{
  "name": "zotero-storage-scanner",
  "version": "5.0.12",
  "description": "Scan attachments for duplicates",
  "scripts": {
    "lint": "eslint . --ext .ts --cache --cache-location .eslintcache/",
    "prebuild": "npm run lint",
    "build": "tsc --noEmit && node esbuild.js",
    "postbuild": "zotero-plugin-zipup build zotero-storage-scanner",
    "release": "zotero-plugin-release",
    "postversion": "git push --follow-tags"
  },
  "repository": {
    "type": "git",
    "url": "https://github.com/retorquere/zotero-storage-scanner.git"
  },
  "author": {
    "name": "Emiliano Heyns",
    "email": "emiliano.heyns@iris-advies.com"
  },
  "bugs": {
    "url": "https://github.com/retorquere/zotero-storage-scanner/issues"
  },
  "homepage": "https://github.com/retorquere/zotero-storage-scanner",
  "dependencies": {
    "@types/mocha": "^9.1.1",
    "@typescript-eslint/eslint-plugin": "^5.25.0",
    "@typescript-eslint/parser": "^5.25.0",
    "esbuild": "^0.14.39",
    "eslint": "^8.16.0",
    "eslint-plugin-import": "^2.26.0",
    "eslint-plugin-jsdoc": "^39.3.0",
    "eslint-plugin-prefer-arrow": "^1.2.3",
    "mkdirp": "^1.0.4",
    "rimraf": "^3.0.2",
    "ts-node": "^10.8.0",
    "typescript": "^4.6.4",
    "zotero-plugin": "^1.4.2"
  },
  "xpi": {
    "name": "Storage Scanner for Zotero",
    "updateLink": "https://github.com/retorquere/zotero-storage-scanner/releases/download/v{version}/zotero-storage-scanner-{version}.xpi",
    "releaseURL": "https://github.com/retorquere/zotero-storage-scanner/releases/download/release/"
  }
}


================================================
FILE: skin/default/overlay.css
================================================


================================================
FILE: tsconfig.json
================================================
{
  "compilerOptions": {
    "importHelpers": true,
    "target": "es2017",
    "disableSizeLimit": true,
    "module": "commonjs",
    "noImplicitAny": false,
    "esModuleInterop": true,
    "removeComments": false,
    "preserveConstEnums": false,
    "sourceMap": false,
    "downlevelIteration": true,
    "lib": [ "es2017", "dom" ],
    "typeRoots": [
      "./typings",
      "./node_modules/@types"
    ]
  },
  "include": [
    "content/**/*",
    "resource/**/*",
    "*.ts"
  ],
  "exclude": [
    "node_modules",
    "**/*.spec.ts",
    "typings"
  ]
}