Repository: zotero/translation-server
Branch: master
Commit: d88a8d538445
Files: 56
Total size: 183.8 KB
Directory structure:
gitextract_c6fz8f0f/
├── .dockerignore
├── .eslintignore
├── .eslintrc
├── .github/
│ └── workflows/
│ └── ci.yml
├── .gitignore
├── .gitmodules
├── COPYING
├── Dockerfile
├── README.md
├── config/
│ ├── custom-environment-variables.json
│ ├── default.json5
│ └── test.json
├── docker-entrypoint.sh
├── lambda_config.env-sample
├── lambda_deploy
├── lambda_local_test
├── lambda_local_test_event.json
├── lambda_package
├── lambda_template.yaml.j2
├── package.json
├── src/
│ ├── args.js
│ ├── cors.js
│ ├── debug.js
│ ├── exportEndpoint.js
│ ├── formats.js
│ ├── http.js
│ ├── importEndpoint.js
│ ├── lambda.js
│ ├── proxy.js
│ ├── searchEndpoint.js
│ ├── server.js
│ ├── testEndpoint.js
│ ├── textSearch.js
│ ├── translation/
│ │ ├── sandboxManager.js
│ │ ├── translate.js
│ │ └── translate_item.js
│ ├── translators.js
│ ├── utilities.js
│ ├── webEndpoint.js
│ ├── webSession.js
│ └── zotero.js
├── test/
│ ├── data/
│ │ ├── bibtex_response.xml
│ │ ├── loc_book1_response.xml
│ │ └── pubmed_article1_response.xml
│ ├── export_test.js
│ ├── import_test.js
│ ├── mocha.opts
│ ├── search_test.js
│ ├── setup.js
│ ├── testTranslators/
│ │ └── testTranslators.js
│ └── web_test.js
├── translate_export
├── translate_search
├── translate_url
├── translate_url_multiple
└── translate_url_single
================================================
FILE CONTENTS
================================================
================================================
FILE: .dockerignore
================================================
.git/
modules/translators/
Dockerfile
docker-compose.*
npm-debug.log
node_modules/
================================================
FILE: .eslintignore
================================================
connectorTypeSchemaData.js
modules
src/rdf/*
================================================
FILE: .eslintrc
================================================
{
"env": {
"es6": true,
"node": true,
"mocha": true
},
"globals": {
"Zotero": false
},
"extends": [
"@zotero"
],
"parserOptions": {
"ecmaVersion": 2018
},
"rules": {}
}
================================================
FILE: .github/workflows/ci.yml
================================================
name: CI
on: [push, pull_request]
jobs:
build:
name: Test
runs-on: ubuntu-latest
timeout-minutes: 5
strategy:
matrix:
node-version: [20, 22, 24, 25]
steps:
- uses: actions/checkout@v2
- run: git submodule update --init --depth=1 modules/translate
- run: git submodule update --init --depth=1 modules/utilities
- run: git submodule update --init --depth=1 modules/zotero-schema
- run: git submodule update --init --depth=1 modules/translators
- name: Install Node ${{ matrix.node-version }}
uses: actions/setup-node@v2
with:
node-version: ${{ matrix.node-version }}
- run: npm ci
- run: npm test
================================================
FILE: .gitignore
================================================
node_modules
config/production.json
config/dev.json
config/local.json
lambda_config*.env
================================================
FILE: .gitmodules
================================================
[submodule "modules/translators"]
path = modules/translators
url = https://github.com/zotero/translators
[submodule "modules/utilities"]
path = modules/utilities
url = https://github.com/zotero/utilities.git
[submodule "modules/translate"]
path = modules/translate
url = https://github.com/zotero/translate.git
[submodule "modules/zotero-schema"]
path = modules/zotero-schema
url = https://github.com/zotero/zotero-schema.git
================================================
FILE: COPYING
================================================
Zotero is Copyright © 2018-2021 Corporation for Digital Scholarship,
Vienna, Virginia, USA http://digitalscholar.org
Copyright © 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017
Roy Rosenzweig Center for History and New Media, George Mason University,
Fairfax, Virginia, USA http://zotero.org
The Corporation for Digital Scholarship distributes the Zotero source code
under the GNU Affero General Public License, version 3 (AGPLv3). The full text
of this license is given below.
The Zotero name is a registered trademark of the Corporation for Digital Scholarship.
See http://zotero.org/trademark for more information.
Third-party copyright in this distribution is noted where applicable.
All rights not expressly granted are reserved.
=========================================================================
GNU AFFERO GENERAL PUBLIC LICENSE
Version 3, 19 November 2007
Copyright (C) 2007 Free Software Foundation, Inc.
Copyright (C)
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see maxLineLength) {
output[i] = Zotero.Utilities.ellipsize(output[i], maxLineLength, false, true);
}
}
}
output = output.join('\n\n');
if (maxChars) {
output = output.substr(maxChars * -1);
// Cut at two newlines
let matches = output.match(/^[\n]*\n\n/);
if (matches) {
output = output.substr(matches[0].length);
}
}
return output;
});
this.setStore = function (enable) {
if (enable) {
this.clear();
}
_store = enable;
this.updateEnabled();
this.storing = _store;
}
this.updateEnabled = function () {
this.enabled = _console || _consoleViewer || _store;
};
this.count = function () {
return _output.length;
}
this.clear = function () {
_output = [];
}
/**
* Format a stack trace for output in the same way that Error.stack does
* @param {Components.stack} stack
* @param {Integer} [lines=5] Number of lines to format
*/
this.stackToString = function (stack, lines) {
if (!lines) lines = 5;
var str = '';
while(stack && lines--) {
str += '\n ' + (stack.name || '') + '@' + stack.filename
+ ':' + stack.lineNumber;
stack = stack.caller;
}
return this.filterStack(str).substr(1);
};
/**
* Strip Bluebird lines from a stack trace
*
* @param {String} stack
*/
this.filterStack = function (stack) {
return stack.split(/\n/).filter(line => line.indexOf('zotero/bluebird') == -1).join('\n');
}
}
module.exports = Zotero.Debug;
================================================
FILE: src/exportEndpoint.js
================================================
/*
***** BEGIN LICENSE BLOCK *****
Copyright © 2018 Corporation for Digital Scholarship
Vienna, Virginia, USA
https://www.zotero.org
This file is part of Zotero.
Zotero is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Zotero is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with Zotero. If not, see
* body - The body of a POST request
* headers - Object of HTTP headers to send with the request
* cookieSandbox - The sandbox from which cookies should be taken
* debug - Log response text and status code
* logBodyLength - Length of request body to log
* timeout - Request timeout specified in milliseconds [default 15000]
* responseType - The response type of the request from the XHR spec
* responseTypeMap - A Map of remote content type ('application/x-bibtex') to
* XHR response type ('text'). 'html' and 'xml' imply isHTML() and isXML() from
* whatwg-mimetype. Use an empty string for the key to set a fallback response type;
* otherwise unspecified content types are rejected.
* successCodes - HTTP status codes that are considered successful, or FALSE to allow all
*
* @return {Promise} A promise resolved with an object containing the following XMLHttpRequest properties:
* - responseText {String}
* - getAllResponseHeaders {Function}
* - getResponseHeader {Function}
* - statusCode {Number}
*/
this.request = async function(method, requestURL, options = {}) {
// Default options
options = Object.assign({
body: null,
headers: {},
cookieSandbox: request.jar(),
debug: false,
logBodyLength: 1024,
timeout: 15000,
responseType: '',
successCodes: null,
maxResponseSize: this.httpMaxResponseSize
}, options);
options.headers = Object.assign({
'User-Agent': process.env.USER_AGENT || config.get('userAgent'),
'Accept': '*/*'
}, options.headers);
let logBody = '';
if (['GET', 'HEAD'].includes(method)) {
if (options.body != null) {
throw new Error(`HTTP ${method} cannot have a request body (${options.body})`)
}
} else if(options.body) {
options.body = typeof options.body == 'string' ? options.body : JSON.stringify(options.body);
if (!options.headers) options.headers = {};
if (!options.headers["Content-Type"]) {
options.headers["Content-Type"] = "application/x-www-form-urlencoded";
}
else if (options.headers["Content-Type"] == 'multipart/form-data') {
// Allow XHR to set Content-Type with boundary for multipart/form-data
delete options.headers["Content-Type"];
}
logBody = `: ${options.body.substr(0, options.logBodyLength)}` +
options.body.length > options.logBodyLength ? '...' : '';
// TODO: make sure below does its job in every API call instance
// Don't display password or session id in console
logBody = logBody.replace(/password":"[^"]+/, 'password":"********');
logBody = logBody.replace(/password=[^&]+/, 'password=********');
}
Zotero.debug(`HTTP ${method} ${requestURL}${logBody}`);
// Parse URL to escape it (e.g., Unicode characters)
var parsedURL = new url.URL(requestURL);
let {response, body} = await customRequest(method, parsedURL, options);
if (options.debug) {
Zotero.debug(`HTTP ${response.statusCode} response: ${body}`);
}
var result = {
responseURL: response.request.uri.href,
status: response.statusCode,
getAllResponseHeaders: () => {
return [...Object.entries(response.headers)].map(([k, v]) => `${k}: ${v}\r\n`).join('');
},
getResponseHeader: (header) => {
return response.headers[header.toLowerCase()] || null;
}
};
var mimeType = new MIMEType(response.headers['content-type']);
var responseType = getResponseType(response.headers['content-type'], options);
result.type = responseType;
if (responseType == 'document') {
body = decodeContent(body, response.headers['content-type']);
let dom = new JSDOM(body, {
url: result.responseURL,
// Inform JSDOM what content type it's parsing,
// so it could reject unsupported content types
contentType: response.headers['content-type']
});
wgxpath.install(dom.window, true);
result.response = dom.window.document;
// Follow meta redirects in HTML files
if (mimeType.isHTML() || mimeType.essence == 'application/xhtml+xml') {
let meta = result.response.querySelector('meta[http-equiv=refresh]');
if (meta && meta.getAttribute('content')) {
let parts = meta.getAttribute('content').split(/;\s*url=/);
// If there's a redirect to another URL in less than 15 seconds,
// follow it
if (parts.length == 2 && parseInt(parts[0]) <= 15) {
let newURL = parts[1].trim().replace(/^'(.+)'/, '$1');
newURL = url.resolve(requestURL, newURL);
Zotero.debug("Meta refresh to " + newURL);
result = Zotero.HTTP.request(method, newURL, options);
}
}
}
}
else if (responseType == 'json') {
result.response = JSON.parse(body.toString());
}
else if (responseType == 'text') {
let charset = mimeType.parameters.get('charset');
// Treat unknown charset as utf-8
if (!charset) {
charset = 'utf8';
}
else if (!iconv.encodingExists(charset)) {
Zotero.debug(`Unknown charset ${charset} -- decoding as UTF-8`);
charset = 'utf8';
}
body = iconv.decode(body, charset);
result.response = body;
result.responseText = body;
}
else {
throw new Error("Invalid responseType");
}
return result;
};
/**
* Load one or more documents
*
* This should stay in sync with the equivalent function in the client
*
* @param {String|String[]} urls - URL(s) of documents to load
* @param {Function} processor - Callback to be executed for each document loaded
* @param {CookieJar} [cookieSandbox] Cookie sandbox object
* @return {Promise} - A promise for an array of results from the processor runs
*/
this.processDocuments = async function (urls, processor, options = {}) {
// Handle old signature: urls, processor, onDone, onError
if (arguments.length > 3) {
Zotero.debug("Zotero.HTTP.processDocuments() now takes only 2 arguments -- update your code");
var onDone = arguments[3];
var onError = arguments[4];
}
var cookieSandbox = options.cookieSandbox;
var headers = options.headers;
if (typeof urls == "string") urls = [urls];
var funcs = urls.map(url => () => {
return Zotero.HTTP.request(
"GET",
url,
{
responseType: 'document',
cookieSandbox,
headers
}
)
.then((req) => {
return processor(req.response, req.responseURL);
});
});
// Run processes serially
// TODO: Add some concurrency?
var f;
var results = [];
while (f = funcs.shift()) {
try {
results.push(await f());
}
catch (e) {
if (onError) {
onError(e);
}
throw e;
}
}
// Deprecated
if (onDone) {
onDone();
}
return results;
}
/**
* Send an HTTP GET request
*
* @deprecated Use {@link Zotero.HTTP.request}
* @param {String} url URL to request
* @param {Function} onDone Callback to be executed upon request completion
* @param {String} responseCharset
* @param {CookieJar} cookieSandbox
* @param {Object} headers HTTP headers to include with the request
* @return {Boolean} True if the request was sent, or false if the browser is offline
*/
this.doGet = function(url, onDone, responseCharset, cookieSandbox, headers) {
Zotero.debug('Zotero.HTTP.doGet is deprecated. Use Zotero.HTTP.request');
this.request('GET', url, {responseCharset, headers, cookieSandbox})
.then(onDone, function(e) {
onDone({status: e.status, responseText: e.responseText});
});
return true;
};
/**
* Send an HTTP POST request
*
* @deprecated Use {@link Zotero.HTTP.request}
* @param {String} url URL to request
* @param {String|Object[]} body Request body
* @param {Function} onDone Callback to be executed upon request completion
* @param {String} headers Request HTTP headers
* @param {String} responseCharset
* @param {CookieJar} cookieSandbox
* @return {Boolean} True if the request was sent, or false if the browser is offline
*/
this.doPost = function(url, body, onDone, headers, responseCharset, cookieSandbox) {
Zotero.debug('Zotero.HTTP.doPost is deprecated. Use Zotero.HTTP.request');
this.request('POST', url, {body, responseCharset, headers, cookieSandbox})
.then(onDone, function(e) {
onDone({status: e.status, responseText: e.responseText});
});
return true;
};
}
/**
* A temporary workaround for JSDOM content decoding issues.
* The code is adapted from jsdom/lib/api.js. All dependencies
* are already used internally in JSDOM
*
* TODO: Remove this code when https://github.com/jsdom/jsdom/issues/2495 will be solved
*/
const sniffHTMLEncoding = require("html-encoding-sniffer");
const whatwgEncoding = require("whatwg-encoding");
function decodeContent(html, contentType) {
let transportLayerEncodingLabel;
if (contentType) {
const mimeType = new MIMEType(contentType);
transportLayerEncodingLabel = mimeType.parameters.get("charset");
}
html = Buffer.from(html.buffer, html.byteOffset, html.byteLength);
let encoding = sniffHTMLEncoding(html, {defaultEncoding: "UTF-8", transportLayerEncodingLabel});
html = whatwgEncoding.decode(html, encoding);
return html;
}
/**
* request.js doesn't support response size limitation, therefore
* we have to do it manually
*
* @param {String} method
* @param {String} requestURL
* @param {Object} options
* @return {Promise} response, body
*/
function customRequest(method, requestURL, options) {
return new Promise(function (resolve, reject) {
let response;
// Make sure resolve/reject is called only once even if request.js
// is emitting events when it shouldn't
let returned = false;
// Store buffers in array, because concatenation operation is is unbelievably slow
let buffers = [];
let bufferLength = 0;
let req = request({
uri: requestURL,
method,
headers: options.headers,
timeout: options.timeout,
body: options.body,
gzip: true,
followAllRedirects: true,
jar: options.cookieSandbox,
encoding: null // Get body in a buffer
})
.on('error', function (err) {
if (returned) return;
reject(err);
})
.on('data', function (chunk) {
if (returned) return;
bufferLength += chunk.length;
buffers.push(chunk);
if (bufferLength > options.maxResponseSize) {
req.abort();
returned = true;
reject(new Zotero.HTTP.ResponseSizeError(requestURL));
}
})
.on('response', function (res) {
if (returned) return;
response = res;
if (!response.headers['content-type']) {
returned = true;
return reject(new Zotero.HTTP.UnsupportedFormatError(requestURL, 'Missing Content-Type header'));
}
// Check if the status code is allowed
// Array of success codes given
if (options.successCodes) {
var success = options.successCodes.includes(response.statusCode);
}
// Explicit FALSE means allow any status code
else if (options.successCodes === false) {
var success = true;
}
// Otherwise, 2xx is success
else {
var success = response.statusCode >= 200 && response.statusCode < 300;
}
if (!success) {
returned = true;
return reject(new Zotero.HTTP.StatusError(requestURL, response.statusCode, response.body));
}
// Check Content-Type before starting the download
let supported = true;
let mimeType = new MIMEType(response.headers['content-type']);
if (options.responseType == 'document') {
supported = mimeType.isHTML() || mimeType.isXML();
}
else if (options.responseTypeMap) {
let map = options.responseTypeMap;
supported = (map.has('html') && mimeType.isHTML())
|| (map.has('xml') && mimeType.isXML())
|| map.has(mimeType.essence)
// An empty string for a key allows unspecified types as text
|| map.has('');
}
if (!supported) {
req.abort();
returned = true;
return reject(new Zotero.HTTP.UnsupportedFormatError(
requestURL,
response.headers['content-type'] + ' is not supported'
));
}
// Content-length doesn't always exists or it can be a length of a gzipped content,
// but it's still worth to do the initial size check
if (
response.headers['content-length'] !== undefined &&
response.headers['content-length'] > options.maxResponseSize
) {
req.abort();
returned = true;
reject(new Zotero.HTTP.ResponseSizeError(requestURL));
}
})
.on('end', function () {
if (returned) return;
returned = true;
resolve({response, body: Buffer.concat(buffers, bufferLength)});
});
});
};
function getResponseType(contentType, options) {
var mimeType = new MIMEType(contentType);
if (options.responseType) {
return options.responseType;
}
if (options.responseTypeMap) {
let map = options.responseTypeMap;
if (map.has('html') && mimeType.isHTML()) {
return map.get('html');
}
if (map.has('xml') && mimeType.isXML()) {
return map.get('xml');
}
if (map.has(mimeType.essence)) {
return map.get(mimeType.essence);
}
}
return 'text';
}
module.exports = Zotero.HTTP;
================================================
FILE: src/importEndpoint.js
================================================
'use strict';
/*
***** BEGIN LICENSE BLOCK *****
Copyright © 2018 Corporation for Digital Scholarship
Vienna, Virginia, USA
https://www.zotero.org
This file is part of Zotero.
Zotero is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Zotero is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with Zotero. If not, see =0; i--) {
var param = this.parameters[i];
re = re.replace(Zotero_Proxy_schemeParameterRegexps[param], "$1"+Zotero_Proxy_schemeParameters[param]);
}
this.regexp = new RegExp(re);
}
/**
* Converts a proxied URL to an unproxied URL using this proxy
*
* @param m {Array} The match from running this proxy's regexp against a URL spec
* @type String
*/
Zotero.Proxy.prototype.toProper = function(m) {
if (!Array.isArray(m)) {
let match = this.regexp.exec(m);
if (!match) {
return m
} else {
m = match;
}
}
let hostIdx = this.parameters.indexOf("%h");
let scheme = m[0].indexOf('https') == 0 ? 'https://' : 'http://';
if (hostIdx != -1) {
var properURL = scheme+m[hostIdx+1]+"/";
} else {
var properURL = scheme+this.hosts[0]+"/";
}
// Replace `-` with `.` in https to support EZProxy HttpsHyphens.
// Potentially troublesome with domains that contain dashes
if (this.dotsToHyphens ||
(this.dotsToHyphens == undefined && scheme == "https://") ||
!properURL.includes('.')) {
properURL = properURL.replace(/-/g, '.');
}
if (this.indices["%p"]) {
properURL += m[this.parameters.indexOf("%p")+1];
} else {
var dir = m[this.parameters.indexOf("%d")+1];
var file = m[this.parameters.indexOf("%f")+1];
if (dir !== "") properURL += dir+"/";
properURL += file;
}
return properURL;
}
/**
* Converts an unproxied URL to a proxied URL using this proxy
*
* @param {Object|String} uri The URI corresponding to the unproxied URL
* @type String
*/
Zotero.Proxy.prototype.toProxy = function(uri) {
if (typeof uri == "string") {
uri = url.parse(uri);
// If there's no path it is set to null, but we need
// at least an empty string to avoid doing many checks
uri.path = uri.path || '';
}
if (this.regexp.exec(uri.href) || Zotero.Proxies._isBlacklisted(uri.host)) {
return uri.href;
}
var proxyURL = this.scheme;
for(var i=this.parameters.length-1; i>=0; i--) {
var param = this.parameters[i];
var value = "";
if (param == "%h") {
value = (this.dotsToHyphens && uri.protocol == 'https:') ? uri.host.replace(/\./g, '-') : uri.host;
} else if (param == "%p") {
value = uri.path.substr(1);
} else if (param == "%d") {
value = uri.path.substr(0, uri.path.lastIndexOf("/"));
} else if (param == "%f") {
value = uri.path.substr(uri.path.lastIndexOf("/")+1)
}
proxyURL = proxyURL.substr(0, this.indices[param])+value+proxyURL.substr(this.indices[param]+2);
}
if (proxyURL.includes('://')) {
return proxyURL;
}
return uri.protocol + '//' + proxyURL;
}
/**
* Generate a display name for the proxy (e.g., "proxy.example.edu (HTTPS)")
*
* @return {String}
*/
Zotero.Proxy.prototype.toDisplayName = function () {
try {
var parts = this.scheme.match(/^(?:(?:[^:]+):\/\/)?([^\/]+)/);
var domain = parts[1]
// Include part after %h, if it's present
.split('%h').pop()
// Trim leading punctuation after the %h
.match(/\W(.+)/)[1];
return domain;
}
catch (e) {
Zotero.logError(`Invalid proxy ${this.scheme}: ${e}`);
return this.scheme;
}
}
module.exports = {Proxies: Zotero.Proxies, Proxy: Zotero.Proxy};
})();
================================================
FILE: src/searchEndpoint.js
================================================
/*
***** BEGIN LICENSE BLOCK *****
Copyright © 2018 Corporation for Digital Scholarship
Vienna, Virginia, USA
https://www.zotero.org
This file is part of Zotero.
Zotero is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Zotero is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with Zotero. If not, see Test Hello';
},
handleSingle: async function (ctx, _next) {
var title = ctx.headers['accept-language'] == 'fr' ? 'Titre' : 'Title';
ctx.response.body = `
${title}
}
*/
handle: async function (ctx, next) {
// If identifier-search is disabled in the request or unavailable, return 501
if (ctx.request.query.text === '0'
|| !config.has('identifierSearchLambda')
|| !config.get("identifierSearchLambda")) {
ctx.throw(501, "No identifiers found", { expose: true });
}
var data = ctx.request.body;
var result = await search(
data,
ctx.query && ctx.query.start
);
// Throw selection if two or more items are found, or the selection flag is marked
if (result.items.length >= 2 || result.items.length >= 1 && result.select) {
let newItems = {};
for (let item of result.items) {
let DOI = item.DOI;
let ISBN = item.ISBN;
if (!DOI && item.extra) {
let m = item.extra.match(/DOI: (.*)/);
if (m) DOI = m[1];
}
if (!ISBN && item.extra) {
let m = item.extra.match(/ISBN: (.*)/);
if (m) ISBN = m[1];
}
let identifier;
// DOI has a priority over ISBN for items that have both
if (DOI) {
identifier = DOI;
}
else if (item.ISBN) {
identifier = ISBN.split(' ')[0];
}
newItems[identifier] = {
itemType: item.itemType,
title: item.title,
description: formatDescription(item),
};
}
// If there were more results, include a link to the next result set
if (result.next) {
ctx.set('Link', `; rel="next"`);
}
ctx.response.status = 300;
//
// TODO: Differentiate from web request 300
//
ctx.response.body = newItems;
return;
}
if (result.items.length === 1) {
ctx.response.body = Zotero.Utilities.Item.itemToAPIJSON(result.items[0]);
return;
}
ctx.response.body = [];
},
// Expose for stubbing in tests
queryLambda: async function (query) {
let params = {
FunctionName: config.get('identifierSearchLambda'),
InvocationType: 'RequestResponse',
Payload: JSON.stringify({query})
};
let result = await Lambda.invoke(params).promise();
if (result.FunctionError) {
throw new Error('Lambda error: ' + result.Payload);
}
identifiers = JSON.parse(result.Payload);
return identifiers;
}
};
async function search(query, start) {
const timeout = config.get('textSearchTimeout') * 1000;
const startTime = new Date();
const numResults = 3;
let identifiers;
let moreResults = false;
try {
identifiers = await module.exports.queryLambda(query);
// If passed a start= parameter, skip ahead
let startPos = 0;
if (start) {
for (let i = 0; i < identifiers.length; i++) {
if (identifierToToken(identifiers[i]) == start) {
startPos = i + 1;
break;
}
}
}
if (identifiers.length > startPos + numResults + 1) {
moreResults = true;
}
identifiers = identifiers.slice(startPos);
} catch(e) {
Zotero.debug(e, 1);
return {select: false, items: []};
}
let items = [];
let nextLastIdentifier = null;
for (let identifier of identifiers) {
let translate = new Zotero.Translate.Search();
try {
translate.setIdentifier(identifier);
let translators = await translate.getTranslators();
if (!translators.length) {
if (new Date() > startTime.getTime() + timeout) {
break;
}
continue;
}
translate.setTranslator(translators);
let newItems = await translate.translate({
libraryID: false
});
if (newItems.length) {
let seq = getLongestCommonSequence(newItems[0].title, query);
if (seq.length >= 6 && seq.split(' ').length >= 2) {
items.push(newItems[0]);
// Keep track of last identifier if we're limiting results
if (moreResults) {
nextLastIdentifier = identifier;
}
if (items.length == numResults) {
break;
}
}
}
}
catch (e) {
if (e !== translate.ERROR_NO_RESULTS) {
Zotero.debug(e, 1);
}
}
if (new Date() > startTime.getTime() + timeout) {
break;
}
}
return {
// Force item selection, even for a single item
select: true,
items,
next: nextLastIdentifier ? identifierToToken(nextLastIdentifier) : null
};
// // Query Crossref and LoC/GBV in parallel to respond faster to the client
// let [crossrefItems, libraryItems] = await Promise.all([queryCrossref(query), queryLibraries(query)]);
//
// // Subtract book reviews from Crossref
// crossrefItems = subtractCrossrefItems(crossrefItems, libraryItems);
//
// let items = crossrefItems.concat(libraryItems);
//
// // Filter out too fuzzy items, by comparing item title (and other metadata) against query
// return await filterResults(items, query);
}
function formatDescription(item) {
let parts = [];
let authors = [];
for (let creator of item.creators) {
if (creator.creatorType === 'author' && creator.lastName) {
authors.push(creator.lastName);
if (authors.length === 3) break;
}
}
if(authors.length) parts.push(authors.join(', '));
if (item.date) {
let m = item.date.toString().match(/[0-9]{4}/);
if (m) parts.push(m[0]);
}
if(item.publicationTitle) {
parts.push(item.publicationTitle);
} else if(item.publisher) {
parts.push(item.publisher);
}
return parts.join(' \u2013 ');
}
function subtractCrossrefItems(crossrefItems, libraryItems) {
let items = [];
for(let crossrefItem of crossrefItems) {
// Keep books and book sections
if(['book', 'bookSection'].includes(crossrefItem.itemType)) {
items.push(crossrefItem);
continue;
}
let crossrefTitle = crossrefItem.title;
// Remove all tags
crossrefTitle = crossrefTitle.replace(/<\/?\w+[^<>]*>/gi, '');
crossrefTitle = crossrefTitle.replace(/:/g, ' ');
// Normalize title, split to words, filter out empty array elements
crossrefTitle = normalize(crossrefTitle).split(' ').filter(x => x).join(' ');
let found = false;
for(let libraryItem of libraryItems) {
let libraryTitle = libraryItem.title;
// Remove all tags
libraryTitle = libraryTitle.replace(/<\/?\w+[^<>]*>/gi, '');
libraryTitle = libraryTitle.replace(/:/g, ' ');
// Normalize title, split to words, filter out empty array elements
libraryTitle = normalize(libraryTitle).split(' ').filter(x => x).join(' ');
if(crossrefTitle.includes(libraryTitle)) {
found = true;
break;
}
}
if(!found) {
items.push(crossrefItem);
}
}
return items;
}
async function queryCrossref(query) {
let items = [];
try {
let translate = new Zotero.Translate.Search();
// Crossref REST
translate.setTranslator("0a61e167-de9a-4f93-a68a-628b48855909");
translate.setSearch({query});
items = await translate.translate({libraryID: false});
}
catch (e) {
Zotero.debug(e, 2);
}
return items;
}
/**
* Queries LoC and if that fails, queries GBV
*/
async function queryLibraries(query) {
let items = [];
try {
let translate = new Zotero.Translate.Search();
// Library of Congress ISBN
translate.setTranslator("c070e5a2-4bfd-44bb-9b3c-4be20c50d0d9");
translate.setSearch({query});
items = await translate.translate({libraryID: false});
}
catch (e) {
Zotero.debug(e, 2);
try {
let translate = new Zotero.Translate.Search();
// Gemeinsamer Bibliotheksverbund ISBN
translate.setTranslator("de0eef58-cb39-4410-ada0-6b39f43383f9");
translate.setSearch({query});
items = await translate.translate({libraryID: false});
}
catch (e) {
Zotero.debug(e, 2);
}
}
return items;
}
/**
* Decomposes all accents and ligatures,
* filters out symbols that aren't space or alphanumeric,
* and lowercases alphabetic symbols.
*/
function normalize(text) {
let rx = XRegExp('[^\\pL 0-9]', 'g');
text = XRegExp.replace(text, rx, '');
text = text.normalize('NFKD');
text = XRegExp.replace(text, rx, '');
text = text.toLowerCase();
return text;
}
/**
* Checks if a given word equals to any of the authors' names
*/
function hasAuthor(authors, word) {
return authors.some(author => {
return (author.firstName && normalize(author.firstName).split(' ').includes(word))
|| (author.lastName && normalize(author.lastName).split(' ').includes(word));
});
}
/**
* Tries to find the longest common words sequence between
* item title and query text. Query text must include title (or part of it)
* from the beginning. If there are leftover query words, it tries to
* validate them against item metadata (currently only authors and year)
*/
async function filterResults(items, query) {
let filteredItems = [];
let select = false;
// Normalize query, split to words, filter out empty array elements
let queryWords = normalize(query).split(' ').filter(x => x);
for (let item of items) {
let DOI = item.DOI;
let ISBN = item.ISBN;
if (!DOI && item.extra) {
let m = item.extra.match(/DOI: (.*)/);
if (m) DOI = m[1];
}
if (!ISBN && item.extra) {
let m = item.extra.match(/ISBN: (.*)/);
if (m) ISBN = m[1];
}
if (!DOI && !ISBN) continue;
let title = item.title;
// Remove all tags
title = title.replace(/<\/?\w+[^<>]*>/gi, '');
title = title.replace(/:/g, ' ');
// Normalize title, split to words, filter out empty array elements
let titleWords = normalize(title).split(' ').filter(x => x);
let longestFrom = 0;
let longestLen = 0;
// Finds the longest common words sequence between query text and item.title
for (let i = 0; i < queryWords.length; i++) {
for (let j = queryWords.length; j > 0; j--) {
let a = queryWords.slice(i, j);
for (let k = 0; k < titleWords.length - a.length + 1; k++) {
let b = titleWords.slice(k, a.length + k);
if (a.length && b.length && a.join(' ') === b.join(' ')) {
if (a.length > longestLen) {
longestFrom = i;
longestLen = b.length;
}
}
}
}
}
// At least two common words sequence must be found between query and title
if (longestLen < 1) continue;
// Longest common sequence of words
let foundPart = queryWords.slice(longestFrom, longestLen);
// Remaining words
let rems = queryWords.slice(0, longestFrom);
rems = rems.concat(queryWords.slice(longestLen));
// If at least one remaining word is left, it tries to compare it against item metadata.
// Otherwise the whole query text is found in the title, and we have a full match
if (rems.length) {
let foundAuthor = false;
let needYear = false;
let foundYear = false;
// Still remaining words
let rems2 = [];
for (let rem of rems) {
// Ignore words
if (['the', 'a', 'an'].indexOf(rem) >= 0) continue;
// If the remaining word has at least 2 chars and exists in metadata authors
if (rem.length >= 2 && hasAuthor(item.creators, rem)) {
foundAuthor = true;
continue;
}
// If the remaining word is a 4 digit number (year)
if (/^[0-9]{4}$/.test(rem)) {
needYear = true;
if (item.date) {
// If the remaining word exists in the item date
let m = item.date.toString().match(/[0-9]{4}/);
if (m && m[0] === rem) {
foundYear = true;
continue;
}
}
}
// Push the word that is still remaining
rems2.push(rem);
}
// If a year exists in the query, but is not matched to the item date
if (needYear && !foundYear) continue;
// If there are still remaining words and none of authors are found
if (rems2.length && !foundAuthor) continue;
}
// If the query part that was found in title is shorter than 30 symbols
if (foundPart.join(' ').length < 30) select = true;
filteredItems.push({
matchedLen: foundPart.join(' ').length,
titleLen: titleWords.join(' ').length,
item
});
}
// Sort results by matched text length
// and how close the matched text length is to title length
filteredItems.sort(function (a, b) {
if (b.matchedLen < a.matchedLen) return -1;
if (b.matchedLen > a.matchedLen) return 1;
return Math.abs(a.matchedLen - a.titleLen) - Math.abs(b.matchedLen - b.titleLen);
});
filteredItems = filteredItems.map(item => item.item);
return {select, items: filteredItems};
}
function getLongestCommonSequence(title, query) {
title = title.replace(/<\/?\w+[^<>]*>/gi, '');
title = title.replace(/:/g, ' ');
query = query.replace(/:/g, ' ');
// Normalize, split to words and filter out empty array elements
let titleWords = normalize(title).split(' ').filter(x => x);
let queryWords = normalize(query).split(' ').filter(x => x);
let longestFrom = 0;
let longestLen = 0;
// Finds the longest common words sequence between query text and item.title
for (let i = 0; i < queryWords.length; i++) {
for (let j = queryWords.length; j > 0; j--) {
let a = queryWords.slice(i, j);
for (let k = 0; k < titleWords.length - a.length + 1; k++) {
let b = titleWords.slice(k, a.length + k);
if (a.length && b.length && a.join(' ') === b.join(' ')) {
if (a.length > longestLen) {
longestFrom = i;
longestLen = b.length;
}
}
}
}
}
return queryWords.slice(longestFrom, longestFrom + longestLen).join(' ');
}
function identifierToToken(identifier) {
return md5(JSON.stringify(identifier));
}
================================================
FILE: src/translation/sandboxManager.js
================================================
/*
***** BEGIN LICENSE BLOCK *****
Copyright © 2009 Center for History and New Media
George Mason University, Fairfax, Virginia, USA
http://zotero.org
This file is part of Zotero.
Zotero is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Zotero is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with Zotero. If not, see b.priority) {
return 1;
}
else if (a.priority < b.priority) {
return -1;
}
}
for(var type in _cache) {
_cache[type].sort(cmp);
}
Zotero.debug(`Translators initialized with ${translators.length} loaded`);
};
this.load = async function() {
var translatorsDirPath = path.resolve(path.resolve(__dirname, '..'), Zotero.Prefs.get("translatorsDirectory"));
if(!await new Promise(resolve => fs.access(translatorsDirPath, (err) => resolve(!err)))) {
throw new Error("Translators directory "+translatorsDirPath+" is not "+
"accessible. Please set this correctly in config.js.\n")
}
var translatorFilePaths = await new Promise((resolve, reject) =>
fs.readdir(translatorsDirPath, (err, files) => err ? reject(err) : resolve(files)));
var translators = [];
for (let filePath of translatorFilePaths) {
if (filePath[0] === '.' || filePath.substr(filePath.length-3) !== ".js") continue;
filePath = path.resolve(translatorsDirPath, filePath);
var data = await new Promise((resolve, reject) =>
fs.readFile(filePath, "utf8", (err, data) => err ? reject(err) : resolve(data)));
// Strip off byte order mark, if one exists
if(data[0] === "\uFEFF") data = data.substr(1);
// We assume lastUpdated is at the end to avoid running the regexp on more than necessary
var lastUpdatedIndex = data.indexOf('"lastUpdated"');
if (lastUpdatedIndex == -1) {
Zotero.debug("Invalid or missing translator metadata JSON object in " + filename);
continue;
}
// Add 50 characters to clear lastUpdated timestamp and final "}"
var header = data.substr(0, lastUpdatedIndex + 50);
var m = infoRe.exec(header);
if (!m) {
Zotero.debug("Invalid or missing translator metadata JSON object in " + filename);
continue;
}
var metadataString = m[0];
try {
var info = JSON.parse(metadataString);
} catch(e) {
Zotero.debug("Invalid or missing translator metadata JSON object in " + filename);
continue;
}
info.code = data;
// We don't ever want to reload from disk again (and don't have the code to do that either)
info.cacheCode = true;
translators.push(info);
}
return translators;
};
/**
* Gets the translator that corresponds to a given ID
*
* @param {String} id The ID of the translator
*/
this.get = function (id) {
if (!_initialized) {
throw new Error("Trying to retrieve a translator without initalizing translator store");
}
var translator = _translators[id];
if (!translator) {
return false;
}
return translator;
};
/**
* Gets all translators for a specific type of translation
* @param {String} type The type of translators to get (import, export, web, or search)
* @param {Boolean} [debugMode] Whether to assume debugging mode. If true, code is included for
* unsupported translators, and code originally retrieved from the
* repo is re-retrieved from Zotero Standalone.
*/
this.getAllForType = async function (type, debugMode) {
if(!_initialized) Zotero.Translators.init()
var translators = _cache[type].slice(0);
var codeGetter = new Zotero.Translators.CodeGetter(translators, debugMode);
return codeGetter.getAll().then(function() {
return translators;
});
};
this.getCodeForTranslator = Zotero.Promise.method(function (translator) {
if (translator.code) {
return translator.code;
} else {
throw new Error(`Code for translator ${translator.translatorID} is missing`);
}
});
/**
* Gets web translators for a specific location
* @param {String} uri The URI for which to look for translators
* @return {Promise} - A promise for a 2-item array containing an array of translators and
* an array of functions for converting URLs from proper to proxied forms
*/
this.getWebTranslatorsForLocation = async function (URI, rootURI) {
var isFrame = URI !== rootURI;
if(!_initialized) Zotero.Translators.init();
var allTranslators = _cache["web"];
var potentialTranslators = [];
var proxies = [];
var rootSearchURIs = Zotero.Proxies.getPotentialProxies(rootURI);
var frameSearchURIs = isFrame ? Zotero.Proxies.getPotentialProxies(URI) : rootSearchURIs;
Zotero.debug("Translators: Looking for translators for "+Object.keys(frameSearchURIs).join(', '));
for(var i=0; i}
*/
WebSession.prototype.handleURL = async function () {
if (typeof this.data == 'object') {
await this.selectDone();
return;
}
var url = this.data;
// Forward supported headers
var headers = {};
for (let header of FORWARDED_HEADERS) {
let lc = header.toLowerCase();
if (this.ctx.headers[lc]) {
headers[header] = this.ctx.headers[lc];
}
}
try {
var parsedURL = urlLib.parse(url);
}
catch (e) {
this.ctx.throw(400, "Invalid URL provided\n");
}
// Check domain
var m = url.match(/https?:\/\/([^/]+)/);
if (m) {
let domain = m[1];
let blacklisted = config.get("blacklistedDomains")
.some(x => x && new RegExp(x).test(domain));
if (blacklisted) {
let doi = this.cleanDOIFromURL(url);
if (!doi) {
this.ctx.throw(500, "An error occurred retrieving the document\n");
}
await SearchEndpoint.handleIdentifier(this.ctx, { DOI: doi });
return;
}
}
var responseTypeMap = new Map([
['html', 'document'],
['application/xhtml+xml', 'document'],
['text/plain', 'text']
]);
// Force all import content types to text
for (let type in CONTENT_TYPES) {
let contentType = CONTENT_TYPES[type];
if (contentType == 'text/html') continue;
if (responseTypeMap.has(contentType)) continue;
responseTypeMap.set(contentType, 'text');
}
var urlsToTry = config.get('deproxifyURLs') ? this.deproxifyURL(url) : [url];
for (let i = 0; i < urlsToTry.length; i++) {
let url = urlsToTry[i];
if (urlsToTry.length > 1) {
Zotero.debug("Trying " + url);
}
/*let runningInstance;
if ((runningInstance = Zotero.Server.Translation.waitingForSelection[data.sessionid])
&& data.items) {
// Already waiting for a items response, so just pass this there
runningInstance._cookieSandbox.setTimeout(SERVER_TRANSLATION_TIMEOUT*1000,
runningInstance.timeout.bind(runningInstance));
runningInstance.sendResponse = sendResponseCallback;
runningInstance.selectDone(data.items);
break;
}*/
// New request
this._cookieSandbox = cookieJar();
let resolve;
let reject;
let promise = new Promise(function () {
resolve = arguments[0];
reject = arguments[1];
});
let translate = new Translate.Web();
let translatePromise;
translate.setHandler("translators", async function (translate, translators) {
// Force single-page saving
if (this.options.single) {
translators = translators.filter(t => t.itemType != 'multiple');
}
try {
translatePromise = this.translate(translate, translators);
await translatePromise;
resolve();
}
catch (e) {
reject(e);
}
}.bind(this));
translate.setHandler("select", (translate, items, callback) => {
try {
this.select(
url,
translate,
items,
callback,
translatePromise
);
}
catch (e) {
Zotero.debug(e, 1);
reject(e);
// Resolve the translate promise
callback([]);
return;
}
resolve();
});
translate.setCookieSandbox(this._cookieSandbox);
translate.setRequestHeaders(headers);
try {
let req = await Zotero.HTTP.request(
"GET",
url,
{
responseTypeMap,
cookieSandbox: this._cookieSandbox,
headers
}
);
if (req.type === 'document') {
translate.setDocument(req.response);
// This could be optimized by only running detect on secondary translators
// if the first fails, but for now just run detect on all
translate.getTranslators(true);
}
else {
Zotero.debug(`Handling ${req.getResponseHeader('content-type')} as import`);
this.ctx.request.body = req.response;
await ImportEndpoint.handle(this.ctx);
return;
}
return promise;
}
catch (e) {
Zotero.debug(e, 1);
if (e instanceof Zotero.HTTP.StatusError && e.status == 404) {
this.ctx.throw(400, "Remote page not found");
}
let doi = this.cleanDOIFromURL(url);
if (doi) {
Zotero.debug(`Error translating page -- continuing with DOI ${doi} from URL`);
await SearchEndpoint.handleIdentifier(this.ctx, { DOI: doi });
return;
}
if (e instanceof Zotero.HTTP.ResponseSizeError) {
this.ctx.throw(400, "Response exceeds max size");
}
if (e instanceof Zotero.HTTP.UnsupportedFormatError) {
this.ctx.throw(400, "The remote document is not in a supported format");
}
// No more URLs to try
if (i == urlsToTry.length - 1) {
this.ctx.throw(500, "An error occurred retrieving the document");
}
}
}
};
/**
* Called when translators are available to perform translation
*
* @return {Promise}
*/
WebSession.prototype.translate = async function (translate, translators) {
// No matching translators
if (!translators.length) {
Zotero.debug("No translators found -- saving as a webpage");
this.saveWebpage(translate);
return;
}
var translator;
var items;
while (translator = translators.shift()) {
translate.setTranslator(translator);
try {
items = await translate.translate({
libraryID: false
});
break;
}
catch (e) {
Zotero.debug("Translation using " + translator.label + " failed", 1);
Zotero.debug(e, 1);
// If no more translators, save as webpage
if (!translators.length) {
this.saveWebpage(translate);
return;
}
// Try next translator
}
}
//this._cookieSandbox.clearTimeout();
// Check for DOI in URL if no results
if (!items.length) {
let doi = this.cleanDOIFromURL(translate.location);
if (doi) {
Zotero.debug(`No results -- continuing with DOI ${doi} from URL`);
await SearchEndpoint.handleIdentifier(this.ctx, { DOI: doi });
return;
}
}
var json = [];
for (let item of items) {
json.push(...Zotero.Utilities.Item.itemToAPIJSON(item));
}
this.ctx.response.status = 200;
this.ctx.response.body = json;
};
/**
* TEMP: Remove once there's a generic webpage translator
*
* @return {undefined}
*/
WebSession.prototype.saveWebpage = function (translate) {
let head = translate.document.documentElement.querySelector('head');
if (!head) {
// XXX better status code?
this.ctx.throw(501, "No translators available\n");
}
// TEMP: Return basic webpage item for HTML
let description = head.querySelector('meta[name=description]');
if (description) {
description = description.getAttribute('content');
}
let data = {
itemType: "webpage",
url: translate.document.location.href,
title: translate.document.title,
abstractNote: description,
accessDate: Zotero.Date.dateToISO(new Date())
};
this.ctx.response.body = Zotero.Utilities.Item.itemToAPIJSON(data);
};
/**
* Called if multiple items are available for selection from the translator
*/
WebSession.prototype.select = function (url, translate, items, callback, promise) {
// Fix for translators that return item list as array rather than object
if (Array.isArray(items)) {
let newItems = {};
for (let i = 0; i < items.length; i++) {
newItems[i] = items[i];
}
items = newItems;
}
// If translator returns objects with 'title' and 'checked' properties (e.g., PubMed),
// extract title
for (let i in items) {
if (items[i].title) {
items[i] = items[i].title;
}
}
this.id = Zotero.Utilities.randomString(15);
this.started = Date.now();
this.url = url;
this.translate = translate;
this.items = items;
this.selectCallback = callback;
this.translatePromise = promise;
// Send "Multiple Choices" HTTP response
//this._cookieSandbox.clearTimeout();
this.ctx.response.status = 300;
this.ctx.response.body = {
url,
session: this.id,
items
};
};
/**
* Called when items have been selected by the client
*/
WebSession.prototype.selectDone = function () {
var url = this.data.url;
var selectedItems = this.data.items;
if (this.url != url) {
this.ctx.throw(409, "'url' does not match URL in session");
}
if (!selectedItems) {
this.ctx.throw(400, "'items' not provided");
}
// Make sure items are actually available
var haveItems = false;
for (let i in selectedItems) {
if (this.items[i] === undefined || this.items[i] !== selectedItems[i]) {
this.selectCallback([]);
this.ctx.throw(409, "Items specified do not match items available");
}
haveItems = true;
}
// Make sure at least one item was specified
if (!haveItems) {
this.selectCallback([]);
this.ctx.throw(400, "No items specified");
}
// Run select callback
this.selectCallback(selectedItems);
// The original translate promise in this.translate() from the first request is stalled while
// waiting for item select from the client. When the follow-up request comes in, the new ctx
// object is swapped in by the endpoint code, and the select callback above allows the
// translate promise to complete and the translated items to be assigned to the new ctx
// response body.
return this.translatePromise;
};
/**
* Called if the request timed out before it could complete
*/
/*WebSession.prototype.timeout = function() {
this.sendResponse(504, "text/plain", "Translation timed out.\n");
};*/
/**
* Try to determine whether the passed URL looks like a proxied URL based on TLDs in the
* middle of the domain and return a list of likely URLs, starting with the longest domain
* and ending with the original one
*
* E.g., https://www-example-co-uk.mutex.gmu.edu ->
*
* [
* 'https://www.example.co.uk',
* 'https://www.example.co',
* 'https://www-example-co-uk.mutex.gmu.edu',
* ]
*
* Based on Zotero.Proxies.getPotentialProxies()
*/
WebSession.prototype.deproxifyURL = function (url) {
var urlToProxy = {
[url]: null
};
// if there is a subdomain that is also a TLD, also test against URI with the domain
// dropped after the TLD
// (i.e., www.nature.com.mutex.gmu.edu => www.nature.com)
var m = /^(https?:\/\/)([^\/]+)/i.exec(url);
if (m) {
// First, drop the 0- if it exists (this is an III invention)
var host = m[2];
if (host.substr(0, 2) === "0-") host = host.substr(2);
var hostnameParts = [host.split(".")];
if (m[1] == 'https://') {
// try replacing hyphens with dots for https protocol
// to account for EZProxy HttpsHypens mode
hostnameParts.push(host.split('.'));
hostnameParts[1].splice(0, 1, ...(hostnameParts[1][0].replace(/-/g, '.').split('.')));
}
for (let i=0; i < hostnameParts.length; i++) {
let parts = hostnameParts[i];
// If hostnameParts has two entries, then the second one is with replaced hyphens
let dotsToHyphens = i == 1;
// skip the lowest level subdomain, domain and TLD
for (let j=1; j b.length - a.length);
urls.push(urls.shift());
return urls;
};
WebSession.prototype.cleanDOIFromURL = function (url) {
let doi = Zotero.Utilities.cleanDOI(decodeURIComponent(url));
if (doi) {
// Stop at query string, ampersand, or hash
doi = doi.replace(/[?&#].*/, '');
}
return doi || null;
};
================================================
FILE: src/zotero.js
================================================
/*
***** BEGIN LICENSE BLOCK *****
Copyright © 2009 Center for History and New Media
George Mason University, Fairfax, Virginia, USA
http://zotero.org
This file is part of Zotero.
Zotero is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Zotero is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with Zotero. If not, see 1.1 1 xml
01691cam a2200361 a 4500
16725802
20120514113518.0
110407s2011 mdua b 001 0 eng
7
cbc
orignew
1
ecip
20
y-gencatlg
acquire
2 shelf copies
policy default
xh09 2011-04-07
xh09 2011-04-07 to Dewey
rd14 2011-04-07
xn02 2011-12-13 1 copy rec'd., to CIP ver.
rf10 2012-02-27 Z-CipVer. telework to BCDD
xn05 2012-05-14 copy 2 added
2011009234
015994368
Uk
1421402831 (hardcover : alk. paper)
9781421402833 (hardcover : alk. paper)
(OCoLC)ocn711959730
DLC
DLC
YDX
YDXCP
BWX
CDX
IXA
UKMGB
ZCU
DLC
pcc
e-fr---
TX719
.T253 2011
641.5944
22
Takats, Sean,
1974-
The expert cook in enlightenment France /
Sean Takats.
Baltimore :
Johns Hopkins University Press,
2011.
x, 203 p. :
ill. ;
24 cm.
The Johns Hopkins University studies in historical and political science, 129th series ;
2
Includes bibliographical references (p.[179]-196) and index.
Defining the cook -- Corrupting spaces -- Pots and pens -- Theorizing the kitchen -- The servant of medicine -- Conclusion.
Cooking, French
History
18th century.
Cooking
France
History
18th century.
Cooks
France
History
18th century.
France
Social life and customs
18th century.
Johns Hopkins University studies in historical and political science ;
129th ser., 2.
1 1.1 bath.ISBN=^9781421402833 1 xml
30571677
2018
12
20
1553-7358
14
12
2018
Dec
PLoS computational biology
PLoS Comput. Biol.
Ten simple rules for documenting scientific software.
e1006561
10.1371/journal.pcbi.1006561
Lee
Benjamin D
BD
http://orcid.org/0000-0002-7133-8397
School of Engineering and Applied Sciences, Harvard University, Cambridge, Massachusetts, United States of America.
eng
Editorial
2018
12
20
United States
PLoS Comput Biol
101238922
1553-734X
The author has declared that no competing interests exist.
2018
12
21
6
0
2018
12
21
6
0
2018
12
21
6
0
epublish
30571677
10.1371/journal.pcbi.1006561
PCOMPBIOL-D-18-01117
================================================
FILE: test/export_test.js
================================================
/* global assert:false, request:false */
var { JSDOM } = require('jsdom');
const dom = new JSDOM('');
const DOMParser = dom.window.DOMParser;
describe("/export", function () {
var json = [{"key":"Q5VYZIG2","version":0,"itemType":"newspaperArticle","creators":[{"firstName":"Keith","lastName":"Collins","creatorType":"author"}],"tags":[{"tag":"Net Neutrality","type":1},{"tag":"Pai, Ajit","type":1},{"tag":"Federal Communications Commission","type":1},{"tag":"Regulation and Deregulation of Industry","type":1},{"tag":"Computers and the Internet","type":1}],"title":"Net Neutrality Has Officially Been Repealed. Here’s How That Could Affect You.","section":"Technology","url":"https://www.nytimes.com/2018/06/11/technology/net-neutrality-repeal.html","abstractNote":"Net Neutrality rules that required internet service providers to offer equal access to all web content are no longer in effect as of Monday.","language":"en-US","libraryCatalog":"NYTimes.com","accessDate":"2018-08-21T08:23:37Z","date":"2018-06-11","publicationTitle":"The New York Times","ISSN":"0362-4331"}];
var jsonMinimal = [{ key: "JZMIXKUW", version: 0, itemType: "webpage", url: "http://example.com/", title: "Example", accessDate: "2019-01-14T20:18:11Z"}]
it("should export to BibTeX (full)", async function () {
var response = await request()
.post('/export?format=bibtex')
.send(json)
.expect(200);
assert.isTrue(response.text.trim().startsWith('@article'));
});
it("should export to BibTeX (minimal)", async function () {
var response = await request()
.post('/export?format=bibtex')
.send(jsonMinimal)
.expect(200);
assert.isTrue(response.text.trim().startsWith('@misc'));
});
it("should set BibTeX access date", async function () {
var response = await request()
.post('/export?format=bibtex')
.send(jsonMinimal)
.expect(200);
assert.match(response.text, /urldate = {2019-01-14}/);
});
it("should export to RIS (full)", async function () {
var response = await request()
.post('/export?format=ris')
.send(json)
.expect(200);
assert.isTrue(response.text.startsWith('TY - NEWS'));
});
it("should export to RIS (minimal)", async function () {
var response = await request()
.post('/export?format=ris')
.send(jsonMinimal)
.expect(200);
console.log(response.text);
assert.isTrue(response.text.startsWith('TY - ELEC'));
});
it("should export COinS (full)", async function () {
var response = await request()
.post('/export?format=coins')
.send(json)
.expect(200)
.expect('Content-Type', 'text/html');
assert.isTrue(response.text.startsWith(" Zotero.Utilities.Item.itemToAPIJSON(item))), // normalize and flatten results
found: response.body,
};
for (const ef of ['expected', 'found']) {
for (const item of items[ef]) {
// inline attached notes -- notes are exported as separate items with a parentItem, but the parentItem is going to be different every time the test is ran
if (!item.notes) {
item.notes = items[ef]
.filter(note => (note.itemType === 'note' && note.parentItem === item.key))
.map(note => ({ note: note.note, itemType: note.itemType}));
}
}
// remove notes now inlined
items[ef] = items[ef].filter(item => (item.itemType !== 'note' || !item.parentItem));
for (const item of items[ef]) {
// remove purely administrative fields -- must be done in 2nd run because the key/parentItem are required in the 1st run for inlining notes
delete item.key;
delete item.version;
delete item.parentItem;
// remove empty array to simplify comparison
if (Array.isArray(item.tags) && !item.tags.length) {
delete item.tags;
}
// sort tags for stable comparison
if (item.tags) {
item.tags.sort((a, b) => `${a.type}::${a.tag}`.localeCompare(`${b.type}::${b.tag}`));
}
}
}
assert.deepEqual(items.expected, items.found);
});
}
function parseJSON(name, json, source) {
try {
return JSON.parse(json);
}
catch (err) {
it(`should import ${name}`, function () {
const m = err.message.match(/(.*) at position ([0-9]+)$/);
if (m) {
const pos = parseInt(m[2]);
const line = source.substring(0, source.indexOf(json) + pos).split('\n').length;
const column = pos - json.lastIndexOf('\n', pos);
throw new Error(`${m[1]} around line ${line}, column ${column}`);
}
throw err;
});
}
return null;
}
(process.env.IMPORT_TESTS ? describe : describe.skip)("/import", function () {
const translators = path.join(__dirname, '../modules/translators');
for (const translator of fs.readdirSync(translators)) {
if (!translator.endsWith('.js')) continue;
const name = translator.replace(/\.js$/, '');
const code = fs.readFileSync(path.join(translators, translator), 'utf-8');
const marker = code.indexOf('/** BEGIN TEST CASES **/');
if (marker === -1) continue;
const header = parseJSON(name, code.replace(/\n}\n[\s\S]*/, '}'), code);
if (!header) continue;
const start = code.indexOf('[', marker);
const end = code.lastIndexOf(']') + 1;
const cases = parseJSON(name, code.substring(Math.max(start, marker), Math.max(end, start, marker)), code);
if (!cases) continue;
let caseNo = 0;
for (const test of cases) {
caseNo += 1;
if (test.type !== 'import') continue;
addTest(`${name} # ${caseNo}`, header.translatorID, test.input, test.items);
}
}
});
================================================
FILE: test/mocha.opts
================================================
--file test/setup.js
================================================
FILE: test/search_test.js
================================================
/* global assert:false, request:false, sinon:false */
const config = require('config');
const HTTP = require('../src/http');
const TextSearch = require('../src/textSearch');
const path = require('path');
const fs = require('fs');
const urlParse = require('url');
describe("/search", function () {
var bookISBN1 = '9781421402833';
var bookTitle1 = 'The expert cook in enlightenment France';
var articlePMID1 = '30571677';
var articleTitle1 = 'Ten simple rules for documenting scientific software';
beforeEach(() => {
var origHTTPRequest = HTTP.request.bind(HTTP);
sinon.stub(HTTP, 'request').callsFake(async function (method, url, options) {
if (url.startsWith('http://127.0.0.1')) {
return origHTTPRequest(method, url, options);
}
Zotero.debug("Mocking request");
if (url.startsWith('https://www.crossref.org')) {
return {
status: 200,
responseText: ''
+ ''
+ 'Either ISSN or Journal title or Proceedings title must be supplied. '
+ '