'use strict'; var obsidian = require('obsidian'); var path = require('path'); function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; } var path__default = /*#__PURE__*/_interopDefaultLegacy(path); /****************************************************************************** Copyright (c) Microsoft Corporation. Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted. THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ***************************************************************************** */ function __awaiter(thisArg, _arguments, P, generator) { function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } return new (P || (P = Promise))(function (resolve, reject) { function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } step((generator = generator.apply(thisArg, _arguments || [])).next()); }); } function downloadImage(url) { return __awaiter(this, void 0, void 0, function* () { const res = yield fetch(url); return { fileContent: yield res.arrayBuffer(), fileExtension: url.slice(url.lastIndexOf('.')), }; }); } /** * Open or create a folderpath if it does not exist * @param vault * @param folderpath */ function checkAndCreateFolder(vault, folderpath) { return __awaiter(this, void 0, void 0, function* () { folderpath = obsidian.normalizePath(folderpath); const folder = vault.getAbstractFileByPath(folderpath); if (folder && folder instanceof obsidian.TFolder) { return; } yield vault.createFolder(folderpath); }); } function isValidUrl(url) { try { new URL(url); } catch (e) { return false; } return true; } function getBaseUrl(url, origin) { const baseURL = new URL(url, origin); return baseURL.href; } function normalizeFilename(fileName) { const illegalSymbols = [':', '#', '/', '\\', '|', '?', '*', '<', '>', '"']; if (illegalSymbols.some((el) => fileName.contains(el))) { illegalSymbols.forEach((ilSymbol) => { fileName = fileName.replace(ilSymbol, ''); }); return fileName; } else { return fileName; } } function pathJoin(dir, subpath) { const result = path__default["default"].join(dir, subpath); // it seems that obsidian do not understand paths with backslashes in Windows, so turn them into forward slashes return obsidian.normalizePath(result.replace(/\\/g, '/')); } var commonjsGlobal = typeof globalThis !== 'undefined' ? globalThis : typeof window !== 'undefined' ? window : typeof global !== 'undefined' ? global : typeof self !== 'undefined' ? self : {}; var sparkMd5 = {exports: {}}; (function (module, exports) { (function (factory) { { // Node/CommonJS module.exports = factory(); } }(function (undefined$1) { /* * Fastest md5 implementation around (JKM md5). * Credits: Joseph Myers * * @see http://www.myersdaily.org/joseph/javascript/md5-text.html * @see http://jsperf.com/md5-shootout/7 */ /* this function is much faster, so if possible we use it. Some IEs are the only ones I know of that need the idiotic second function, generated by an if clause. */ var hex_chr = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']; function md5cycle(x, k) { var a = x[0], b = x[1], c = x[2], d = x[3]; a += (b & c | ~b & d) + k[0] - 680876936 | 0; a = (a << 7 | a >>> 25) + b | 0; d += (a & b | ~a & c) + k[1] - 389564586 | 0; d = (d << 12 | d >>> 20) + a | 0; c += (d & a | ~d & b) + k[2] + 606105819 | 0; c = (c << 17 | c >>> 15) + d | 0; b += (c & d | ~c & a) + k[3] - 1044525330 | 0; b = (b << 22 | b >>> 10) + c | 0; a += (b & c | ~b & d) + k[4] - 176418897 | 0; a = (a << 7 | a >>> 25) + b | 0; d += (a & b | ~a & c) + k[5] + 1200080426 | 0; d = (d << 12 | d >>> 20) + a | 0; c += (d & a | ~d & b) + k[6] - 1473231341 | 0; c = (c << 17 | c >>> 15) + d | 0; b += (c & d | ~c & a) + k[7] - 45705983 | 0; b = (b << 22 | b >>> 10) + c | 0; a += (b & c | ~b & d) + k[8] + 1770035416 | 0; a = (a << 7 | a >>> 25) + b | 0; d += (a & b | ~a & c) + k[9] - 1958414417 | 0; d = (d << 12 | d >>> 20) + a | 0; c += (d & a | ~d & b) + k[10] - 42063 | 0; c = (c << 17 | c >>> 15) + d | 0; b += (c & d | ~c & a) + k[11] - 1990404162 | 0; b = (b << 22 | b >>> 10) + c | 0; a += (b & c | ~b & d) + k[12] + 1804603682 | 0; a = (a << 7 | a >>> 25) + b | 0; d += (a & b | ~a & c) + k[13] - 40341101 | 0; d = (d << 12 | d >>> 20) + a | 0; c += (d & a | ~d & b) + k[14] - 1502002290 | 0; c = (c << 17 | c >>> 15) + d | 0; b += (c & d | ~c & a) + k[15] + 1236535329 | 0; b = (b << 22 | b >>> 10) + c | 0; a += (b & d | c & ~d) + k[1] - 165796510 | 0; a = (a << 5 | a >>> 27) + b | 0; d += (a & c | b & ~c) + k[6] - 1069501632 | 0; d = (d << 9 | d >>> 23) + a | 0; c += (d & b | a & ~b) + k[11] + 643717713 | 0; c = (c << 14 | c >>> 18) + d | 0; b += (c & a | d & ~a) + k[0] - 373897302 | 0; b = (b << 20 | b >>> 12) + c | 0; a += (b & d | c & ~d) + k[5] - 701558691 | 0; a = (a << 5 | a >>> 27) + b | 0; d += (a & c | b & ~c) + k[10] + 38016083 | 0; d = (d << 9 | d >>> 23) + a | 0; c += (d & b | a & ~b) + k[15] - 660478335 | 0; c = (c << 14 | c >>> 18) + d | 0; b += (c & a | d & ~a) + k[4] - 405537848 | 0; b = (b << 20 | b >>> 12) + c | 0; a += (b & d | c & ~d) + k[9] + 568446438 | 0; a = (a << 5 | a >>> 27) + b | 0; d += (a & c | b & ~c) + k[14] - 1019803690 | 0; d = (d << 9 | d >>> 23) + a | 0; c += (d & b | a & ~b) + k[3] - 187363961 | 0; c = (c << 14 | c >>> 18) + d | 0; b += (c & a | d & ~a) + k[8] + 1163531501 | 0; b = (b << 20 | b >>> 12) + c | 0; a += (b & d | c & ~d) + k[13] - 1444681467 | 0; a = (a << 5 | a >>> 27) + b | 0; d += (a & c | b & ~c) + k[2] - 51403784 | 0; d = (d << 9 | d >>> 23) + a | 0; c += (d & b | a & ~b) + k[7] + 1735328473 | 0; c = (c << 14 | c >>> 18) + d | 0; b += (c & a | d & ~a) + k[12] - 1926607734 | 0; b = (b << 20 | b >>> 12) + c | 0; a += (b ^ c ^ d) + k[5] - 378558 | 0; a = (a << 4 | a >>> 28) + b | 0; d += (a ^ b ^ c) + k[8] - 2022574463 | 0; d = (d << 11 | d >>> 21) + a | 0; c += (d ^ a ^ b) + k[11] + 1839030562 | 0; c = (c << 16 | c >>> 16) + d | 0; b += (c ^ d ^ a) + k[14] - 35309556 | 0; b = (b << 23 | b >>> 9) + c | 0; a += (b ^ c ^ d) + k[1] - 1530992060 | 0; a = (a << 4 | a >>> 28) + b | 0; d += (a ^ b ^ c) + k[4] + 1272893353 | 0; d = (d << 11 | d >>> 21) + a | 0; c += (d ^ a ^ b) + k[7] - 155497632 | 0; c = (c << 16 | c >>> 16) + d | 0; b += (c ^ d ^ a) + k[10] - 1094730640 | 0; b = (b << 23 | b >>> 9) + c | 0; a += (b ^ c ^ d) + k[13] + 681279174 | 0; a = (a << 4 | a >>> 28) + b | 0; d += (a ^ b ^ c) + k[0] - 358537222 | 0; d = (d << 11 | d >>> 21) + a | 0; c += (d ^ a ^ b) + k[3] - 722521979 | 0; c = (c << 16 | c >>> 16) + d | 0; b += (c ^ d ^ a) + k[6] + 76029189 | 0; b = (b << 23 | b >>> 9) + c | 0; a += (b ^ c ^ d) + k[9] - 640364487 | 0; a = (a << 4 | a >>> 28) + b | 0; d += (a ^ b ^ c) + k[12] - 421815835 | 0; d = (d << 11 | d >>> 21) + a | 0; c += (d ^ a ^ b) + k[15] + 530742520 | 0; c = (c << 16 | c >>> 16) + d | 0; b += (c ^ d ^ a) + k[2] - 995338651 | 0; b = (b << 23 | b >>> 9) + c | 0; a += (c ^ (b | ~d)) + k[0] - 198630844 | 0; a = (a << 6 | a >>> 26) + b | 0; d += (b ^ (a | ~c)) + k[7] + 1126891415 | 0; d = (d << 10 | d >>> 22) + a | 0; c += (a ^ (d | ~b)) + k[14] - 1416354905 | 0; c = (c << 15 | c >>> 17) + d | 0; b += (d ^ (c | ~a)) + k[5] - 57434055 | 0; b = (b << 21 |b >>> 11) + c | 0; a += (c ^ (b | ~d)) + k[12] + 1700485571 | 0; a = (a << 6 | a >>> 26) + b | 0; d += (b ^ (a | ~c)) + k[3] - 1894986606 | 0; d = (d << 10 | d >>> 22) + a | 0; c += (a ^ (d | ~b)) + k[10] - 1051523 | 0; c = (c << 15 | c >>> 17) + d | 0; b += (d ^ (c | ~a)) + k[1] - 2054922799 | 0; b = (b << 21 |b >>> 11) + c | 0; a += (c ^ (b | ~d)) + k[8] + 1873313359 | 0; a = (a << 6 | a >>> 26) + b | 0; d += (b ^ (a | ~c)) + k[15] - 30611744 | 0; d = (d << 10 | d >>> 22) + a | 0; c += (a ^ (d | ~b)) + k[6] - 1560198380 | 0; c = (c << 15 | c >>> 17) + d | 0; b += (d ^ (c | ~a)) + k[13] + 1309151649 | 0; b = (b << 21 |b >>> 11) + c | 0; a += (c ^ (b | ~d)) + k[4] - 145523070 | 0; a = (a << 6 | a >>> 26) + b | 0; d += (b ^ (a | ~c)) + k[11] - 1120210379 | 0; d = (d << 10 | d >>> 22) + a | 0; c += (a ^ (d | ~b)) + k[2] + 718787259 | 0; c = (c << 15 | c >>> 17) + d | 0; b += (d ^ (c | ~a)) + k[9] - 343485551 | 0; b = (b << 21 | b >>> 11) + c | 0; x[0] = a + x[0] | 0; x[1] = b + x[1] | 0; x[2] = c + x[2] | 0; x[3] = d + x[3] | 0; } function md5blk(s) { var md5blks = [], i; /* Andy King said do it this way. */ for (i = 0; i < 64; i += 4) { md5blks[i >> 2] = s.charCodeAt(i) + (s.charCodeAt(i + 1) << 8) + (s.charCodeAt(i + 2) << 16) + (s.charCodeAt(i + 3) << 24); } return md5blks; } function md5blk_array(a) { var md5blks = [], i; /* Andy King said do it this way. */ for (i = 0; i < 64; i += 4) { md5blks[i >> 2] = a[i] + (a[i + 1] << 8) + (a[i + 2] << 16) + (a[i + 3] << 24); } return md5blks; } function md51(s) { var n = s.length, state = [1732584193, -271733879, -1732584194, 271733878], i, length, tail, tmp, lo, hi; for (i = 64; i <= n; i += 64) { md5cycle(state, md5blk(s.substring(i - 64, i))); } s = s.substring(i - 64); length = s.length; tail = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; for (i = 0; i < length; i += 1) { tail[i >> 2] |= s.charCodeAt(i) << ((i % 4) << 3); } tail[i >> 2] |= 0x80 << ((i % 4) << 3); if (i > 55) { md5cycle(state, tail); for (i = 0; i < 16; i += 1) { tail[i] = 0; } } // Beware that the final length might not fit in 32 bits so we take care of that tmp = n * 8; tmp = tmp.toString(16).match(/(.*?)(.{0,8})$/); lo = parseInt(tmp[2], 16); hi = parseInt(tmp[1], 16) || 0; tail[14] = lo; tail[15] = hi; md5cycle(state, tail); return state; } function md51_array(a) { var n = a.length, state = [1732584193, -271733879, -1732584194, 271733878], i, length, tail, tmp, lo, hi; for (i = 64; i <= n; i += 64) { md5cycle(state, md5blk_array(a.subarray(i - 64, i))); } // Not sure if it is a bug, however IE10 will always produce a sub array of length 1 // containing the last element of the parent array if the sub array specified starts // beyond the length of the parent array - weird. // https://connect.microsoft.com/IE/feedback/details/771452/typed-array-subarray-issue a = (i - 64) < n ? a.subarray(i - 64) : new Uint8Array(0); length = a.length; tail = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; for (i = 0; i < length; i += 1) { tail[i >> 2] |= a[i] << ((i % 4) << 3); } tail[i >> 2] |= 0x80 << ((i % 4) << 3); if (i > 55) { md5cycle(state, tail); for (i = 0; i < 16; i += 1) { tail[i] = 0; } } // Beware that the final length might not fit in 32 bits so we take care of that tmp = n * 8; tmp = tmp.toString(16).match(/(.*?)(.{0,8})$/); lo = parseInt(tmp[2], 16); hi = parseInt(tmp[1], 16) || 0; tail[14] = lo; tail[15] = hi; md5cycle(state, tail); return state; } function rhex(n) { var s = '', j; for (j = 0; j < 4; j += 1) { s += hex_chr[(n >> (j * 8 + 4)) & 0x0F] + hex_chr[(n >> (j * 8)) & 0x0F]; } return s; } function hex(x) { var i; for (i = 0; i < x.length; i += 1) { x[i] = rhex(x[i]); } return x.join(''); } // In some cases the fast add32 function cannot be used.. if (hex(md51('hello')) !== '5d41402abc4b2a76b9719d911017c592') ; // --------------------------------------------------- /** * ArrayBuffer slice polyfill. * * @see https://github.com/ttaubert/node-arraybuffer-slice */ if (typeof ArrayBuffer !== 'undefined' && !ArrayBuffer.prototype.slice) { (function () { function clamp(val, length) { val = (val | 0) || 0; if (val < 0) { return Math.max(val + length, 0); } return Math.min(val, length); } ArrayBuffer.prototype.slice = function (from, to) { var length = this.byteLength, begin = clamp(from, length), end = length, num, target, targetArray, sourceArray; if (to !== undefined$1) { end = clamp(to, length); } if (begin > end) { return new ArrayBuffer(0); } num = end - begin; target = new ArrayBuffer(num); targetArray = new Uint8Array(target); sourceArray = new Uint8Array(this, begin, num); targetArray.set(sourceArray); return target; }; })(); } // --------------------------------------------------- /** * Helpers. */ function toUtf8(str) { if (/[\u0080-\uFFFF]/.test(str)) { str = unescape(encodeURIComponent(str)); } return str; } function utf8Str2ArrayBuffer(str, returnUInt8Array) { var length = str.length, buff = new ArrayBuffer(length), arr = new Uint8Array(buff), i; for (i = 0; i < length; i += 1) { arr[i] = str.charCodeAt(i); } return returnUInt8Array ? arr : buff; } function arrayBuffer2Utf8Str(buff) { return String.fromCharCode.apply(null, new Uint8Array(buff)); } function concatenateArrayBuffers(first, second, returnUInt8Array) { var result = new Uint8Array(first.byteLength + second.byteLength); result.set(new Uint8Array(first)); result.set(new Uint8Array(second), first.byteLength); return returnUInt8Array ? result : result.buffer; } function hexToBinaryString(hex) { var bytes = [], length = hex.length, x; for (x = 0; x < length - 1; x += 2) { bytes.push(parseInt(hex.substr(x, 2), 16)); } return String.fromCharCode.apply(String, bytes); } // --------------------------------------------------- /** * SparkMD5 OOP implementation. * * Use this class to perform an incremental md5, otherwise use the * static methods instead. */ function SparkMD5() { // call reset to init the instance this.reset(); } /** * Appends a string. * A conversion will be applied if an utf8 string is detected. * * @param {String} str The string to be appended * * @return {SparkMD5} The instance itself */ SparkMD5.prototype.append = function (str) { // Converts the string to utf8 bytes if necessary // Then append as binary this.appendBinary(toUtf8(str)); return this; }; /** * Appends a binary string. * * @param {String} contents The binary string to be appended * * @return {SparkMD5} The instance itself */ SparkMD5.prototype.appendBinary = function (contents) { this._buff += contents; this._length += contents.length; var length = this._buff.length, i; for (i = 64; i <= length; i += 64) { md5cycle(this._hash, md5blk(this._buff.substring(i - 64, i))); } this._buff = this._buff.substring(i - 64); return this; }; /** * Finishes the incremental computation, reseting the internal state and * returning the result. * * @param {Boolean} raw True to get the raw string, false to get the hex string * * @return {String} The result */ SparkMD5.prototype.end = function (raw) { var buff = this._buff, length = buff.length, i, tail = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], ret; for (i = 0; i < length; i += 1) { tail[i >> 2] |= buff.charCodeAt(i) << ((i % 4) << 3); } this._finish(tail, length); ret = hex(this._hash); if (raw) { ret = hexToBinaryString(ret); } this.reset(); return ret; }; /** * Resets the internal state of the computation. * * @return {SparkMD5} The instance itself */ SparkMD5.prototype.reset = function () { this._buff = ''; this._length = 0; this._hash = [1732584193, -271733879, -1732584194, 271733878]; return this; }; /** * Gets the internal state of the computation. * * @return {Object} The state */ SparkMD5.prototype.getState = function () { return { buff: this._buff, length: this._length, hash: this._hash.slice() }; }; /** * Gets the internal state of the computation. * * @param {Object} state The state * * @return {SparkMD5} The instance itself */ SparkMD5.prototype.setState = function (state) { this._buff = state.buff; this._length = state.length; this._hash = state.hash; return this; }; /** * Releases memory used by the incremental buffer and other additional * resources. If you plan to use the instance again, use reset instead. */ SparkMD5.prototype.destroy = function () { delete this._hash; delete this._buff; delete this._length; }; /** * Finish the final calculation based on the tail. * * @param {Array} tail The tail (will be modified) * @param {Number} length The length of the remaining buffer */ SparkMD5.prototype._finish = function (tail, length) { var i = length, tmp, lo, hi; tail[i >> 2] |= 0x80 << ((i % 4) << 3); if (i > 55) { md5cycle(this._hash, tail); for (i = 0; i < 16; i += 1) { tail[i] = 0; } } // Do the final computation based on the tail and length // Beware that the final length may not fit in 32 bits so we take care of that tmp = this._length * 8; tmp = tmp.toString(16).match(/(.*?)(.{0,8})$/); lo = parseInt(tmp[2], 16); hi = parseInt(tmp[1], 16) || 0; tail[14] = lo; tail[15] = hi; md5cycle(this._hash, tail); }; /** * Performs the md5 hash on a string. * A conversion will be applied if utf8 string is detected. * * @param {String} str The string * @param {Boolean} [raw] True to get the raw string, false to get the hex string * * @return {String} The result */ SparkMD5.hash = function (str, raw) { // Converts the string to utf8 bytes if necessary // Then compute it using the binary function return SparkMD5.hashBinary(toUtf8(str), raw); }; /** * Performs the md5 hash on a binary string. * * @param {String} content The binary string * @param {Boolean} [raw] True to get the raw string, false to get the hex string * * @return {String} The result */ SparkMD5.hashBinary = function (content, raw) { var hash = md51(content), ret = hex(hash); return raw ? hexToBinaryString(ret) : ret; }; // --------------------------------------------------- /** * SparkMD5 OOP implementation for array buffers. * * Use this class to perform an incremental md5 ONLY for array buffers. */ SparkMD5.ArrayBuffer = function () { // call reset to init the instance this.reset(); }; /** * Appends an array buffer. * * @param {ArrayBuffer} arr The array to be appended * * @return {SparkMD5.ArrayBuffer} The instance itself */ SparkMD5.ArrayBuffer.prototype.append = function (arr) { var buff = concatenateArrayBuffers(this._buff.buffer, arr, true), length = buff.length, i; this._length += arr.byteLength; for (i = 64; i <= length; i += 64) { md5cycle(this._hash, md5blk_array(buff.subarray(i - 64, i))); } this._buff = (i - 64) < length ? new Uint8Array(buff.buffer.slice(i - 64)) : new Uint8Array(0); return this; }; /** * Finishes the incremental computation, reseting the internal state and * returning the result. * * @param {Boolean} raw True to get the raw string, false to get the hex string * * @return {String} The result */ SparkMD5.ArrayBuffer.prototype.end = function (raw) { var buff = this._buff, length = buff.length, tail = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], i, ret; for (i = 0; i < length; i += 1) { tail[i >> 2] |= buff[i] << ((i % 4) << 3); } this._finish(tail, length); ret = hex(this._hash); if (raw) { ret = hexToBinaryString(ret); } this.reset(); return ret; }; /** * Resets the internal state of the computation. * * @return {SparkMD5.ArrayBuffer} The instance itself */ SparkMD5.ArrayBuffer.prototype.reset = function () { this._buff = new Uint8Array(0); this._length = 0; this._hash = [1732584193, -271733879, -1732584194, 271733878]; return this; }; /** * Gets the internal state of the computation. * * @return {Object} The state */ SparkMD5.ArrayBuffer.prototype.getState = function () { var state = SparkMD5.prototype.getState.call(this); // Convert buffer to a string state.buff = arrayBuffer2Utf8Str(state.buff); return state; }; /** * Gets the internal state of the computation. * * @param {Object} state The state * * @return {SparkMD5.ArrayBuffer} The instance itself */ SparkMD5.ArrayBuffer.prototype.setState = function (state) { // Convert string to buffer state.buff = utf8Str2ArrayBuffer(state.buff, true); return SparkMD5.prototype.setState.call(this, state); }; SparkMD5.ArrayBuffer.prototype.destroy = SparkMD5.prototype.destroy; SparkMD5.ArrayBuffer.prototype._finish = SparkMD5.prototype._finish; /** * Performs the md5 hash on an array buffer. * * @param {ArrayBuffer} arr The array buffer * @param {Boolean} [raw] True to get the raw string, false to get the hex one * * @return {String} The result */ SparkMD5.ArrayBuffer.hash = function (arr, raw) { var hash = md51_array(new Uint8Array(arr)), ret = hex(hash); return raw ? hexToBinaryString(ret) : ret; }; return SparkMD5; })); } (sparkMd5)); class LinkHashes { constructor() { this.linksInfo = {}; } ensureHashGenerated(link, data) { if (!this.linksInfo[link]) { this.linksInfo[link] = sparkMd5.exports.ArrayBuffer.hash(data); } } isSame(link, data) { const fileHash = sparkMd5.exports.ArrayBuffer.hash(data); return this.linksInfo[link] == fileHash; } } const linkHashes = new LinkHashes(); const EXTERNAL_MEDIA_LINK_PATTERN = /\!\[(?.*?)\]\((?.+?)\)/g; function replaceImages(app, content, assetsDir) { return __awaiter(this, void 0, void 0, function* () { return yield replaceAsync(content, EXTERNAL_MEDIA_LINK_PATTERN, imageTagProcessor(app, assetsDir)); }); } function replaceAsync(string, searchValue, replacer) { try { if (typeof replacer === 'function') { // 1. Run fake pass of `replace`, collect values from `replacer` calls // 2. Resolve them with `Promise.all` // 3. Run `replace` with resolved values const values = []; String.prototype.replace.call(string, searchValue, function () { values.push(replacer.apply(undefined, arguments)); return ''; }); return Promise.all(values).then(function (resolvedValues) { return String.prototype.replace.call(string, searchValue, function () { return resolvedValues.shift(); }); }); } else { return Promise.resolve(String.prototype.replace.call(string, searchValue, replacer)); } } catch (error) { return Promise.reject(error); } } const FILENAME_ATTEMPTS = 5; function imageTagProcessor(app, mediaDir) { return function processImageTag(match, anchor, link) { return __awaiter(this, void 0, void 0, function* () { if (!isValidUrl(link)) { return match; } yield checkAndCreateFolder(app.vault, mediaDir); try { const { fileContent, fileExtension } = yield downloadImage(link); let attempt = 0; while (attempt < FILENAME_ATTEMPTS) { try { const { fileName, needWrite } = yield chooseFileName(app.vault.adapter, mediaDir, anchor, link, fileContent, fileExtension); if (needWrite && fileName) { yield app.vault.createBinary(fileName, fileContent); } if (fileName) { const maskedFilename = fileName.replace(/\s/g, '%20'); return `![${anchor}](${maskedFilename})`; } else { return match; } } catch (error) { if (error.message === 'File already exists.') { attempt++; } else { throw error; } } } return match; } catch (error) { console.warn('Image processing failed: ', error); return match; } }); }; } const FILENAME_TEMPLATE = 'media'; const MAX_FILENAME_INDEX = 1000; function chooseFileName(adapter, dir, baseName, link, contentData, fileExtension) { return __awaiter(this, void 0, void 0, function* () { if (!fileExtension) { return { fileName: '', needWrite: false }; } // if there is no anchor try get file name from url if (!baseName) { const parsedUrl = new URL(link); baseName = path.basename(parsedUrl.pathname); } // if there is no part for file name from url use name template if (!baseName) { baseName = FILENAME_TEMPLATE; } // if filename already ends with correct extension, remove it to work with base name if (baseName.endsWith(`.${fileExtension}`)) { baseName = baseName.slice(0, -1 * (fileExtension.length + 1)); } baseName = normalizeFilename(baseName); let fileName = ''; let needWrite = true; let index = 0; while (!fileName && index < MAX_FILENAME_INDEX) { const suggestedName = index ? pathJoin(dir, `${baseName}-${index}.${fileExtension}`) : pathJoin(dir, `${baseName}.${fileExtension}`); if (yield adapter.exists(suggestedName, false)) { linkHashes.ensureHashGenerated(link, contentData); const fileData = yield adapter.readBinary(suggestedName); if (linkHashes.isSame(link, fileData)) { fileName = suggestedName; needWrite = false; } } else { fileName = suggestedName; } index++; } if (!fileName) { throw new Error('Failed to generate file name for media file.'); } linkHashes.ensureHashGenerated(link, contentData); return { fileName, needWrite }; }); } const DEFAULT_SETTINGS = { inboxDir: 'ReadItLater Inbox', assetsDir: 'ReadItLater Inbox/assets', openNewNote: false, youtubeNoteTitle: 'Youtube - %title%', youtubeNote: `[[ReadItLater]] [[Youtube]]\n\n# [%videoTitle%](%videoURL%)\n\n%videoPlayer%`, twitterNoteTitle: 'Tweet from %tweetAuthorName% (%date%)', twitterNote: `[[ReadItLater]] [[Tweet]]\n\n# [%tweetAuthorName%](%tweetURL%)\n\n%tweetContent%`, parseableArticleNoteTitle: '%title%', parsableArticleNote: `[[ReadItLater]] [[Article]]\n\n# [%articleTitle%](%articleURL%)\n\n%articleContent%`, notParseableArticleNoteTitle: 'Article %date%', notParsableArticleNote: `[[ReadItLater]] [[Article]]\n\n[%articleURL%](%articleURL%)`, textSnippetNoteTitle: 'Notice %date%', textSnippetNote: `[[ReadItLater]] [[Textsnippet]]\n\n%content%`, downloadImages: true, dateTitleFmt: 'YYYY-MM-DD HH-mm-ss', dateContentFmt: 'YYYY-MM-DD', }; class Note { constructor(fileName, content) { this.fileName = fileName; this.content = content; } } class Parser { constructor(app, settings) { this.app = app; this.settings = settings; } isValidUrl(url) { try { new URL(url); } catch (e) { return false; } return true; } getFormattedDateForFilename() { const date = new Date(); return obsidian.moment(date).format(this.settings.dateTitleFmt); } getFormattedDateForContent() { const date = new Date(); return obsidian.moment(date).format(this.settings.dateContentFmt); } } class YoutubeParser extends Parser { constructor(app, settings) { super(app, settings); this.PATTERN = /(youtube.com|youtu.be)\/(watch)?(\?v=)?(\S+)?/; } test(url) { return this.isValidUrl(url) && this.PATTERN.test(url); } prepareNote(url) { return __awaiter(this, void 0, void 0, function* () { const response = yield obsidian.request({ method: 'GET', url }); const videoHTML = new DOMParser().parseFromString(response, 'text/html'); const videoTitle = videoHTML.querySelector("[property~='og:title']").getAttribute('content'); const videoId = this.PATTERN.exec(url)[4]; const videoPlayer = ``; const content = this.settings.youtubeNote .replace(/%date%/g, this.getFormattedDateForContent()) .replace(/%videoTitle%/g, videoTitle) .replace(/%videoURL%/g, url) .replace(/%videoId%/g, videoId) .replace(/%videoPlayer%/g, videoPlayer); const fileNameTemplate = this.settings.youtubeNoteTitle.replace(/%title%/g, videoTitle); const fileName = `${fileNameTemplate}.md`; return new Note(fileName, content); }); } } function extend (destination) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) { if (source.hasOwnProperty(key)) destination[key] = source[key]; } } return destination } function repeat (character, count) { return Array(count + 1).join(character) } function trimLeadingNewlines (string) { return string.replace(/^\n*/, '') } function trimTrailingNewlines (string) { // avoid match-at-end regexp bottleneck, see #370 var indexEnd = string.length; while (indexEnd > 0 && string[indexEnd - 1] === '\n') indexEnd--; return string.substring(0, indexEnd) } var blockElements = [ 'ADDRESS', 'ARTICLE', 'ASIDE', 'AUDIO', 'BLOCKQUOTE', 'BODY', 'CANVAS', 'CENTER', 'DD', 'DIR', 'DIV', 'DL', 'DT', 'FIELDSET', 'FIGCAPTION', 'FIGURE', 'FOOTER', 'FORM', 'FRAMESET', 'H1', 'H2', 'H3', 'H4', 'H5', 'H6', 'HEADER', 'HGROUP', 'HR', 'HTML', 'ISINDEX', 'LI', 'MAIN', 'MENU', 'NAV', 'NOFRAMES', 'NOSCRIPT', 'OL', 'OUTPUT', 'P', 'PRE', 'SECTION', 'TABLE', 'TBODY', 'TD', 'TFOOT', 'TH', 'THEAD', 'TR', 'UL' ]; function isBlock (node) { return is(node, blockElements) } var voidElements = [ 'AREA', 'BASE', 'BR', 'COL', 'COMMAND', 'EMBED', 'HR', 'IMG', 'INPUT', 'KEYGEN', 'LINK', 'META', 'PARAM', 'SOURCE', 'TRACK', 'WBR' ]; function isVoid (node) { return is(node, voidElements) } function hasVoid (node) { return has(node, voidElements) } var meaningfulWhenBlankElements = [ 'A', 'TABLE', 'THEAD', 'TBODY', 'TFOOT', 'TH', 'TD', 'IFRAME', 'SCRIPT', 'AUDIO', 'VIDEO' ]; function isMeaningfulWhenBlank (node) { return is(node, meaningfulWhenBlankElements) } function hasMeaningfulWhenBlank (node) { return has(node, meaningfulWhenBlankElements) } function is (node, tagNames) { return tagNames.indexOf(node.nodeName) >= 0 } function has (node, tagNames) { return ( node.getElementsByTagName && tagNames.some(function (tagName) { return node.getElementsByTagName(tagName).length }) ) } var rules$1 = {}; rules$1.paragraph = { filter: 'p', replacement: function (content) { return '\n\n' + content + '\n\n' } }; rules$1.lineBreak = { filter: 'br', replacement: function (content, node, options) { return options.br + '\n' } }; rules$1.heading = { filter: ['h1', 'h2', 'h3', 'h4', 'h5', 'h6'], replacement: function (content, node, options) { var hLevel = Number(node.nodeName.charAt(1)); if (options.headingStyle === 'setext' && hLevel < 3) { var underline = repeat((hLevel === 1 ? '=' : '-'), content.length); return ( '\n\n' + content + '\n' + underline + '\n\n' ) } else { return '\n\n' + repeat('#', hLevel) + ' ' + content + '\n\n' } } }; rules$1.blockquote = { filter: 'blockquote', replacement: function (content) { content = content.replace(/^\n+|\n+$/g, ''); content = content.replace(/^/gm, '> '); return '\n\n' + content + '\n\n' } }; rules$1.list = { filter: ['ul', 'ol'], replacement: function (content, node) { var parent = node.parentNode; if (parent.nodeName === 'LI' && parent.lastElementChild === node) { return '\n' + content } else { return '\n\n' + content + '\n\n' } } }; rules$1.listItem = { filter: 'li', replacement: function (content, node, options) { content = content .replace(/^\n+/, '') // remove leading newlines .replace(/\n+$/, '\n') // replace trailing newlines with just a single one .replace(/\n/gm, '\n '); // indent var prefix = options.bulletListMarker + ' '; var parent = node.parentNode; if (parent.nodeName === 'OL') { var start = parent.getAttribute('start'); var index = Array.prototype.indexOf.call(parent.children, node); prefix = (start ? Number(start) + index : index + 1) + '. '; } return ( prefix + content + (node.nextSibling && !/\n$/.test(content) ? '\n' : '') ) } }; rules$1.indentedCodeBlock = { filter: function (node, options) { return ( options.codeBlockStyle === 'indented' && node.nodeName === 'PRE' && node.firstChild && node.firstChild.nodeName === 'CODE' ) }, replacement: function (content, node, options) { return ( '\n\n ' + node.firstChild.textContent.replace(/\n/g, '\n ') + '\n\n' ) } }; rules$1.fencedCodeBlock = { filter: function (node, options) { return ( options.codeBlockStyle === 'fenced' && node.nodeName === 'PRE' && node.firstChild && node.firstChild.nodeName === 'CODE' ) }, replacement: function (content, node, options) { var className = node.firstChild.getAttribute('class') || ''; var language = (className.match(/language-(\S+)/) || [null, ''])[1]; var code = node.firstChild.textContent; var fenceChar = options.fence.charAt(0); var fenceSize = 3; var fenceInCodeRegex = new RegExp('^' + fenceChar + '{3,}', 'gm'); var match; while ((match = fenceInCodeRegex.exec(code))) { if (match[0].length >= fenceSize) { fenceSize = match[0].length + 1; } } var fence = repeat(fenceChar, fenceSize); return ( '\n\n' + fence + language + '\n' + code.replace(/\n$/, '') + '\n' + fence + '\n\n' ) } }; rules$1.horizontalRule = { filter: 'hr', replacement: function (content, node, options) { return '\n\n' + options.hr + '\n\n' } }; rules$1.inlineLink = { filter: function (node, options) { return ( options.linkStyle === 'inlined' && node.nodeName === 'A' && node.getAttribute('href') ) }, replacement: function (content, node) { var href = node.getAttribute('href'); var title = cleanAttribute(node.getAttribute('title')); if (title) title = ' "' + title + '"'; return '[' + content + '](' + href + title + ')' } }; rules$1.referenceLink = { filter: function (node, options) { return ( options.linkStyle === 'referenced' && node.nodeName === 'A' && node.getAttribute('href') ) }, replacement: function (content, node, options) { var href = node.getAttribute('href'); var title = cleanAttribute(node.getAttribute('title')); if (title) title = ' "' + title + '"'; var replacement; var reference; switch (options.linkReferenceStyle) { case 'collapsed': replacement = '[' + content + '][]'; reference = '[' + content + ']: ' + href + title; break case 'shortcut': replacement = '[' + content + ']'; reference = '[' + content + ']: ' + href + title; break default: var id = this.references.length + 1; replacement = '[' + content + '][' + id + ']'; reference = '[' + id + ']: ' + href + title; } this.references.push(reference); return replacement }, references: [], append: function (options) { var references = ''; if (this.references.length) { references = '\n\n' + this.references.join('\n') + '\n\n'; this.references = []; // Reset references } return references } }; rules$1.emphasis = { filter: ['em', 'i'], replacement: function (content, node, options) { if (!content.trim()) return '' return options.emDelimiter + content + options.emDelimiter } }; rules$1.strong = { filter: ['strong', 'b'], replacement: function (content, node, options) { if (!content.trim()) return '' return options.strongDelimiter + content + options.strongDelimiter } }; rules$1.code = { filter: function (node) { var hasSiblings = node.previousSibling || node.nextSibling; var isCodeBlock = node.parentNode.nodeName === 'PRE' && !hasSiblings; return node.nodeName === 'CODE' && !isCodeBlock }, replacement: function (content) { if (!content) return '' content = content.replace(/\r?\n|\r/g, ' '); var extraSpace = /^`|^ .*?[^ ].* $|`$/.test(content) ? ' ' : ''; var delimiter = '`'; var matches = content.match(/`+/gm) || []; while (matches.indexOf(delimiter) !== -1) delimiter = delimiter + '`'; return delimiter + extraSpace + content + extraSpace + delimiter } }; rules$1.image = { filter: 'img', replacement: function (content, node) { var alt = cleanAttribute(node.getAttribute('alt')); var src = node.getAttribute('src') || ''; var title = cleanAttribute(node.getAttribute('title')); var titlePart = title ? ' "' + title + '"' : ''; return src ? '![' + alt + ']' + '(' + src + titlePart + ')' : '' } }; function cleanAttribute (attribute) { return attribute ? attribute.replace(/(\n+\s*)+/g, '\n') : '' } /** * Manages a collection of rules used to convert HTML to Markdown */ function Rules (options) { this.options = options; this._keep = []; this._remove = []; this.blankRule = { replacement: options.blankReplacement }; this.keepReplacement = options.keepReplacement; this.defaultRule = { replacement: options.defaultReplacement }; this.array = []; for (var key in options.rules) this.array.push(options.rules[key]); } Rules.prototype = { add: function (key, rule) { this.array.unshift(rule); }, keep: function (filter) { this._keep.unshift({ filter: filter, replacement: this.keepReplacement }); }, remove: function (filter) { this._remove.unshift({ filter: filter, replacement: function () { return '' } }); }, forNode: function (node) { if (node.isBlank) return this.blankRule var rule; if ((rule = findRule(this.array, node, this.options))) return rule if ((rule = findRule(this._keep, node, this.options))) return rule if ((rule = findRule(this._remove, node, this.options))) return rule return this.defaultRule }, forEach: function (fn) { for (var i = 0; i < this.array.length; i++) fn(this.array[i], i); } }; function findRule (rules, node, options) { for (var i = 0; i < rules.length; i++) { var rule = rules[i]; if (filterValue(rule, node, options)) return rule } return void 0 } function filterValue (rule, node, options) { var filter = rule.filter; if (typeof filter === 'string') { if (filter === node.nodeName.toLowerCase()) return true } else if (Array.isArray(filter)) { if (filter.indexOf(node.nodeName.toLowerCase()) > -1) return true } else if (typeof filter === 'function') { if (filter.call(rule, node, options)) return true } else { throw new TypeError('`filter` needs to be a string, array, or function') } } /** * The collapseWhitespace function is adapted from collapse-whitespace * by Luc Thevenard. * * The MIT License (MIT) * * Copyright (c) 2014 Luc Thevenard * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. */ /** * collapseWhitespace(options) removes extraneous whitespace from an the given element. * * @param {Object} options */ function collapseWhitespace (options) { var element = options.element; var isBlock = options.isBlock; var isVoid = options.isVoid; var isPre = options.isPre || function (node) { return node.nodeName === 'PRE' }; if (!element.firstChild || isPre(element)) return var prevText = null; var keepLeadingWs = false; var prev = null; var node = next(prev, element, isPre); while (node !== element) { if (node.nodeType === 3 || node.nodeType === 4) { // Node.TEXT_NODE or Node.CDATA_SECTION_NODE var text = node.data.replace(/[ \r\n\t]+/g, ' '); if ((!prevText || / $/.test(prevText.data)) && !keepLeadingWs && text[0] === ' ') { text = text.substr(1); } // `text` might be empty at this point. if (!text) { node = remove(node); continue } node.data = text; prevText = node; } else if (node.nodeType === 1) { // Node.ELEMENT_NODE if (isBlock(node) || node.nodeName === 'BR') { if (prevText) { prevText.data = prevText.data.replace(/ $/, ''); } prevText = null; keepLeadingWs = false; } else if (isVoid(node) || isPre(node)) { // Avoid trimming space around non-block, non-BR void elements and inline PRE. prevText = null; keepLeadingWs = true; } else if (prevText) { // Drop protection if set previously. keepLeadingWs = false; } } else { node = remove(node); continue } var nextNode = next(prev, node, isPre); prev = node; node = nextNode; } if (prevText) { prevText.data = prevText.data.replace(/ $/, ''); if (!prevText.data) { remove(prevText); } } } /** * remove(node) removes the given node from the DOM and returns the * next node in the sequence. * * @param {Node} node * @return {Node} node */ function remove (node) { var next = node.nextSibling || node.parentNode; node.parentNode.removeChild(node); return next } /** * next(prev, current, isPre) returns the next node in the sequence, given the * current and previous nodes. * * @param {Node} prev * @param {Node} current * @param {Function} isPre * @return {Node} */ function next (prev, current, isPre) { if ((prev && prev.parentNode === current) || isPre(current)) { return current.nextSibling || current.parentNode } return current.firstChild || current.nextSibling || current.parentNode } /* * Set up window for Node.js */ var root = (typeof window !== 'undefined' ? window : {}); /* * Parsing HTML strings */ function canParseHTMLNatively () { var Parser = root.DOMParser; var canParse = false; // Adapted from https://gist.github.com/1129031 // Firefox/Opera/IE throw errors on unsupported types try { // WebKit returns null on unsupported types if (new Parser().parseFromString('', 'text/html')) { canParse = true; } } catch (e) {} return canParse } function createHTMLParser () { var Parser = function () {}; { if (shouldUseActiveX()) { Parser.prototype.parseFromString = function (string) { var doc = new window.ActiveXObject('htmlfile'); doc.designMode = 'on'; // disable on-page scripts doc.open(); doc.write(string); doc.close(); return doc }; } else { Parser.prototype.parseFromString = function (string) { var doc = document.implementation.createHTMLDocument(''); doc.open(); doc.write(string); doc.close(); return doc }; } } return Parser } function shouldUseActiveX () { var useActiveX = false; try { document.implementation.createHTMLDocument('').open(); } catch (e) { if (window.ActiveXObject) useActiveX = true; } return useActiveX } var HTMLParser = canParseHTMLNatively() ? root.DOMParser : createHTMLParser(); function RootNode (input, options) { var root; if (typeof input === 'string') { var doc = htmlParser().parseFromString( // DOM parsers arrange elements in the and . // Wrapping in a custom element ensures elements are reliably arranged in // a single element. '' + input + '', 'text/html' ); root = doc.getElementById('turndown-root'); } else { root = input.cloneNode(true); } collapseWhitespace({ element: root, isBlock: isBlock, isVoid: isVoid, isPre: options.preformattedCode ? isPreOrCode : null }); return root } var _htmlParser; function htmlParser () { _htmlParser = _htmlParser || new HTMLParser(); return _htmlParser } function isPreOrCode (node) { return node.nodeName === 'PRE' || node.nodeName === 'CODE' } function Node (node, options) { node.isBlock = isBlock(node); node.isCode = node.nodeName === 'CODE' || node.parentNode.isCode; node.isBlank = isBlank(node); node.flankingWhitespace = flankingWhitespace(node, options); return node } function isBlank (node) { return ( !isVoid(node) && !isMeaningfulWhenBlank(node) && /^\s*$/i.test(node.textContent) && !hasVoid(node) && !hasMeaningfulWhenBlank(node) ) } function flankingWhitespace (node, options) { if (node.isBlock || (options.preformattedCode && node.isCode)) { return { leading: '', trailing: '' } } var edges = edgeWhitespace(node.textContent); // abandon leading ASCII WS if left-flanked by ASCII WS if (edges.leadingAscii && isFlankedByWhitespace('left', node, options)) { edges.leading = edges.leadingNonAscii; } // abandon trailing ASCII WS if right-flanked by ASCII WS if (edges.trailingAscii && isFlankedByWhitespace('right', node, options)) { edges.trailing = edges.trailingNonAscii; } return { leading: edges.leading, trailing: edges.trailing } } function edgeWhitespace (string) { var m = string.match(/^(([ \t\r\n]*)(\s*))[\s\S]*?((\s*?)([ \t\r\n]*))$/); return { leading: m[1], // whole string for whitespace-only strings leadingAscii: m[2], leadingNonAscii: m[3], trailing: m[4], // empty for whitespace-only strings trailingNonAscii: m[5], trailingAscii: m[6] } } function isFlankedByWhitespace (side, node, options) { var sibling; var regExp; var isFlanked; if (side === 'left') { sibling = node.previousSibling; regExp = / $/; } else { sibling = node.nextSibling; regExp = /^ /; } if (sibling) { if (sibling.nodeType === 3) { isFlanked = regExp.test(sibling.nodeValue); } else if (options.preformattedCode && sibling.nodeName === 'CODE') { isFlanked = false; } else if (sibling.nodeType === 1 && !isBlock(sibling)) { isFlanked = regExp.test(sibling.textContent); } } return isFlanked } var reduce = Array.prototype.reduce; var escapes = [ [/\\/g, '\\\\'], [/\*/g, '\\*'], [/^-/g, '\\-'], [/^\+ /g, '\\+ '], [/^(=+)/g, '\\$1'], [/^(#{1,6}) /g, '\\$1 '], [/`/g, '\\`'], [/^~~~/g, '\\~~~'], [/\[/g, '\\['], [/\]/g, '\\]'], [/^>/g, '\\>'], [/_/g, '\\_'], [/^(\d+)\. /g, '$1\\. '] ]; function TurndownService (options) { if (!(this instanceof TurndownService)) return new TurndownService(options) var defaults = { rules: rules$1, headingStyle: 'setext', hr: '* * *', bulletListMarker: '*', codeBlockStyle: 'indented', fence: '```', emDelimiter: '_', strongDelimiter: '**', linkStyle: 'inlined', linkReferenceStyle: 'full', br: ' ', preformattedCode: false, blankReplacement: function (content, node) { return node.isBlock ? '\n\n' : '' }, keepReplacement: function (content, node) { return node.isBlock ? '\n\n' + node.outerHTML + '\n\n' : node.outerHTML }, defaultReplacement: function (content, node) { return node.isBlock ? '\n\n' + content + '\n\n' : content } }; this.options = extend({}, defaults, options); this.rules = new Rules(this.options); } TurndownService.prototype = { /** * The entry point for converting a string or DOM node to Markdown * @public * @param {String|HTMLElement} input The string or DOM node to convert * @returns A Markdown representation of the input * @type String */ turndown: function (input) { if (!canConvert(input)) { throw new TypeError( input + ' is not a string, or an element/document/fragment node.' ) } if (input === '') return '' var output = process.call(this, new RootNode(input, this.options)); return postProcess.call(this, output) }, /** * Add one or more plugins * @public * @param {Function|Array} plugin The plugin or array of plugins to add * @returns The Turndown instance for chaining * @type Object */ use: function (plugin) { if (Array.isArray(plugin)) { for (var i = 0; i < plugin.length; i++) this.use(plugin[i]); } else if (typeof plugin === 'function') { plugin(this); } else { throw new TypeError('plugin must be a Function or an Array of Functions') } return this }, /** * Adds a rule * @public * @param {String} key The unique key of the rule * @param {Object} rule The rule * @returns The Turndown instance for chaining * @type Object */ addRule: function (key, rule) { this.rules.add(key, rule); return this }, /** * Keep a node (as HTML) that matches the filter * @public * @param {String|Array|Function} filter The unique key of the rule * @returns The Turndown instance for chaining * @type Object */ keep: function (filter) { this.rules.keep(filter); return this }, /** * Remove a node that matches the filter * @public * @param {String|Array|Function} filter The unique key of the rule * @returns The Turndown instance for chaining * @type Object */ remove: function (filter) { this.rules.remove(filter); return this }, /** * Escapes Markdown syntax * @public * @param {String} string The string to escape * @returns A string with Markdown syntax escaped * @type String */ escape: function (string) { return escapes.reduce(function (accumulator, escape) { return accumulator.replace(escape[0], escape[1]) }, string) } }; /** * Reduces a DOM node down to its Markdown string equivalent * @private * @param {HTMLElement} parentNode The node to convert * @returns A Markdown representation of the node * @type String */ function process (parentNode) { var self = this; return reduce.call(parentNode.childNodes, function (output, node) { node = new Node(node, self.options); var replacement = ''; if (node.nodeType === 3) { replacement = node.isCode ? node.nodeValue : self.escape(node.nodeValue); } else if (node.nodeType === 1) { replacement = replacementForNode.call(self, node); } return join(output, replacement) }, '') } /** * Appends strings as each rule requires and trims the output * @private * @param {String} output The conversion output * @returns A trimmed version of the ouput * @type String */ function postProcess (output) { var self = this; this.rules.forEach(function (rule) { if (typeof rule.append === 'function') { output = join(output, rule.append(self.options)); } }); return output.replace(/^[\t\r\n]+/, '').replace(/[\t\r\n\s]+$/, '') } /** * Converts an element node to its Markdown equivalent * @private * @param {HTMLElement} node The node to convert * @returns A Markdown representation of the node * @type String */ function replacementForNode (node) { var rule = this.rules.forNode(node); var content = process.call(this, node); var whitespace = node.flankingWhitespace; if (whitespace.leading || whitespace.trailing) content = content.trim(); return ( whitespace.leading + rule.replacement(content, node, this.options) + whitespace.trailing ) } /** * Joins replacement to the current output with appropriate number of new lines * @private * @param {String} output The current conversion output * @param {String} replacement The string to append to the output * @returns Joined output * @type String */ function join (output, replacement) { var s1 = trimTrailingNewlines(output); var s2 = trimLeadingNewlines(replacement); var nls = Math.max(output.length - s1.length, replacement.length - s2.length); var separator = '\n\n'.substring(0, nls); return s1 + separator + s2 } /** * Determines whether an input can be converted * @private * @param {String|HTMLElement} input Describe this parameter * @returns Describe what it returns * @type String|Object|Array|Boolean|Number */ function canConvert (input) { return ( input != null && ( typeof input === 'string' || (input.nodeType && ( input.nodeType === 1 || input.nodeType === 9 || input.nodeType === 11 )) ) ) } var turndownPluginGfm_cjs = {}; Object.defineProperty(turndownPluginGfm_cjs, '__esModule', { value: true }); var highlightRegExp = /highlight-(?:text|source)-([a-z0-9]+)/; function highlightedCodeBlock (turndownService) { turndownService.addRule('highlightedCodeBlock', { filter: function (node) { var firstChild = node.firstChild; return ( node.nodeName === 'DIV' && highlightRegExp.test(node.className) && firstChild && firstChild.nodeName === 'PRE' ) }, replacement: function (content, node, options) { var className = node.className || ''; var language = (className.match(highlightRegExp) || [null, ''])[1]; return ( '\n\n' + options.fence + language + '\n' + node.firstChild.textContent + '\n' + options.fence + '\n\n' ) } }); } function strikethrough (turndownService) { turndownService.addRule('strikethrough', { filter: ['del', 's', 'strike'], replacement: function (content) { return '~' + content + '~' } }); } var indexOf = Array.prototype.indexOf; var rules = {}; rules.tableCell = { filter: ['th', 'td'], replacement: function (content, node) { return cell(content, node) + spannedCells(node, '') } }; rules.tableRow = { filter: 'tr', replacement: function (content, node) { var borderCells = ''; var alignMap = { left: ':--', right: '--:', center: ':-:' }; if (isHeadingRow(node)) { for (var i = 0; i < node.childNodes.length; i++) { var border = '---'; var align = ( node.childNodes[i].getAttribute('align') || '' ).toLowerCase(); if (align) border = alignMap[align] || border; borderCells += cell(border, node.childNodes[i]) + spannedCells(node.childNodes[i], border); } } return '\n' + content + (borderCells ? '\n' + borderCells : '') } }; rules.table = { // Only convert tables that are not nested in another table, they are kept using `keep` (see below). // TODO: nested tables should be converted to plain text in a strict (non HTML) gfm filter: function (node) { return node.nodeName === 'TABLE' && !isNestedTable(node) }, replacement: function (content) { // Ensure there are no blank lines content = content.replace('\n\n', '\n'); return '\n\n' + content + '\n\n' } }; rules.tableSection = { filter: ['thead', 'tbody', 'tfoot'], replacement: function (content) { return content } }; rules.captionSection = { // only return content if caption if the first node immediately after TABLE filter: 'caption', replacement: function (content, node) { if (node.parentNode.nodeName === 'TABLE' && node.parentNode.childNodes[0] === node) return content return '' } }; function isHeadingRow (tr) { var parentNode = tr.parentNode; var tableNode = parentNode; if (parentNode.nodeName === 'THEAD' || parentNode.nodeName === 'TFOOT' || parentNode.nodeName === 'TBODY') { tableNode = parentNode.parentNode; } return (tableNode.nodeName === 'TABLE' && tableNode.rows[0] === tr) } function cell (content, node) { var index = indexOf.call(node.parentNode.childNodes, node); var prefix = ' '; if (index === 0) prefix = '| '; // Ensure single line per cell (both windows and unix EoL) // TODO: allow gfm non-strict mode to replace new lines by `
` content = content.replace(/\r\n/g, '\n').replace(/\n/g, ' '); // | must be escaped as \| content = content.replace(/\|/g, '\\|'); return prefix + content + ' |' } function spannedCells (node, spannedCellContent) { var colspan = node.getAttribute('colspan') || 1; if (colspan <= 1) return '' return (' ' + spannedCellContent + ' |').repeat(colspan - 1) } function isNestedTable (tableNode) { var currentNode = tableNode.parentNode; while (currentNode) { if (currentNode.nodeName === 'TABLE') return true currentNode = currentNode.parentNode; } return false } function tables (turndownService) { turndownService.keep(function (node) { return node.nodeName === 'TABLE' && isNestedTable(node) }); for (var key in rules) turndownService.addRule(key, rules[key]); } function taskListItems (turndownService) { turndownService.addRule('taskListItems', { filter: function (node) { return node.type === 'checkbox' && node.parentNode.nodeName === 'LI' }, replacement: function (content, node) { return (node.checked ? '[x]' : '[ ]') + ' ' } }); } function gfm (turndownService) { turndownService.use([ highlightedCodeBlock, strikethrough, tables, taskListItems ]); } var gfm_1 = turndownPluginGfm_cjs.gfm = gfm; turndownPluginGfm_cjs.highlightedCodeBlock = highlightedCodeBlock; turndownPluginGfm_cjs.strikethrough = strikethrough; turndownPluginGfm_cjs.tables = tables; turndownPluginGfm_cjs.taskListItems = taskListItems; function parseHtmlContent(content) { return __awaiter(this, void 0, void 0, function* () { const gfm = gfm_1; const turndownService = new TurndownService({ headingStyle: 'atx', hr: '---', bulletListMarker: '-', codeBlockStyle: 'fenced', emDelimiter: '*', }); turndownService.use(gfm); const articleContent = turndownService.turndown(content); return articleContent; }); } class TwitterParser extends Parser { constructor(app, settings) { super(app, settings); this.PATTERN = /(https:\/\/twitter.com\/([a-zA-Z0-9_]+\/)([a-zA-Z0-9_]+\/[a-zA-Z0-9_]+))/; } test(url) { return this.isValidUrl(url) && this.PATTERN.test(url); } prepareNote(url) { return __awaiter(this, void 0, void 0, function* () { const response = JSON.parse(yield obsidian.request({ method: 'GET', contentType: 'application/json', url: `https://publish.twitter.com/oembed?url=${url}`, })); const tweetAuthorName = response.author_name; const content = yield parseHtmlContent(response.html); const processedContent = this.settings.twitterNote .replace(/%date%/g, this.getFormattedDateForContent()) .replace(/%tweetAuthorName%/g, tweetAuthorName) .replace(/%tweetURL%/g, response.url) .replace(/%tweetContent%/g, content); const fileNameTemplate = this.settings.twitterNoteTitle .replace(/%tweetAuthorName%/g, tweetAuthorName) .replace(/%date%/g, this.getFormattedDateForFilename()); const fileName = `${fileNameTemplate}.md`; return new Note(fileName, processedContent); }); } } var Readability$1 = {exports: {}}; /*eslint-env es6:false*/ (function (module) { /* * Copyright (c) 2010 Arc90 Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * This code is heavily based on Arc90's readability.js (1.7.1) script * available at: http://code.google.com/p/arc90labs-readability */ /** * Public constructor. * @param {HTMLDocument} doc The document to parse. * @param {Object} options The options object. */ function Readability(doc, options) { // In some older versions, people passed a URI as the first argument. Cope: if (options && options.documentElement) { doc = options; options = arguments[2]; } else if (!doc || !doc.documentElement) { throw new Error("First argument to Readability constructor should be a document object."); } options = options || {}; this._doc = doc; this._docJSDOMParser = this._doc.firstChild.__JSDOMParser__; this._articleTitle = null; this._articleByline = null; this._articleDir = null; this._articleSiteName = null; this._attempts = []; // Configurable options this._debug = !!options.debug; this._maxElemsToParse = options.maxElemsToParse || this.DEFAULT_MAX_ELEMS_TO_PARSE; this._nbTopCandidates = options.nbTopCandidates || this.DEFAULT_N_TOP_CANDIDATES; this._charThreshold = options.charThreshold || this.DEFAULT_CHAR_THRESHOLD; this._classesToPreserve = this.CLASSES_TO_PRESERVE.concat(options.classesToPreserve || []); this._keepClasses = !!options.keepClasses; this._serializer = options.serializer || function(el) { return el.innerHTML; }; this._disableJSONLD = !!options.disableJSONLD; // Start with all flags set this._flags = this.FLAG_STRIP_UNLIKELYS | this.FLAG_WEIGHT_CLASSES | this.FLAG_CLEAN_CONDITIONALLY; // Control whether log messages are sent to the console if (this._debug) { let logNode = function(node) { if (node.nodeType == node.TEXT_NODE) { return `${node.nodeName} ("${node.textContent}")`; } let attrPairs = Array.from(node.attributes || [], function(attr) { return `${attr.name}="${attr.value}"`; }).join(" "); return `<${node.localName} ${attrPairs}>`; }; this.log = function () { if (typeof dump !== "undefined") { var msg = Array.prototype.map.call(arguments, function(x) { return (x && x.nodeName) ? logNode(x) : x; }).join(" "); dump("Reader: (Readability) " + msg + "\n"); } else if (typeof console !== "undefined") { let args = Array.from(arguments, arg => { if (arg && arg.nodeType == this.ELEMENT_NODE) { return logNode(arg); } return arg; }); args.unshift("Reader: (Readability)"); console.log.apply(console, args); } }; } else { this.log = function () {}; } } Readability.prototype = { FLAG_STRIP_UNLIKELYS: 0x1, FLAG_WEIGHT_CLASSES: 0x2, FLAG_CLEAN_CONDITIONALLY: 0x4, // https://developer.mozilla.org/en-US/docs/Web/API/Node/nodeType ELEMENT_NODE: 1, TEXT_NODE: 3, // Max number of nodes supported by this parser. Default: 0 (no limit) DEFAULT_MAX_ELEMS_TO_PARSE: 0, // The number of top candidates to consider when analysing how // tight the competition is among candidates. DEFAULT_N_TOP_CANDIDATES: 5, // Element tags to score by default. DEFAULT_TAGS_TO_SCORE: "section,h2,h3,h4,h5,h6,p,td,pre".toUpperCase().split(","), // The default number of chars an article must have in order to return a result DEFAULT_CHAR_THRESHOLD: 500, // All of the regular expressions in use within readability. // Defined up here so we don't instantiate them repeatedly in loops. REGEXPS: { // NOTE: These two regular expressions are duplicated in // Readability-readerable.js. Please keep both copies in sync. unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote/i, okMaybeItsACandidate: /and|article|body|column|content|main|shadow/i, positive: /article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story/i, negative: /-ad-|hidden|^hid$| hid$| hid |^hid |banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget/i, extraneous: /print|archive|comment|discuss|e[\-]?mail|share|reply|all|login|sign|single|utility/i, byline: /byline|author|dateline|writtenby|p-author/i, replaceFonts: /<(\/?)font[^>]*>/gi, normalize: /\s{2,}/g, videos: /\/\/(www\.)?((dailymotion|youtube|youtube-nocookie|player\.vimeo|v\.qq)\.com|(archive|upload\.wikimedia)\.org|player\.twitch\.tv)/i, shareElements: /(\b|_)(share|sharedaddy)(\b|_)/i, nextLink: /(next|weiter|continue|>([^\|]|$)|»([^\|]|$))/i, prevLink: /(prev|earl|old|new|<|«)/i, tokenize: /\W+/g, whitespace: /^\s*$/, hasContent: /\S$/, hashUrl: /^#.+/, srcsetUrl: /(\S+)(\s+[\d.]+[xw])?(\s*(?:,|$))/g, b64DataUrl: /^data:\s*([^\s;,]+)\s*;\s*base64\s*,/i, // See: https://schema.org/Article jsonLdArticleTypes: /^Article|AdvertiserContentArticle|NewsArticle|AnalysisNewsArticle|AskPublicNewsArticle|BackgroundNewsArticle|OpinionNewsArticle|ReportageNewsArticle|ReviewNewsArticle|Report|SatiricalArticle|ScholarlyArticle|MedicalScholarlyArticle|SocialMediaPosting|BlogPosting|LiveBlogPosting|DiscussionForumPosting|TechArticle|APIReference$/ }, UNLIKELY_ROLES: [ "menu", "menubar", "complementary", "navigation", "alert", "alertdialog", "dialog" ], DIV_TO_P_ELEMS: new Set([ "BLOCKQUOTE", "DL", "DIV", "IMG", "OL", "P", "PRE", "TABLE", "UL" ]), ALTER_TO_DIV_EXCEPTIONS: ["DIV", "ARTICLE", "SECTION", "P"], PRESENTATIONAL_ATTRIBUTES: [ "align", "background", "bgcolor", "border", "cellpadding", "cellspacing", "frame", "hspace", "rules", "style", "valign", "vspace" ], DEPRECATED_SIZE_ATTRIBUTE_ELEMS: [ "TABLE", "TH", "TD", "HR", "PRE" ], // The commented out elements qualify as phrasing content but tend to be // removed by readability when put into paragraphs, so we ignore them here. PHRASING_ELEMS: [ // "CANVAS", "IFRAME", "SVG", "VIDEO", "ABBR", "AUDIO", "B", "BDO", "BR", "BUTTON", "CITE", "CODE", "DATA", "DATALIST", "DFN", "EM", "EMBED", "I", "IMG", "INPUT", "KBD", "LABEL", "MARK", "MATH", "METER", "NOSCRIPT", "OBJECT", "OUTPUT", "PROGRESS", "Q", "RUBY", "SAMP", "SCRIPT", "SELECT", "SMALL", "SPAN", "STRONG", "SUB", "SUP", "TEXTAREA", "TIME", "VAR", "WBR" ], // These are the classes that readability sets itself. CLASSES_TO_PRESERVE: [ "page" ], // These are the list of HTML entities that need to be escaped. HTML_ESCAPE_MAP: { "lt": "<", "gt": ">", "amp": "&", "quot": '"', "apos": "'", }, /** * Run any post-process modifications to article content as necessary. * * @param Element * @return void **/ _postProcessContent: function(articleContent) { // Readability cannot open relative uris so we convert them to absolute uris. this._fixRelativeUris(articleContent); this._simplifyNestedElements(articleContent); if (!this._keepClasses) { // Remove classes. this._cleanClasses(articleContent); } }, /** * Iterates over a NodeList, calls `filterFn` for each node and removes node * if function returned `true`. * * If function is not passed, removes all the nodes in node list. * * @param NodeList nodeList The nodes to operate on * @param Function filterFn the function to use as a filter * @return void */ _removeNodes: function(nodeList, filterFn) { // Avoid ever operating on live node lists. if (this._docJSDOMParser && nodeList._isLiveNodeList) { throw new Error("Do not pass live node lists to _removeNodes"); } for (var i = nodeList.length - 1; i >= 0; i--) { var node = nodeList[i]; var parentNode = node.parentNode; if (parentNode) { if (!filterFn || filterFn.call(this, node, i, nodeList)) { parentNode.removeChild(node); } } } }, /** * Iterates over a NodeList, and calls _setNodeTag for each node. * * @param NodeList nodeList The nodes to operate on * @param String newTagName the new tag name to use * @return void */ _replaceNodeTags: function(nodeList, newTagName) { // Avoid ever operating on live node lists. if (this._docJSDOMParser && nodeList._isLiveNodeList) { throw new Error("Do not pass live node lists to _replaceNodeTags"); } for (const node of nodeList) { this._setNodeTag(node, newTagName); } }, /** * Iterate over a NodeList, which doesn't natively fully implement the Array * interface. * * For convenience, the current object context is applied to the provided * iterate function. * * @param NodeList nodeList The NodeList. * @param Function fn The iterate function. * @return void */ _forEachNode: function(nodeList, fn) { Array.prototype.forEach.call(nodeList, fn, this); }, /** * Iterate over a NodeList, and return the first node that passes * the supplied test function * * For convenience, the current object context is applied to the provided * test function. * * @param NodeList nodeList The NodeList. * @param Function fn The test function. * @return void */ _findNode: function(nodeList, fn) { return Array.prototype.find.call(nodeList, fn, this); }, /** * Iterate over a NodeList, return true if any of the provided iterate * function calls returns true, false otherwise. * * For convenience, the current object context is applied to the * provided iterate function. * * @param NodeList nodeList The NodeList. * @param Function fn The iterate function. * @return Boolean */ _someNode: function(nodeList, fn) { return Array.prototype.some.call(nodeList, fn, this); }, /** * Iterate over a NodeList, return true if all of the provided iterate * function calls return true, false otherwise. * * For convenience, the current object context is applied to the * provided iterate function. * * @param NodeList nodeList The NodeList. * @param Function fn The iterate function. * @return Boolean */ _everyNode: function(nodeList, fn) { return Array.prototype.every.call(nodeList, fn, this); }, /** * Concat all nodelists passed as arguments. * * @return ...NodeList * @return Array */ _concatNodeLists: function() { var slice = Array.prototype.slice; var args = slice.call(arguments); var nodeLists = args.map(function(list) { return slice.call(list); }); return Array.prototype.concat.apply([], nodeLists); }, _getAllNodesWithTag: function(node, tagNames) { if (node.querySelectorAll) { return node.querySelectorAll(tagNames.join(",")); } return [].concat.apply([], tagNames.map(function(tag) { var collection = node.getElementsByTagName(tag); return Array.isArray(collection) ? collection : Array.from(collection); })); }, /** * Removes the class="" attribute from every element in the given * subtree, except those that match CLASSES_TO_PRESERVE and * the classesToPreserve array from the options object. * * @param Element * @return void */ _cleanClasses: function(node) { var classesToPreserve = this._classesToPreserve; var className = (node.getAttribute("class") || "") .split(/\s+/) .filter(function(cls) { return classesToPreserve.indexOf(cls) != -1; }) .join(" "); if (className) { node.setAttribute("class", className); } else { node.removeAttribute("class"); } for (node = node.firstElementChild; node; node = node.nextElementSibling) { this._cleanClasses(node); } }, /** * Converts each and uri in the given element to an absolute URI, * ignoring #ref URIs. * * @param Element * @return void */ _fixRelativeUris: function(articleContent) { var baseURI = this._doc.baseURI; var documentURI = this._doc.documentURI; function toAbsoluteURI(uri) { // Leave hash links alone if the base URI matches the document URI: if (baseURI == documentURI && uri.charAt(0) == "#") { return uri; } // Otherwise, resolve against base URI: try { return new URL(uri, baseURI).href; } catch (ex) { // Something went wrong, just return the original: } return uri; } var links = this._getAllNodesWithTag(articleContent, ["a"]); this._forEachNode(links, function(link) { var href = link.getAttribute("href"); if (href) { // Remove links with javascript: URIs, since // they won't work after scripts have been removed from the page. if (href.indexOf("javascript:") === 0) { // if the link only contains simple text content, it can be converted to a text node if (link.childNodes.length === 1 && link.childNodes[0].nodeType === this.TEXT_NODE) { var text = this._doc.createTextNode(link.textContent); link.parentNode.replaceChild(text, link); } else { // if the link has multiple children, they should all be preserved var container = this._doc.createElement("span"); while (link.firstChild) { container.appendChild(link.firstChild); } link.parentNode.replaceChild(container, link); } } else { link.setAttribute("href", toAbsoluteURI(href)); } } }); var medias = this._getAllNodesWithTag(articleContent, [ "img", "picture", "figure", "video", "audio", "source" ]); this._forEachNode(medias, function(media) { var src = media.getAttribute("src"); var poster = media.getAttribute("poster"); var srcset = media.getAttribute("srcset"); if (src) { media.setAttribute("src", toAbsoluteURI(src)); } if (poster) { media.setAttribute("poster", toAbsoluteURI(poster)); } if (srcset) { var newSrcset = srcset.replace(this.REGEXPS.srcsetUrl, function(_, p1, p2, p3) { return toAbsoluteURI(p1) + (p2 || "") + p3; }); media.setAttribute("srcset", newSrcset); } }); }, _simplifyNestedElements: function(articleContent) { var node = articleContent; while (node) { if (node.parentNode && ["DIV", "SECTION"].includes(node.tagName) && !(node.id && node.id.startsWith("readability"))) { if (this._isElementWithoutContent(node)) { node = this._removeAndGetNext(node); continue; } else if (this._hasSingleTagInsideElement(node, "DIV") || this._hasSingleTagInsideElement(node, "SECTION")) { var child = node.children[0]; for (var i = 0; i < node.attributes.length; i++) { child.setAttribute(node.attributes[i].name, node.attributes[i].value); } node.parentNode.replaceChild(child, node); node = child; continue; } } node = this._getNextNode(node); } }, /** * Get the article title as an H1. * * @return string **/ _getArticleTitle: function() { var doc = this._doc; var curTitle = ""; var origTitle = ""; try { curTitle = origTitle = doc.title.trim(); // If they had an element with id "title" in their HTML if (typeof curTitle !== "string") curTitle = origTitle = this._getInnerText(doc.getElementsByTagName("title")[0]); } catch (e) {/* ignore exceptions setting the title. */} var titleHadHierarchicalSeparators = false; function wordCount(str) { return str.split(/\s+/).length; } // If there's a separator in the title, first remove the final part if ((/ [\|\-\\\/>»] /).test(curTitle)) { titleHadHierarchicalSeparators = / [\\\/>»] /.test(curTitle); curTitle = origTitle.replace(/(.*)[\|\-\\\/>»] .*/gi, "$1"); // If the resulting title is too short (3 words or fewer), remove // the first part instead: if (wordCount(curTitle) < 3) curTitle = origTitle.replace(/[^\|\-\\\/>»]*[\|\-\\\/>»](.*)/gi, "$1"); } else if (curTitle.indexOf(": ") !== -1) { // Check if we have an heading containing this exact string, so we // could assume it's the full title. var headings = this._concatNodeLists( doc.getElementsByTagName("h1"), doc.getElementsByTagName("h2") ); var trimmedTitle = curTitle.trim(); var match = this._someNode(headings, function(heading) { return heading.textContent.trim() === trimmedTitle; }); // If we don't, let's extract the title out of the original title string. if (!match) { curTitle = origTitle.substring(origTitle.lastIndexOf(":") + 1); // If the title is now too short, try the first colon instead: if (wordCount(curTitle) < 3) { curTitle = origTitle.substring(origTitle.indexOf(":") + 1); // But if we have too many words before the colon there's something weird // with the titles and the H tags so let's just use the original title instead } else if (wordCount(origTitle.substr(0, origTitle.indexOf(":"))) > 5) { curTitle = origTitle; } } } else if (curTitle.length > 150 || curTitle.length < 15) { var hOnes = doc.getElementsByTagName("h1"); if (hOnes.length === 1) curTitle = this._getInnerText(hOnes[0]); } curTitle = curTitle.trim().replace(this.REGEXPS.normalize, " "); // If we now have 4 words or fewer as our title, and either no // 'hierarchical' separators (\, /, > or ») were found in the original // title or we decreased the number of words by more than 1 word, use // the original title. var curTitleWordCount = wordCount(curTitle); if (curTitleWordCount <= 4 && (!titleHadHierarchicalSeparators || curTitleWordCount != wordCount(origTitle.replace(/[\|\-\\\/>»]+/g, "")) - 1)) { curTitle = origTitle; } return curTitle; }, /** * Prepare the HTML document for readability to scrape it. * This includes things like stripping javascript, CSS, and handling terrible markup. * * @return void **/ _prepDocument: function() { var doc = this._doc; // Remove all style tags in head this._removeNodes(this._getAllNodesWithTag(doc, ["style"])); if (doc.body) { this._replaceBrs(doc.body); } this._replaceNodeTags(this._getAllNodesWithTag(doc, ["font"]), "SPAN"); }, /** * Finds the next node, starting from the given node, and ignoring * whitespace in between. If the given node is an element, the same node is * returned. */ _nextNode: function (node) { var next = node; while (next && (next.nodeType != this.ELEMENT_NODE) && this.REGEXPS.whitespace.test(next.textContent)) { next = next.nextSibling; } return next; }, /** * Replaces 2 or more successive
elements with a single

. * Whitespace between
elements are ignored. For example: *

foo
bar


abc
* will become: *
foo
bar

abc

*/ _replaceBrs: function (elem) { this._forEachNode(this._getAllNodesWithTag(elem, ["br"]), function(br) { var next = br.nextSibling; // Whether 2 or more
elements have been found and replaced with a //

block. var replaced = false; // If we find a
chain, remove the
s until we hit another node // or non-whitespace. This leaves behind the first
in the chain // (which will be replaced with a

later). while ((next = this._nextNode(next)) && (next.tagName == "BR")) { replaced = true; var brSibling = next.nextSibling; next.parentNode.removeChild(next); next = brSibling; } // If we removed a
chain, replace the remaining
with a

. Add // all sibling nodes as children of the

until we hit another
// chain. if (replaced) { var p = this._doc.createElement("p"); br.parentNode.replaceChild(p, br); next = p.nextSibling; while (next) { // If we've hit another

, we're done adding children to this

. if (next.tagName == "BR") { var nextElem = this._nextNode(next.nextSibling); if (nextElem && nextElem.tagName == "BR") break; } if (!this._isPhrasingContent(next)) break; // Otherwise, make this node a child of the new

. var sibling = next.nextSibling; p.appendChild(next); next = sibling; } while (p.lastChild && this._isWhitespace(p.lastChild)) { p.removeChild(p.lastChild); } if (p.parentNode.tagName === "P") this._setNodeTag(p.parentNode, "DIV"); } }); }, _setNodeTag: function (node, tag) { this.log("_setNodeTag", node, tag); if (this._docJSDOMParser) { node.localName = tag.toLowerCase(); node.tagName = tag.toUpperCase(); return node; } var replacement = node.ownerDocument.createElement(tag); while (node.firstChild) { replacement.appendChild(node.firstChild); } node.parentNode.replaceChild(replacement, node); if (node.readability) replacement.readability = node.readability; for (var i = 0; i < node.attributes.length; i++) { try { replacement.setAttribute(node.attributes[i].name, node.attributes[i].value); } catch (ex) { /* it's possible for setAttribute() to throw if the attribute name * isn't a valid XML Name. Such attributes can however be parsed from * source in HTML docs, see https://github.com/whatwg/html/issues/4275, * so we can hit them here and then throw. We don't care about such * attributes so we ignore them. */ } } return replacement; }, /** * Prepare the article node for display. Clean out any inline styles, * iframes, forms, strip extraneous

tags, etc. * * @param Element * @return void **/ _prepArticle: function(articleContent) { this._cleanStyles(articleContent); // Check for data tables before we continue, to avoid removing items in // those tables, which will often be isolated even though they're // visually linked to other content-ful elements (text, images, etc.). this._markDataTables(articleContent); this._fixLazyImages(articleContent); // Clean out junk from the article content this._cleanConditionally(articleContent, "form"); this._cleanConditionally(articleContent, "fieldset"); this._clean(articleContent, "object"); this._clean(articleContent, "embed"); this._clean(articleContent, "footer"); this._clean(articleContent, "link"); this._clean(articleContent, "aside"); // Clean out elements with little content that have "share" in their id/class combinations from final top candidates, // which means we don't remove the top candidates even they have "share". var shareElementThreshold = this.DEFAULT_CHAR_THRESHOLD; this._forEachNode(articleContent.children, function (topCandidate) { this._cleanMatchedNodes(topCandidate, function (node, matchString) { return this.REGEXPS.shareElements.test(matchString) && node.textContent.length < shareElementThreshold; }); }); this._clean(articleContent, "iframe"); this._clean(articleContent, "input"); this._clean(articleContent, "textarea"); this._clean(articleContent, "select"); this._clean(articleContent, "button"); this._cleanHeaders(articleContent); // Do these last as the previous stuff may have removed junk // that will affect these this._cleanConditionally(articleContent, "table"); this._cleanConditionally(articleContent, "ul"); this._cleanConditionally(articleContent, "div"); // replace H1 with H2 as H1 should be only title that is displayed separately this._replaceNodeTags(this._getAllNodesWithTag(articleContent, ["h1"]), "h2"); // Remove extra paragraphs this._removeNodes(this._getAllNodesWithTag(articleContent, ["p"]), function (paragraph) { var imgCount = paragraph.getElementsByTagName("img").length; var embedCount = paragraph.getElementsByTagName("embed").length; var objectCount = paragraph.getElementsByTagName("object").length; // At this point, nasty iframes have been removed, only remain embedded video ones. var iframeCount = paragraph.getElementsByTagName("iframe").length; var totalCount = imgCount + embedCount + objectCount + iframeCount; return totalCount === 0 && !this._getInnerText(paragraph, false); }); this._forEachNode(this._getAllNodesWithTag(articleContent, ["br"]), function(br) { var next = this._nextNode(br.nextSibling); if (next && next.tagName == "P") br.parentNode.removeChild(br); }); // Remove single-cell tables this._forEachNode(this._getAllNodesWithTag(articleContent, ["table"]), function(table) { var tbody = this._hasSingleTagInsideElement(table, "TBODY") ? table.firstElementChild : table; if (this._hasSingleTagInsideElement(tbody, "TR")) { var row = tbody.firstElementChild; if (this._hasSingleTagInsideElement(row, "TD")) { var cell = row.firstElementChild; cell = this._setNodeTag(cell, this._everyNode(cell.childNodes, this._isPhrasingContent) ? "P" : "DIV"); table.parentNode.replaceChild(cell, table); } } }); }, /** * Initialize a node with the readability object. Also checks the * className/id for special names to add to its score. * * @param Element * @return void **/ _initializeNode: function(node) { node.readability = {"contentScore": 0}; switch (node.tagName) { case "DIV": node.readability.contentScore += 5; break; case "PRE": case "TD": case "BLOCKQUOTE": node.readability.contentScore += 3; break; case "ADDRESS": case "OL": case "UL": case "DL": case "DD": case "DT": case "LI": case "FORM": node.readability.contentScore -= 3; break; case "H1": case "H2": case "H3": case "H4": case "H5": case "H6": case "TH": node.readability.contentScore -= 5; break; } node.readability.contentScore += this._getClassWeight(node); }, _removeAndGetNext: function(node) { var nextNode = this._getNextNode(node, true); node.parentNode.removeChild(node); return nextNode; }, /** * Traverse the DOM from node to node, starting at the node passed in. * Pass true for the second parameter to indicate this node itself * (and its kids) are going away, and we want the next node over. * * Calling this in a loop will traverse the DOM depth-first. */ _getNextNode: function(node, ignoreSelfAndKids) { // First check for kids if those aren't being ignored if (!ignoreSelfAndKids && node.firstElementChild) { return node.firstElementChild; } // Then for siblings... if (node.nextElementSibling) { return node.nextElementSibling; } // And finally, move up the parent chain *and* find a sibling // (because this is depth-first traversal, we will have already // seen the parent nodes themselves). do { node = node.parentNode; } while (node && !node.nextElementSibling); return node && node.nextElementSibling; }, // compares second text to first one // 1 = same text, 0 = completely different text // works the way that it splits both texts into words and then finds words that are unique in second text // the result is given by the lower length of unique parts _textSimilarity: function(textA, textB) { var tokensA = textA.toLowerCase().split(this.REGEXPS.tokenize).filter(Boolean); var tokensB = textB.toLowerCase().split(this.REGEXPS.tokenize).filter(Boolean); if (!tokensA.length || !tokensB.length) { return 0; } var uniqTokensB = tokensB.filter(token => !tokensA.includes(token)); var distanceB = uniqTokensB.join(" ").length / tokensB.join(" ").length; return 1 - distanceB; }, _checkByline: function(node, matchString) { if (this._articleByline) { return false; } if (node.getAttribute !== undefined) { var rel = node.getAttribute("rel"); var itemprop = node.getAttribute("itemprop"); } if ((rel === "author" || (itemprop && itemprop.indexOf("author") !== -1) || this.REGEXPS.byline.test(matchString)) && this._isValidByline(node.textContent)) { this._articleByline = node.textContent.trim(); return true; } return false; }, _getNodeAncestors: function(node, maxDepth) { maxDepth = maxDepth || 0; var i = 0, ancestors = []; while (node.parentNode) { ancestors.push(node.parentNode); if (maxDepth && ++i === maxDepth) break; node = node.parentNode; } return ancestors; }, /*** * grabArticle - Using a variety of metrics (content score, classname, element types), find the content that is * most likely to be the stuff a user wants to read. Then return it wrapped up in a div. * * @param page a document to run upon. Needs to be a full document, complete with body. * @return Element **/ _grabArticle: function (page) { this.log("**** grabArticle ****"); var doc = this._doc; var isPaging = page !== null; page = page ? page : this._doc.body; // We can't grab an article if we don't have a page! if (!page) { this.log("No body found in document. Abort."); return null; } var pageCacheHtml = page.innerHTML; while (true) { this.log("Starting grabArticle loop"); var stripUnlikelyCandidates = this._flagIsActive(this.FLAG_STRIP_UNLIKELYS); // First, node prepping. Trash nodes that look cruddy (like ones with the // class name "comment", etc), and turn divs into P tags where they have been // used inappropriately (as in, where they contain no other block level elements.) var elementsToScore = []; var node = this._doc.documentElement; let shouldRemoveTitleHeader = true; while (node) { if (node.tagName === "HTML") { this._articleLang = node.getAttribute("lang"); } var matchString = node.className + " " + node.id; if (!this._isProbablyVisible(node)) { this.log("Removing hidden node - " + matchString); node = this._removeAndGetNext(node); continue; } // Check to see if this node is a byline, and remove it if it is. if (this._checkByline(node, matchString)) { node = this._removeAndGetNext(node); continue; } if (shouldRemoveTitleHeader && this._headerDuplicatesTitle(node)) { this.log("Removing header: ", node.textContent.trim(), this._articleTitle.trim()); shouldRemoveTitleHeader = false; node = this._removeAndGetNext(node); continue; } // Remove unlikely candidates if (stripUnlikelyCandidates) { if (this.REGEXPS.unlikelyCandidates.test(matchString) && !this.REGEXPS.okMaybeItsACandidate.test(matchString) && !this._hasAncestorTag(node, "table") && !this._hasAncestorTag(node, "code") && node.tagName !== "BODY" && node.tagName !== "A") { this.log("Removing unlikely candidate - " + matchString); node = this._removeAndGetNext(node); continue; } if (this.UNLIKELY_ROLES.includes(node.getAttribute("role"))) { this.log("Removing content with role " + node.getAttribute("role") + " - " + matchString); node = this._removeAndGetNext(node); continue; } } // Remove DIV, SECTION, and HEADER nodes without any content(e.g. text, image, video, or iframe). if ((node.tagName === "DIV" || node.tagName === "SECTION" || node.tagName === "HEADER" || node.tagName === "H1" || node.tagName === "H2" || node.tagName === "H3" || node.tagName === "H4" || node.tagName === "H5" || node.tagName === "H6") && this._isElementWithoutContent(node)) { node = this._removeAndGetNext(node); continue; } if (this.DEFAULT_TAGS_TO_SCORE.indexOf(node.tagName) !== -1) { elementsToScore.push(node); } // Turn all divs that don't have children block level elements into p's if (node.tagName === "DIV") { // Put phrasing content into paragraphs. var p = null; var childNode = node.firstChild; while (childNode) { var nextSibling = childNode.nextSibling; if (this._isPhrasingContent(childNode)) { if (p !== null) { p.appendChild(childNode); } else if (!this._isWhitespace(childNode)) { p = doc.createElement("p"); node.replaceChild(p, childNode); p.appendChild(childNode); } } else if (p !== null) { while (p.lastChild && this._isWhitespace(p.lastChild)) { p.removeChild(p.lastChild); } p = null; } childNode = nextSibling; } // Sites like http://mobile.slate.com encloses each paragraph with a DIV // element. DIVs with only a P element inside and no text content can be // safely converted into plain P elements to avoid confusing the scoring // algorithm with DIVs with are, in practice, paragraphs. if (this._hasSingleTagInsideElement(node, "P") && this._getLinkDensity(node) < 0.25) { var newNode = node.children[0]; node.parentNode.replaceChild(newNode, node); node = newNode; elementsToScore.push(node); } else if (!this._hasChildBlockElement(node)) { node = this._setNodeTag(node, "P"); elementsToScore.push(node); } } node = this._getNextNode(node); } /** * Loop through all paragraphs, and assign a score to them based on how content-y they look. * Then add their score to their parent node. * * A score is determined by things like number of commas, class names, etc. Maybe eventually link density. **/ var candidates = []; this._forEachNode(elementsToScore, function(elementToScore) { if (!elementToScore.parentNode || typeof(elementToScore.parentNode.tagName) === "undefined") return; // If this paragraph is less than 25 characters, don't even count it. var innerText = this._getInnerText(elementToScore); if (innerText.length < 25) return; // Exclude nodes with no ancestor. var ancestors = this._getNodeAncestors(elementToScore, 5); if (ancestors.length === 0) return; var contentScore = 0; // Add a point for the paragraph itself as a base. contentScore += 1; // Add points for any commas within this paragraph. contentScore += innerText.split(",").length; // For every 100 characters in this paragraph, add another point. Up to 3 points. contentScore += Math.min(Math.floor(innerText.length / 100), 3); // Initialize and score ancestors. this._forEachNode(ancestors, function(ancestor, level) { if (!ancestor.tagName || !ancestor.parentNode || typeof(ancestor.parentNode.tagName) === "undefined") return; if (typeof(ancestor.readability) === "undefined") { this._initializeNode(ancestor); candidates.push(ancestor); } // Node score divider: // - parent: 1 (no division) // - grandparent: 2 // - great grandparent+: ancestor level * 3 if (level === 0) var scoreDivider = 1; else if (level === 1) scoreDivider = 2; else scoreDivider = level * 3; ancestor.readability.contentScore += contentScore / scoreDivider; }); }); // After we've calculated scores, loop through all of the possible // candidate nodes we found and find the one with the highest score. var topCandidates = []; for (var c = 0, cl = candidates.length; c < cl; c += 1) { var candidate = candidates[c]; // Scale the final candidates score based on link density. Good content // should have a relatively small link density (5% or less) and be mostly // unaffected by this operation. var candidateScore = candidate.readability.contentScore * (1 - this._getLinkDensity(candidate)); candidate.readability.contentScore = candidateScore; this.log("Candidate:", candidate, "with score " + candidateScore); for (var t = 0; t < this._nbTopCandidates; t++) { var aTopCandidate = topCandidates[t]; if (!aTopCandidate || candidateScore > aTopCandidate.readability.contentScore) { topCandidates.splice(t, 0, candidate); if (topCandidates.length > this._nbTopCandidates) topCandidates.pop(); break; } } } var topCandidate = topCandidates[0] || null; var neededToCreateTopCandidate = false; var parentOfTopCandidate; // If we still have no top candidate, just use the body as a last resort. // We also have to copy the body node so it is something we can modify. if (topCandidate === null || topCandidate.tagName === "BODY") { // Move all of the page's children into topCandidate topCandidate = doc.createElement("DIV"); neededToCreateTopCandidate = true; // Move everything (not just elements, also text nodes etc.) into the container // so we even include text directly in the body: while (page.firstChild) { this.log("Moving child out:", page.firstChild); topCandidate.appendChild(page.firstChild); } page.appendChild(topCandidate); this._initializeNode(topCandidate); } else if (topCandidate) { // Find a better top candidate node if it contains (at least three) nodes which belong to `topCandidates` array // and whose scores are quite closed with current `topCandidate` node. var alternativeCandidateAncestors = []; for (var i = 1; i < topCandidates.length; i++) { if (topCandidates[i].readability.contentScore / topCandidate.readability.contentScore >= 0.75) { alternativeCandidateAncestors.push(this._getNodeAncestors(topCandidates[i])); } } var MINIMUM_TOPCANDIDATES = 3; if (alternativeCandidateAncestors.length >= MINIMUM_TOPCANDIDATES) { parentOfTopCandidate = topCandidate.parentNode; while (parentOfTopCandidate.tagName !== "BODY") { var listsContainingThisAncestor = 0; for (var ancestorIndex = 0; ancestorIndex < alternativeCandidateAncestors.length && listsContainingThisAncestor < MINIMUM_TOPCANDIDATES; ancestorIndex++) { listsContainingThisAncestor += Number(alternativeCandidateAncestors[ancestorIndex].includes(parentOfTopCandidate)); } if (listsContainingThisAncestor >= MINIMUM_TOPCANDIDATES) { topCandidate = parentOfTopCandidate; break; } parentOfTopCandidate = parentOfTopCandidate.parentNode; } } if (!topCandidate.readability) { this._initializeNode(topCandidate); } // Because of our bonus system, parents of candidates might have scores // themselves. They get half of the node. There won't be nodes with higher // scores than our topCandidate, but if we see the score going *up* in the first // few steps up the tree, that's a decent sign that there might be more content // lurking in other places that we want to unify in. The sibling stuff // below does some of that - but only if we've looked high enough up the DOM // tree. parentOfTopCandidate = topCandidate.parentNode; var lastScore = topCandidate.readability.contentScore; // The scores shouldn't get too low. var scoreThreshold = lastScore / 3; while (parentOfTopCandidate.tagName !== "BODY") { if (!parentOfTopCandidate.readability) { parentOfTopCandidate = parentOfTopCandidate.parentNode; continue; } var parentScore = parentOfTopCandidate.readability.contentScore; if (parentScore < scoreThreshold) break; if (parentScore > lastScore) { // Alright! We found a better parent to use. topCandidate = parentOfTopCandidate; break; } lastScore = parentOfTopCandidate.readability.contentScore; parentOfTopCandidate = parentOfTopCandidate.parentNode; } // If the top candidate is the only child, use parent instead. This will help sibling // joining logic when adjacent content is actually located in parent's sibling node. parentOfTopCandidate = topCandidate.parentNode; while (parentOfTopCandidate.tagName != "BODY" && parentOfTopCandidate.children.length == 1) { topCandidate = parentOfTopCandidate; parentOfTopCandidate = topCandidate.parentNode; } if (!topCandidate.readability) { this._initializeNode(topCandidate); } } // Now that we have the top candidate, look through its siblings for content // that might also be related. Things like preambles, content split by ads // that we removed, etc. var articleContent = doc.createElement("DIV"); if (isPaging) articleContent.id = "readability-content"; var siblingScoreThreshold = Math.max(10, topCandidate.readability.contentScore * 0.2); // Keep potential top candidate's parent node to try to get text direction of it later. parentOfTopCandidate = topCandidate.parentNode; var siblings = parentOfTopCandidate.children; for (var s = 0, sl = siblings.length; s < sl; s++) { var sibling = siblings[s]; var append = false; this.log("Looking at sibling node:", sibling, sibling.readability ? ("with score " + sibling.readability.contentScore) : ""); this.log("Sibling has score", sibling.readability ? sibling.readability.contentScore : "Unknown"); if (sibling === topCandidate) { append = true; } else { var contentBonus = 0; // Give a bonus if sibling nodes and top candidates have the example same classname if (sibling.className === topCandidate.className && topCandidate.className !== "") contentBonus += topCandidate.readability.contentScore * 0.2; if (sibling.readability && ((sibling.readability.contentScore + contentBonus) >= siblingScoreThreshold)) { append = true; } else if (sibling.nodeName === "P") { var linkDensity = this._getLinkDensity(sibling); var nodeContent = this._getInnerText(sibling); var nodeLength = nodeContent.length; if (nodeLength > 80 && linkDensity < 0.25) { append = true; } else if (nodeLength < 80 && nodeLength > 0 && linkDensity === 0 && nodeContent.search(/\.( |$)/) !== -1) { append = true; } } } if (append) { this.log("Appending node:", sibling); if (this.ALTER_TO_DIV_EXCEPTIONS.indexOf(sibling.nodeName) === -1) { // We have a node that isn't a common block level element, like a form or td tag. // Turn it into a div so it doesn't get filtered out later by accident. this.log("Altering sibling:", sibling, "to div."); sibling = this._setNodeTag(sibling, "DIV"); } articleContent.appendChild(sibling); // Fetch children again to make it compatible // with DOM parsers without live collection support. siblings = parentOfTopCandidate.children; // siblings is a reference to the children array, and // sibling is removed from the array when we call appendChild(). // As a result, we must revisit this index since the nodes // have been shifted. s -= 1; sl -= 1; } } if (this._debug) this.log("Article content pre-prep: " + articleContent.innerHTML); // So we have all of the content that we need. Now we clean it up for presentation. this._prepArticle(articleContent); if (this._debug) this.log("Article content post-prep: " + articleContent.innerHTML); if (neededToCreateTopCandidate) { // We already created a fake div thing, and there wouldn't have been any siblings left // for the previous loop, so there's no point trying to create a new div, and then // move all the children over. Just assign IDs and class names here. No need to append // because that already happened anyway. topCandidate.id = "readability-page-1"; topCandidate.className = "page"; } else { var div = doc.createElement("DIV"); div.id = "readability-page-1"; div.className = "page"; while (articleContent.firstChild) { div.appendChild(articleContent.firstChild); } articleContent.appendChild(div); } if (this._debug) this.log("Article content after paging: " + articleContent.innerHTML); var parseSuccessful = true; // Now that we've gone through the full algorithm, check to see if // we got any meaningful content. If we didn't, we may need to re-run // grabArticle with different flags set. This gives us a higher likelihood of // finding the content, and the sieve approach gives us a higher likelihood of // finding the -right- content. var textLength = this._getInnerText(articleContent, true).length; if (textLength < this._charThreshold) { parseSuccessful = false; page.innerHTML = pageCacheHtml; if (this._flagIsActive(this.FLAG_STRIP_UNLIKELYS)) { this._removeFlag(this.FLAG_STRIP_UNLIKELYS); this._attempts.push({articleContent: articleContent, textLength: textLength}); } else if (this._flagIsActive(this.FLAG_WEIGHT_CLASSES)) { this._removeFlag(this.FLAG_WEIGHT_CLASSES); this._attempts.push({articleContent: articleContent, textLength: textLength}); } else if (this._flagIsActive(this.FLAG_CLEAN_CONDITIONALLY)) { this._removeFlag(this.FLAG_CLEAN_CONDITIONALLY); this._attempts.push({articleContent: articleContent, textLength: textLength}); } else { this._attempts.push({articleContent: articleContent, textLength: textLength}); // No luck after removing flags, just return the longest text we found during the different loops this._attempts.sort(function (a, b) { return b.textLength - a.textLength; }); // But first check if we actually have something if (!this._attempts[0].textLength) { return null; } articleContent = this._attempts[0].articleContent; parseSuccessful = true; } } if (parseSuccessful) { // Find out text direction from ancestors of final top candidate. var ancestors = [parentOfTopCandidate, topCandidate].concat(this._getNodeAncestors(parentOfTopCandidate)); this._someNode(ancestors, function(ancestor) { if (!ancestor.tagName) return false; var articleDir = ancestor.getAttribute("dir"); if (articleDir) { this._articleDir = articleDir; return true; } return false; }); return articleContent; } } }, /** * Check whether the input string could be a byline. * This verifies that the input is a string, and that the length * is less than 100 chars. * * @param possibleByline {string} - a string to check whether its a byline. * @return Boolean - whether the input string is a byline. */ _isValidByline: function(byline) { if (typeof byline == "string" || byline instanceof String) { byline = byline.trim(); return (byline.length > 0) && (byline.length < 100); } return false; }, /** * Converts some of the common HTML entities in string to their corresponding characters. * * @param str {string} - a string to unescape. * @return string without HTML entity. */ _unescapeHtmlEntities: function(str) { if (!str) { return str; } var htmlEscapeMap = this.HTML_ESCAPE_MAP; return str.replace(/&(quot|amp|apos|lt|gt);/g, function(_, tag) { return htmlEscapeMap[tag]; }).replace(/&#(?:x([0-9a-z]{1,4})|([0-9]{1,4}));/gi, function(_, hex, numStr) { var num = parseInt(hex || numStr, hex ? 16 : 10); return String.fromCharCode(num); }); }, /** * Try to extract metadata from JSON-LD object. * For now, only Schema.org objects of type Article or its subtypes are supported. * @return Object with any metadata that could be extracted (possibly none) */ _getJSONLD: function (doc) { var scripts = this._getAllNodesWithTag(doc, ["script"]); var metadata; this._forEachNode(scripts, function(jsonLdElement) { if (!metadata && jsonLdElement.getAttribute("type") === "application/ld+json") { try { // Strip CDATA markers if present var content = jsonLdElement.textContent.replace(/^\s*\s*$/g, ""); var parsed = JSON.parse(content); if ( !parsed["@context"] || !parsed["@context"].match(/^https?\:\/\/schema\.org$/) ) { return; } if (!parsed["@type"] && Array.isArray(parsed["@graph"])) { parsed = parsed["@graph"].find(function(it) { return (it["@type"] || "").match( this.REGEXPS.jsonLdArticleTypes ); }); } if ( !parsed || !parsed["@type"] || !parsed["@type"].match(this.REGEXPS.jsonLdArticleTypes) ) { return; } metadata = {}; if (typeof parsed.name === "string" && typeof parsed.headline === "string" && parsed.name !== parsed.headline) { // we have both name and headline element in the JSON-LD. They should both be the same but some websites like aktualne.cz // put their own name into "name" and the article title to "headline" which confuses Readability. So we try to check if either // "name" or "headline" closely matches the html title, and if so, use that one. If not, then we use "name" by default. var title = this._getArticleTitle(); var nameMatches = this._textSimilarity(parsed.name, title) > 0.75; var headlineMatches = this._textSimilarity(parsed.headline, title) > 0.75; if (headlineMatches && !nameMatches) { metadata.title = parsed.headline; } else { metadata.title = parsed.name; } } else if (typeof parsed.name === "string") { metadata.title = parsed.name.trim(); } else if (typeof parsed.headline === "string") { metadata.title = parsed.headline.trim(); } if (parsed.author) { if (typeof parsed.author.name === "string") { metadata.byline = parsed.author.name.trim(); } else if (Array.isArray(parsed.author) && parsed.author[0] && typeof parsed.author[0].name === "string") { metadata.byline = parsed.author .filter(function(author) { return author && typeof author.name === "string"; }) .map(function(author) { return author.name.trim(); }) .join(", "); } } if (typeof parsed.description === "string") { metadata.excerpt = parsed.description.trim(); } if ( parsed.publisher && typeof parsed.publisher.name === "string" ) { metadata.siteName = parsed.publisher.name.trim(); } return; } catch (err) { this.log(err.message); } } }); return metadata ? metadata : {}; }, /** * Attempts to get excerpt and byline metadata for the article. * * @param {Object} jsonld — object containing any metadata that * could be extracted from JSON-LD object. * * @return Object with optional "excerpt" and "byline" properties */ _getArticleMetadata: function(jsonld) { var metadata = {}; var values = {}; var metaElements = this._doc.getElementsByTagName("meta"); // property is a space-separated list of values var propertyPattern = /\s*(dc|dcterm|og|twitter)\s*:\s*(author|creator|description|title|site_name)\s*/gi; // name is a single value var namePattern = /^\s*(?:(dc|dcterm|og|twitter|weibo:(article|webpage))\s*[\.:]\s*)?(author|creator|description|title|site_name)\s*$/i; // Find description tags. this._forEachNode(metaElements, function(element) { var elementName = element.getAttribute("name"); var elementProperty = element.getAttribute("property"); var content = element.getAttribute("content"); if (!content) { return; } var matches = null; var name = null; if (elementProperty) { matches = elementProperty.match(propertyPattern); if (matches) { // Convert to lowercase, and remove any whitespace // so we can match below. name = matches[0].toLowerCase().replace(/\s/g, ""); // multiple authors values[name] = content.trim(); } } if (!matches && elementName && namePattern.test(elementName)) { name = elementName; if (content) { // Convert to lowercase, remove any whitespace, and convert dots // to colons so we can match below. name = name.toLowerCase().replace(/\s/g, "").replace(/\./g, ":"); values[name] = content.trim(); } } }); // get title metadata.title = jsonld.title || values["dc:title"] || values["dcterm:title"] || values["og:title"] || values["weibo:article:title"] || values["weibo:webpage:title"] || values["title"] || values["twitter:title"]; if (!metadata.title) { metadata.title = this._getArticleTitle(); } // get author metadata.byline = jsonld.byline || values["dc:creator"] || values["dcterm:creator"] || values["author"]; // get description metadata.excerpt = jsonld.excerpt || values["dc:description"] || values["dcterm:description"] || values["og:description"] || values["weibo:article:description"] || values["weibo:webpage:description"] || values["description"] || values["twitter:description"]; // get site name metadata.siteName = jsonld.siteName || values["og:site_name"]; // in many sites the meta value is escaped with HTML entities, // so here we need to unescape it metadata.title = this._unescapeHtmlEntities(metadata.title); metadata.byline = this._unescapeHtmlEntities(metadata.byline); metadata.excerpt = this._unescapeHtmlEntities(metadata.excerpt); metadata.siteName = this._unescapeHtmlEntities(metadata.siteName); return metadata; }, /** * Check if node is image, or if node contains exactly only one image * whether as a direct child or as its descendants. * * @param Element **/ _isSingleImage: function(node) { if (node.tagName === "IMG") { return true; } if (node.children.length !== 1 || node.textContent.trim() !== "") { return false; } return this._isSingleImage(node.children[0]); }, /** * Find all