Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
043496c
implement aliases for `getVars` method
nleush Apr 20, 2026
bff3859
`plugins.always` try run plugin every call
nleush Apr 22, 2026
ab002ce
use `options.dataMode` and `/data/` plugins
nleush Apr 24, 2026
4750e4c
fix loading data plugins
nleush Apr 24, 2026
c8ee082
debugger: allow data request
nleush Apr 24, 2026
1a081f5
simplify `getDomainOptions` call
nleush Apr 27, 2026
bd561ef
config: add `GET_VARS_ALIASES`
nleush Apr 27, 2026
e934e31
fix undefined
nleush Apr 27, 2026
26034fa
show `vars` in debugger meta section
nleush Apr 27, 2026
2c5067d
remove unnecessary entities
iparamonau Apr 30, 2026
1804b6c
html escape query params (#647)
nleush Apr 22, 2026
9214b72
[Domains] dailymail.co.uk is dailymail.com now (#648)
siniakinaa Apr 23, 2026
c515cfe
Meta: use `decodeHTMLStrict` instead of `decodeHTML5` (#652)
nleush Apr 30, 2026
cd86766
Merge branch 'dev' into feature/data-vars
iparamonau Apr 30, 2026
0e5c0e3
`htmlparser` can return `plainresponse`
nleush May 4, 2026
76eb9f9
detect `BIG_CONTEXT` by `abortController`; use `__nonHtmlContentRespo…
nleush May 4, 2026
77a6a3b
move `__allowHtmlparser` to `cachedMeta`
nleush May 5, 2026
060c238
use `options.requestHeaders` to customize main request headers
nleush May 5, 2026
309d725
fix meta for __allowHtmlparser
nleush May 5, 2026
49817c0
meta: allow customize cache prefix
nleush May 6, 2026
3cf7e01
Update audit.log
nleush May 6, 2026
74cddea
htmlparser: add `provides` `__nonHtmlContentResponse`
nleush May 6, 2026
534b242
simplify matchContentTypeHeaders
iparamonau May 7, 2026
a43aed5
allow for content-encoding in content-type header
iparamonau May 7, 2026
99ab5fb
correct headers check
iparamonau May 7, 2026
41183b8
debug: show positive plugin timings
nleush May 7, 2026
0e8c280
cleanup
nleush May 7, 2026
06a9d2e
debug: faster BIG_CONTEXT check
nleush May 7, 2026
12861cb
debug: remove obsolete hides
nleush May 7, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 16 additions & 2 deletions audit.log
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,20 @@
│ More info │ https://github.com/advisories/GHSA-j3q9-mxjg-w52f │
└─────────────────────┴────────────────────────────────────────────────────────┘
┌─────────────────────┬────────────────────────────────────────────────────────┐
│ high │ Mongoose's Improper Sanitization of $nor in │
│ │ sanitizeFilter May Allow NoSQL Injection │
├─────────────────────┼────────────────────────────────────────────────────────┤
│ Package │ mongoose │
├─────────────────────┼────────────────────────────────────────────────────────┤
│ Vulnerable versions │ >=8.0.0 <=8.22.0 │
├─────────────────────┼────────────────────────────────────────────────────────┤
│ Patched versions │ >=8.22.1 │
├─────────────────────┼────────────────────────────────────────────────────────┤
│ Paths │ .>mongoose │
├─────────────────────┼────────────────────────────────────────────────────────┤
│ More info │ https://github.com/advisories/GHSA-wpg9-53fq-2r8h │
└─────────────────────┴────────────────────────────────────────────────────────┘
┌─────────────────────┬────────────────────────────────────────────────────────┐
│ moderate │ brace-expansion: Zero-step sequence causes process │
│ │ hang and memory exhaustion │
├─────────────────────┼────────────────────────────────────────────────────────┤
Expand Down Expand Up @@ -68,5 +82,5 @@
├─────────────────────┼────────────────────────────────────────────────────────┤
│ More info │ https://github.com/advisories/GHSA-qj8w-gfj5-8c6v │
└─────────────────────┴────────────────────────────────────────────────────────┘
5 vulnerabilities found
Severity: 4 moderate | 1 high
6 vulnerabilities found
Severity: 4 moderate | 2 high
8 changes: 7 additions & 1 deletion config.js
Original file line number Diff line number Diff line change
Expand Up @@ -367,7 +367,13 @@
'ENET',
'HPE_INVALID_',
'ERR_SSL_'
]
],

GET_VARS_METHODS: {
'getSignals': 'signals',
'getPolicy': 'policy',
'getSources': 'sources'
}
};

// Providers config loader.
Expand Down
55 changes: 37 additions & 18 deletions lib/core.js
Original file line number Diff line number Diff line change
Expand Up @@ -458,13 +458,19 @@
// Use all generic plugins.
for(var i = 0; i < pluginsList.length; i++) {
var plugin = pluginsList[i];
if (!plugin.domain && !plugin.custom) {
if (!plugin.domain
&& !plugin.custom
// dataMode - enables data plugins
&& (options.dataMode || !plugin.data)) {

var match = plugin.pluginReMatchesUrl(domain, uri);
if (match) {
pluginsUrlMatches[plugin.id] = match;
}

// false - has re, no match.
// match = result; - url matched plugin.re.
// match = undefined; - domain has no plugin.re.
// match = false - plugin has re, and no match.
if (match !== false) {
initialPlugins.push(plugin);
}
Expand All @@ -473,7 +479,7 @@
}

// In domain debug: add all plugins before domain plugins to make them low priority.
if (options.mixAllWithDomainPlugin) {
if (options.mixAllWithDomainPlugin || options.dataMode) {
addAllGeneric();
}

Expand Down Expand Up @@ -513,6 +519,15 @@
addAllGeneric();
}

function addOneMorePlugin(pluginId) {
var exists = initialPlugins.find(function(plugin) {
return plugin.id === pluginId;
});
if (!exists) {
initialPlugins.push(plugins[pluginId]);
}
}

if (options.forceParams) {
// TODO: replace forEach
options.forceParams.forEach(function(param) {
Expand All @@ -525,13 +540,7 @@

for(var k = 0; k < paramPlugins.length; k++) {
var foundPluginId = paramPlugins[k];

var exists = initialPlugins.find(function(plugin) {
return plugin.id === foundPluginId;
});
if (!exists) {
initialPlugins.push(plugins[foundPluginId]);
}
addOneMorePlugin(foundPluginId);
}
}
});
Expand Down Expand Up @@ -943,9 +952,20 @@
}
}
}
} else if (r.method.name === "getVars") {
for(var key in r.data) {
var v = r.data[key];
} else if (r.method.name === "getVars" || CONFIG.GET_VARS_METHODS && r.method.name in CONFIG.GET_VARS_METHODS) {

var result_data = r.data;

// Set specific alias as key.
if (CONFIG.GET_VARS_METHODS && r.method.name in CONFIG.GET_VARS_METHODS) {
var key = CONFIG.GET_VARS_METHODS[r.method.name];
result_data = {
[key]: r.data
};
}

for(var key in result_data) {
var v = result_data[key];

// TODO: work with arrays?
if (v !== '' && v !== null && ((typeof v === 'string' && !/^\s+$/.test(v)) || typeof v === 'number' || typeof v === 'object')) {
Expand Down Expand Up @@ -1134,7 +1154,7 @@
return hasDomainData;
}

const BIG_CONTEXT = ['htmlparser', 'readability', 'decode', 'cheerio'];
const BIG_CONTEXT = ['readability', 'decode', 'cheerio'];

function prepareResultData(uri, result, options) {

Expand Down Expand Up @@ -1312,10 +1332,9 @@
if (allData) {
for(var i = 0; i < allData.length; i++) {
var r = allData[i];
for(var j = 0; j < BIG_CONTEXT.length; j++) {
var d = BIG_CONTEXT[j];
if (r.data && r.data[d]) {
r.data[d] = 'BIG_CONTEXT';
for(var key in r.data) {
if (r.data[key].abortController || BIG_CONTEXT.indexOf(key) > -1) {
r.data[key] = 'BIG_CONTEXT';
}
}
}
Expand Down
5 changes: 5 additions & 0 deletions lib/loader/pluginLoader.js
Original file line number Diff line number Diff line change
Expand Up @@ -370,6 +370,9 @@
// Plugins in folder 'custom' or 'core' will be run only on explicit dependency.
pluginDeclaration.custom = (bits.indexOf('custom') > -1 || bits.indexOf('system') > -1) && !plugin.generic;

// Plugins in folder 'data' will be run only in data_mode.
pluginDeclaration.data = bits.indexOf('data') > -1;

var stat = fs.statSync(pluginPath);
pluginDeclaration.modified = new Date(stat.mtime);
pluginDeclaration.getPluginLastModifiedDate = getPluginLastModifiedDate;
Expand Down Expand Up @@ -665,6 +668,7 @@
await loadPlugins(pluginsRoot, 'custom');
await loadPlugins(pluginsRoot, 'links');
await loadPlugins(pluginsRoot, 'meta');
await loadPlugins(pluginsRoot, 'data');
await loadPlugins(pluginsRoot, 'templates');

// TODO: if has multiple modules_listing - CUSTOM_PLUGINS_PATH will be loaded multiple times.
Expand All @@ -674,6 +678,7 @@
await loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'custom');
await loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'links');
await loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'meta');
await loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'data');
await loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'templates');
} else {
console.warn('Custom plugin folder "' + CONFIG.CUSTOM_PLUGINS_PATH + '" not found.');
Expand Down
8 changes: 8 additions & 0 deletions lib/loader/utils.js
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import CONFIG from '../../config.loader.js';

export const DEFAULT_PARAMS = [
"url",
"urlMatch",
Expand Down Expand Up @@ -43,6 +45,12 @@
"prepareLink"
];


// Adds vars methods to PLUGIN_METHODS` as `getSignals`.
CONFIG.GET_VARS_METHODS && Object.keys(CONFIG.GET_VARS_METHODS).forEach(methodName => {
PLUGIN_METHODS.push(methodName);
});

export const PLUGIN_FIELDS = PLUGIN_METHODS.concat([
"mixins"
]);
1 change: 1 addition & 0 deletions lib/plugins/system/htmlparser/__allowHtmlparser.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ export default {
provides: [
// Run for all who requests htmlparser or meta.
'htmlparser',
'meta',
'__allowHtmlparser'
],

Expand Down
31 changes: 20 additions & 11 deletions lib/plugins/system/htmlparser/htmlparser.js
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,11 @@ export default {
provides: [
'self',
'__nonHtmlContentData',
'__nonHtmlContentResponse',
'__statusCode'
],

getData: function(url, whitelistRecord, options, __noCachedHtmlparserFallback, __allowHtmlparser, cb) {
getData: function(url, whitelistRecord, options, __noCachedHtmlparserFallback, cb) {

var options2 = {...options, ...{
followRedirect: !!options.followHTTPRedirect,
Expand Down Expand Up @@ -159,17 +160,25 @@ export default {
}
}

if('content-type' in headers && !/text\/html|application\/xhtml\+xml/gi.test(headers['content-type'])){
abortController.abort();
if ('content-type' in headers && !/text\/html|application\/xhtml\+xml/gi.test(headers['content-type'])) {

return cacheAndRespond(null, {
__nonHtmlContentData: {
type: headers['content-type'],
content_length: headers['content-length'],
'set-cookie': headers['set-cookie']
},
headers: headers
});
if (options.requestHeaders?.['Accept']?.indexOf(headers['content-type'].split(';')[0]) > -1) {
return cb(null, {
__nonHtmlContentResponse: resp
});

} else {
abortController.abort();

return cacheAndRespond(null, {
__nonHtmlContentData: {
type: headers['content-type'],
content_length: headers['content-length'],
'set-cookie': headers['set-cookie']
},
headers: headers
});
}
}

// Init htmlparser handler.
Expand Down
2 changes: 1 addition & 1 deletion lib/plugins/system/meta/cachedMeta.js
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ export default {
'__statusCode'
],

getData: function(url, options, whitelistRecord, cb) {
getData: function(url, __allowHtmlparser, options, whitelistRecord, cb) {

// Ignore proxy.cache_ttl, if options.cache_ttl === 0 - do not read from cache.
if (options.refresh || options.cache_ttl === 0) {
Expand Down
10 changes: 8 additions & 2 deletions lib/plugins/system/meta/utils.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,12 @@
export function getMetaCacheKey(url, whitelistRecord, options) {

var meta_key = 'meta:' + url;
var meta_key = 'meta';

if (options.metaKeyPrefix) {
meta_key += ':' + options.metaKeyPrefix;
}

meta_key += ':' + url;

var whitelistHash = whitelistRecord && whitelistRecord.getRecordHash();
if (whitelistHash) {
Expand All @@ -15,4 +21,4 @@ export function getMetaCacheKey(url, whitelistRecord, options) {
return meta_key;
};

export const notPlugin = true;
export const notPlugin = true;
8 changes: 6 additions & 2 deletions lib/utils.js
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ export function prepareRequestOptions(request_options, options) {
}
}

// Some calls (like oembed) use basic options without `getDomainOptions`.
var enable_domain_prerender = options?.getDomainOptions && options.getDomainOptions('meta.prerender');
if (enable_domain_prerender) {
setPrerender(enable_domain_prerender);
Expand Down Expand Up @@ -241,7 +242,7 @@ export function prepareRequestOptions(request_options, options) {
follow = 0;
}

// Custom redirect logit for cookies.
// Custom redirect logic for cookies.
if (options.followRedirect && options.reuseCookies) {
redirect = 'manual';
follow = 0;
Expand All @@ -254,7 +255,10 @@ export function prepareRequestOptions(request_options, options) {
uri: url,
method: 'GET',
headers: {
'Accept': '*/*'
...{
'Accept': '*/*'
},
...options.requestHeaders
},
timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
redirect: redirect,
Expand Down
1 change: 1 addition & 0 deletions modules/api/views.js
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@ export default function(app) {
debug: getBooleanParam(req, 'debug'),
returnProviderOptionsUsage: getBooleanParam(req, 'debug'),
mixAllWithDomainPlugin: getBooleanParam(req, 'mixAllWithDomainPlugin'),
dataMode: getBooleanParam(req, 'dataMode'),
forceParams: req.query.meta === "true" ? CONFIG.DEBUG_CONTEXTS : null,
whitelist: getBooleanParam(req, 'whitelist'),
readability: getBooleanParam(req, 'readability'),
Expand Down
6 changes: 4 additions & 2 deletions modules/debug/views.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,14 @@ export default function(app) {
DEBUG = {"on":1, "true":1}[req.query.debug];
}

res.render('debug',{
res.render('debug', {
uri: req.query.uri,
mixAllWithDomainPlugin: !!{"on":1, "true":1}[req.query.mixAllWithDomainPlugin],
dataMode: !!{"on":1, "true":1}[req.query.dataMode],
refresh: !!{"on":1, "true":1}[req.query.refresh],
DEBUG: DEBUG,
DATA_DEBUG_ENABLED: CONFIG.DATA_DEBUG_ENABLED,
QUERY: getProviderOptionsQuery(req.query)
});
});
};
};
Loading
Loading