USER
1. Review this whole codes throughly and fix all of the potential errors, issues, edge cases, bottle necks, resources consumption, possible errors or issues, might give a error or issue, bugs you can find.
2. Make all of the things precise, careful, appropriate, working, practical, client side only, fast, effective, efficient, reliable.
Send the fully working full complete codes.
---
/*!
* ScraperLib.js v6.5.0
* A fully stable, practical, and robust client-side web scraping library ready for production use
* License: MIT
*/
(function (global) {
'use strict';
/**
* Utility functions for validation and processing
*/
const Utils = {
// Validates if a string is a valid URL
isValidURL(url) {
if (typeof url !== 'string') return false;
try {
new URL(url);
return true;
} catch {
return false;
}
},
// Sanitizes a string to prevent XSS
sanitize(str) {
if (typeof str !== 'string') return '';
const temp = document.createElement('div');
temp.textContent = str;
return temp.textContent;
},
// Initializes the cache if available
async initCache() {
if ('caches' in global) {
try {
return await caches.open('scraper-lib-cache');
} catch (e) {
console.error('Cache initialization failed:', e);
return null;
}
}
return null;
},
// Generates a cache key based on URL and options
generateCacheKey(url, options = {}) {
const cacheOptions = {
useProxy: options.useProxy,
proxyUrl: options.useProxy ? options.proxyUrl : undefined,
selectorsHash: Utils.hashSelectors(options.selectors),
userAgent: options.userAgent,
acceptLanguage: options.acceptLanguage,
};
return `${url}-${btoa(JSON.stringify(cacheOptions))}`;
},
// Generates a hash for selectors to include in cache key
hashSelectors(selectors) {
if (!selectors) return '';
try {
return btoa(JSON.stringify(selectors));
} catch {
return '';
}
},
// Validates and normalizes CSS selectors
normalizeSelector(selector) {
if (typeof selector !== 'string') throw new Error('Selector must be a string');
try {
document.createDocumentFragment().querySelector(selector);
return selector;
} catch {
throw new Error(`Invalid CSS selector: "${selector}"`);
}
},
// Deep merge of objects
deepMerge(target, ...sources) {
target = target || {};
sources.forEach(src => {
if (src && typeof src === 'object') {
Object.keys(src).forEach(key => {
const value = src[key];
if (value && typeof value === 'object' && !Array.isArray(value)) {
target[key] = Utils.deepMerge(target[key], value);
} else {
target[key] = value;
}
});
}
});
return target;
},
// Extracts text from an element, optionally preserving formatting
extractText(element, options = {}) {
if (!element) return '';
const clone = element.cloneNode(true);
// Remove scripts and styles
clone.querySelectorAll('script, style, noscript').forEach(el => el.remove());
if (options.preserveFormatting) {
// Replace <br> with newline characters
clone.querySelectorAll('br').forEach(br => {
const textNode = document.createTextNode('\n');
br.parentNode.replaceChild(textNode, br);
});
// Add newline after </p> elements
clone.querySelectorAll('p').forEach(p => {
const textNode = document.createTextNode('\n');
p.appendChild(textNode);
});
}
return clone.textContent.trim();
},
// Removes scripts and styles from HTML string
removeScriptsAndStyles(html) {
try {
const doc = new DOMParser().parseFromString(html, 'text/html');
doc.querySelectorAll('script, style, noscript').forEach(el => el.remove());
return doc.documentElement.outerHTML;
} catch (e) {
console.error('Error parsing HTML for script/style removal:', e);
return html
.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
.replace(/<noscript[^>]*>[\s\S]*?<\/noscript>/gi, '');
}
},
// Chunks an array into smaller arrays of specified size
chunkArray(array, size) {
if (!Array.isArray(array)) throw new Error('Input must be an array');
if (!Number.isInteger(size) || size <= 0) throw new Error('Chunk size must be a positive integer');
const result = [];
for (let i = 0; i < array.length; i += size) {
result.push(array.slice(i, i + size));
}
return result;
},
// Parses a string to JSON safely
safeParseJSON(str) {
try {
return JSON.parse(str);
} catch {
return null;
}
},
// Converts a NodeList to an Array
nodeListToArray(nodeList) {
return Array.prototype.slice.call(nodeList);
},
// Function to delay execution
delay(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
},
// Check for anti-scraping mechanisms (e.g., CAPTCHA)
detectAntiScraping(html) {
// Simple keyword detection for common anti-scraping messages
const antiScrapingKeywords = ['captcha', 'verify you are human', 'access denied', 'bot detection'];
const lowerHtml = html.toLowerCase();
return antiScrapingKeywords.some(keyword => lowerHtml.includes(keyword));
},
// Built-in filters for common operations
builtInFilters: {
trim: value => (typeof value === 'string' ? value.trim() : value),
toUpperCase: value => (typeof value === 'string' ? value.toUpperCase() : value),
toLowerCase: value => (typeof value === 'string' ? value.toLowerCase() : value),
parseInt: value => parseInt(value, 10),
parseFloat: value => parseFloat(value),
// Additional built-in filters
replace: (value, args) => {
if (typeof value !== 'string' || !Array.isArray(args) || args.length < 2) return value;
return value.replace(args[0], args[1]);
},
substring: (value, args) => {
if (typeof value !== 'string' || !Array.isArray(args)) return value;
return value.substring(args[0], args[1]);
},
split: (value, delimiter) => {
if (typeof value !== 'string') return value;
return value.split(delimiter);
},
join: (value, delimiter) => {
if (!Array.isArray(value)) return value;
return value.join(delimiter);
},
},
};
/**
* Handles storing scraped data
*/
class StorageHandler {
constructor() {
this.dbPromise = null;
this.storeName = 'scrapedData';
if ('indexedDB' in global) {
this.storageType = 'indexeddb';
this.dbPromise = this.initIndexedDB();
} else if ('localStorage' in global && this._isLocalStorageAccessible()) {
this.storageType = 'localstorage';
} else {
this.storageType = 'memory';
this.memoryStorage = new Map();
}
}
_isLocalStorageAccessible() {
try {
const testKey = '__test__';
localStorage.setItem(testKey, testKey);
localStorage.removeItem(testKey);
return true;
} catch {
console.warn('localStorage is not accessible. Using in-memory storage.');
return false;
}
}
async initIndexedDB(dbName = 'ScraperLibDB') {
return new Promise((resolve, reject) => {
const request = indexedDB.open(dbName, 1);
request.onerror = () => reject(new Error('IndexedDB initialization failed'));
request.onupgradeneeded = event => {
const db = event.target.result;
if (!db.objectStoreNames.contains(this.storeName)) {
const store = db.createObjectStore(this.storeName, { keyPath: 'id', autoIncrement: true });
store.createIndex('url', 'url', { unique: false });
store.createIndex('timestamp', 'timestamp', { unique: false });
}
};
request.onsuccess = event => {
const db = event.target.result;
db.onerror = event => {
console.error('Database error:', event.target.error);
};
resolve(db);
};
});
}
async store(url, data, format, options = {}) {
const timestamp = Date.now();
const storeData = { url, data, timestamp };
switch (format) {
case 'indexeddb':
if (this.dbPromise) {
const db = await this.dbPromise;
return new Promise((resolve, reject) => {
const transaction = db.transaction(this.storeName, 'readwrite');
const store = transaction.objectStore(this.storeName);
const request = store.add(storeData);
request.onsuccess = () => resolve(true);
request.onerror = () => reject(request.error);
});
}
throw new Error('IndexedDB is not supported or initialized');
case 'localstorage':
try {
const key = `scraper-${url}-${timestamp}`;
localStorage.setItem(key, JSON.stringify(storeData));
return true;
} catch (e) {
if (e.name === 'QuotaExceededError' || e.name === 'NS_ERROR_DOM_QUOTA_REACHED') {
this._cleanupLocalStorage();
const key = `scraper-${url}-${timestamp}`;
localStorage.setItem(key, JSON.stringify(storeData));
return true;
}
throw e;
}
case 'memory':
this.memoryStorage.set(`${url}-${timestamp}`, storeData);
return true;
case 'custom':
if (options.customStore && typeof options.customStore.save === 'function') {
await options.customStore.save(url, data);
return true;
}
throw new Error('Invalid custom storage handler');
default:
throw new Error(`Unsupported storage format: ${format}`);
}
}
_cleanupLocalStorage() {
const keys = Object.keys(localStorage).filter(key => key.startsWith('scraper-'));
if (keys.length > 100) {
keys.sort((a, b) => {
try {
const itemA = JSON.parse(localStorage.getItem(a));
const itemB = JSON.parse(localStorage.getItem(b));
return itemA.timestamp - itemB.timestamp;
} catch (e) {
localStorage.removeItem(a);
localStorage.removeItem(b);
return 0;
}
});
for (let i = 0; i < keys.length - 100; i++) {
localStorage.removeItem(keys[i]);
}
}
}
async retrieve(url, format) {
switch (format) {
case 'indexeddb':
if (this.dbPromise) {
const db = await this.dbPromise;
return new Promise((resolve, reject) => {
const transaction = db.transaction(this.storeName, 'readonly');
const store = transaction.objectStore(this.storeName);
const index = store.index('url');
const request = index.getAll(url);
request.onsuccess = () => {
const results = request.result;
if (results && results.length > 0) {
results.sort((a, b) => b.timestamp - a.timestamp);
resolve(results[0].data);
} else {
resolve(null);
}
};
request.onerror = () => reject(request.error);
});
}
throw new Error('IndexedDB is not supported or initialized');
case 'localstorage':
const keys = Object.keys(localStorage)
.filter(k => k.startsWith(`scraper-${url}-`))
.sort((a, b) => {
const itemA = Utils.safeParseJSON(localStorage.getItem(a));
const itemB = Utils.safeParseJSON(localStorage.getItem(b));
return (itemB?.timestamp || 0) - (itemA?.timestamp || 0);
});
if (keys.length > 0) {
const item = Utils.safeParseJSON(localStorage.getItem(keys[0]));
return item ? item.data : null;
}
return null;
case 'memory':
const memoryItems = Array.from(this.memoryStorage.entries())
.filter(([key]) => key.startsWith(`${url}-`))
.sort((a, b) => b[1].timestamp - a[1].timestamp);
if (memoryItems.length > 0) {
return memoryItems[0][1].data;
}
return null;
default:
throw new Error(`Unsupported storage format: ${format}`);
}
}
}
/**
* Handles rate limiting to respect website policies
*/
class RateLimiter {
constructor(requestsPerSecond = 2) {
this.requestsPerSecond = Math.max(1, requestsPerSecond);
this.interval = 1000 / this.requestsPerSecond;
this.queue = [];
this.lastRequestTime = 0;
this.isProcessing = false;
}
async acquireToken() {
return new Promise(resolve => {
this.queue.push(resolve);
this._processQueue();
});
}
_processQueue() {
if (this.isProcessing || this.queue.length === 0) return;
this.isProcessing = true;
const now = Date.now();
const elapsed = now - this.lastRequestTime;
const delay = Math.max(this.interval - elapsed, 0);
setTimeout(() => {
this.lastRequestTime = Date.now();
const resolve = this.queue.shift();
if (resolve) resolve();
this.isProcessing = false;
if (this.queue.length > 0) {
this._processQueue();
}
}, delay);
}
}
/**
* Main ScraperLib class
*/
class ScraperLib {
constructor(options = {}) {
this.defaultSettings = {
rateLimitRequests: 2,
maxConcurrent: 5,
timeout: 30000,
retries: 3,
retryDelay: 1000,
maxRetryDelay: 30000,
cacheTime: 3600000,
outputFormat: 'json',
useCache: false,
headers: {},
preserveFormatting: false,
validateSelectors: true,
maxResponseSize: 10 * 1024 * 1024,
useProxy: false,
proxyUrl: '',
sinceModified: false,
filters: {}, // Global filters
userAgent: navigator.userAgent,
acceptLanguage: navigator.language || 'en-US',
concurrencyStrategy: 'auto', // 'auto', 'parallel', 'sequential'
logger: null, // Custom logger function
customFetch: null, // Custom fetch function
politenessDelay: 0, // Delay between requests to the same domain
maxRetriesOnAntiScraping: 2, // Max retries if anti-scraping measures detected
builtInFilters: Utils.builtInFilters, // Reference to built-in filters
enableLogging: true, // Enable or disable logging
};
// Merge and validate default settings with user options
this.settings = Utils.deepMerge({}, this.defaultSettings, options);
this._validateSettings(this.settings);
// Initialize per-URL settings
this.urlSettings = new Map();
this.storage = new StorageHandler();
this.rateLimiter = new RateLimiter(this.settings.rateLimitRequests);
this.cache = null;
this.errorHandlers = new Map();
// Initialize cache and wait for initialization
if (this.settings.useCache) {
this.cachePromise = Utils.initCache().then(cache => {
this.cache = cache;
});
} else {
this.cachePromise = Promise.resolve();
}
// Initialize logger
this.logger = this.settings.logger || console;
// Hostname to last request time mapping for politeness
this.hostLastRequestTime = new Map();
}
_validateSettings(settings) {
const positiveNumberProps = [
'rateLimitRequests',
'maxConcurrent',
'retries',
'retryDelay',
'maxRetryDelay',
'cacheTime',
'timeout',
'maxResponseSize',
'politenessDelay',
'maxRetriesOnAntiScraping',
];
positiveNumberProps.forEach(prop => {
if (typeof settings[prop] !== 'number' || settings[prop] < 0) {
throw new Error(`Invalid setting "${prop}": must be a positive number`);
}
});
const allowedStrategies = ['auto', 'parallel', 'sequential'];
if (!allowedStrategies.includes(settings.concurrencyStrategy)) {
throw new Error(
`Invalid concurrency strategy: "${settings.concurrencyStrategy}". Allowed values are ${allowedStrategies.join(
', '
)}`
);
}
}
/**
* Main scraping method
*/
async scrape(configs, globalOptions = {}) {
await this.cachePromise;
if (typeof configs !== 'object' || configs === null || Array.isArray(configs)) {
throw new Error('Configs must be an object with URL keys and configuration values');
}
// Merge global options into settings
const globalSettings = Utils.deepMerge({}, this.settings, globalOptions);
this._validateSettings(globalSettings);
const results = {};
const urls = Object.keys(configs);
if (urls.length === 0) {
return results;
}
await this._log('Starting scraping process', { urls });
// Prepare per-domain request queues to respect politeness
this.domainQueues = {};
switch (globalSettings.concurrencyStrategy) {
case 'parallel':
await Promise.all(
urls.map(url => this._scrapeUrlWrapper(url, configs[url], globalSettings, results))
);
break;
case 'sequential':
for (const url of urls) {
await this._scrapeUrlWrapper(url, configs[url], globalSettings, results);
}
break;
case 'auto':
default:
const chunks = Utils.chunkArray(urls, globalSettings.maxConcurrent);
for (const chunk of chunks) {
await Promise.all(
chunk.map(url => this._scrapeUrlWrapper(url, configs[url], globalSettings, results))
);
}
break;
}
await this._log('Completed scraping process', { results });
return results;
}
/**
* Wrapper for processing a URL, handles logging and error handling
*/
async _scrapeUrlWrapper(url, config, globalSettings, results) {
try {
const data = await this._processUrl(url, config, globalSettings);
results[url] = data;
} catch (error) {
const handler = this.errorHandlers.get(url) || this.errorHandlers.get('default');
if (handler) {
try {
const data = await handler(error, url);
results[url] = data;
} catch (e) {
results[url] = { error: e.message };
}
} else {
results[url] = { error: error.message };
}
await this._log(`Error scraping ${url}: ${error.message}`, { error });
}
}
/**
* Process a single URL with retries and error handling
*/
async _processUrl(url, config, globalSettings) {
if (!Utils.isValidURL(url)) {
throw new Error(`Invalid URL: ${url}`);
}
// Merge global settings with per-URL settings using deep merge
const settings = Utils.deepMerge({}, globalSettings, config.options);
this._validateSettings(settings);
this.urlSettings.set(url, settings);
if (settings.validateSelectors) {
this._validateSelectors(config.selectors);
}
// Wait for cache initialization if needed
if (settings.useCache) {
await this.cachePromise;
if (this.cache) {
const cachedData = await this._getFromCache(url, settings);
if (cachedData) {
await this._log(`Using cached data for ${url}`);
return cachedData;
}
}
}
let attempts = 0;
let antiScrapingAttempts = 0;
while (attempts < settings.retries) {
attempts += 1;
await this._respectPoliteness(url, settings);
await this.rateLimiter.acquireToken();
const options = this._prepareFetchOptions(config, settings);
try {
const response = await this._fetchWithTimeout(url, options, settings);
if (!response.ok) {
if (
response.status === 304 &&
settings.useCache &&
this.cache &&
settings.sinceModified
) {
const cachedData = await this._getFromCache(url, settings);
if (cachedData) {
return cachedData;
}
}
throw new Error(`HTTP error: ${response.status} ${response.statusText}`);
}
const contentType = response.headers.get('Content-Type') || '';
if (!contentType.includes('text/html')) {
throw new Error(`Invalid Content-Type: ${contentType}`);
}
const data = await this._extractDataFromResponse(response, config.selectors, settings);
if (settings.outputFormat !== 'json') {
await this.storage.store(url, data, settings.outputFormat, { customStore: settings.customStore });
}
if (settings.useCache && this.cache) {
await this._storeInCache(url, data, settings);
}
return data;
} catch (error) {
if (
Utils.detectAntiScraping(error.message) &&
antiScrapingAttempts < settings.maxRetriesOnAntiScraping
) {
antiScrapingAttempts += 1;
await this._log(
`Detected anti-scraping measures on ${url}. Attempt ${antiScrapingAttempts}`,
{ error }
);
// Delay before retrying
await Utils.delay(settings.retryDelay * antiScrapingAttempts);
continue;
}
if (attempts >= settings.retries) {
throw error;
}
// Exponential backoff with jitter
const delay = Math.min(
settings.retryDelay * Math.pow(2, attempts - 1) * (0.5 + Math.random() * 0.5),
settings.maxRetryDelay
);
await this._log(`Retrying ${url} after delay of ${delay}ms`, { attempt: attempts, error });
await Utils.delay(delay);
}
}
}
/**
* Respects politeness policies by delaying requests to the same domain
*/
async _respectPoliteness(url, settings) {
const urlObj = new URL(url);
const hostname = urlObj.hostname;
const now = Date.now();
const lastRequestTime = this.hostLastRequestTime.get(hostname) || 0;
const elapsed = now - lastRequestTime;
if (elapsed < settings.politenessDelay) {
const delay = settings.politenessDelay - elapsed;
await this._log(`Delaying request to ${url} by ${delay}ms for politeness`);
await Utils.delay(delay);
}
this.hostLastRequestTime.set(hostname, Date.now());
}
/**
* Validates the CSS selectors
*/
_validateSelectors(selectors) {
if (typeof selectors !== 'object' || selectors === null) {
throw new Error('Selectors must be an object');
}
for (const key of Object.keys(selectors)) {
const sel = selectors[key];
if (typeof sel === 'string') {
Utils.normalizeSelector(sel);
} else if (sel && typeof sel.selector === 'string') {
Utils.normalizeSelector(sel.selector);
} else if (sel && typeof sel.customSelector === 'function') {
// Custom selector function, assume valid
} else if (sel && typeof sel.xpath === 'string') {
// Validate XPath if possible
} else {
throw new Error(`Invalid selector configuration for key "${key}"`);
}
}
}
/**
* Prepares fetch options including headers
*/
_prepareFetchOptions(config, settings) {
const defaultHeaders = {
'Accept-Language': settings.acceptLanguage,
'User-Agent': settings.userAgent,
};
// Remove disallowed headers
const disallowedHeaders = [
'Referer',
'Origin',
'Cookie',
'Host',
'Accept-Encoding',
'Connection',
'Upgrade-Insecure-Requests',
];
const headersInit = Utils.deepMerge({}, defaultHeaders, settings.headers, config.headers);
disallowedHeaders.forEach(header => {
delete headersInit[header];
});
const headers = new Headers(headersInit);
if (settings.sinceModified) {
const lastModified = new Date(Date.now() - settings.cacheTime).toUTCString();
headers.set('If-Modified-Since', lastModified);
}
const options = {
method: 'GET',
headers: headers,
mode: 'cors',
credentials: 'omit',
redirect: 'follow',
};
return options;
}
/**
* Performs a fetch request with a timeout, using proxy if configured
*/
async _fetchWithTimeout(url, options, settings) {
const controller = new AbortController();
options.signal = controller.signal;
const timeoutId = setTimeout(() => {
controller.abort();
}, settings.timeout);
try {
let fetchUrl = url;
if (settings.useProxy) {
if (!settings.proxyUrl) {
throw new Error('Proxy URL is not set');
}
fetchUrl = `${settings.proxyUrl}?url=${encodeURIComponent(url)}`;
}
const fetchFunction = settings.customFetch || fetch;
const response = await fetchFunction(fetchUrl, options);
clearTimeout(timeoutId);
// Check if response size exceeds maxResponseSize
const contentLength = response.headers.get('Content-Length');
if (contentLength && parseInt(contentLength, 10) > settings.maxResponseSize) {
throw new Error('Response size exceeds maximum allowed');
}
return response;
} catch (error) {
clearTimeout(timeoutId);
if (error.name === 'AbortError') {
throw new Error('Request timed out');
}
throw error;
}
}
/**
* Enhanced data extraction with advanced filters
*/
async _extractDataFromResponse(response, selectors, settings) {
try {
const decoder = new TextDecoder('utf-8');
const reader = response.body.getReader();
let receivedLength = 0;
let chunks = [];
while (true) {
const { done, value } = await reader.read();
if (done) break;
receivedLength += value.length;
if (receivedLength > settings.maxResponseSize) {
throw new Error('Response size exceeds maximum allowed');
}
chunks.push(value);
}
const htmlText = decoder.decode(this._concatChunks(chunks));
// Check for anti-scraping mechanisms
if (Utils.detectAntiScraping(htmlText)) {
throw new Error('Detected anti-scraping measures in the response');
}
// Remove scripts and styles to prevent XSS and reduce parsing overhead
const sanitizedHtml = Utils.removeScriptsAndStyles(htmlText);
const parser = new DOMParser();
const doc = parser.parseFromString(sanitizedHtml, 'text/html');
const data = {};
for (const key of Object.keys(selectors)) {
const selConfig = selectors[key];
let elements;
if (typeof selConfig === 'string') {
elements = doc.querySelectorAll(selConfig);
} else if (selConfig && typeof selConfig.selector === 'string') {
elements = doc.querySelectorAll(selConfig.selector);
} else if (selConfig && typeof selConfig.customSelector === 'function') {
elements = selConfig.customSelector(doc);
if (!Array.isArray(elements)) {
throw new Error(`customSelector for key "${key}" must return an array of elements`);
}
} else if (selConfig && typeof selConfig.xpath === 'string') {
elements = this._selectXPath(doc, selConfig.xpath);
} else {
throw new Error(`Invalid selector configuration for key "${key}"`);
}
if (!elements || elements.length === 0) {
data[key] = [];
continue;
}
data[key] = await Promise.all(
Array.from(elements).map(async element => {
let value;
if (selConfig.attr) {
value = element.getAttribute(selConfig.attr) || '';
} else if (selConfig.html) {
value = element.innerHTML;
} else {
value = Utils.extractText(element, { preserveFormatting: settings.preserveFormatting });
}
// Apply per-selector filters (support chaining)
if (selConfig.filters) {
value = await this._applyFilters(value, selConfig.filters, element, {
key,
settings,
});
}
// Apply global filters if no per-selector filters
else if (settings.filters && settings.filters[key]) {
value = await this._applyFilters(value, settings.filters[key], element, {
key,
settings,
});
}
return value;
})
);
data[key] = data[key].filter(val => val !== undefined && val !== null);
if (selConfig.single) {
data[key] = data[key][0] || null;
}
}
return data;
} catch (e) {
throw new Error(`Failed to extract data: ${e.message}`);
}
}
/**
* Apply filters to a value, supporting chaining and built-in filters
*/
async _applyFilters(value, filters, element, context) {
if (!filters) return value;
const filtersArray = Array.isArray(filters) ? filters : [filters];
for (let filter of filtersArray) {
if (typeof filter === 'string') {
// Built-in filter by name
const builtInFilter = this._parseFilterString(filter);
if (typeof builtInFilter === 'function') {
value = builtInFilter(value, element, context);
} else {
console.warn(`Built-in filter "${filter}" not found`);
}
} else if (typeof filter === 'function') {
try {
const filterResult = filter(value, element, context);
value = filterResult instanceof Promise ? await filterResult : filterResult;
} catch (e) {
console.error(`Error in filter for key "${context.key}":`, e);
value = null;
break;
}
} else if (filter instanceof RegExp) {
// Apply regular expression
const match = value.match(filter);
value = match ? match[0] : null;
} else {
console.warn(`Invalid filter type for key "${context.key}"`);
}
// If value becomes null or undefined, stop processing filters
if (value == null) {
break;
}
}
return value;
}
/**
* Parse filter string to support filter arguments
*/
_parseFilterString(filterStr) {
// Example filter string: 'replace(/foo/g, "bar")'
const filterPattern = /^(\w+)\((.*)\)$/;
const match = filterStr.match(filterPattern);
if (match) {
const filterName = match[1];
const argsStr = match[2];
const args = this._parseFilterArgs(argsStr);
const filterFunc = this.settings.builtInFilters[filterName];
if (typeof filterFunc === 'function') {
return (value, element, context) => filterFunc(value, args, element, context);
}
} else {
const filterFunc = this.settings.builtInFilters[filterStr];
if (typeof filterFunc === 'function') {
return filterFunc;
}
}
return null;
}
/**
* Parse filter arguments from string
*/
_parseFilterArgs(argsStr) {
try {
// Use Function constructor to parse arguments safely
const argsFunc = new Function(`return [${argsStr}];`);
return argsFunc();
} catch (e) {
console.error('Failed to parse filter arguments:', e);
return [];
}
}
/**
* Select elements using XPath
*/
_selectXPath(doc, xpathExpression) {
const iterator = doc.evaluate(xpathExpression, doc, null, XPathResult.ORDERED_NODE_ITERATOR_TYPE, null);
const nodes = [];
let node = iterator.iterateNext();
while (node) {
nodes.push(node);
node = iterator.iterateNext();
}
return nodes;
}
/**
* Concatenates array of Uint8Arrays into one Uint8Array
*/
_concatChunks(chunks) {
let totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
let result = new Uint8Array(totalLength);
let position = 0;
for (let chunk of chunks) {
result.set(chunk, position);
position += chunk.length;
}
return result;
}
/**
* Gets data from the cache if available and not expired
*/
async _getFromCache(url, settings) {
if (!this.cache) return null;
const cacheKey = Utils.generateCacheKey(url, settings);
const cachedResponse = await this.cache.match(cacheKey);
if (cachedResponse) {
try {
const cachedData = await cachedResponse.json();
const age = Date.now() - cachedData.timestamp;
if (age < settings.cacheTime) {
return cachedData.data;
}
} catch (e) {
console.warn('Error reading from cache:', e);
}
}
return null;
}
/**
* Stores data in the cache
*/
async _storeInCache(url, data, settings) {
if (!this.cache) return;
const cacheKey = Utils.generateCacheKey(url, settings);
const cachedResponse = new Response(
JSON.stringify({ data, timestamp: Date.now() }),
{
headers: { 'Content-Type': 'application/json' },
}
);
try {
await this.cache.put(cacheKey, cachedResponse);
} catch (e) {
console.error('Failed to store data in cache:', e);
}
}
/**
* Registers a custom error handler
*/
onError(urlOrDefault, handler) {
if (typeof handler !== 'function') {
throw new Error('Error handler must be a function');
}
this.errorHandlers.set(urlOrDefault, handler);
}
/**
* Logs messages using the provided logger
*/
async _log(message, data = {}) {
if (!this.settings.enableLogging) return;
if (typeof this.logger === 'function') {
await this.logger(message, data);
} else if (this.logger && typeof this.logger.log === 'function') {
await this.logger.log(message, data);
}
}
}
// Attach ScraperLib to the global object
global.ScraperLib = ScraperLib;
})(typeof window !== 'undefined' ? window : this);ASSISTANT
```javascript
/*!
* ScraperLib.js v6.5.1
* A fully stable, practical, and robust client-side web scraping library ready for production use
* License: MIT
*/
(function (global) {
'use strict';
/**
* Utility functions for validation and processing
*/
const Utils = {
// Validates if a string is a valid URL
isValidURL(url) {
if (typeof url !== 'string') return false;
try {
new URL(url);
return true;
} catch {
return false;
}
},
// Sanitizes a string to prevent XSS
sanitize(str) {
if (typeof str !== 'string') return '';
const temp = document.createElement('div');
temp.textContent = str;
return temp.innerHTML;
},
// Initializes the cache if available
async initCache() {
if ('caches' in global) {
try {
return await caches.open('scraper-lib-cache');
} catch (e) {
console.error('Cache initialization failed:', e);
return null;
}
}
return null;
},
// Generates a cache key based on URL and options
generateCacheKey(url, options = {}) {
const cacheOptions = {
useProxy: options.useProxy,
proxyUrl: options.useProxy ? options.proxyUrl : undefined,
selectorsHash: Utils.hashSelectors(options.selectors),
userAgent: options.userAgent,
acceptLanguage: options.acceptLanguage,
};
return `${url}-${Utils.hashString(JSON.stringify(cacheOptions))}`;
},
// Generates a hash for selectors to include in cache key
hashSelectors(selectors) {
if (!selectors) return '';
try {
return Utils.hashString(JSON.stringify(selectors));
} catch {
return '';
}
},
// Generates a simple hash from a string
hashString(str) {
let hash = 0,
i,
chr;
if (str.length === 0) return hash;
for (i = 0; i < str.length; i++) {
chr = str.charCodeAt(i);
hash = (hash << 5) - hash + chr;
hash |= 0; // Convert to 32bit integer
}
return hash.toString();
},
// Validates and normalizes CSS selectors
normalizeSelector(selector) {
if (typeof selector !== 'string') throw new Error('Selector must be a string');
try {
document.createDocumentFragment().querySelector(selector);
return selector;
} catch {
throw new Error(`Invalid CSS selector: "${selector}"`);
}
},
// Deep merge of objects
deepMerge(target, ...sources) {
target = target || {};
sources.forEach(src => {
if (src && typeof src === 'object') {
Object.keys(src).forEach(key => {
const value = src[key];
if (
value &&
typeof value === 'object' &&
!Array.isArray(value) &&
!(value instanceof RegExp)
) {
target[key] = Utils.deepMerge(target[key], value);
} else {
target[key] = value;
}
});
}
});
return target;
},
// Extracts text from an element, optionally preserving formatting
extractText(element, options = {}) {
if (!element) return '';
const clone = element.cloneNode(true);
// Remove scripts and styles
clone.querySelectorAll('script, style, noscript').forEach(el => el.remove());
if (options.preserveFormatting) {
// Replace <br> with newline characters
clone.querySelectorAll('br').forEach(br => {
const textNode = document.createTextNode('\n');
br.parentNode.replaceChild(textNode, br);
});
// Add newline after </p> elements
clone.querySelectorAll('p').forEach(p => {
const textNode = document.createTextNode('\n');
p.appendChild(textNode);
});
}
return clone.textContent.trim();
},
// Removes scripts and styles from HTML string
removeScriptsAndStyles(html) {
try {
const doc = new DOMParser().parseFromString(html, 'text/html');
doc.querySelectorAll('script, style, noscript').forEach(el => el.remove());
return doc.documentElement.outerHTML;
} catch (e) {
console.error('Error parsing HTML for script/style removal:', e);
return html
.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
.replace(/<noscript[^>]*>[\s\S]*?<\/noscript>/gi, '');
}
},
// Chunks an array into smaller arrays of specified size
chunkArray(array, size) {
if (!Array.isArray(array)) throw new Error('Input must be an array');
if (!Number.isInteger(size) || size <= 0) throw new Error('Chunk size must be a positive integer');
const result = [];
for (let i = 0; i < array.length; i += size) {
result.push(array.slice(i, i + size));
}
return result;
},
// Parses a string to JSON safely
safeParseJSON(str) {
try {
return JSON.parse(str);
} catch {
return null;
}
},
// Converts a NodeList to an Array
nodeListToArray(nodeList) {
return Array.from(nodeList);
},
// Function to delay execution
delay(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
},
// Check for anti-scraping mechanisms (e.g., CAPTCHA)
detectAntiScraping(html) {
// Simple keyword detection for common anti-scraping messages
const antiScrapingKeywords = ['captcha', 'verify you are human', 'access denied', 'bot detection'];
const lowerHtml = html.toLowerCase();
return antiScrapingKeywords.some(keyword => lowerHtml.includes(keyword));
},
// Built-in filters for common operations
builtInFilters: {
trim: value => (typeof value === 'string' ? value.trim() : value),
toUpperCase: value => (typeof value === 'string' ? value.toUpperCase() : value),
toLowerCase: value => (typeof value === 'string' ? value.toLowerCase() : value),
parseInt: value => parseInt(value, 10),
parseFloat: value => parseFloat(value),
// Additional built-in filters
replace: (value, args) => {
if (typeof value !== 'string' || !Array.isArray(args) || args.length < 2) return value;
const [pattern, replacement] = args;
let regex;
if (pattern instanceof RegExp) {
regex = pattern;
} else if (typeof pattern === 'string') {
regex = new RegExp(pattern, 'g');
} else {
return value;
}
return value.replace(regex, replacement);
},
substring: (value, args) => {
if (typeof value !== 'string' || !Array.isArray(args)) return value;
return value.substring(args[0], args[1]);
},
split: (value, args) => {
if (typeof value !== 'string') return value;
return value.split(args[0]);
},
join: (value, args) => {
if (!Array.isArray(value)) return value;
return value.join(args[0]);
},
},
};
/**
* Handles storing scraped data
*/
class StorageHandler {
constructor() {
this.dbPromise = null;
this.storeName = 'scrapedData';
if ('indexedDB' in global) {
this.storageType = 'indexeddb';
this.dbPromise = this.initIndexedDB();
} else if ('localStorage' in global && this._isLocalStorageAccessible()) {
this.storageType = 'localstorage';
} else {
this.storageType = 'memory';
this.memoryStorage = new Map();
}
}
_isLocalStorageAccessible() {
try {
const testKey = '__test__';
localStorage.setItem(testKey, testKey);
localStorage.removeItem(testKey);
return true;
} catch {
console.warn('localStorage is not accessible. Using in-memory storage.');
return false;
}
}
async initIndexedDB(dbName = 'ScraperLibDB') {
return new Promise((resolve, reject) => {
const request = indexedDB.open(dbName, 1);
request.onerror = () => reject(new Error('IndexedDB initialization failed'));
request.onupgradeneeded = event => {
const db = event.target.result;
if (!db.objectStoreNames.contains(this.storeName)) {
const store = db.createObjectStore(this.storeName, { keyPath: 'id', autoIncrement: true });
store.createIndex('url', 'url', { unique: false });
store.createIndex('timestamp', 'timestamp', { unique: false });
}
};
request.onsuccess = event => {
const db = event.target.result;
db.onerror = event => {
console.error('Database error:', event.target.error);
};
resolve(db);
};
});
}
async store(url, data, format, options = {}) {
const timestamp = Date.now();
const storeData = { url, data, timestamp };
switch (format) {
case 'indexeddb':
if (this.dbPromise) {
const db = await this.dbPromise;
return new Promise((resolve, reject) => {
const transaction = db.transaction(this.storeName, 'readwrite');
const store = transaction.objectStore(this.storeName);
const request = store.add(storeData);
request.onsuccess = () => resolve(true);
request.onerror = () => reject(request.error);
});
}
throw new Error('IndexedDB is not supported or initialized');
case 'localstorage':
try {
const key = `scraper-${url}-${timestamp}`;
localStorage.setItem(key, JSON.stringify(storeData));
return true;
} catch (e) {
if (e.name === 'QuotaExceededError' || e.name === 'NS_ERROR_DOM_QUOTA_REACHED') {
this._cleanupLocalStorage();
const key = `scraper-${url}-${timestamp}`;
localStorage.setItem(key, JSON.stringify(storeData));
return true;
}
throw e;
}
case 'memory':
this.memoryStorage.set(`${url}-${timestamp}`, storeData);
return true;
case 'custom':
if (options.customStore && typeof options.customStore.save === 'function') {
await options.customStore.save(url, data);
return true;
}
throw new Error('Invalid custom storage handler');
default:
throw new Error(`Unsupported storage format: ${format}`);
}
}
_cleanupLocalStorage() {
const keys = Object.keys(localStorage).filter(key => key.startsWith('scraper-'));
if (keys.length > 100) {
keys.sort((a, b) => {
try {
const itemA = JSON.parse(localStorage.getItem(a));
const itemB = JSON.parse(localStorage.getItem(b));
return itemA.timestamp - itemB.timestamp;
} catch (e) {
localStorage.removeItem(a);
localStorage.removeItem(b);
return 0;
}
});
for (let i = 0; i < keys.length - 100; i++) {
localStorage.removeItem(keys[i]);
}
}
}
async retrieve(url, format) {
switch (format) {
case 'indexeddb':
if (this.dbPromise) {
const db = await this.dbPromise;
return new Promise((resolve, reject) => {
const transaction = db.transaction(this.storeName, 'readonly');
const store = transaction.objectStore(this.storeName);
const index = store.index('url');
const request = index.getAll(url);
request.onsuccess = () => {
const results = request.result;
if (results && results.length > 0) {
results.sort((a, b) => b.timestamp - a.timestamp);
resolve(results[0].data);
} else {
resolve(null);
}
};
request.onerror = () => reject(request.error);
});
}
throw new Error('IndexedDB is not supported or initialized');
case 'localstorage':
const keys = Object.keys(localStorage)
.filter(k => k.startsWith(`scraper-${url}-`))
.sort((a, b) => {
const itemA = Utils.safeParseJSON(localStorage.getItem(a));
const itemB = Utils.safeParseJSON(localStorage.getItem(b));
return (itemB?.timestamp || 0) - (itemA?.timestamp || 0);
});
if (keys.length > 0) {
const item = Utils.safeParseJSON(localStorage.getItem(keys[0]));
return item ? item.data : null;
}
return null;
case 'memory':
const memoryItems = Array.from(this.memoryStorage.entries())
.filter(([key]) => key.startsWith(`${url}-`))
.sort((a, b) => b[1].timestamp - a[1].timestamp);
if (memoryItems.length > 0) {
return memoryItems[0][1].data;
}
return null;
default:
throw new Error(`Unsupported storage format: ${format}`);
}
}
}
/**
* Handles rate limiting to respect website policies
*/
class RateLimiter {
constructor(requestsPerSecond = 2) {
this.requestsPerSecond = Math.max(1, requestsPerSecond);
this.interval = 1000 / this.requestsPerSecond;
this.queue = [];
this.lastRequestTime = 0;
this.isProcessing = false;
}
async acquireToken() {
return new Promise(resolve => {
this.queue.push(resolve);
this._processQueue();
});
}
_processQueue() {
if (this.isProcessing || this.queue.length === 0) return;
this.isProcessing = true;
const now = Date.now();
const elapsed = now - this.lastRequestTime;
const delay = Math.max(this.interval - elapsed, 0);
setTimeout(() => {
this.lastRequestTime = Date.now();
const resolve = this.queue.shift();
if (resolve) resolve();
this.isProcessing = false;
if (this.queue.length > 0) {
this._processQueue();
}
}, delay);
}
}
/**
* Main ScraperLib class
*/
class ScraperLib {
constructor(options = {}) {
this.defaultSettings = {
rateLimitRequests: 2,
maxConcurrent: 5,
timeout: 30000,
retries: 3,
retryDelay: 1000,
maxRetryDelay: 30000,
cacheTime: 3600000,
outputFormat: 'json',
useCache: false,
headers: {},
preserveFormatting: false,
validateSelectors: true,
maxResponseSize: 10 * 1024 * 1024,
useProxy: false,
proxyUrl: '',
sinceModified: false,
filters: {}, // Global filters
userAgent: '',
acceptLanguage: '',
concurrencyStrategy: 'auto', // 'auto', 'parallel', 'sequential'
logger: null, // Custom logger function
customFetch: null, // Custom fetch function
politenessDelay: 0, // Delay between requests to the same domain
maxRetriesOnAntiScraping: 2, // Max retries if anti-scraping measures detected
builtInFilters: Utils.builtInFilters, // Reference to built-in filters
enableLogging: true, // Enable or disable logging
};
// Remove forbidden headers from default headers
this.forbiddenHeaders = new Set([
'Accept-Charset',
'Accept-Encoding',
'Access-Control-Request-Headers',
'Access-Control-Request-Method',
'Connection',
'Content-Length',
'Cookie',
'Cookie2',
'Date',
'DNT',
'Expect',
'Host',
'Keep-Alive',
'Origin',
'Referer',
'TE',
'Trailer',
'Transfer-Encoding',
'Upgrade',
'Via',
'Proxy-',
'Sec-',
'User-Agent',
'Accept-Language',
]);
// Merge and validate default settings with user options
this.settings = Utils.deepMerge({}, this.defaultSettings, options);
this._validateSettings(this.settings);
// Initialize per-URL settings
this.urlSettings = new Map();
this.storage = new StorageHandler();
this.rateLimiter = new RateLimiter(this.settings.rateLimitRequests);
this.cache = null;
this.errorHandlers = new Map();
// Initialize cache and wait for initialization
if (this.settings.useCache) {
this.cachePromise = Utils.initCache().then(cache => {
this.cache = cache;
});
} else {
this.cachePromise = Promise.resolve();
}
// Initialize logger
this.logger = this.settings.logger || console;
// Hostname to last request time mapping for politeness
this.hostLastRequestTime = new Map();
}
_validateSettings(settings) {
const positiveNumberProps = [
'rateLimitRequests',
'maxConcurrent',
'retries',
'retryDelay',
'maxRetryDelay',
'cacheTime',
'timeout',
'maxResponseSize',
'politenessDelay',
'maxRetriesOnAntiScraping',
];
positiveNumberProps.forEach(prop => {
if (typeof settings[prop] !== 'number' || settings[prop] < 0) {
throw new Error(`Invalid setting "${prop}": must be a positive number`);
}
});
const allowedStrategies = ['auto', 'parallel', 'sequential'];
if (!allowedStrategies.includes(settings.concurrencyStrategy)) {
throw new Error(
`Invalid concurrency strategy: "${settings.concurrencyStrategy}". Allowed values are ${allowedStrategies.join(
', '
)}`
);
}
}
/**
* Main scraping method
*/
async scrape(configs, globalOptions = {}) {
await this.cachePromise;
if (typeof configs !== 'object' || configs === null || Array.isArray(configs)) {
throw new Error('Configs must be an object with URL keys and configuration values');
}
// Merge global options into settings
const globalSettings = Utils.deepMerge({}, this.settings, globalOptions);
this._validateSettings(globalSettings);
const results = {};
const urls = Object.keys(configs);
if (urls.length === 0) {
return results;
}
await this._log('Starting scraping process', { urls });
// Prepare per-domain request queues to respect politeness
this.domainQueues = {};
switch (globalSettings.concurrencyStrategy) {
case 'parallel':
await Promise.all(
urls.map(url => this._scrapeUrlWrapper(url, configs[url], globalSettings, results))
);
break;
case 'sequential':
for (const url of urls) {
await this._scrapeUrlWrapper(url, configs[url], globalSettings, results);
}
break;
case 'auto':
default:
const chunks = Utils.chunkArray(urls, globalSettings.maxConcurrent);
for (const chunk of chunks) {
await Promise.all(
chunk.map(url => this._scrapeUrlWrapper(url, configs[url], globalSettings, results))
);
}
break;
}
await this._log('Completed scraping process', { results });
return results;
}
/**
* Wrapper for processing a URL, handles logging and error handling
*/
async _scrapeUrlWrapper(url, config, globalSettings, results) {
try {
const data = await this._processUrl(url, config, globalSettings);
results[url] = data;
} catch (error) {
const handler = this.errorHandlers.get(url) || this.errorHandlers.get('default');
if (handler) {
try {
const data = await handler(error, url);
results[url] = data;
} catch (e) {
results[url] = { error: e.message };
}
} else {
results[url] = { error: error.message };
}
await this._log(`Error scraping ${url}: ${error.message}`, { error });
}
}
/**
* Process a single URL with retries and error handling
*/
async _processUrl(url, config, globalSettings) {
if (!Utils.isValidURL(url)) {
throw new Error(`Invalid URL: ${url}`);
}
// Merge global settings with per-URL settings using deep merge
const settings = Utils.deepMerge({}, globalSettings, config.options);
this._validateSettings(settings);
this.urlSettings.set(url, settings);
if (settings.validateSelectors) {
this._validateSelectors(config.selectors);
}
// Wait for cache initialization if needed
if (settings.useCache) {
await this.cachePromise;
if (this.cache) {
const cachedData = await this._getFromCache(url, settings);
if (cachedData) {
await this._log(`Using cached data for ${url}`);
return cachedData;
}
}
}
let attempts = 0;
let antiScrapingAttempts = 0;
while (attempts < settings.retries) {
attempts += 1;
await this._respectPoliteness(url, settings);
await this.rateLimiter.acquireToken();
const options = this._prepareFetchOptions(config, settings);
try {
const response = await this._fetchWithTimeout(url, options, settings);
if (!response.ok) {
if (
response.status === 304 &&
settings.useCache &&
this.cache &&
settings.sinceModified
) {
const cachedData = await this._getFromCache(url, settings);
if (cachedData) {
return cachedData;
}
}
throw new Error(`HTTP error: ${response.status} ${response.statusText}`);
}
const contentType = response.headers.get('Content-Type') || '';
if (!contentType.includes('text/html')) {
throw new Error(`Invalid Content-Type: ${contentType}`);
}
const data = await this._extractDataFromResponse(response, config.selectors, settings);
if (settings.outputFormat !== 'json') {
await this.storage.store(url, data, settings.outputFormat, { customStore: settings.customStore });
}
if (settings.useCache && this.cache) {
await this._storeInCache(url, data, settings);
}
return data;
} catch (error) {
if (
Utils.detectAntiScraping(error.message) &&
antiScrapingAttempts < settings.maxRetriesOnAntiScraping
) {
antiScrapingAttempts += 1;
await this._log(
`Detected anti-scraping measures on ${url}. Attempt ${antiScrapingAttempts}`,
{ error }
);
// Delay before retrying
await Utils.delay(settings.retryDelay * antiScrapingAttempts);
continue;
}
if (attempts >= settings.retries) {
throw error;
}
// Exponential backoff with jitter
const delay = Math.min(
settings.retryDelay * Math.pow(2, attempts - 1) * (0.5 + Math.random() * 0.5),
settings.maxRetryDelay
);
await this._log(`Retrying ${url} after delay of ${delay}ms`, { attempt: attempts, error });
await Utils.delay(delay);
}
}
}
/**
* Respects politeness policies by delaying requests to the same domain
*/
async _respectPoliteness(url, settings) {
const urlObj = new URL(url);
const hostname = urlObj.hostname;
const now = Date.now();
const lastRequestTime = this.hostLastRequestTime.get(hostname) || 0;
const elapsed = now - lastRequestTime;
if (elapsed < settings.politenessDelay) {
const delay = settings.politenessDelay - elapsed;
await this._log(`Delaying request to ${url} by ${delay}ms for politeness`);
await Utils.delay(delay);
}
this.hostLastRequestTime.set(hostname, Date.now());
}
/**
* Validates the CSS selectors
*/
_validateSelectors(selectors) {
if (typeof selectors !== 'object' || selectors === null) {
throw new Error('Selectors must be an object');
}
for (const key of Object.keys(selectors)) {
const sel = selectors[key];
if (typeof sel === 'string') {
Utils.normalizeSelector(sel);
} else if (sel && typeof sel.selector === 'string') {
Utils.normalizeSelector(sel.selector);
} else if (sel && typeof sel.customSelector === 'function') {
// Custom selector function, assume valid
} else if (sel && typeof sel.xpath === 'string') {
// Could validate XPath here if needed
} else {
throw new Error(`Invalid selector configuration for key "${key}"`);
}
}
}
/**
* Prepares fetch options including headers
*/
_prepareFetchOptions(config, settings) {
const defaultHeaders = {
// Cannot set User-Agent or Accept-Language in browser fetch API
};
// Merge headers, prioritizing per-config and settings over defaults
const headersInit = Utils.deepMerge({}, defaultHeaders, settings.headers, config.headers);
// Remove forbidden headers
Object.keys(headersInit).forEach(header => {
if (
this.forbiddenHeaders.has(header) ||
Array.from(this.forbiddenHeaders).some(fh => fh.endsWith('-') && header.startsWith(fh))
) {
delete headersInit[header];
}
});
const headers = new Headers(headersInit);
if (settings.sinceModified) {
const lastModified = new Date(Date.now() - settings.cacheTime).toUTCString();
headers.set('If-Modified-Since', lastModified);
}
const options = {
method: 'GET',
headers: headers,
mode: 'cors',
credentials: 'omit',
redirect: 'follow',
};
return options;
}
/**
* Performs a fetch request with a timeout, using proxy if configured
*/
async _fetchWithTimeout(url, options, settings) {
const controller = new AbortController();
options.signal = controller.signal;
const timeoutId = setTimeout(() => {
controller.abort();
}, settings.timeout);
try {
let fetchUrl = url;
if (settings.useProxy) {
if (!settings.proxyUrl) {
throw new Error('Proxy URL is not set');
}
fetchUrl = `${settings.proxyUrl}?url=${encodeURIComponent(url)}`;
}
const fetchFunction = settings.customFetch || fetch;
const response = await fetchFunction(fetchUrl, options);
clearTimeout(timeoutId);
// Check if response size exceeds maxResponseSize
const contentLength = response.headers.get('Content-Length');
if (contentLength && parseInt(contentLength, 10) > settings.maxResponseSize) {
throw new Error('Response size exceeds maximum allowed');
}
return response;
} catch (error) {
clearTimeout(timeoutId);
if (error.name === 'AbortError') {
throw new Error('Request timed out');
}
throw error;
}
}
/**
* Enhanced data extraction with advanced filters
*/
async _extractDataFromResponse(response, selectors, settings) {
try {
const decoder = new TextDecoder('utf-8');
const reader = response.body.getReader();
let receivedLength = 0;
let chunks = [];
while (true) {
const { done, value } = await reader.read();
if (done) break;
receivedLength += value.length;
if (receivedLength > settings.maxResponseSize) {
throw new Error('Response size exceeds maximum allowed');
}
chunks.push(value);
}
const htmlText = decoder.decode(Utils._concatChunks(chunks));
// Check for anti-scraping mechanisms
if (Utils.detectAntiScraping(htmlText)) {
throw new Error('Detected anti-scraping measures in the response');
}
// Remove scripts and styles to prevent XSS and reduce parsing overhead
const sanitizedHtml = Utils.removeScriptsAndStyles(htmlText);
const parser = new DOMParser();
const doc = parser.parseFromString(sanitizedHtml, 'text/html');
const data = {};
for (const key of Object.keys(selectors)) {
const selConfig = selectors[key];
let elements;
if (typeof selConfig === 'string') {
elements = doc.querySelectorAll(selConfig);
} else if (selConfig && typeof selConfig.selector === 'string') {
elements = doc.querySelectorAll(selConfig.selector);
} else if (selConfig && typeof selConfig.customSelector === 'function') {
elements = selConfig.customSelector(doc);
if (!Array.isArray(elements)) {
throw new Error(`customSelector for key "${key}" must return an array of elements`);
}
} else if (selConfig && typeof selConfig.xpath === 'string') {
elements = this._selectXPath(doc, selConfig.xpath);
} else {
throw new Error(`Invalid selector configuration for key "${key}"`);
}
if (!elements || elements.length === 0) {
data[key] = [];
continue;
}
data[key] = await Promise.all(
Array.from(elements).map(async element => {
let value;
if (selConfig.attr) {
value = element.getAttribute(selConfig.attr) || '';
} else if (selConfig.html) {
value = element.innerHTML;
} else {
value = Utils.extractText(element, { preserveFormatting: settings.preserveFormatting });
}
// Apply per-selector filters (support chaining)
if (selConfig.filters) {
value = await this._applyFilters(value, selConfig.filters, element, {
key,
settings,
});
}
// Apply global filters if no per-selector filters
else if (settings.filters && settings.filters[key]) {
value = await this._applyFilters(value, settings.filters[key], element, {
key,
settings,
});
}
return value;
})
);
data[key] = data[key].filter(val => val !== undefined && val !== null);
if (selConfig.single) {
data[key] = data[key][0] || null;
}
}
return data;
} catch (e) {
throw new Error(`Failed to extract data: ${e.message}`);
}
}
/**
* Apply filters to a value, supporting chaining and built-in filters
*/
async _applyFilters(value, filters, element, context) {
if (!filters) return value;
const filtersArray = Array.isArray(filters) ? filters : [filters];
for (let filter of filtersArray) {
if (typeof filter === 'string') {
// Built-in filter by name
const builtInFilter = this._parseFilterString(filter);
if (typeof builtInFilter === 'function') {
value = builtInFilter(value, element, context);
} else {
console.warn(`Built-in filter "${filter}" not found`);
}
} else if (typeof filter === 'function') {
try {
const filterResult = filter(value, element, context);
value = filterResult instanceof Promise ? await filterResult : filterResult;
} catch (e) {
console.error(`Error in filter for key "${context.key}":`, e);
value = null;
break;
}
} else if (filter instanceof RegExp) {
// Apply regular expression
const match = value.match(filter);
value = match ? match[0] : null;
} else {
console.warn(`Invalid filter type for key "${context.key}"`);
}
// If value becomes null or undefined, stop processing filters
if (value == null) {
break;
}
}
return value;
}
/**
* Parse filter string to support filter arguments safely
*/
_parseFilterString(filterStr) {
// Example filter string: 'replace(/foo/g, "bar")'
const filterPattern = /^(\w+)\((.*)\)$/;
const match = filterStr.match(filterPattern);
if (match) {
const filterName = match[1];
const argsStr = match[2];
const args = this._parseFilterArgs(argsStr);
const filterFunc = this.settings.builtInFilters[filterName];
if (typeof filterFunc === 'function') {
return (value, element, context) => filterFunc(value, args, element, context);
}
} else {
const filterFunc = this.settings.builtInFilters[filterStr];
if (typeof filterFunc === 'function') {
return filterFunc;
}
}
return null;
}
/**
* Parse filter arguments from string safely
*/
_parseFilterArgs(argsStr) {
try {
const args = [];
let currentArg = '';
let inQuotes = false;
let quoteChar = '';
let escapeNext = false;
for (let i = 0; i < argsStr.length; i++) {
const char = argsStr.charAt(i);
if (escapeNext) {
currentArg += char;
escapeNext = false;
continue;
}
if (char === '\\') {
escapeNext = true;
continue;
}
if (inQuotes) {
if (char === quoteChar) {
inQuotes = false;
}
currentArg += char;
continue;
}
if (char === '"' || char === "'") {
inQuotes = true;
quoteChar = char;
currentArg += char;
continue;
}
if (char === ',' && !inQuotes) {
args.push(this._parseArgument(currentArg.trim()));
currentArg = '';
continue;
}
currentArg += char;
}
if (currentArg) {
args.push(this._parseArgument(currentArg.trim()));
}
return args;
} catch (e) {
console.error('Failed to parse filter arguments:', e);
return [];
}
}
_parseArgument(arg) {
if ((arg.startsWith('"') && arg.endsWith('"')) || (arg.startsWith("'") && arg.endsWith("'"))) {
// String argument
return arg.slice(1, -1).replace(/\\'/g, "'").replace(/\\"/g, '"');
}
// Try to parse as number
const num = Number(arg);
if (!isNaN(num)) {
return num;
}
// For regex arguments, e.g., /pattern/flags
if (arg.startsWith('/') && arg.lastIndexOf('/') > 0) {
const lastSlash = arg.lastIndexOf('/');
const pattern = arg.slice(1, lastSlash);
const flags = arg.slice(lastSlash + 1);
return new RegExp(pattern, flags);
}
// Otherwise, return the arg as string
return arg;
}
/**
* Select elements using XPath
*/
_selectXPath(doc, xpathExpression) {
const iterator = doc.evaluate(xpathExpression, doc, null, XPathResult.ORDERED_NODE_ITERATOR_TYPE, null);
const nodes = [];
let node = iterator.iterateNext();
while (node) {
nodes.push(node);
node = iterator.iterateNext();
}
return nodes;
}
/**
* Concatenates array of Uint8Arrays into one Uint8Array
*/
static _concatChunks(chunks) {
let totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
let result = new Uint8Array(totalLength);
let position = 0;
for (let chunk of chunks) {
result.set(chunk, position);
position += chunk.length;
}
return result;
}
/**
* Gets data from the cache if available and not expired
*/
async _getFromCache(url, settings) {
if (!this.cache) return null;
const cacheKey = Utils.generateCacheKey(url, settings);
const cachedResponse = await this.cache.match(cacheKey);
if (cachedResponse) {
try {
const cachedData = await cachedResponse.json();
const age = Date.now() - cachedData.timestamp;
if (age < settings.cacheTime) {
return cachedData.data;
}
} catch (e) {
console.warn('Error reading from cache:', e);
}
}
return null;
}
/**
* Stores data in the cache
*/
async _storeInCache(url, data, settings) {
if (!this.cache) return;
const cacheKey = Utils.generateCacheKey(url, settings);
const cachedResponse = new Response(
JSON.stringify({ data, timestamp: Date.now() }),
{
headers: { 'Content-Type': 'application/json' },
}
);
try {
await this.cache.put(cacheKey, cachedResponse);
} catch (e) {
console.error('Failed to store data in cache:', e);
}
}
/**
* Registers a custom error handler
*/
onError(urlOrDefault, handler) {
if (typeof handler !== 'function') {
throw new Error('Error handler must be a function');
}
this.errorHandlers.set(urlOrDefault, handler);
}
/**
* Logs messages using the provided logger
*/
async _log(message, data = {}) {
if (!this.settings.enableLogging) return;
if (typeof this.logger === 'function') {
await this.logger(message, data);
} else if (this.logger && typeof this.logger.log === 'function') {
await this.logger.log(message, data);
}
}
}
// Attach ScraperLib to the global object
global.ScraperLib = ScraperLib;
})(typeof window !== 'undefined' ? window : this);
```