ASSISTANT
```javascript
// ==UserScript==
// @name Advanced Web Crawler
// @namespace http://tampermonkey.net/
// @version 1.0
// @description Comprehensive web crawler with UI for data collection and management
// @author Your Name
// @match *://*/*
// @grant GM_setValue
// @grant GM_getValue
// @grant GM_addStyle
// @grant GM_xmlhttpRequest
// @connect *
// @require https://code.jquery.com/jquery-3.6.0.min.js
// ==/UserScript==
(function () {
'use strict';
// =========================
// Utility Functions
// =========================
// Log Function
function log(message, type = 'info') {
const timestamp = new Date().toLocaleString();
const logEntry = `[${timestamp}] [${type.toUpperCase()}]: ${message}`;
$('#crawler-log').append(`<div>${logEntry}</div>`);
// Save to persistent storage if needed
}
// Duplicate Check
function isDuplicate(url) {
const visited = GM_getValue('visited', []);
return visited.includes(url);
}
function markVisited(url) {
let visited = GM_getValue('visited', []);
if (!visited.includes(url)) {
visited.push(url);
GM_setValue('visited', visited);
}
}
// Delay Function
function delay(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
// =========================
// UI Creation
// =========================
// Append Styles
GM_addStyle(`
#crawler-ui {
position: fixed;
top: 10px;
right: 10px;
width: 400px;
height: 600px;
background: #f9f9f9;
border: 2px solid #333;
z-index: 10000;
font-family: Arial, sans-serif;
display: flex;
flex-direction: column;
}
#crawler-tabs {
display: flex;
background: #333;
}
.crawler-tab {
flex: 1;
padding: 10px;
color: #fff;
text-align: center;
cursor: pointer;
}
.crawler-tab.active {
background: #555;
}
#crawler-content {
flex: 1;
padding: 10px;
overflow: auto;
background: #fff;
}
.hidden { display: none; }
#crawler-progress {
width: 100%;
background: #ddd;
height: 20px;
margin: 10px 0;
border-radius: 5px;
overflow: hidden;
}
#crawler-progress-bar {
height: 100%;
width: 0%;
background: #4caf50;
transition: width 0.3s;
}
table {
width: 100%;
border-collapse: collapse;
}
table, th, td {
border: 1px solid #aaa;
}
th, td {
padding: 5px;
text-align: left;
}
#crawler-data th {
background: #f1f1f1;
}
/* Scrollbar Styling */
#crawler-content::-webkit-scrollbar {
width: 8px;
}
#crawler-content::-webkit-scrollbar-thumb {
background: #888;
border-radius: 4px;
}
#crawler-content::-webkit-scrollbar-thumb:hover {
background: #555;
}
/* Modal Styles */
.modal {
display: none;
position: fixed;
z-index: 10001;
left: 0;
top: 0;
width: 100%;
height: 100%;
overflow: auto;
background-color: rgba(0,0,0,0.4);
}
.modal-content {
background-color: #fefefe;
margin: 10% auto;
padding: 20px;
border: 1px solid #888;
width: 80%;
height: 80%;
overflow: auto;
}
.close-modal {
color: #aaa;
float: right;
font-size: 28px;
font-weight: bold;
cursor: pointer;
}
`);
// Create UI Elements
const uiHTML = `
<div id="crawler-ui">
<div id="crawler-tabs">
<div class="crawler-tab active" data-tab="log">📜 로그</div>
<div class="crawler-tab" data-tab="data">📊 데이터</div>
<div class="crawler-tab" data-tab="settings">⚙️ 설정</div>
</div>
<div id="crawler-content">
<div id="log-content" class="tab-content">
<div id="crawler-log" style="max-height: 550px; overflow-y: auto;"></div>
</div>
<div id="data-content" class="tab-content hidden">
<table id="crawler-data">
<thead>
<tr>
<th>사이트 이름</th>
<th>주소</th>
<th>설명</th>
<th>HTML 내용</th>
<th>텍스트 내용</th>
<th>분류</th>
</tr>
</thead>
<tbody></tbody>
</table>
</div>
<div id="settings-content" class="tab-content hidden">
<div>
<label for="crawl-delay">크롤링 딜레이 (ms):</label>
<input type="number" id="crawl-delay" value="3000" min="1000">
</div>
<div>
<label for="blacklist">블랙리스트 도메인 (쉼표로 구분):</label>
<input type="text" id="blacklist" placeholder="예: ads.example.com,tracker.com">
</div>
<div>
<label for="whitelist">화이트리스트 도메인 (쉼표로 구분):</label>
<input type="text" id="whitelist" placeholder="예: trusted.com,educational.org">
</div>
<button id="save-settings">설정 저장 (Ctrl+S)</button>
</div>
</div>
<div id="crawler-progress">
<div id="crawler-progress-bar"></div>
</div>
<div style="display: flex; justify-content: space-around; padding: 5px;">
<button id="start-crawl" title="크롤링 시작 (Ctrl+P)">시작</button>
<button id="stop-crawl" title="크롤링 중지 (Ctrl+Q)">중지</button>
<button id="reset-crawl" title="크롤링 초기화">초기화</button>
<button id="export-csv" title="CSV 다운로드 (Ctrl+D)">CSV</button>
<button id="export-json" title="JSON 다운로드 (Ctrl+J)">JSON</button>
</div>
</div>
<!-- Modal for HTML Content -->
<div id="html-modal" class="modal">
<div class="modal-content">
<span class="close-modal">×</span>
<pre id="modal-html-content" style="white-space: pre-wrap;"></pre>
</div>
</div>
`;
$('body').append(uiHTML);
// =========================
// Tab Functionality
// =========================
$('.crawler-tab').on('click', function () {
$('.crawler-tab').removeClass('active');
$(this).addClass('active');
const selectedTab = $(this).data('tab');
$('.tab-content').addClass('hidden');
$(`#${selectedTab}-content`).removeClass('hidden');
});
// =========================
// Modal Functionality
// =========================
// Open Modal
$(document).on('click', '#crawler-data td.html-cell', function () {
const htmlContent = $(this).data('html');
$('#modal-html-content').text(htmlContent);
$('#html-modal').css('display', 'block');
});
// Close Modal
$('.close-modal').on('click', function () {
$('#html-modal').css('display', 'none');
});
// Close Modal on Outside Click
$(window).on('click', function (event) {
if (event.target.id === 'html-modal') {
$('#html-modal').css('display', 'none');
}
});
// =========================
// Settings Functionality
// =========================
$('#save-settings').on('click', function () {
const delay = parseInt($('#crawl-delay').val());
const blacklist = $('#blacklist').val().split(',').map(s => s.trim()).filter(s => s);
const whitelist = $('#whitelist').val().split(',').map(s => s.trim()).filter(s => s);
GM_setValue('crawl-delay', delay);
GM_setValue('blacklist', blacklist);
GM_setValue('whitelist', whitelist);
log('설정이 저장되었습니다.', 'info');
});
// =========================
// Crawling Logic
// =========================
let isCrawling = false;
let crawlQueue = [];
let totalToCrawl = 0;
let crawledCount = 0;
$('#start-crawl').on('click', startCrawling);
$('#stop-crawl').on('click', stopCrawling);
$('#reset-crawl').on('click', resetCrawling);
$('#export-csv').on('click', exportCSV);
$('#export-json').on('click', exportJSON);
// Keyboard Shortcuts
$(document).on('keydown', function (e) {
if (e.ctrlKey && e.key === 'p') {
startCrawling();
e.preventDefault();
}
if (e.ctrlKey && e.key === 'q') {
stopCrawling();
e.preventDefault();
}
if (e.ctrlKey && e.key === 's') {
$('#save-settings').click();
e.preventDefault();
}
if (e.ctrlKey && e.key === 'd') {
exportCSV();
e.preventDefault();
}
if (e.ctrlKey && e.key === 'j') {
exportJSON();
e.preventDefault();
}
if (e.ctrlKey && e.key === '1') {
$('[data-tab="log"]').click();
e.preventDefault();
}
if (e.ctrlKey && e.key === '2') {
$('[data-tab="data"]').click();
e.preventDefault();
}
if (e.ctrlKey && e.key === '3') {
$('[data-tab="settings"]').click();
e.preventDefault();
}
});
async function startCrawling() {
if (isCrawling) {
log('크롤링이 이미 진행 중입니다.', 'warning');
return;
}
const query = prompt('검색어를 입력하세요:');
if (!query) {
log('검색어가 입력되지 않았습니다.', 'error');
return;
}
isCrawling = true;
crawledCount = 0;
$('#crawler-data tbody').empty();
crawlQueue = [];
totalToCrawl = 0;
log(`크롤링을 시작합니다: "${query}"`, 'info');
// Example: Replace with actual search URL and parsing logic
// For demonstration, we'll assume a hypothetical search result structure
let currentPage = 1;
const maxPages = 5; // Define how many pages to crawl
while (isCrawling && currentPage <= maxPages) {
const searchUrl = `https://www.example.com/search?q=${encodeURIComponent(query)}&page=${currentPage}`;
log(`검색 페이지 로드 중: ${searchUrl}`, 'info');
// Fetch search results page
GM_xmlhttpRequest({
method: 'GET',
url: searchUrl,
onload: function (response) {
if (response.status !== 200) {
log(`검색 페이지 로드 실패: ${searchUrl}`, 'error');
return;
}
// Parse HTML response
const parser = new DOMParser();
const doc = parser.parseFromString(response.responseText, 'text/html');
// Extract search results - modify selectors based on actual structure
const results = doc.querySelectorAll('.search-result');
results.forEach(result => {
const title = result.querySelector('.result-title') ? result.querySelector('.result-title').innerText : 'N/A';
const link = result.querySelector('.result-link') ? result.querySelector('.result-link').href : 'N/A';
const description = result.querySelector('.result-description') ? result.querySelector('.result-description').innerText : 'N/A';
crawlQueue.push({ title, link, description });
});
},
onerror: function () {
log(`검색 페이지 요청 에러: ${searchUrl}`, 'error');
}
});
// Wait for the request to complete
await delay(GM_getValue('crawl-delay', 3000));
currentPage++;
}
totalToCrawl = crawlQueue.length;
log(`총 크롤링 할 사이트 수: ${totalToCrawl}`, 'info');
updateProgress();
// Start crawling the queue
while (isCrawling && crawlQueue.length > 0) {
const site = crawlQueue.shift();
if (isBlacklisted(site.link) || !isWhitelisted(site.link)) {
log(`블랙리스트에 포함되거나 화이트리스트에 없는 사이트: ${site.link}`, 'warning');
continue;
}
if (isDuplicate(site.link)) {
log(`중복 방문: ${site.link}`, 'warning');
continue;
}
log(`사이트 방문 중: ${site.link}`, 'info');
// Fetch site HTML
GM_xmlhttpRequest({
method: 'GET',
url: site.link,
onload: function (response) {
if (response.status !== 200) {
log(`사이트 로드 실패: ${site.link}`, 'error');
return;
}
const siteHTML = response.responseText;
const siteDoc = new DOMParser().parseFromString(siteHTML, 'text/html');
const siteText = siteDoc.body ? siteDoc.body.innerText.substring(0, 500) : 'N/A'; // Truncate for display
// Evaluate content quality
const quality = evaluateContentQuality(siteText);
// Classify keywords
const category = classifyKeywords(site.description);
// Add to table
addDataRow(site.title, site.link, site.description, siteHTML, siteText, category);
markVisited(site.link);
crawledCount++;
updateProgress();
},
onerror: function () {
log(`사이트 요청 에러: ${site.link}`, 'error');
}
});
// Delay between requests
await delay(GM_getValue('crawl-delay', 3000));
}
if (crawledCount >= totalToCrawl) {
log('크롤링이 완료되었습니다.', 'success');
isCrawling = false;
}
}
function stopCrawling() {
if (!isCrawling) {
log('크롤링이 진행 중이지 않습니다.', 'warning');
return;
}
isCrawling = false;
log('크롤링이 중지되었습니다.', 'info');
}
function resetCrawling() {
stopCrawling();
crawledCount = 0;
totalToCrawl = 0;
crawlQueue = [];
$('#crawler-data tbody').empty();
GM_setValue('visited', []);
log('크롤링 초기화가 완료되었습니다.', 'info');
updateProgress();
}
function updateProgress() {
const percent = totalToCrawl === 0 ? 0 : ((crawledCount / totalToCrawl) * 100).toFixed(2);
$('#crawler-progress-bar').css('width', `${percent}%`);
}
function addDataRow(name, url, description, html, text, category) {
const row = `<tr>
<td>${escapeHtml(name)}</td>
<td><a href="${escapeHtml(url)}" target="_blank">${escapeHtml(url)}</a></td>
<td>${escapeHtml(description)}</td>
<td class="html-cell" data-html="${escapeHtml(html)}">보기</td>
<td>${escapeHtml(text)}</td>
<td>${escapeHtml(category)}</td>
</tr>`;
$('#crawler-data tbody').append(row);
}
function escapeHtml(text) {
return text.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">");
}
// =========================
// Additional Functionalities
// =========================
function isBlacklisted(url) {
const blacklist = GM_getValue('blacklist', []);
return blacklist.some(domain => url.includes(domain));
}
function isWhitelisted(url) {
const whitelist = GM_getValue('whitelist', []);
if (whitelist.length === 0) return true; // If whitelist is empty, allow all
return whitelist.some(domain => url.includes(domain));
}
function evaluateContentQuality(text) {
const wordCount = text.split(/\s+/).length;
if (wordCount > 1000) return 'High';
if (wordCount > 500) return 'Medium';
return 'Low';
}
function classifyKeywords(description) {
const keywords = {
'Education': ['school', 'university', 'course', 'learning'],
'Technology': ['tech', 'software', 'hardware', 'AI'],
'Commerce': ['shop', 'buy', 'store', 'market'],
// Add more categories and keywords as needed
};
for (const [category, kwList] of Object.entries(keywords)) {
for (const kw of kwList) {
if (description.toLowerCase().includes(kw)) {
return category;
}
}
}
return 'Uncategorized';
}
// =========================
// Export Functions
// =========================
function exportCSV() {
let csv = '사이트 이름,주소,설명,HTML 내용,텍스트 내용,분류\n';
$('#crawler-data tbody tr').each(function () {
const cols = $(this).find('td');
const row = [];
cols.each(function () {
let text = $(this).text().replace(/"/g, '""');
row.push(`"${text}"`);
});
csv += row.join(',') + '\n';
});
downloadFile(csv, 'data.csv', 'text/csv');
}
function exportJSON() {
const data = [];
$('#crawler-data tbody tr').each(function () {
const cols = $(this).find('td');
data.push({
name: $(cols[0]).text(),
url: $(cols[1]).find('a').attr('href'),
description: $(cols[2]).text(),
html: $(cols[3]).data('html'),
text: $(cols[4]).text(),
category: $(cols[5]).text(),
});
});
const json = JSON.stringify(data, null, 2);
downloadFile(json, 'data.json', 'application/json');
}
function downloadFile(content, filename, mimeType) {
const blob = new Blob([content], { type: mimeType });
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = filename;
a.click();
URL.revokeObjectURL(url);
}
// =========================
// Initialization
// =========================
// Load settings if any
$(document).ready(function () {
$('#crawl-delay').val(GM_getValue('crawl-delay', 3000));
$('#blacklist').val(GM_getValue('blacklist', []).join(', '));
$('#whitelist').val(GM_getValue('whitelist', []).join(', '));
});
// =========================
// Prevent Crawler Reset on Refresh
// =========================
window.addEventListener('beforeunload', function (e) {
if (isCrawling) {
e.preventDefault();
e.returnValue = '';
// Optionally save state here
}
});
// Optionally, reset crawling on page load
resetCrawling();
})();
```