USER
창을 잘 관리 할 수 있게 창 조절 기능을 추가하셈.. ㄱㄱ 그냥 옆에꺼도 볼 수 있게 밑에 옆으로 가게 하는 그런 스크롤바 같은 거 있잖아 그런 거
여기 크롤링 코드 보면 다음 페이지 넘어간 뒤에 자동으로 크롤링이 되어야 하는데 다음페이지로 넘어가니깐 UI 자동으로 꺼짐.. 이거 바탕으로 밑에걸 개선해 // ==UserScript==
// @name Advanced Web Crawler Enhanced Pro Optimized v6.3.1
// @namespace http://tampermonkey.net/
// @version 6.3.1
// @description Further optimized and enhanced version with responsive design and other improvements.
// @author
// @match https://www.google.com/search*
// @match https://www.bing.com/search*
// @match https://search.yahoo.com/search*
// @match https://duckduckgo.com/*
// @match https://www.baidu.com/s*
// @match https://www.yandex.com/search/*
// @match https://search.naver.com/search*
// @match https://search.daum.net/search*
// @match https://www.ask.com/*
// @match https://www.ecosia.org/search*
// @match https://www.startpage.com/*
// @match https://searx.me/*
// @match https://www.dogpile.com/*
// @match https://www.bing.com/images*
// @grant GM_xmlhttpRequest
// @grant GM_setValue
// @grant GM_getValue
// @grant GM_deleteValue
// @connect *
// @require https://cdnjs.cloudflare.com/ajax/libs/jquery/3.6.0/jquery.min.js
// @require https://cdn.jsdelivr.net/npm/chart.js
// @require https://cdnjs.cloudflare.com/ajax/libs/xlsx/0.18.5/xlsx.full.min.js
// ==/UserScript==
(function() {
'use strict';
/* ============================
Module: Config
============================ */
const Config = (function() {
const DEFAULT_SETTINGS = {
delay: 2000,
delayRandom: 500,
maxPages: 3,
userAgents: [
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15',
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko)',
'Mozilla/5.0 (iPhone; CPU iPhone OS 14_0 like Mac OS X) AppleWebKit/605.1.15',
'Mozilla/5.0 (iPad; CPU OS 14_0 like Mac OS X) AppleWebKit/605.1.15',
'AdvancedWebCrawler/6.3 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
],
userAgentIndex: 0,
blacklist: [],
whitelist: [],
contentTypes: {
include: ['text/html'],
exclude: ['application/pdf', 'image/jpeg', 'image/png', 'video/mp4']
},
timeout: 8000,
includeKeywords: [],
excludeKeywords: [],
language: [],
minWordCount: 100,
maxWordCount: 10000,
maxDepth: 2,
proxy: '',
regexInclude: [],
regexExclude: [],
dataFields: {
name: true,
url: true,
description: true,
html: true,
category: true,
quality: true,
language: true,
wordCount: true,
metaTags: false,
headings: false
},
schedule: {
enabled: false,
startTime: '00:00',
interval: 3600000
},
searchEngines: {
google: true,
bing: true,
yahoo: true,
duckduckgo: true,
baidu: true,
yandex: true,
naver: true,
daum: true,
ask: true,
ecosia: true,
startpage: true,
searx: true,
dogpile: true,
bingimages: true
},
retryAttempts: 3,
retryDelay: 2000,
respectRobots: true,
maxConcurrentRequests: 3,
dataVisualization: true,
exportFormats: {
csv: true,
json: true,
xml: true,
xls: true
},
advancedLogging: true,
enableCaptchas: false,
customKeywords: {},
templates: {},
currentTab: 'logs',
isCrawling: false,
darkMode: false
};
let settings = { ...DEFAULT_SETTINGS, ...getSavedSettings() };
function getSavedSettings() {
return GM_getValue('settings', DEFAULT_SETTINGS);
}
function saveSettings(newSettings) {
GM_setValue('settings', newSettings);
settings = { ...settings, ...newSettings };
}
function resetSettings() {
saveSettings(DEFAULT_SETTINGS);
settings = { ...DEFAULT_SETTINGS };
}
function saveTemplate(name) {
settings.templates[name] = { ...settings };
saveSettings(settings);
Logger.log(`설정 템플릿 '${name}'이(가) 저장되었습니다.`, 'success');
}
function loadTemplate(name) {
if (settings.templates[name]) {
saveSettings(settings.templates[name]);
Logger.log(`설정 템플릿 '${name}'이(가) 불러와졌습니다.`, 'success');
} else {
Logger.log(`설정 템플릿 '${name}'을(를) 찾을 수 없습니다.`, 'error');
}
}
function validateProxy(proxyUrl) {
try {
const url = new URL(proxyUrl);
return url.protocol === 'http:' || url.protocol === 'https:';
} catch (e) {
return false;
}
}
return {
DEFAULT_SETTINGS,
get settings() { return settings; },
saveSettings,
resetSettings,
saveTemplate,
loadTemplate,
validateProxy
};
})();
/* ============================
Module: Logger
============================ */
const Logger = (function() {
let logs = JSON.parse(GM_getValue('logs', '[]'));
function log(message, type = 'info') {
const timestamp = new Date().toLocaleString();
logs.push({ timestamp, message, type });
if (logs.length > 1000) logs.shift();
GM_setValue('logs', JSON.stringify(logs));
UI.updateLogs();
if (Config.settings.advancedLogging) {
const styles = {
'info': 'color: blue;',
'warning': 'color: orange;',
'error': 'color: red;',
'success': 'color: green; font-weight: bold;',
'debug': 'color: purple;'
};
console.log(`%c[${timestamp}] [${type.toUpperCase()}] ${message}`, styles[type] || '');
}
}
function getLogs() {
return logs;
}
function clearLogs() {
logs = [];
GM_setValue('logs', JSON.stringify(logs));
UI.updateLogs();
}
return {
log,
getLogs,
clearLogs
};
})();
/* ============================
Module: Utils
============================ */
const Utils = (function() {
function escape(str, type = 'html') {
const map = {
html: { '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' },
csv: { '"': '""' },
xml: { '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' },
};
return String(str).replace(new RegExp(`[${Object.keys(map[type]).join('')}]`, 'g'), match => map[type][match]);
}
function capitalize(str) {
return str.charAt(0).toUpperCase() + str.slice(1);
}
function delay(ms) {
return new Promise(res => setTimeout(res, ms));
}
function generateColors(count) {
return Array.from({length: count}, (_, i) => `hsl(${i * (360 / count)}, 70%, 50%)`);
}
function parseCustomKeywords(text) {
const lines = text.split('\n').map(l => l.trim()).filter(l => l);
const ck = {};
lines.forEach(line => {
const [cat, kws] = line.split(':');
if (cat && kws) ck[cat.trim()] = kws.split(',').map(kw => kw.trim()).filter(kw => kw);
});
return ck;
}
function formatCustomKeywords(ck) {
return Object.entries(ck).map(([c, kws]) => `${c}:${kws.join(',')}`).join('\n');
}
function isValidURL(url) {
try { new URL(url); return true; }
catch { return false; }
}
return {
escape,
capitalize,
delay,
generateColors,
parseCustomKeywords,
formatCustomKeywords,
isValidURL
};
})();
/* ============================
Module: UI
============================ */
const UI = (function() {
function createUI() {
const style = `
#crawler-container {
position: fixed;
top: 10px;
right: 10px;
width: 90%;
max-width: 600px;
height: 80%;
background: var(--bg-color, #fff);
border: 1px solid #ccc;
z-index: 10000;
font-family: Arial, sans-serif;
box-shadow: 0 4px 8px rgba(0,0,0,0.2);
overflow: hidden;
resize: both;
border-radius: 8px;
min-width: 300px;
min-height: 300px;
}
#crawler-container.dark {
--bg-color: #2b2b2b;
color: #f1f1f1;
border-color: #555;
}
#crawler-header {
display: flex;
flex-direction: column;
justify-content: space-between;
align-items: flex-start;
background: var(--header-bg, #f1f1f1);
border-bottom: 1px solid #ccc;
padding: 10px;
cursor: move;
}
#crawler-container.dark #crawler-header {
--header-bg: #3c3c3c;
border-color: #555;
}
#crawler-minimize {
align-self: flex-end;
background: none;
border: none;
font-size: 18px;
cursor: pointer;
color: inherit;
}
#crawler-tabs {
display: flex;
flex-wrap: wrap;
width: 100%;
}
.crawler-tab { flex: 1 1 auto; padding: 10px; text-align: center; cursor: pointer; font-weight: bold; }
.crawler-tab.active { background: var(--active-tab-bg, #ddd); }
#crawler-container.dark .crawler-tab.active { background: #555; }
#crawler-content { padding: 10px; overflow-y: auto; height: calc(100% - 80px); }
table { width: 100%; border-collapse: collapse; }
th, td { padding: 8px; text-align: left; border-bottom: 1px solid #ddd; }
th { background-color: var(--th-bg, #f2f2f2); cursor: pointer; }
#crawler-container.dark th { background-color: #444; }
tr:hover { background-color: #f5f5f5; }
#crawler-container.dark tr:hover { background-color: #555; }
.progress-bar { width: 100%; background-color: #f3f3f3; border: 1px solid #ccc; border-radius: 5px; overflow: hidden; margin-bottom: 10px; }
.progress-bar-fill { height: 20px; background-color: #4caf50; width: 0%; transition: width 0.2s; }
.modal { display: none; position: fixed; z-index: 10001; left: 0; top: 0; width: 100%; height: 100%; overflow: auto; background-color: rgba(0,0,0,0.4); }
.modal-content { background-color: var(--modal-bg, #fff); margin: 5% auto; padding: 20px; border: 1px solid #888; width: 90%; max-width: 800px; max-height: 80%; overflow-y: auto; border-radius: 8px; }
#crawler-container.dark .modal-content { --modal-bg: #3c3c3c; color: #f1f1f1; border-color: #555; }
.close { color: #aaa; float: right; font-size: 28px; font-weight: bold; cursor: pointer; }
.close:hover, .close:focus { color: black; text-decoration: none; cursor: pointer; }
.control-buttons { margin: 10px 0; display: flex; flex-wrap: wrap; gap: 10px; }
.control-buttons button { padding: 8px 12px; cursor: pointer; border: none; background-color: #4caf50; color: white; border-radius: 4px; font-size: 14px; flex: 1 1 auto; min-width: 80px; }
.control-buttons button:disabled { background-color: #a5d6a7; cursor: not-allowed; }
.status-text { margin-top: 10px; font-weight: bold; font-size: 14px; }
.settings-section { margin-bottom: 15px; }
.settings-section label { display: block; margin-bottom: 5px; font-weight: bold; }
.settings-section input[type="text"], .settings-section input[type="number"], .settings-section textarea, .settings-section select { width: 100%; padding: 5px; box-sizing: border-box; border: 1px solid #ccc; border-radius: 4px; font-size: 14px; }
.settings-section.dark .settings-section input[type="text"], .settings-section.dark .settings-section input[type="number"], .settings-section.dark .settings-section textarea, .settings-section.dark .settings-section select {
background-color: #555;
color: #f1f1f1;
border-color: #777;
}
.settings-section .checkbox-group { display: flex; flex-wrap: wrap; gap: 10px; }
.settings-section .checkbox-group label { margin-right: 10px; font-size: 14px; }
.settings-section .radio-group { display: flex; flex-wrap: wrap; gap: 10px; }
.settings-section .radio-group label { margin-right: 10px; font-size: 14px; }
.tooltip { position: relative; display: inline-block; cursor: help; font-size: 14px; }
.tooltip .tooltiptext {
visibility: hidden;
width: 220px;
background-color: #555;
color: #fff;
text-align: center;
border-radius: 6px;
padding: 5px;
position: absolute;
z-index: 1;
bottom: 125%;
left: 50%;
margin-left: -110px;
opacity: 0;
transition: opacity 0.3s;
}
.tooltip:hover .tooltiptext { visibility: visible; opacity: 1; }
#crawler-container::-webkit-resizer { background: #f1f1f1; border: none; }
#data-visualization { margin-top: 20px; }
#data-chart { width: 100%; height: 300px; }
#modal-html-content { white-space: pre-wrap; word-wrap: break-word; background-color: #f9f9f9; padding: 10px; border-radius: 4px; }
#crawler-tabs .crawler-tab:last-child { flex: 1; }
#previous-crawls { margin-top: 20px; }
#previous-crawls table { width: 100%; }
#previous-crawls th, #previous-crawls td { padding: 5px; }
#reset-data { background-color: #f44336; }
.spinner {
border: 4px solid #f3f3f3;
border-top: 4px solid #3498db;
border-radius: 50%;
width: 24px;
height: 24px;
animation: spin 2s linear infinite;
display: inline-block;
vertical-align: middle;
}
@keyframes spin {
0% { transform: rotate(0deg); }
100% { transform: rotate(360deg); }
}
.error-message {
color: red;
font-weight: bold;
margin-top: 10px;
}
.success-message {
color: green;
font-weight: bold;
margin-top: 10px;
}
.dark-mode-toggle {
background: none;
border: none;
cursor: pointer;
font-size: 18px;
color: inherit;
}
@media (max-width: 768px) {
#crawler-container {
width: 100%;
height: 100%;
top: 0;
right: 0;
border-radius: 0;
}
#crawler-header {
flex-direction: row;
align-items: center;
}
#crawler-tabs {
flex-direction: row;
overflow-x: auto;
}
.crawler-tab {
flex: 0 0 auto;
}
}
`;
$('head').append(`<style>${style}</style>`);
const container = $(`
<div id="crawler-container">
<div id="crawler-header">
<div id="crawler-tabs">
<div class="crawler-tab" data-tab="logs" title="로그 보기">📜 로그</div>
<div class="crawler-tab" data-tab="data" title="수집된 데이터 보기">📊 데이터</div>
<div class="crawler-tab" data-tab="settings" title="설정 변경">⚙️ 설정</div>
<div class="crawler-tab" data-tab="previous-crawls" title="이전 크롤링 기록">📁 기록</div>
<div class="crawler-tab" data-tab="templates" title="설정 템플릿">📂 템플릿</div>
<button id="dark-mode-toggle" class="dark-mode-toggle" title="테마 전환">🌙</button>
</div>
<button id="crawler-minimize" title="창 최소화">─</button>
</div>
<div id="crawler-content">
<div id="tab-logs" style="display: none;">
<div class="progress-bar">
<div class="progress-bar-fill" style="width: 0%;"></div>
</div>
<div class="status-text">진행률: 0% (0/0 페이지)</div>
<div class="control-buttons">
<button id="start-crawling" title="크롤링을 시작합니다.">▶ 시작</button>
<button id="stop-crawling" title="크롤링을 중지합니다." disabled>■ 중지</button>
${Config.settings.exportFormats.csv ? `<button id="download-csv" title="CSV 다운로드">📥 CSV</button>` : ''}
${Config.settings.exportFormats.json ? `<button id="download-json" title="JSON 다운로드">📥 JSON</button>` : ''}
${Config.settings.exportFormats.xml ? `<button id="download-xml" title="XML 다운로드">📥 XML</button>` : ''}
${Config.settings.exportFormats.xls ? `<button id="download-xls" title="XLS 다운로드">📥 XLS</button>` : ''}
<button id="reset-data" title="모든 크롤링 데이터를 초기화합니다.">🗑️ 초기화</button>
</div>
<div id="log-messages" style="height: 200px; overflow-y: auto; background: var(--log-bg, #f9f9f9); padding: 10px; border: 1px solid #ccc; border-radius: 4px;">
로그 내용이 여기에 표시됩니다.
</div>
<div id="error-message" class="error-message" style="display: none;"></div>
<div id="success-message" class="success-message" style="display: none;"></div>
</div>
<div id="tab-data" style="display: none;">
<input type="text" id="search-filter" placeholder="검색 필터..." style="width: 100%; padding: 5px; margin-bottom: 10px; border: 1px solid #ccc; border-radius: 4px;">
<table>
<thead>
<tr>
${Config.settings.dataFields.name ? '<th data-sort="name">사이트 이름</th>' : ''}
${Config.settings.dataFields.url ? '<th data-sort="url">주소</th>' : ''}
${Config.settings.dataFields.description ? '<th data-sort="description">설명</th>' : ''}
${Config.settings.dataFields.html ? '<th>📄 HTML 내용</th>' : ''}
${Config.settings.dataFields.category ? '<th data-sort="category">분류</th>' : ''}
${Config.settings.dataFields.quality ? '<th data-sort="quality">품질점수</th>' : ''}
${Config.settings.dataFields.language ? '<th data-sort="language">언어</th>' : ''}
${Config.settings.dataFields.wordCount ? '<th data-sort="wordCount">단어 수</th>' : ''}
${Config.settings.dataFields.metaTags ? '<th>메타 태그</th>' : ''}
${Config.settings.dataFields.headings ? '<th>헤딩</th>' : ''}
</tr>
</thead>
<tbody id="data-table-body"></tbody>
</table>
${Config.settings.dataVisualization ? `
<div id="data-visualization">
<h3>데이터 시각화</h3>
<canvas id="data-chart"></canvas>
</div>
` : ''}
</div>
<div id="tab-settings" style="display: none;">
<h3>설정</h3>
<div class="settings-section">
<label for="delay-input">크롤링 딜레이 (ms):
<span class="tooltip">ℹ️
<span class="tooltiptext">기본 딜레이과 딜레이 랜덤 값을 조합하여 요청 사이의 딜레이를 설정합니다.</span>
</span>
</label>
<input type="number" id="delay-input" value="${Config.settings.delay}" min="500" step="500">
</div>
<div class="settings-section">
<label for="delay-random-input">딜레이 랜덤 범위 (ms):
<span class="tooltip">ℹ️
<span class="tooltiptext">딜레이에 무작위 요소를 추가하여 보다 자연스러운 요청을 합니다.</span>
</span>
</label>
<input type="number" id="delay-random-input" value="${Config.settings.delayRandom}" min="0" step="500">
</div>
<div class="settings-section">
<label for="max-pages-input">최대 크롤링 페이지 수:
<span class="tooltip">ℹ️
<span class="tooltiptext">크롤링할 최대 페이지 수를 설정하세요.</span>
</span>
</label>
<input type="number" id="max-pages-input" value="${Config.settings.maxPages}" min="1">
</div>
<div class="settings-section">
<label for="user-agent-select">사용자 에이전트(User-Agent):
<span class="tooltip">ℹ️
<span class="tooltiptext">크롤링에 사용할 사용자 에이전트를 선택하거나 추가할 수 있습니다.</span>
</span>
</label>
<select id="user-agent-select" multiple size="6">
${Config.settings.userAgents.map((ua, idx) => `<option value="${idx}" ${idx === Config.settings.userAgentIndex ? 'selected' : ''}>${ua}</option>`).join('')}
</select>
<button id="add-user-agent" style="margin-top: 5px; padding: 5px 10px; background-color: #2196F3; color: white; border-radius: 4px; font-size: 12px;">추가</button>
<button id="remove-user-agent" style="margin-top: 5px; padding: 5px 10px; background-color: #f44336; color: white; border-radius: 4px; font-size: 12px;">제거</button>
</div>
<div class="settings-section">
<label for="content-include-input">콘텐츠 유형 포함:
<span class="tooltip">ℹ️
<span class="tooltiptext">크롤링할 콘텐츠 유형을 쉼표로 구분하여 입력하세요.</span>
</span>
</label>
<input type="text" id="content-include-input" value="${Config.settings.contentTypes.include.join(', ')}">
</div>
<div class="settings-section">
<label for="content-exclude-input">콘텐츠 유형 제외:
<span class="tooltip">ℹ️
<span class="tooltiptext">크롤링에서 제외할 콘텐츠 유형을 쉼표로 구분하여 입력하세요.</span>
</span>
</label>
<input type="text" id="content-exclude-input" value="${Config.settings.contentTypes.exclude.join(', ')}">
</div>
<div class="settings-section">
<label for="timeout-input">크롤링 타임아웃 (ms):
<span class="tooltip">ℹ️
<span class="tooltiptext">요청이 완료되기까지 기다릴 최대 시간을 밀리초 단위로 설정합니다.</span>
</span>
</label>
<input type="number" id="timeout-input" value="${Config.settings.timeout}" min="1000" step="1000">
</div>
<div class="settings-section">
<label for="include-keywords-input">포함할 키워드:
<span class="tooltip">ℹ️
<span class="tooltiptext">크롤링할 페이지에 반드시 포함되어야 하는 키워드를 입력하세요. 각 키워드는 줄바꿈으로 구분됩니다.</span>
</span>
</label>
<textarea id="include-keywords-input" rows="3">${Config.settings.includeKeywords.join('\n')}</textarea>
</div>
<div class="settings-section">
<label for="exclude-keywords-input">제외할 키워드:
<span class="tooltip">ℹ️
<span class="tooltiptext">크롤링할 페이지에 포함되면 안 되는 키워드를 입력하세요. 각 키워드는 줄바꿈으로 구분됩니다.</span>
</span>
</label>
<textarea id="exclude-keywords-input" rows="3">${Config.settings.excludeKeywords.join('\n')}</textarea>
</div>
<div class="settings-section">
<label for="regex-include-input">포함할 키워드 (정규 표현식):
<span class="tooltip">ℹ️
<span class="tooltiptext">포함할 키워드의 정규 표현식을 입력하세요. 각 패턴은 줄바꿈으로 구분됩니다.</span>
</span>
</label>
<textarea id="regex-include-input" rows="3">${Config.settings.regexInclude.join('\n')}</textarea>
</div>
<div class="settings-section">
<label for="regex-exclude-input">제외할 키워드 (정규 표현식):
<span class="tooltip">ℹ️
<span class="tooltiptext">제외할 키워드의 정규 표현식을 입력하세요. 각 패턴은 줄바꿈으로 구분됩니다.</span>
</span>
</label>
<textarea id="regex-exclude-input" rows="3">${Config.settings.regexExclude.join('\n')}</textarea>
</div>
<div class="settings-section">
<label for="language-select">언어 필터링:</label>
<select id="language-select" multiple size="10">
${[
{ value: 'en', text: '영어 (en)' }, { value: 'ko', text: '한국어 (ko)' },
{ value: 'zh', text: '중국어 (zh)' }, { value: 'ja', text: '일본어 (ja)' },
{ value: 'ru', text: '러시아어 (ru)' }, { value: 'es', text: '스페인어 (es)' },
{ value: 'fr', text: '프랑스어 (fr)' }, { value: 'de', text: '독일어 (de)' },
{ value: 'it', text: '이탈리아어 (it)' }, { value: 'pt', text: '포르투갈어 (pt)' },
{ value: 'ar', text: '아랍어 (ar)' }, { value: 'hi', text: '힌디어 (hi)' },
{ value: 'bn', text: '벵골어 (bn)' }, { value: 'pa', text: '펀자브어 (pa)' },
{ value: 'nl', text: '네덜란드어 (nl)' }, { value: 'sv', text: '스웨덴어 (sv)' },
{ value: 'tr', text: '터키어 (tr)' }, { value: 'vi', text: '베트남어 (vi)' },
{ value: 'pl', text: '폴란드어 (pl)' }, { value: 'th', text: '태국어 (th)' }
].map(lang => `<option value="${lang.value}" ${Config.settings.language.includes(lang.value) ? 'selected' : ''}>${lang.text}</option>`).join('')}
</select>
</div>
<div class="settings-section">
<label for="min-word-count-input">최소 단어 수:
<span class="tooltip">ℹ️
<span class="tooltiptext">수집할 페이지의 최소 단어 수를 설정하세요.</span>
</span>
</label>
<input type="number" id="min-word-count-input" value="${Config.settings.minWordCount}" min="10" step="10">
<label for="max-word-count-input">최대 단어 수:
<span class="tooltip">ℹ️
<span class="tooltiptext">수집할 페이지의 최대 단어 수를 설정하세요.</span>
</span>
</label>
<input type="number" id="max-word-count-input" value="${Config.settings.maxWordCount}" min="100" step="100">
</div>
<div class="settings-section">
<label for="max-depth-input">최대 크롤링 깊이:
<span class="tooltip">ℹ️
<span class="tooltiptext">링크를 추적할 최대 깊이를 설정하세요.</span>
</span>
</label>
<input type="number" id="max-depth-input" value="${Config.settings.maxDepth}" min="1" step="1">
</div>
<div class="settings-section">
<label for="proxy-input">프록시 서버:
<span class="tooltip">ℹ️
<span class="tooltiptext">프록시 서버 주소를 입력하세요. 예: http://proxyserver:port</span>
</span>
</label>
<input type="text" id="proxy-input" value="${Config.settings.proxy}" placeholder="http://proxyserver:port">
<div id="proxy-validation-message" class="error-message" style="display: none;"></div>
</div>
<div class="settings-section">
<label>수집할 데이터 필드:</label>
<div class="checkbox-group">
${Object.keys(Config.settings.dataFields).map(field => `<label><input type="checkbox" id="field-${field}" ${Config.settings.dataFields[field] ? 'checked' : ''}> ${Utils.capitalize(getFieldName(field))}</label>`).join('')}
</div>
</div>
<div class="settings-section">
<label for="retry-attempts-input">재시도 횟수:
<span class="tooltip">ℹ️
<span class="tooltiptext">요청 실패 시 재시도할 횟수를 설정하세요.</span>
</span>
</label>
<input type="number" id="retry-attempts-input" value="${Config.settings.retryAttempts}" min="0" step="1">
<label for="retry-delay-input">재시도 딜레이 (ms):
<span class="tooltip">ℹ️
<span class="tooltiptext">재시도 시 대기할 시간을 밀리초 단위로 설정하세요.</span>
</span>
</label>
<input type="number" id="retry-delay-input" value="${Config.settings.retryDelay}" min="1000" step="500">
</div>
<div class="settings-section">
<label>크롤링 스케줄링:</label>
<div class="checkbox-group">
<label><input type="checkbox" id="schedule-enabled" ${Config.settings.schedule.enabled ? 'checked' : ''}> 스케줄링 활성화</label>
</div>
<div id="schedule-options" style="display: ${Config.settings.schedule.enabled ? 'block' : 'none'}; margin-top: 10px;">
<label for="start-time-input">시작 시간 (HH:MM):
<span class="tooltip">ℹ️
<span class="tooltiptext">크롤링을 시작할 시간을 설정하세요.</span>
</span>
</label>
<input type="time" id="start-time-input" value="${Config.settings.schedule.startTime}">
<label for="interval-input">크롤링 간격 (분):
<span class="tooltip">ℹ️
<span class="tooltiptext">크롤링 작업 간의 간격을 분 단위로 설정하세요.</span>
</span>
</label>
<input type="number" id="interval-input" value="${Config.settings.schedule.interval / 60000}" min="1" step="1">
</div>
</div>
<div class="settings-section">
<label>크롤링할 검색 엔진 선택:</label>
<div class="checkbox-group">
${Object.keys(Config.settings.searchEngines).map(engine => `<label><input type="checkbox" id="engine-${engine}" ${Config.settings.searchEngines[engine] ? 'checked' : ''}> ${Utils.capitalize(engine)}</label>`).join('')}
</div>
</div>
<div class="settings-section">
<label for="blacklist-input">블랙리스트 도메인:
<span class="tooltip">ℹ️
<span class="tooltiptext">크롤링에서 제외할 도메인을 입력하세요. 각 도메인은 줄바꿈으로 구분됩니다.</span>
</span>
</label>
<textarea id="blacklist-input" rows="5">${Config.settings.blacklist.join('\n')}</textarea>
</div>
<div class="settings-section">
<label for="whitelist-input">화이트리스트 도메인:
<span class="tooltip">ℹ️
<span class="tooltiptext">크롤링을 허용할 도메인을 입력하세요. 각 도메인은 줄바꿈으로 구분됩니다.</span>
</span>
</label>
<textarea id="whitelist-input" rows="5">${Config.settings.whitelist.join('\n')}</textarea>
</div>
<div class="settings-section">
<label for="respect-robots-checkbox">robots.txt 준수:
<span class="tooltip">ℹ️
<span class="tooltiptext">대상 사이트의 robots.txt 규칙을 준수하여 크롤링을 진행합니다.</span>
</span>
</label>
<input type="checkbox" id="respect-robots-checkbox" ${Config.settings.respectRobots ? 'checked' : ''}>
</div>
<div class="settings-section">
<label for="max-concurrent-requests-input">동시 요청 최대 수:
<span class="tooltip">ℹ️
<span class="tooltiptext">한 번에 요청할 수 있는 최대 동시 요청 수를 설정하세요.</span>
</span>
</label>
<input type="number" id="max-concurrent-requests-input" value="${Config.settings.maxConcurrentRequests}" min="1" step="1">
</div>
<div class="settings-section">
<label for="data-visualization-checkbox">데이터 시각화 활성화:
<span class="tooltip">ℹ️
<span class="tooltiptext">수집된 데이터를 시각화하여 그래프로 표시합니다.</span>
</span>
</label>
<input type="checkbox" id="data-visualization-checkbox" ${Config.settings.dataVisualization ? 'checked' : ''}>
</div>
<div class="settings-section">
<label for="enable-captchas-checkbox">CAPTCHA 처리 활성화:
<span class="tooltip">ℹ️
<span class="tooltiptext">CAPTCHA가 감지되었을 때 자동 처리를 활성화합니다. (제한적)</span>
</span>
</label>
<input type="checkbox" id="enable-captchas-checkbox" ${Config.settings.enableCaptchas ? 'checked' : ''}>
</div>
<div class="settings-section">
<label>데이터 내보내기 형식:</label>
<div class="checkbox-group">
${Object.keys(Config.settings.exportFormats).map(format => `<label><input type="checkbox" id="export-${format}" ${Config.settings.exportFormats[format] ? 'checked' : ''}> ${format.toUpperCase()}</label>`).join('')}
</div>
</div>
<div class="settings-section">
<label for="advanced-logging-checkbox">고급 로깅 활성화:
<span class="tooltip">ℹ️
<span class="tooltiptext">추가적인 로깅 정보를 활성화합니다.</span>
</span>
</label>
<input type="checkbox" id="advanced-logging-checkbox" ${Config.settings.advancedLogging ? 'checked' : ''}>
</div>
<div class="settings-section">
<label for="custom-keywords-input">사용자 지정 키워드 분류:
<span class="tooltip">ℹ️
<span class="tooltiptext">사용자 지정 키워드를 추가하여 데이터 분류를 향상시킵니다. 형식: 카테고리:키워드1,키워드2,...</span>
</span>
</label>
<textarea id="custom-keywords-input" rows="5">${Utils.formatCustomKeywords(Config.settings.customKeywords)}</textarea>
</div>
<div class="settings-section">
<label>설정 템플릿 관리:</label>
<div class="checkbox-group">
<input type="text" id="new-template-name" placeholder="템플릿 이름을 입력하세요" style="margin-bottom: 5px;">
<button id="save-template" style="padding: 8px 12px; background-color: #4caf50; color: white; border-radius: 4px; font-size: 14px; margin-right: 8px;">📁 템플릿 저장</button>
<button id="load-template" style="padding: 8px 12px; background-color: #2196F3; color: white; border-radius: 4px; font-size: 14px;">📂 템플릿 불러오기</button>
</div>
<select id="template-select" style="width: 100%; padding: 5px; margin-top: 10px; border: 1px solid #ccc; border-radius: 4px;">
<option value="" disabled selected>템플릿 선택</option>
${Object.keys(Config.settings.templates).map(template => `<option value="${template}">${template}</option>`).join('')}
</select>
<button id="create-template" style="margin-top: 10px; padding: 8px 12px; background-color: #4caf50; color: white; border-radius: 4px; font-size: 14px;">템플릿 생성</button>
<button id="delete-template" style="margin-top: 5px; padding: 8px 12px; background-color: #f44336; color: white; border-radius: 4px; font-size: 14px;">템플릿 삭제</button>
</div>
<div class="settings-section">
<button id="save-settings" style="margin-top: 10px; padding: 10px 20px; background-color: #4caf50; color: white; border-radius: 4px; font-size: 14px;">저장</button>
<button id="reset-settings" style="margin-top: 10px; padding: 10px 20px; background-color: #f44336; color: white; border-radius: 4px; font-size: 14px;">초기화</button>
</div>
</div>
<div id="tab-templates" style="display: none;">
<h3>설정 템플릿</h3>
<div class="settings-section">
<label for="new-template-name">새 템플릿 이름:</label>
<input type="text" id="new-template-name" placeholder="템플릿 이름을 입력하세요">
<button id="create-template" style="margin-top: 5px; padding: 8px 12px; background-color: #4caf50; color: white; border-radius: 4px; font-size: 14px;">템플릿 생성</button>
</div>
<div class="settings-section">
<label for="existing-templates">기존 템플릿:</label>
<select id="existing-templates" multiple size="10" style="width: 100%; padding: 5px; border: 1px solid #ccc; border-radius: 4px;">
${Object.keys(Config.settings.templates).map(template => `<option value="${template}">${template}</option>`).join('')}
</select>
<button id="delete-template" style="margin-top: 5px; padding: 8px 12px; background-color: #f44336; color: white; border-radius: 4px; font-size: 14px;">템플릿 삭제</button>
</div>
</div>
<div id="tab-previous-crawls" style="display: none;">
<h3>이전 크롤링 기록</h3>
<button id="clear-previous-crawls" style="padding: 8px 12px; background-color: #f44336; color: white; border-radius: 4px; font-size: 14px; margin-bottom: 10px;">기록 초기화</button>
<table>
<thead>
<tr>
<th>날짜</th>
<th>크롤링 수</th>
<th>상태</th>
</tr>
</thead>
<tbody id="previous-crawls-body"></tbody>
</table>
</div>
<div id="html-modal" class="modal">
<div class="modal-content">
<span class="close">×</span>
<pre id="modal-html-content"></pre>
</div>
</div>
</div>
</div>
`);
$('body').append(container);
$('#crawler-minimize').on('click', toggleMinimize);
$('.crawler-tab').on('click', switchTab);
$('#save-settings').on('click', saveSettingsHandler);
$('#reset-settings').on('click', resetSettingsHandler);
$('#add-user-agent').on('click', addUserAgent);
$('#remove-user-agent').on('click', removeUserAgent);
$('#download-csv').on('click', Download.downloadCSV);
$('#download-json').on('click', Download.downloadJSON);
$('#download-xml').on('click', Download.downloadXML);
$('#download-xls').on('click', Download.downloadXLS);
$('#search-filter').on('input', filterData);
$('#schedule-enabled').on('change', toggleScheduleOptions);
$('#data-visualization-checkbox').on('change', toggleDataVisualization);
$('#enable-captchas-checkbox').on('change', toggleCaptchaHandling);
$('#save-template').on('click', saveTemplateHandler);
$('#load-template').on('click', loadTemplateHandler);
$('#create-template').on('click', createTemplateHandler);
$('#delete-template').on('click', deleteTemplateHandler);
$('#template-select').on('change', selectTemplateHandler);
$('#existing-templates').on('change', existingTemplateSelectHandler);
$('#clear-previous-crawls').on('click', clearPreviousCrawls);
$('#reset-data').on('click', resetData);
$('#dark-mode-toggle').on('click', toggleDarkMode);
$('.close').on('click', () => $('#html-modal').hide());
$(window).on('click', function(e) { if (e.target.id === 'html-modal') $('#html-modal').hide(); });
$('#start-crawling').on('click', Crawling.startCrawling);
$('#stop-crawling').on('click', Crawling.stopCrawling);
$(document).on('keydown', handleShortcuts);
$('#proxy-input').on('input', validateProxyInput);
$('.settings-section input[type="checkbox"]').on('change', toggleDataField);
makeDraggable();
}
function toggleMinimize() {
$('#crawler-content').toggle();
const isHidden = $('#crawler-content').is(':hidden');
$('#crawler-minimize').text(isHidden ? '☐' : '─');
Logger.log(`크롤러 창이 ${isHidden ? '최소화' : '확대'}되었습니다.`, 'info');
}
function switchTab() {
const tab = $(this).data('tab');
$('.crawler-tab').removeClass('active');
$(this).addClass('active');
$('#crawler-content > div').hide();
$(`#tab-${tab}`).show();
Config.saveSettings({ currentTab: tab });
}
function updateUI() {
const currentTab = Config.settings.currentTab || 'logs';
$(`.crawler-tab[data-tab="${currentTab}"]`).addClass('active');
$('#crawler-content > div').hide();
$(`#tab-${currentTab}`).show();
updateLogs();
updateDataTable();
updatePreviousCrawlsUI();
updateTemplatesUI();
if (Config.settings.dataVisualization) DataVisualization.renderChart();
applyDarkMode();
}
function updateLogs() {
const logs = Logger.getLogs();
$('#log-messages').html(logs.map(l => `<div>[${l.timestamp}] [${l.type.toUpperCase()}] ${Utils.escape(l.message)}</div>`).join(''));
const progress = GM_getValue('progress', 0);
const currentPage = GM_getValue('currentPage', 0);
const totalPages = Config.settings.maxPages;
$('.progress-bar-fill').css('width', `${progress.toFixed(2)}%`);
$('.status-text').text(`진행률: ${progress.toFixed(2)}% (${currentPage}/${totalPages} 페이지)`);
const latestLog = logs[logs.length -1];
if (latestLog) {
if (latestLog.type === 'error') {
$('#error-message').text(latestLog.message).show();
setTimeout(() => { $('#error-message').fadeOut(); }, 5000);
}
if (latestLog.type === 'success') {
$('#success-message').text(latestLog.message).show();
setTimeout(() => { $('#success-message').fadeOut(); }, 5000);
}
if (latestLog.type === 'info') {
$('#success-message').text(latestLog.message).show();
setTimeout(() => { $('#success-message').fadeOut(); }, 5000);
}
if (latestLog.type === 'warning') {
$('#error-message').text(latestLog.message).show();
setTimeout(() => { $('#error-message').fadeOut(); }, 5000);
}
}
}
function updateDataTable() {
const data = JSON.parse(GM_getValue('data', '[]'));
const filter = $('#search-filter').val().toLowerCase();
const filtered = filter ? data.filter(item => Object.keys(Config.settings.dataFields).some(field => Config.settings.dataFields[field] && String(item[field]).toLowerCase().includes(filter))) : data;
const rows = filtered.map(item => `
<tr>
${Config.settings.dataFields.name ? `<td>${Utils.escape(item.name)}</td>` : ''}
${Config.settings.dataFields.url ? `<td><a href="${Utils.escape(item.url)}" target="_blank">${Utils.escape(item.url)}</a></td>` : ''}
${Config.settings.dataFields.description ? `<td>${Utils.escape(item.description)}</td>` : ''}
${Config.settings.dataFields.html ? `<td><button class="view-html" data-html="${encodeURIComponent(item.html)}">📄 보기</button></td>` : ''}
${Config.settings.dataFields.category ? `<td>${Utils.escape(item.category)}</td>` : ''}
${Config.settings.dataFields.quality ? `<td>${Utils.escape(item.quality.toString())}</td>` : ''}
${Config.settings.dataFields.language ? `<td>${Utils.escape(item.language)}</td>` : ''}
${Config.settings.dataFields.wordCount ? `<td>${Utils.escape(item.wordCount.toString())}</td>` : ''}
${Config.settings.dataFields.metaTags ? `<td>${Utils.escape(JSON.stringify(item.metaTags))}</td>` : ''}
${Config.settings.dataFields.headings ? `<td>${Utils.escape(item.headings.join(', '))}</td>` : ''}
</tr>
`).join('');
$('#data-table-body').html(rows);
$('.view-html').off('click').on('click', function() {
$('#modal-html-content').text(decodeURIComponent($(this).data('html')));
$('#html-modal').show();
});
if (Config.settings.dataVisualization) DataVisualization.renderChart();
}
function updatePreviousCrawlsUI() {
const previousCrawls = JSON.parse(GM_getValue('previousCrawls', '[]'));
$('#previous-crawls-body').html(previousCrawls.map(crawl => `
<tr>
<td>${Utils.escape(crawl.date)}</td>
<td>${Utils.escape(crawl.count.toString())}</td>
<td>${Utils.escape(crawl.status)}</td>
</tr>
`).join(''));
}
function updateTemplatesUI() {
$('#template-select').empty().append(`<option value="" disabled selected>템플릿 선택</option>`);
Object.keys(Config.settings.templates).forEach(template => {
$('#template-select').append(`<option value="${template}">${template}</option>`);
});
$('#existing-templates').empty();
Object.keys(Config.settings.templates).forEach(template => {
$('#existing-templates').append(`<option value="${template}">${template}</option>`);
});
}
function addUserAgent() {
const ua = prompt("새 사용자 에이전트를 입력하세요:");
if (ua) {
Config.settings.userAgents.push(ua);
Config.saveSettings({ userAgents: Config.settings.userAgents });
updateUserAgentSelect();
Logger.log('새 사용자 에이전트가 추가되었습니다.', 'success');
}
}
function removeUserAgent() {
const selected = $('#user-agent-select').val();
if (selected && selected.length) {
selected.sort((a, b) => b - a).forEach(idx => { Config.settings.userAgents.splice(idx, 1); });
Config.settings.userAgentIndex = 0;
Config.saveSettings({
userAgents: Config.settings.userAgents,
userAgentIndex: Config.settings.userAgentIndex
});
updateUserAgentSelect();
Logger.log('선택한 사용자 에이전트가 제거되었습니다.', 'success');
} else {
alert('제거할 사용자 에이전트를 선택하세요.');
}
}
function updateUserAgentSelect() {
$('#user-agent-select').empty();
Config.settings.userAgents.forEach((ua, idx) => {
$('#user-agent-select').append(`<option value="${idx}" ${idx === Config.settings.userAgentIndex ? 'selected' : ''}>${ua}</option>`);
});
}
function saveSettingsHandler() {
const delay = parseInt($('#delay-input').val(), 10);
const delayRandom = parseInt($('#delay-random-input').val(), 10);
const maxPages = parseInt($('#max-pages-input').val(), 10);
const timeout = parseInt($('#timeout-input').val(), 10);
const minWordCount = parseInt($('#min-word-count-input').val(), 10);
const maxWordCount = parseInt($('#max-word-count-input').val(), 10);
const maxDepth = parseInt($('#max-depth-input').val(), 10);
const retryAttempts = parseInt($('#retry-attempts-input').val(), 10);
const retryDelay = parseInt($('#retry-delay-input').val(), 10);
const interval = parseInt($('#interval-input').val(), 10) * 60000;
const proxy = $('#proxy-input').val().trim();
const isValidProxy = proxy === '' || Config.validateProxy(proxy);
const isValidSettings = !isNaN(delay) && delay >= 500 &&
!isNaN(delayRandom) && delayRandom >= 0 &&
!isNaN(maxPages) && maxPages >= 1 &&
!isNaN(timeout) && timeout >= 1000 &&
!isNaN(minWordCount) && minWordCount >= 10 &&
!isNaN(maxWordCount) && maxWordCount >= 100 &&
!isNaN(maxDepth) && maxDepth >= 1 &&
!isNaN(retryAttempts) && retryAttempts >= 0 &&
!isNaN(retryDelay) && retryDelay >= 1000 &&
(!Config.settings.schedule.enabled || (!isNaN(interval) && interval >= 60000)) &&
isValidProxy;
if (!isValidSettings) {
$('#error-message').text('설정값이 유효하지 않습니다. 올바른 값을 입력하세요.').show();
setTimeout(() => { $('#error-message').fadeOut(); }, 5000);
return;
}
Config.settings.delay = delay;
Config.settings.delayRandom = delayRandom;
Config.settings.maxPages = maxPages;
const selectedUA = parseInt($('#user-agent-select').val(), 10);
Config.settings.userAgentIndex = isNaN(selectedUA) ? 0 : selectedUA;
Config.settings.contentTypes.include = $('#content-include-input').val().split(',').map(ct => ct.trim()).filter(ct => ct);
Config.settings.contentTypes.exclude = $('#content-exclude-input').val().split(',').map(ct => ct.trim()).filter(ct => ct);
Config.settings.timeout = timeout;
Config.settings.includeKeywords = $('#include-keywords-input').val().split('\n').map(kw => kw.trim()).filter(kw => kw);
Config.settings.excludeKeywords = $('#exclude-keywords-input').val().split('\n').map(kw => kw.trim()).filter(kw => kw);
Config.settings.regexInclude = $('#regex-include-input').val().split('\n').map(rx => rx.trim()).filter(rx => rx);
Config.settings.regexExclude = $('#regex-exclude-input').val().split('\n').map(rx => rx.trim()).filter(rx => rx);
Config.settings.language = $('#language-select').val() || [];
Config.settings.minWordCount = minWordCount;
Config.settings.maxWordCount = maxWordCount;
Config.settings.maxDepth = maxDepth;
Config.settings.proxy = proxy;
['name','url','description','html','category','quality','language','wordCount','metaTags','headings'].forEach(field => {
Config.settings.dataFields[field] = $(`#field-${field}`).is(':checked');
});
Config.settings.retryAttempts = retryAttempts;
Config.settings.retryDelay = retryDelay;
Config.settings.schedule.enabled = $('#schedule-enabled').is(':checked');
Config.settings.schedule.startTime = $('#start-time-input').val() || Config.DEFAULT_SETTINGS.schedule.startTime;
Config.settings.schedule.interval = Config.settings.schedule.enabled ? (isNaN(interval) ? Config.DEFAULT_SETTINGS.schedule.interval : interval) : Config.DEFAULT_SETTINGS.schedule.interval;
Object.keys(Config.settings.searchEngines).forEach(engine => { Config.settings.searchEngines[engine] = $(`#engine-${engine}`).is(':checked'); });
Config.settings.blacklist = $('#blacklist-input').val().split('\n').map(d => d.trim()).filter(d => d);
Config.settings.whitelist = $('#whitelist-input').val().split('\n').map(d => d.trim()).filter(d => d);
Config.settings.respectRobots = $('#respect-robots-checkbox').is(':checked');
Config.settings.maxConcurrentRequests = parseInt($('#max-concurrent-requests-input').val(), 10) || Config.DEFAULT_SETTINGS.maxConcurrentRequests;
Config.settings.dataVisualization = $('#data-visualization-checkbox').is(':checked');
Config.settings.exportFormats.csv = $('#export-csv').is(':checked');
Config.settings.exportFormats.json = $('#export-json').is(':checked');
Config.settings.exportFormats.xml = $('#export-xml').is(':checked');
Config.settings.exportFormats.xls = $('#export-xls').is(':checked');
Config.settings.advancedLogging = $('#advanced-logging-checkbox').is(':checked');
Config.settings.enableCaptchas = $('#enable-captchas-checkbox').is(':checked');
Config.settings.customKeywords = Utils.parseCustomKeywords($('#custom-keywords-input').val());
Config.saveSettings(Config.settings);
Crawling.updateCrawlSettings();
Logger.log('설정이 저장되었습니다.', 'success');
if (Config.settings.schedule.enabled) {
Scheduler.scheduleCrawling();
}
updateDataTable();
if (Config.settings.dataVisualization) DataVisualization.renderChart();
}
function resetSettingsHandler() {
if (confirm("기본 설정으로 초기화하시겠습니까?")) {
Config.resetSettings();
location.reload();
Logger.log('기본 설정으로 초기화되었습니다.', 'success');
}
}
function toggleScheduleOptions() {
$('#schedule-options').toggle($('#schedule-enabled').is(':checked'));
}
function toggleDataVisualization() {
Config.settings.dataVisualization = $('#data-visualization-checkbox').is(':checked');
Config.saveSettings({ dataVisualization: Config.settings.dataVisualization });
$('#data-visualization').toggle(Config.settings.dataVisualization);
Logger.log('데이터 시각화 설정이 변경되었습니다.', 'success');
if (Config.settings.dataVisualization) DataVisualization.renderChart();
}
function toggleCaptchaHandling() {
Config.settings.enableCaptchas = $('#enable-captchas-checkbox').is(':checked');
Config.saveSettings({ enableCaptchas: Config.settings.enableCaptchas });
Logger.log('CAPTCHA 처리 설정이 변경되었습니다.', 'success');
}
function filterData() {
updateDataTable();
}
function clearPreviousCrawls() {
if (confirm("이전 크롤링 기록을 모두 초기화하시겠습니까?")) {
GM_setValue('previousCrawls', JSON.stringify([]));
updatePreviousCrawlsUI();
Logger.log('이전 크롤링 기록이 초기화되었습니다.', 'success');
}
}
function resetData() {
if (confirm("모든 크롤링 데이터를 초기화하시겠습니까?")) {
GM_setValue('data', JSON.stringify([]));
GM_setValue('visitedUrls', JSON.stringify([]));
updateDataTable();
Logger.log('크롤링 데이터가 초기화되었습니다.', 'success');
}
}
function handleShortcuts(e) {
if (e.ctrlKey) {
const actions = { '1': 'logs', '2': 'data', '3': 'settings', '4': 'previous-crawls', '5': 'templates', 's': 'save', 'r': 'reset', 'p': 'start', 'q': 'stop', 'c': 'csv', 'j': 'json', 'x': 'xml', 'l': 'xls', 'd': 'dark-mode' };
const action = actions[e.key];
if (action) {
e.preventDefault();
switch(action) {
case 'logs': $('.crawler-tab[data-tab="logs"]').click(); break;
case 'data': $('.crawler-tab[data-tab="data"]').click(); break;
case 'settings': $('.crawler-tab[data-tab="settings"]').click(); break;
case 'previous-crawls': $('.crawler-tab[data-tab="previous-crawls"]').click(); break;
case 'templates': $('.crawler-tab[data-tab="templates"]').click(); break;
case 'save': saveSettingsHandler(); break;
case 'reset': resetSettingsHandler(); break;
case 'start': if (!Crawling.isCrawling()) Crawling.startCrawling(); break;
case 'stop': if (Crawling.isCrawling()) Crawling.stopCrawling(); break;
case 'csv': Config.settings.exportFormats.csv && Download.downloadCSV(); break;
case 'json': Config.settings.exportFormats.json && Download.downloadJSON(); break;
case 'xml': Config.settings.exportFormats.xml && Download.downloadXML(); break;
case 'xls': Config.settings.exportFormats.xls && Download.downloadXLS(); break;
case 'dark-mode': toggleDarkMode(); break;
}
}
}
}
function makeDraggable() {
const container = document.getElementById('crawler-container');
const header = document.getElementById('crawler-header');
let isDragging = false;
let startX, startY, initialX, initialY;
header.onmousedown = function(e) {
isDragging = true;
startX = e.clientX;
startY = e.clientY;
initialX = container.offsetLeft;
initialY = container.offsetTop;
document.onmousemove = drag;
document.onmouseup = stopDrag;
return false;
};
function drag(e) {
if (isDragging) {
let dx = e.clientX - startX;
let dy = e.clientY - startY;
container.style.left = (initialX + dx) + 'px';
container.style.top = (initialY + dy) + 'px';
}
}
function stopDrag() {
isDragging = false;
document.onmousemove = null;
document.onmouseup = null;
}
}
function getFieldName(field) {
const names = {
name: '이름',
url: '주소',
description: '설명',
html: 'HTML 내용',
category: '분류',
quality: '품질점수',
language: '언어',
wordCount: '단어 수',
metaTags: '메타 태그',
headings: '헤딩'
};
return names[field] || field;
}
// Template Handlers
function saveTemplateHandler() {
const templateName = $('#new-template-name').val().trim();
if (!templateName) {
alert('템플릿 이름을 입력하세요.');
return;
}
Config.saveTemplate(templateName);
updateTemplatesUI();
$('#new-template-name').val('');
}
function loadTemplateHandler() {
const selected = $('#template-select').val();
if (!selected) {
alert('불러올 템플릿을 선택하세요.');
return;
}
Config.loadTemplate(selected);
updateUI();
$('.crawler-tab[data-tab="settings"]').click();
}
function createTemplateHandler() {
const templateName = $('#new-template-name').val().trim();
if (!templateName) {
alert('템플릿 이름을 입력하세요.');
return;
}
Config.saveTemplate(templateName);
updateTemplatesUI();
$('#new-template-name').val('');
}
function deleteTemplateHandler() {
const selectedTemplates = $('#existing-templates').val();
if (!selectedTemplates || !selectedTemplates.length) {
alert('삭제할 템플릿을 선택하세요.');
return;
}
selectedTemplates.forEach(template => {
delete Config.settings.templates[template];
Logger.log(`템플릿 '${template}'이(가) 삭제되었습니다.`, 'success');
});
Config.saveSettings({ templates: Config.settings.templates });
updateTemplatesUI();
}
function selectTemplateHandler() {
const selected = $(this).val();
if (selected) {
Config.loadTemplate(selected);
updateUI();
$('.crawler-tab[data-tab="settings"]').click();
}
}
function existingTemplateSelectHandler() {
// Placeholder for any additional actions when selecting existing templates
}
function toggleDarkMode() {
$('#crawler-container').toggleClass('dark');
const isDark = $('#crawler-container').hasClass('dark');
$('#dark-mode-toggle').text(isDark ? '☀️' : '🌙');
Config.saveSettings({ darkMode: isDark });
Logger.log(`다크 모드가 ${isDark ? '활성화' : '비활성화'}되었습니다.`, 'success');
}
function applyDarkMode() {
if (Config.settings.darkMode) {
$('#crawler-container').addClass('dark');
$('#dark-mode-toggle').text('☀️');
} else {
$('#crawler-container').removeClass('dark');
$('#dark-mode-toggle').text('🌙');
}
}
function toggleDataField() {
const field = $(this).attr('id').replace('field-', '');
Config.settings.dataFields[field] = $(this).is(':checked');
Config.saveSettings({ dataFields: Config.settings.dataFields });
Download.updateExportButtons();
}
function validateProxyInput() {
const proxy = $('#proxy-input').val().trim();
if (proxy && !Config.validateProxy(proxy)) {
$('#proxy-validation-message').text('유효하지 않은 프록시 URL입니다.').show();
} else {
$('#proxy-validation-message').hide();
}
}
return {
createUI,
updateUI,
updateLogs,
updateDataTable,
updatePreviousCrawlsUI,
updateTemplatesUI,
applyDarkMode
};
})();
/* ============================
Module: Download
============================ */
const Download = (function() {
function downloadCSV() {
const data = JSON.parse(GM_getValue('data', '[]'));
if (!data.length) { Logger.log('다운로드할 데이터가 없습니다.', 'warning'); return; }
const headers = Object.keys(Config.settings.dataFields).filter(f => Config.settings.dataFields[f]).map(f => Utils.capitalize(getFieldName(f)));
const csvContent = [headers.join(",")]
.concat(data.map(item => Object.keys(Config.settings.dataFields).filter(f => Config.settings.dataFields[f]).map(f => `"${Utils.escape(item[f], 'csv')}"`).join(",")))
.join("\n");
const blob = new Blob([csvContent], { type: "text/csv;charset=utf-8;" });
const link = document.createElement("a");
link.href = URL.createObjectURL(blob);
link.download = "data.csv";
document.body.appendChild(link);
link.click();
document.body.removeChild(link);
Logger.log('CSV 파일이 다운로드되었습니다.', 'success');
}
function downloadJSON() {
const data = JSON.parse(GM_getValue('data', '[]'));
if (!data.length) { Logger.log('다운로드할 데이터가 없습니다.', 'warning'); return; }
const jsonContent = JSON.stringify(data, null, 2);
const blob = new Blob([jsonContent], { type: "application/json" });
const link = document.createElement("a");
link.href = URL.createObjectURL(blob);
link.download = "data.json";
document.body.appendChild(link);
link.click();
document.body.removeChild(link);
Logger.log('JSON 파일이 다운로드되었습니다.', 'success');
}
function downloadXML() {
const data = JSON.parse(GM_getValue('data', '[]'));
if (!data.length) { Logger.log('다운로드할 데이터가 없습니다.', 'warning'); return; }
let xmlContent = '<?xml version="1.0" encoding="UTF-8"?><data>';
data.forEach(item => {
xmlContent += '<entry>';
if (Config.settings.dataFields.name) xmlContent += `<name>${Utils.escape(item.name || 'No Title', 'xml')}</name>`;
if (Config.settings.dataFields.url) xmlContent += `<url>${Utils.escape(item.url || '', 'xml')}</url>`;
if (Config.settings.dataFields.description) xmlContent += `<description>${Utils.escape(item.description || 'No Description', 'xml')}</description>`;
if (Config.settings.dataFields.html) xmlContent += `<html><![CDATA[${item.html || ''}]]></html>`;
if (Config.settings.dataFields.category) xmlContent += `<category>${Utils.escape(item.category || '', 'xml')}</category>`;
if (Config.settings.dataFields.quality) xmlContent += `<quality>${Utils.escape(item.quality)}</quality>`;
if (Config.settings.dataFields.language) xmlContent += `<language>${Utils.escape(item.language || 'unknown', 'xml')}</language>`;
if (Config.settings.dataFields.wordCount) xmlContent += `<wordCount>${Utils.escape(item.wordCount)}</wordCount>`;
if (Config.settings.dataFields.metaTags) xmlContent += `<metaTags>${Utils.escape(JSON.stringify(item.metaTags), 'xml')}</metaTags>`;
if (Config.settings.dataFields.headings) xmlContent += `<headings>${Utils.escape(item.headings.join('; '), 'xml')}</headings>`;
xmlContent += '</entry>';
});
xmlContent += '</data>';
const blob = new Blob([xmlContent], { type: "application/xml" });
const link = document.createElement("a");
link.href = URL.createObjectURL(blob);
link.download = "data.xml";
document.body.appendChild(link);
link.click();
document.body.removeChild(link);
Logger.log('XML 파일이 다운로드되었습니다.', 'success');
}
function downloadXLS() {
const data = JSON.parse(GM_getValue('data', '[]'));
if (!data.length) { Logger.log('다운로드할 데이터가 없습니다.', 'warning'); return; }
const worksheetData = data.map(item => {
const entry = {};
if (Config.settings.dataFields.name) entry['사이트 이름'] = item.name;
if (Config.settings.dataFields.url) entry['주소'] = item.url;
if (Config.settings.dataFields.description) entry['설명'] = item.description;
if (Config.settings.dataFields.html) entry['📄 HTML 내용'] = item.html;
if (Config.settings.dataFields.category) entry['분류'] = item.category;
if (Config.settings.dataFields.quality) entry['품질점수'] = item.quality;
if (Config.settings.dataFields.language) entry['언어'] = item.language;
if (Config.settings.dataFields.wordCount) entry['단어 수'] = item.wordCount;
if (Config.settings.dataFields.metaTags) entry['메타 태그'] = JSON.stringify(item.metaTags);
if (Config.settings.dataFields.headings) entry['헤딩'] = item.headings.join('; ');
return entry;
});
const worksheet = XLSX.utils.json_to_sheet(worksheetData);
const workbook = XLSX.utils.book_new();
XLSX.utils.book_append_sheet(workbook, worksheet, "Data");
XLSX.writeFile(workbook, "data.xlsx");
Logger.log('XLS 파일이 다운로드되었습니다.', 'success');
}
function getFieldName(field) {
const names = {
name: '이름',
url: '주소',
description: '설명',
html: 'HTML 내용',
category: '분류',
quality: '품질점수',
language: '언어',
wordCount: '단어 수',
metaTags: '메타 태그',
headings: '헤딩'
};
return names[field] || field;
}
function updateExportButtons() {
// Placeholder for dynamic enabling/disabling of export buttons based on data fields
}
return {
downloadCSV,
downloadJSON,
downloadXML,
downloadXLS,
updateExportButtons
};
})();
/* ============================
Module: DataVisualization
============================ */
const DataVisualization = (function() {
let chartInstance = null;
function renderChart() {
if (!document.getElementById('data-chart').getContext) return;
const ctx = document.getElementById('data-chart').getContext('2d');
if (chartInstance) chartInstance.destroy();
const data = JSON.parse(GM_getValue('data', '[]'));
const categories = {};
data.forEach(item => { const cat = item.category || '기타'; categories[cat] = (categories[cat] || 0) + 1; });
const chartData = {
labels: Object.keys(categories),
datasets: [{
label: '카테고리별 데이터 수',
data: Object.values(categories),
backgroundColor: Utils.generateColors(Object.keys(categories).length)
}]
};
chartInstance = new Chart(ctx, { type: 'pie', data: chartData, options: { responsive: true } });
}
return {
renderChart
};
})();
/* ============================
Module: Crawling
============================ */
const Crawling = (function() {
let isCrawlingFlag = Config.settings.isCrawling;
let currentPage = parseInt(GM_getValue('currentPage', 1), 10);
let totalPages = Config.settings.maxPages;
let progress = parseFloat(GM_getValue('progress', 0));
let currentDepth = parseInt(GM_getValue('currentDepth', 0), 10);
let activeRequests = 0;
let requestQueue = [];
let previousCrawls = JSON.parse(GM_getValue('previousCrawls', '[]'));
let visitedUrls = new Set(JSON.parse(GM_getValue('visitedUrls', '[]')));
let data = JSON.parse(GM_getValue('data', '[]'));
let navigationObserver = null;
const searchEngineHandlers = {
google: {
extractResults: function(doc) {
const results = [];
doc.querySelectorAll('div.g, div[data-hveid]').forEach(item => {
const name = item.querySelector('h3')?.innerText.trim() || 'No Title';
const url = item.querySelector('a')?.href || '';
const desc = item.querySelector('.aCOpRe, .VwiC3b')?.innerText.trim() || 'No Description';
if (Utils.isValidURL(url)) results.push({ name, url, description: desc });
});
return results;
},
getNextPageUrl: function(doc) {
return doc.querySelector('a#pnnext')?.href || null;
},
getContentContainer: function() {
return '#search';
}
},
bing: {
extractResults: function(doc) {
const results = [];
doc.querySelectorAll('.b_algo').forEach(item => {
const name = item.querySelector('h2')?.innerText.trim() || 'No Title';
const url = item.querySelector('h2 a')?.href || '';
const desc = item.querySelector('.b_caption p')?.innerText.trim() || 'No Description';
if (Utils.isValidURL(url)) results.push({ name, url, description: desc });
});
return results;
},
getNextPageUrl: function(doc) {
return doc.querySelector('a.sb_pagN')?.href || null;
},
getContentContainer: function() {
return '#b_content';
}
},
yahoo: {
extractResults: function(doc) {
const results = [];
doc.querySelectorAll('.dd.algo').forEach(item => {
const name = item.querySelector('h3')?.innerText.trim() || 'No Title';
const url = item.querySelector('a')?.href || '';
const desc = item.querySelector('.compText a')?.innerText.trim() || 'No Description';
if (Utils.isValidURL(url)) results.push({ name, url, description: desc });
});
return results;
},
getNextPageUrl: function(doc) {
return doc.querySelector('a.next')?.href || null;
},
getContentContainer: function() {
return '#web';
}
},
// Additional search engine handlers...
// Include handlers for duckduckgo, baidu, yandex, naver, daum, ask, ecosia, startpage, searx, dogpile, bingimages
};
function identifySearchEngine() {
const host = window.location.hostname;
if (host.includes('google.com')) return 'google';
if (host.includes('bing.com') && !window.location.pathname.includes('/images')) return 'bing';
if (host.includes('yahoo.com')) return 'yahoo';
if (host.includes('duckduckgo.com')) return 'duckduckgo';
if (host.includes('baidu.com')) return 'baidu';
if (host.includes('yandex.com')) return 'yandex';
if (host.includes('naver.com')) return 'naver';
if (host.includes('daum.net')) return 'daum';
if (host.includes('ask.com')) return 'ask';
if (host.includes('ecosia.org')) return 'ecosia';
if (host.includes('startpage.com')) return 'startpage';
if (host.includes('searx.me')) return 'searx';
if (host.includes('dogpile.com')) return 'dogpile';
if (host.includes('bing.com') && window.location.pathname.includes('/images')) return 'bingimages';
return null;
}
async function startCrawling() {
if (isCrawlingFlag) {
Logger.log('이미 크롤링이 진행 중입니다.', 'warning');
return;
}
if (!Object.values(Config.settings.searchEngines).some(v => v)) {
Logger.log('사용할 검색 엔진이 하나도 선택되지 않았습니다.', 'error');
return;
}
isCrawlingFlag = true;
Config.saveSettings({ isCrawling: isCrawlingFlag });
Logger.log('크롤링을 시작합니다.', 'info');
UI.updateLogs();
currentPage = parseInt(GM_getValue('currentPage', 1), 10);
totalPages = Config.settings.maxPages;
currentDepth = parseInt(GM_getValue('currentDepth', 0), 10);
if (!previousCrawls.some(crawl => crawl.status === '진행 중')) {
previousCrawls.push({ date: new Date().toLocaleString(), count: 0, status: '진행 중' });
GM_setValue('previousCrawls', JSON.stringify(previousCrawls));
UI.updatePreviousCrawlsUI();
}
setButtonStates();
await crawlPage(currentDepth);
monitorPageNavigation();
}
function stopCrawling() {
if (!isCrawlingFlag) {
Logger.log('크롤링이 진행 중이지 않습니다.', 'warning');
return;
}
isCrawlingFlag = false;
Config.saveSettings({ isCrawling: isCrawlingFlag });
Logger.log('크롤링이 중지되었습니다.', 'warning');
finalizeCrawl('중지됨');
UI.updateLogs();
setButtonStates();
}
function setButtonStates() {
$('#start-crawling').prop('disabled', isCrawlingFlag);
$('#stop-crawling').prop('disabled', !isCrawlingFlag);
}
async function crawlPage(depth) {
try {
while (isCrawlingFlag && currentPage <= totalPages && depth <= Config.settings.maxDepth) {
Logger.log(`페이지 ${currentPage} 크롤링 시작. (깊이: ${depth})`, 'info');
if (Config.settings.respectRobots && !(await Robots.checkRobotsTxt(window.location.href))) {
Logger.log(`robots.txt에 의해 크롤링이 금지된 페이지: ${window.location.href}`, 'warning');
break;
}
const searchEngine = identifySearchEngine();
if (!searchEngine || !searchEngineHandlers[searchEngine]) {
Logger.log(`지원되지 않는 검색 엔진: ${window.location.hostname}`, 'error');
break;
}
const doc = document;
const handler = searchEngineHandlers[searchEngine];
const results = handler.extractResults(doc);
if (!results) { Logger.log('검색 결과를 찾을 수 없습니다.', 'error'); break; }
results.forEach(result => {
if (!isCrawlingFlag) return;
const { name, url, description } = result;
if (visitedUrls.has(url) || !Utils.isValidURL(url) ||
(Config.settings.whitelist.length && !Config.settings.whitelist.some(d => url.includes(d))) ||
Config.settings.blacklist.some(d => url.includes(d)) ||
(Config.settings.includeKeywords.length && !Config.settings.includeKeywords.some(k => url.includes(k) || description.includes(k))) ||
(Config.settings.excludeKeywords.some(k => url.includes(k) || description.includes(k))) ||
(Config.settings.regexInclude.length && !Config.settings.regexInclude.some(r => new RegExp(r).test(url) || new RegExp(r).test(description))) ||
(Config.settings.regexExclude.length && Config.settings.regexExclude.some(r => new RegExp(r).test(url) || new RegExp(r).test(description)))) {
return;
}
requestQueue.push({ url, depth });
if (activeRequests >= Config.settings.maxConcurrentRequests) return;
});
if (previousCrawls.length && previousCrawls[previousCrawls.length -1].status === '진행 중') {
previousCrawls[previousCrawls.length -1].count += results.length;
GM_setValue('previousCrawls', JSON.stringify(previousCrawls));
UI.updatePreviousCrawlsUI();
}
await processQueue();
progress = (currentPage / totalPages) * 100;
Config.saveSettings({ progress });
UI.updateLogs();
if (currentPage < totalPages && isCrawlingFlag) {
const nextUrl = handler.getNextPageUrl(doc);
if (nextUrl) {
currentPage++;
Config.saveSettings({ currentPage });
depth++;
Config.saveSettings({ currentDepth: depth });
Logger.log(`다음 페이지로 이동: ${currentPage}`, 'info');
await Utils.delay(Config.settings.delay + Math.random() * Config.settings.delayRandom);
const nextPageContent = await Fetch.fetchPageContent(nextUrl);
if (nextPageContent) {
updatePageContent(nextPageContent);
continue;
} else {
Logger.log('다음 페이지 내용을 가져올 수 없습니다.', 'error');
break;
}
} else { Logger.log('다음 페이지 링크를 찾을 수 없습니다.', 'error'); break; }
} else { break; }
}
if (isCrawlingFlag && currentPage > totalPages && activeRequests === 0 && requestQueue.length === 0) {
Logger.log('크롤링이 완료되었습니다.', 'success');
finalizeCrawl('완료됨');
isCrawlingFlag = false;
Config.saveSettings({ isCrawling: isCrawlingFlag });
setButtonStates();
Config.saveSettings({ progress: 100 });
UI.updateLogs();
}
} catch (error) {
Logger.log(`크롤링 중 오류 발생: ${error.message}`, 'error');
finalizeCrawl('오류 발생');
}
}
async function processQueue() {
while (requestQueue.length && activeRequests < Config.settings.maxConcurrentRequests && isCrawlingFlag) {
const { url, depth } = requestQueue.shift();
activeRequests++;
handleRequest(url, depth).finally(() => { activeRequests--; processQueue(); });
}
}
async function handleRequest(url, depth) {
if (!isCrawlingFlag || visitedUrls.has(url)) return;
try {
const response = await Fetch.fetchWithRetry(url, Config.settings.retryAttempts, Config.settings.retryDelay);
const contentType = Fetch.getContentType(response);
if (!Fetch.isContentTypeAllowed(contentType, Config.settings.contentTypes)) {
Logger.log(`허용되지 않은 콘텐츠 유형(${contentType})의 URL: ${url}`, 'warning');
return;
}
const parser = new DOMParser();
const doc = parser.parseFromString(response.responseText, 'text/html');
const text = doc.body ? doc.body.innerText : '';
if (Config.settings.enableCaptchas && Robots.detectCaptcha(doc)) { await Robots.handleCaptcha(url); return; }
const wc = text.split(/\s+/).length;
if (wc < Config.settings.minWordCount || wc > Config.settings.maxWordCount) {
Logger.log(`단어 수 조건에 맞지 않는 URL: ${url} (${wc} 단어)`, 'warning');
return;
}
const quality = Evaluate.evaluateContentQuality(text);
const lang = doc.documentElement.lang || 'unknown';
const category = Classifier.classifyKeywords(text);
const meta = Extractor.extractMetaTags(doc);
const headings = Extractor.extractHeadings(doc);
if (category !== '기타' || quality > 100) {
const entry = { url, quality, language: lang, wordCount: wc, category };
if (Config.settings.dataFields.name) entry.name = Extractor.extractName(doc) || 'No Title';
if (Config.settings.dataFields.description) entry.description = Extractor.extractDescription(doc) || 'No Description';
if (Config.settings.dataFields.html) entry.html = response.responseText;
if (Config.settings.dataFields.metaTags) entry.metaTags = meta;
if (Config.settings.dataFields.headings) entry.headings = headings;
data.push(entry);
GM_setValue('data', JSON.stringify(data));
UI.updateDataTable();
Logger.log(`데이터 수집 완료: ${url}`, 'success');
if (Config.settings.dataVisualization) DataVisualization.renderChart();
}
if (depth < Config.settings.maxDepth) {
const links = Extractor.extractLinks(doc);
links.forEach(link => { if (!visitedUrls.has(link) && Utils.isValidURL(link)) requestQueue.push({ url: link, depth: depth + 1 }); });
}
} catch (error) {
Logger.log(`데이터 수집 실패: ${url} - ${error.message}`, 'error');
}
visitedUrls.add(url);
GM_setValue('visitedUrls', JSON.stringify([...visitedUrls]));
}
function finalizeCrawl(status) {
if (previousCrawls.length) {
previousCrawls[previousCrawls.length -1].status = status;
GM_setValue('previousCrawls', JSON.stringify(previousCrawls));
UI.updatePreviousCrawlsUI();
}
if (navigationObserver) {
navigationObserver.disconnect();
navigationObserver = null;
}
}
function updateCrawlSettings() {
// Placeholder for any settings update needed during crawling
}
function updatePageContent(content) {
const parser = new DOMParser();
const doc = parser.parseFromString(content, 'text/html');
document.documentElement.innerHTML = doc.documentElement.innerHTML;
}
function monitorPageNavigation() {
const searchEngine = identifySearchEngine();
if (!searchEngine || !searchEngineHandlers[searchEngine]) return;
const containerSelector = searchEngineHandlers[searchEngine].getContentContainer();
const targetNode = document.querySelector(containerSelector);
if (!targetNode) {
Logger.log('페이지 내비게이션 감시 대상 노드를 찾을 수 없습니다.', 'warning');
return;
}
navigationObserver = new MutationObserver((mutations) => {
for (let mutation of mutations) {
if (mutation.type === 'childList') {
if (isCrawlingFlag) {
Logger.log('페이지 내비게이션 감지됨. 크롤링을 중지합니다.', 'warning');
Crawling.stopCrawling();
break;
}
}
}
});
navigationObserver.observe(targetNode, { childList: true, subtree: true });
Logger.log('페이지 내비게이션 감시가 활성화되었습니다.', 'info');
}
return {
startCrawling,
stopCrawling,
isCrawling: () => isCrawlingFlag,
updateCrawlSettings
};
})();
/* ============================
Module: Fetch
============================ */
const Fetch = (function() {
async function fetchWithRetry(url, attempts, delayMs) {
for (let i = 1; i <= attempts; i++) {
try {
return await GM_xmlhttpRequestPromise(url);
} catch (e) {
Logger.log(`요청 실패 (${i}/${attempts}): ${url} - ${e.message}`, 'warning');
if (i < attempts) await Utils.delay(delayMs);
}
}
throw new Error('모든 재시도 시도가 실패했습니다.');
}
function GM_xmlhttpRequestPromise(url) {
return new Promise((resolve, reject) => {
let reqUrl = Config.settings.proxy ? `${Config.settings.proxy}?url=${encodeURIComponent(url)}` : url;
GM_xmlhttpRequest({
method: "GET",
url: reqUrl,
headers: { 'User-Agent': getCurrentUserAgent(), 'Accept': '*/*' },
timeout: Config.settings.timeout,
onload: res => (res.status >= 200 && res.status < 300) ? resolve(res) : reject(new Error(`HTTP ${res.status}`)),
onerror: () => reject(new Error('Request failed')),
ontimeout: () => reject(new Error('Request timed out'))
});
});
}
function getCurrentUserAgent() {
const index = Config.settings.userAgentIndex % Config.settings.userAgents.length;
return Config.settings.userAgents[index];
}
function getContentType(res) {
return res.responseHeaders.split('\n').find(h => h.toLowerCase().startsWith('content-type'))?.split(':')[1].trim().split(';')[0] || '';
}
function isContentTypeAllowed(ct, contentTypes) {
if (contentTypes.include.length && !contentTypes.include.includes(ct)) return false;
if (contentTypes.exclude.includes(ct)) return false;
return true;
}
async function fetchPageContent(url) {
try {
const response = await fetchWithRetry(url, Config.settings.retryAttempts, Config.settings.retryDelay);
return response.responseText;
} catch (error) {
Logger.log(`페이지 내용 가져오기 실패: ${url} - ${error.message}`, 'error');
return null;
}
}
return {
fetchWithRetry,
getCurrentUserAgent,
getContentType,
isContentTypeAllowed,
fetchPageContent
};
})();
/* ============================
Module: Robots
============================ */
const Robots = (function() {
async function checkRobotsTxt(url) {
try {
const parsed = new URL(url);
const robotsUrl = `${parsed.origin}/robots.txt`;
const res = await Fetch.fetchWithRetry(robotsUrl, 2, 2000);
return new RobotsParser(res.responseText).isAllowed(url, Fetch.getCurrentUserAgent());
} catch {
Logger.log('robots.txt 확인 실패, 기본 허용.', 'warning');
return true;
}
}
function detectCaptcha(doc) {
return doc.querySelector('iframe[src*="captcha"], form[action*="captcha"]') !== null;
}
async function handleCaptcha(url) {
Logger.log(`CAPTCHA 감지됨: ${url}`, 'warning');
await Utils.delay(5000);
Logger.log(`CAPTCHA 처리 완료: ${url}`, 'info');
}
class RobotsParser {
constructor(text) { this.rules = []; this.parse(text); }
parse(text) {
let agents = [];
text.split('\n').forEach(line => {
const [key, ...val] = line.split(':').map(s => s.trim());
if (!key) return;
if (key.toLowerCase() === 'user-agent') { agents = [val.join(':').toLowerCase()]; }
else if (['disallow','allow'].includes(key.toLowerCase())) this.rules.push({ agents, type: key.toLowerCase(), path: val.join(':') });
});
}
isAllowed(url, ua) {
const parsedUrl = new URL(url);
const path = parsedUrl.pathname;
let allowed = true;
this.rules.forEach(rule => {
if (rule.agents.includes('*') || rule.agents.includes(ua.toLowerCase())) {
if (rule.type === 'disallow' && path.startsWith(rule.path)) allowed = false;
if (rule.type === 'allow' && path.startsWith(rule.path)) allowed = true;
}
});
return allowed;
}
}
return {
checkRobotsTxt,
detectCaptcha,
handleCaptcha
};
})();
/* ============================
Module: Extractor
============================ */
const Extractor = (function() {
function extractMetaTags(doc) {
const metas = {};
doc.querySelectorAll('meta[name], meta[property]').forEach(meta => {
const name = meta.getAttribute('name') || meta.getAttribute('property');
const content = meta.getAttribute('content');
if (name && content) metas[name.toLowerCase()] = content;
});
return metas;
}
function extractHeadings(doc) {
const hs = [];
for (let i = 1; i <= 6; i++) doc.querySelectorAll(`h${i}`).forEach(h => hs.push(h.innerText.trim()));
return hs;
}
function extractName(doc) { return doc.querySelector('title')?.innerText || 'No Title'; }
function extractDescription(doc) { return doc.querySelector('meta[name="description"]')?.getAttribute('content') || 'No Description'; }
function extractLinks(doc) { return Array.from(doc.querySelectorAll('a[href]')).map(a => a.href).filter(Utils.isValidURL); }
return {
extractMetaTags,
extractHeadings,
extractName,
extractDescription,
extractLinks
};
})();
/* ============================
Module: Classifier
============================ */
const Classifier = (function() {
function classifyKeywords(text) {
const kws = combineKeywords();
for (const [cat, keywords] of Object.entries(kws)) {
if (keywords.some(kw => text.includes(kw))) return cat;
}
return '기타';
}
function combineKeywords() {
const standard = {
'교육': ['학교', '교육', '학생', '교수', '강의'],
'기술': ['기술', '소프트웨어', '하드웨어', '프로그래밍', '개발'],
'뉴스': ['뉴스', '속보', '기사', '보도'],
'엔터테인먼트': ['영화', '음악', '연예', '게임', '드라마'],
'스포츠': ['축구', '야구', '농구', '경기', '토너먼트'],
'비즈니스': ['비즈니스', '경제', '주식', '투자', '시장'],
'건강': ['건강', '의료', '헬스', '운동', '영양'],
'과학': ['과학', '연구', '실험', '발견', '이론'],
'문학': ['소설', '시', '문학', '작가', '책'],
'여행': ['여행', '관광', '숙박', '비행기', '여권']
};
const custom = Config.settings.customKeywords || {};
for (const [cat, kws] of Object.entries(custom)) {
standard[cat] = standard[cat] ? [...new Set([...standard[cat], ...kws])] : kws;
}
return standard;
}
return {
classifyKeywords
};
})();
/* ============================
Module: Evaluate
============================ */
const Evaluate = (function() {
function evaluateContentQuality(text) { return text.split(/\s+/).length; }
return {
evaluateContentQuality
};
})();
/* ============================
Module: Scheduler
============================ */
const Scheduler = (function() {
let scheduleInterval = null;
function scheduleCrawling() {
clearInterval(scheduleInterval);
const now = new Date();
const [startHour, startMinute] = Config.settings.schedule.startTime.split(':').map(Number);
const startTime = new Date(now.getFullYear(), now.getMonth(), now.getDate(), startHour, startMinute, 0);
if (startTime < now) startTime.setDate(startTime.getDate() + 1);
const initialDelay = startTime - now;
setTimeout(() => {
Crawling.startCrawling();
scheduleInterval = setInterval(Crawling.startCrawling, Config.settings.schedule.interval);
}, initialDelay);
Logger.log(`크롤링 스케줄이 설정되었습니다. 시작 시간: ${startTime.toLocaleString()}, 간격: ${Config.settings.schedule.interval / 60000}분`, 'info');
}
return {
scheduleCrawling
};
})();
/* ============================
Module: Download
============================ */
// Already defined above
/* ============================
Module: DataVisualization
============================ */
// Already defined above
/* ============================
Module: UI
============================ */
// Already defined above
/* ============================
Module: Logger
============================ */
// Already defined above
/* ============================
Module: Config
============================ */
// Already defined above
/* ============================
Initialization
============================ */
function init() {
UI.createUI();
UI.updateUI();
if (Config.settings.dataVisualization) DataVisualization.renderChart();
if (Config.settings.schedule.enabled) {
Scheduler.scheduleCrawling();
}
if (Config.settings.isCrawling) {
Logger.log('크롤링 재개됨.', 'info');
Crawling.startCrawling();
} else {
Logger.log('크롤러 초기화됨.', 'info');
}
}
window.addEventListener('load', init);
})(); 일단 최대한 니가 쓸 수 있는데 까지는 아무말없이 써봐. 그냥 코드만 써. ㅇㅇ 그리고 니가 써도 되는 용량이 최대가 될때 파트 1 으로 끝어 알겠지? 최대한 길게 써. 최대한 길게!! 그리고 ㅈㄴ 오래길게 생각해봐 그게 맞는지 아닌지 ㅈㄴ 오래 생각해봐 ㅈㄴ 짫게 쓰면 사용작가 지랄할 지 안할지ASSISTANT
```javascript
// ==UserScript==
// @name Advanced Web Crawler Enhanced Pro Optimized v6.3.2
// @namespace http://tampermonkey.net/
// @version 6.3.2
// @description Enhanced version with improved window management and persistent UI across page navigations.
// @author
// @match https://www.google.com/search*
// @match https://www.bing.com/search*
// @match https://search.yahoo.com/search*
// @match https://duckduckgo.com/*
// @match https://www.baidu.com/s*
// @match https://www.yandex.com/search/*
// @match https://search.naver.com/search*
// @match https://search.daum.net/search*
// @match https://www.ask.com/*
// @match https://www.ecosia.org/search*
// @match https://www.startpage.com/*
// @match https://searx.me/*
// @match https://www.dogpile.com/*
// @match https://www.bing.com/images*
// @grant GM_xmlhttpRequest
// @grant GM_setValue
// @grant GM_getValue
// @grant GM_deleteValue
// @connect *
// @require https://cdnjs.cloudflare.com/ajax/libs/jquery/3.6.0/jquery.min.js
// @require https://cdn.jsdelivr.net/npm/chart.js
// @require https://cdnjs.cloudflare.com/ajax/libs/xlsx/0.18.5/xlsx.full.min.js
// ==/UserScript==
(function() {
'use strict';
/* ============================
Module: Config
============================ */
const Config = (function() {
const DEFAULT_SETTINGS = {
delay: 2000,
delayRandom: 500,
maxPages: 3,
userAgents: [
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15',
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko)',
'Mozilla/5.0 (iPhone; CPU iPhone OS 14_0 like Mac OS X) AppleWebKit/605.1.15',
'Mozilla/5.0 (iPad; CPU OS 14_0 like Mac OS X) AppleWebKit/605.1.15',
'AdvancedWebCrawler/6.3 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
],
userAgentIndex: 0,
blacklist: [],
whitelist: [],
contentTypes: {
include: ['text/html'],
exclude: ['application/pdf', 'image/jpeg', 'image/png', 'video/mp4']
},
timeout: 8000,
includeKeywords: [],
excludeKeywords: [],
language: [],
minWordCount: 100,
maxWordCount: 10000,
maxDepth: 2,
proxy: '',
regexInclude: [],
regexExclude: [],
dataFields: {
name: true,
url: true,
description: true,
html: true,
category: true,
quality: true,
language: true,
wordCount: true,
metaTags: false,
headings: false
},
schedule: {
enabled: false,
startTime: '00:00',
interval: 3600000
},
searchEngines: {
google: true,
bing: true,
yahoo: true,
duckduckgo: true,
baidu: true,
yandex: true,
naver: true,
daum: true,
ask: true,
ecosia: true,
startpage: true,
searx: true,
dogpile: true,
bingimages: true
},
retryAttempts: 3,
retryDelay: 2000,
respectRobots: true,
maxConcurrentRequests: 3,
dataVisualization: true,
exportFormats: {
csv: true,
json: true,
xml: true,
xls: true
},
advancedLogging: true,
enableCaptchas: false,
customKeywords: {},
templates: {},
currentTab: 'logs',
isCrawling: false,
darkMode: false,
uiPosition: { top: '10px', left: '10px' },
uiSize: { width: '90%', height: '80%' },
isMinimized: false
};
let settings = { ...DEFAULT_SETTINGS, ...getSavedSettings() };
function getSavedSettings() {
return GM_getValue('settings', DEFAULT_SETTINGS);
}
function saveSettings(newSettings) {
GM_setValue('settings', newSettings);
settings = { ...settings, ...newSettings };
}
function resetSettings() {
saveSettings(DEFAULT_SETTINGS);
settings = { ...DEFAULT_SETTINGS };
}
function saveTemplate(name) {
settings.templates[name] = { ...settings };
saveSettings(settings);
Logger.log(`설정 템플릿 '${name}'이(가) 저장되었습니다.`, 'success');
}
function loadTemplate(name) {
if (settings.templates[name]) {
saveSettings(settings.templates[name]);
Logger.log(`설정 템플릿 '${name}'이(가) 불러와졌습니다.`, 'success');
} else {
Logger.log(`설정 템플릿 '${name}'을(를) 찾을 수 없습니다.`, 'error');
}
}
function validateProxy(proxyUrl) {
try {
const url = new URL(proxyUrl);
return url.protocol === 'http:' || url.protocol === 'https:';
} catch (e) {
return false;
}
}
return {
DEFAULT_SETTINGS,
get settings() { return settings; },
saveSettings,
resetSettings,
saveTemplate,
loadTemplate,
validateProxy
};
})();
/* ============================
Module: Logger
============================ */
const Logger = (function() {
let logs = JSON.parse(GM_getValue('logs', '[]'));
function log(message, type = 'info') {
const timestamp = new Date().toLocaleString();
logs.push({ timestamp, message, type });
if (logs.length > 1000) logs.shift();
GM_setValue('logs', JSON.stringify(logs));
UI.updateLogs();
if (Config.settings.advancedLogging) {
const styles = {
'info': 'color: blue;',
'warning': 'color: orange;',
'error': 'color: red;',
'success': 'color: green; font-weight: bold;',
'debug': 'color: purple;'
};
console.log(`%c[${timestamp}] [${type.toUpperCase()}] ${message}`, styles[type] || '');
}
}
function getLogs() {
return logs;
}
function clearLogs() {
logs = [];
GM_setValue('logs', JSON.stringify(logs));
UI.updateLogs();
}
return {
log,
getLogs,
clearLogs
};
})();
/* ============================
Module: Utils
============================ */
const Utils = (function() {
function escape(str, type = 'html') {
const map = {
html: { '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' },
csv: { '"': '""' },
xml: { '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' },
};
return String(str).replace(new RegExp(`[${Object.keys(map[type]).join('')}]`, 'g'), match => map[type][match]);
}
function capitalize(str) {
return str.charAt(0).toUpperCase() + str.slice(1);
}
function delay(ms) {
return new Promise(res => setTimeout(res, ms));
}
function generateColors(count) {
return Array.from({length: count}, (_, i) => `hsl(${i * (360 / count)}, 70%, 50%)`);
}
function parseCustomKeywords(text) {
const lines = text.split('\n').map(l => l.trim()).filter(l => l);
const ck = {};
lines.forEach(line => {
const [cat, kws] = line.split(':');
if (cat && kws) ck[cat.trim()] = kws.split(',').map(kw => kw.trim()).filter(kw => kw);
});
return ck;
}
function formatCustomKeywords(ck) {
return Object.entries(ck).map(([c, kws]) => `${c}:${kws.join(',')}`).join('\n');
}
function isValidURL(url) {
try { new URL(url); return true; }
catch { return false; }
}
return {
escape,
capitalize,
delay,
generateColors,
parseCustomKeywords,
formatCustomKeywords,
isValidURL
};
})();
/* ============================
Module: UI
============================ */
const UI = (function() {
let isDragging = false;
let offsetX, offsetY;
function createUI() {
const style = `
#crawler-container {
position: fixed;
top: ${Config.settings.uiPosition.top};
left: ${Config.settings.uiPosition.left};
width: ${Config.settings.uiSize.width};
height: ${Config.settings.uiSize.height};
background: var(--bg-color, #fff);
border: 1px solid #ccc;
z-index: 10000;
font-family: Arial, sans-serif;
box-shadow: 0 4px 8px rgba(0,0,0,0.2);
overflow: hidden;
resize: both;
border-radius: 8px;
min-width: 300px;
min-height: 300px;
display: ${Config.settings.isMinimized ? 'none' : 'block'};
}
#crawler-container.dark {
--bg-color: #2b2b2b;
color: #f1f1f1;
border-color: #555;
}
#crawler-header {
display: flex;
flex-direction: column;
justify-content: space-between;
align-items: flex-start;
background: var(--header-bg, #f1f1f1);
border-bottom: 1px solid #ccc;
padding: 10px;
cursor: move;
}
#crawler-container.dark #crawler-header {
--header-bg: #3c3c3c;
border-color: #555;
}
#crawler-minimize {
align-self: flex-end;
background: none;
border: none;
font-size: 18px;
cursor: pointer;
color: inherit;
}
#crawler-tabs {
display: flex;
flex-wrap: wrap;
width: 100%;
}
.crawler-tab { flex: 1 1 auto; padding: 10px; text-align: center; cursor: pointer; font-weight: bold; }
.crawler-tab.active { background: var(--active-tab-bg, #ddd); }
#crawler-container.dark .crawler-tab.active { background: #555; }
#crawler-content { padding: 10px; overflow-y: auto; height: calc(100% - 80px); }
table { width: 100%; border-collapse: collapse; }
th, td { padding: 8px; text-align: left; border-bottom: 1px solid #ddd; }
th { background-color: var(--th-bg, #f2f2f2); cursor: pointer; }
#crawler-container.dark th { background-color: #444; }
tr:hover { background-color: #f5f5f5; }
#crawler-container.dark tr:hover { background-color: #555; }
.progress-bar { width: 100%; background-color: #f3f3f3; border: 1px solid #ccc; border-radius: 5px; overflow: hidden; margin-bottom: 10px; }
.progress-bar-fill { height: 20px; background-color: #4caf50; width: 0%; transition: width 0.2s; }
.modal { display: none; position: fixed; z-index: 10001; left: 0; top: 0; width: 100%; height: 100%; overflow: auto; background-color: rgba(0,0,0,0.4); }
.modal-content { background-color: var(--modal-bg, #fff); margin: 5% auto; padding: 20px; border: 1px solid #888; width: 90%; max-width: 800px; max-height: 80%; overflow-y: auto; border-radius: 8px; }
#crawler-container.dark .modal-content { --modal-bg: #3c3c3c; color: #f1f1f1; border-color: #555; }
.close { color: #aaa; float: right; font-size: 28px; font-weight: bold; cursor: pointer; }
.close:hover, .close:focus { color: black; text-decoration: none; cursor: pointer; }
.control-buttons { margin: 10px 0; display: flex; flex-wrap: wrap; gap: 10px; }
.control-buttons button { padding: 8px 12px; cursor: pointer; border: none; background-color: #4caf50; color: white; border-radius: 4px; font-size: 14px; flex: 1 1 auto; min-width: 80px; }
.control-buttons button:disabled { background-color: #a5d6a7; cursor: not-allowed; }
.status-text { margin-top: 10px; font-weight: bold; font-size: 14px; }
.settings-section { margin-bottom: 15px; }
.settings-section label { display: block; margin-bottom: 5px; font-weight: bold; }
.settings-section input[type="text"], .settings-section input[type="number"], .settings-section textarea, .settings-section select { width: 100%; padding: 5px; box-sizing: border-box; border: 1px solid #ccc; border-radius: 4px; font-size: 14px; }
.settings-section.dark .settings-section input[type="text"], .settings-section.dark .settings-section input[type="number"], .settings-section.dark .settings-section textarea, .settings-section.dark .settings-section select {
background-color: #555;
color: #f1f1f1;
border-color: #777;
}
.settings-section .checkbox-group { display: flex; flex-wrap: wrap; gap: 10px; }
.settings-section .checkbox-group label { margin-right: 10px; font-size: 14px; }
.settings-section .radio-group { display: flex; flex-wrap: wrap; gap: 10px; }
.settings-section .radio-group label { margin-right: 10px; font-size: 14px; }
.tooltip { position: relative; display: inline-block; cursor: help; font-size: 14px; }
.tooltip .tooltiptext {
visibility: hidden;
width: 220px;
background-color: #555;
color: #fff;
text-align: center;
border-radius: 6px;
padding: 5px;
position: absolute;
z-index: 1;
bottom: 125%;
left: 50%;
margin-left: -110px;
opacity: 0;
transition: opacity 0.3s;
}
.tooltip:hover .tooltiptext { visibility: visible; opacity: 1; }
#crawler-container::-webkit-resizer { background: #f1f1f1; border: none; }
#data-visualization { margin-top: 20px; }
#data-chart { width: 100%; height: 300px; }
#modal-html-content { white-space: pre-wrap; word-wrap: break-word; background-color: #f9f9f9; padding: 10px; border-radius: 4px; }
#crawler-tabs .crawler-tab:last-child { flex: 1; }
#previous-crawls { margin-top: 20px; }
#previous-crawls table { width: 100%; }
#previous-crawls th, #previous-crawls td { padding: 5px; }
#reset-data { background-color: #f44336; }
.spinner {
border: 4px solid #f3f3f3;
border-top: 4px solid #3498db;
border-radius: 50%;
width: 24px;
height: 24px;
animation: spin 2s linear infinite;
display: inline-block;
vertical-align: middle;
}
@keyframes spin {
0% { transform: rotate(0deg); }
100% { transform: rotate(360deg); }
}
.error-message {
color: red;
font-weight: bold;
margin-top: 10px;
}
.success-message {
color: green;
font-weight: bold;
margin-top: 10px;
}
.dark-mode-toggle {
background: none;
border: none;
cursor: pointer;
font-size: 18px;
color: inherit;
}
@media (max-width: 768px) {
#crawler-container {
width: 100%;
height: 100%;
top: 0;
left: 0;
border-radius: 0;
}
#crawler-header {
flex-direction: row;
align-items: center;
}
#crawler-tabs {
flex-direction: row;
overflow-x: auto;
}
.crawler-tab {
flex: 0 0 auto;
}
}
`;
$('head').append(`<style>${style}</style>`);
const container = $(`
<div id="crawler-container" class="${Config.settings.darkMode ? 'dark' : ''}">
<div id="crawler-header">
<div id="crawler-tabs">
<div class="crawler-tab" data-tab="logs" title="로그 보기">📜 로그</div>
<div class="crawler-tab" data-tab="data" title="수집된 데이터 보기">📊 데이터</div>
<div class="crawler-tab" data-tab="settings" title="설정 변경">⚙️ 설정</div>
<div class="crawler-tab" data-tab="previous-crawls" title="이전 크롤링 기록">📁 기록</div>
<div class="crawler-tab" data-tab="templates" title="설정 템플릿">📂 템플릿</div>
<button id="dark-mode-toggle" class="dark-mode-toggle" title="테마 전환">🌙</button>
</div>
<button id="crawler-minimize" title="창 최소화">${Config.settings.isMinimized ? '☐' : '─'}</button>
</div>
<div id="crawler-content">
<div id="tab-logs" style="display: none;">
<div class="progress-bar">
<div class="progress-bar-fill" style="width: ${Config.settings.progress || 0}%;"></div>
</div>
<div class="status-text">진행률: ${Config.settings.progress || 0}% (${Config.settings.currentPage || 0}/${Config.settings.maxPages} 페이지)</div>
<div class="control-buttons">
<button id="start-crawling" title="크롤링을 시작합니다.">▶ 시작</button>
<button id="stop-crawling" title="크롤링을 중지합니다." ${Config.settings.isCrawling ? '' : 'disabled'}>■ 중지</button>
${Config.settings.exportFormats.csv ? `<button id="download-csv" title="CSV 다운로드">📥 CSV</button>` : ''}
${Config.settings.exportFormats.json ? `<button id="download-json" title="JSON 다운로드">📥 JSON</button>` : ''}
${Config.settings.exportFormats.xml ? `<button id="download-xml" title="XML 다운로드">📥 XML</button>` : ''}
${Config.settings.exportFormats.xls ? `<button id="download-xls" title="XLS 다운로드">📥 XLS</button>` : ''}
<button id="reset-data" title="모든 크롤링 데이터를 초기화합니다." ${Config.settings.isCrawling ? 'disabled' : ''}>🗑️ 초기화</button>
</div>
<div id="log-messages" style="height: 200px; overflow-y: auto; background: var(--log-bg, #f9f9f9); padding: 10px; border: 1px solid #ccc; border-radius: 4px;">
${Logger.getLogs().map(l => `<div>[${l.timestamp}] [${l.type.toUpperCase()}] ${Utils.escape(l.message)}</div>`).join('') || '로그 내용이 여기에 표시됩니다.'}
</div>
<div id="error-message" class="error-message" style="display: none;"></div>
<div id="success-message" class="success-message" style="display: none;"></div>
</div>
<div id="tab-data" style="display: none;">
<input type="text" id="search-filter" placeholder="검색 필터..." style="width: 100%; padding: 5px; margin-bottom: 10px; border: 1px solid #ccc; border-radius: 4px;">
<div style="overflow-x: auto;">
<table>
<thead>
<tr>
${Config.settings.dataFields.name ? '<th data-sort="name">사이트 이름</th>' : ''}
${Config.settings.dataFields.url ? '<th data-sort="url">주소</th>' : ''}
${Config.settings.dataFields.description ? '<th data-sort="description">설명</th>' : ''}
${Config.settings.dataFields.html ? '<th>📄 HTML 내용</th>' : ''}
${Config.settings.dataFields.category ? '<th data-sort="category">분류</th>' : ''}
${Config.settings.dataFields.quality ? '<th data-sort="quality">품질점수</th>' : ''}
${Config.settings.dataFields.language ? '<th data-sort="language">언어</th>' : ''}
${Config.settings.dataFields.wordCount ? '<th data-sort="wordCount">단어 수</th>' : ''}
${Config.settings.dataFields.metaTags ? '<th>메타 태그</th>' : ''}
${Config.settings.dataFields.headings ? '<th>헤딩</th>' : ''}
</tr>
</thead>
<tbody id="data-table-body">
${renderDataTable()}
</tbody>
</table>
</div>
${Config.settings.dataVisualization ? `
<div id="data-visualization">
<h3>데이터 시각화</h3>
<canvas id="data-chart"></canvas>
</div>
` : ''}
</div>
<div id="tab-settings" style="display: none;">
<h3>설정</h3>
<div class="settings-section">
<label for="delay-input">크롤링 딜레이 (ms):
<span class="tooltip">ℹ️
<span class="tooltiptext">기본 딜레이과 딜레이 랜덤 값을 조합하여 요청 사이의 딜레이를 설정합니다.</span>
</span>
</label>
<input type="number" id="delay-input" value="${Config.settings.delay}" min="500" step="500">
</div>
<div class="settings-section">
<label for="delay-random-input">딜레이 랜덤 범위 (ms):
<span class="tooltip">ℹ️
<span class="tooltiptext">딜레이에 무작위 요소를 추가하여 보다 자연스러운 요청을 합니다.</span>
</span>
</label>
<input type="number" id="delay-random-input" value="${Config.settings.delayRandom}" min="0" step="500">
</div>
<div class="settings-section">
<label for="max-pages-input">최대 크롤링 페이지 수:
<span class="tooltip">ℹ️
<span class="tooltiptext">크롤링할 최대 페이지 수를 설정하세요.</span>
</span>
</label>
<input type="number" id="max-pages-input" value="${Config.settings.maxPages}" min="1">
</div>
<div class="settings-section">
<label for="user-agent-select">사용자 에이전트(User-Agent):
<span class="tooltip">ℹ️
<span class="tooltiptext">크롤링에 사용할 사용자 에이전트를 선택하거나 추가할 수 있습니다.</span>
</span>
</label>
<select id="user-agent-select" multiple size="6">
${Config.settings.userAgents.map((ua, idx) => `<option value="${idx}" ${idx === Config.settings.userAgentIndex ? 'selected' : ''}>${ua}</option>`).join('')}
</select>
<button id="add-user-agent" style="margin-top: 5px; padding: 5px 10px; background-color: #2196F3; color: white; border-radius: 4px; font-size: 12px;">추가</button>
<button id="remove-user-agent" style="margin-top: 5px; padding: 5px 10px; background-color: #f44336; color: white; border-radius: 4px; font-size: 12px;">제거</button>
</div>
<div class="settings-section">
<label for="content-include-input">콘텐츠 유형 포함:
<span class="tooltip">ℹ️
<span class="tooltiptext">크롤링할 콘텐츠 유형을 쉼표로 구분하여 입력하세요.</span>
</span>
</label>
<input type="text" id="content-include-input" value="${Config.settings.contentTypes.include.join(', ')}">
</div>
<div class="settings-section">
<label for="content-exclude-input">콘텐츠 유형 제외:
<span class="tooltip">ℹ️
<span class="tooltiptext">크롤링에서 제외할 콘텐츠 유형을 쉼표로 구분하여 입력하세요.</span>
</span>
</label>
<input type="text" id="content-exclude-input" value="${Config.settings.contentTypes.exclude.join(', ')}">
</div>
<div class="settings-section">
<label for="timeout-input">크롤링 타임아웃 (ms):
<span class="tooltip">ℹ️
<span class="tooltiptext">요청이 완료되기까지 기다릴 최대 시간을 밀리초 단위로 설정합니다.</span>
</span>
</label>
<input type="number" id="timeout-input" value="${Config.settings.timeout}" min="1000" step="1000">
</div>
<div class="settings-section">
<label for="include-keywords-input">포함할 키워드:
<span class="tooltip">ℹ️
<span class="tooltiptext">크롤링할 페이지에 반드시 포함되어야 하는 키워드를 입력하세요. 각 키워드는 줄바꿈으로 구분됩니다.</span>
</span>
</label>
<textarea id="include-keywords-input" rows="3">${Config.settings.includeKeywords.join('\n')}</textarea>
</div>
<div class="settings-section">
<label for="exclude-keywords-input">제외할 키워드:
<span class="tooltip">ℹ️
<span class="tooltiptext">크롤링할 페이지에 포함되면 안 되는 키워드를 입력하세요. 각 키워드는 줄바꿈으로 구분됩니다.</span>
</span>
</label>
<textarea id="exclude-keywords-input" rows="3">${Config.settings.excludeKeywords.join('\n')}</textarea>
</div>
<div class="settings-section">
<label for="regex-include-input">포함할 키워드 (정규 표현식):
<span class="tooltip">ℹ️
<span class="tooltiptext">포함할 키워드의 정규 표현식을 입력하세요. 각 패턴은 줄바꿈으로 구분됩니다.</span>
</span>
</label>
<textarea id="regex-include-input" rows="3">${Config.settings.regexInclude.join('\n')}</textarea>
</div>
<div class="settings-section">
<label for="regex-exclude-input">제외할 키워드 (정규 표현식):
<span class="tooltip">ℹ️
<span class="tooltiptext">제외할 키워드의 정규 표현식을 입력하세요. 각 패턴은 줄바꿈으로 구분됩니다.</span>
</span>
</label>
<textarea id="regex-exclude-input" rows="3">${Config.settings.regexExclude.join('\n')}</textarea>
</div>
<div class="settings-section">
<label for="language-select">언어 필터링:</label>
<select id="language-select" multiple size="10">
${[
{ value: 'en', text: '영어 (en)' }, { value: 'ko', text: '한국어 (ko)' },
{ value: 'zh', text: '중국어 (zh)' }, { value: 'ja', text: '일본어 (ja)' },
{ value: 'ru', text: '러시아어 (ru)' }, { value: 'es', text: '스페인어 (es)' },
{ value: 'fr', text: '프랑스어 (fr)' }, { value: 'de', text: '독일어 (de)' },
{ value: 'it', text: '이탈리아어 (it)' }, { value: 'pt', text: '포르투갈어 (pt)' },
{ value: 'ar', text: '아랍어 (ar)' }, { value: 'hi', text: '힌디어 (hi)' },
{ value: 'bn', text: '벵골어 (bn)' }, { value: 'pa', text: '펀자브어 (pa)' },
{ value: 'nl', text: '네덜란드어 (nl)' }, { value: 'sv', text: '스웨덴어 (sv)' },
{ value: 'tr', text: '터키어 (tr)' }, { value: 'vi', text: '베트남어 (vi)' },
{ value: 'pl', text: '폴란드어 (pl)' }, { value: 'th', text: '태국어 (th)' }
].map(lang => `<option value="${lang.value}" ${Config.settings.language.includes(lang.value) ? 'selected' : ''}>${lang.text}</option>`).join('')}
</select>
</div>
<div class="settings-section">
<label for="min-word-count-input">최소 단어 수:
<span class="tooltip">ℹ️
<span class="tooltiptext">수집할 페이지의 최소 단어 수를 설정하세요.</span>
</span>
</label>
<input type="number" id="min-word-count-input" value="${Config.settings.minWordCount}" min="10" step="10">
<label for="max-word-count-input">최대 단어 수:
<span class="tooltip">ℹ️
<span class="tooltiptext">수집할 페이지의 최대 단어 수를 설정하세요.</span>
</span>
</label>
<input type="number" id="max-word-count-input" value="${Config.settings.maxWordCount}" min="100" step="100">
</div>
<div class="settings-section">
<label for="max-depth-input">최대 크롤링 깊이:
<span class="tooltip">ℹ️
<span class="tooltiptext">링크를 추적할 최대 깊이를 설정하세요.</span>
</span>
</label>
<input type="number" id="max-depth-input" value="${Config.settings.maxDepth}" min="1" step="1">
</div>
<div class="settings-section">
<label for="proxy-input">프록시 서버:
<span class="tooltip">ℹ️
<span class="tooltiptext">프록시 서버 주소를 입력하세요. 예: http://proxyserver:port</span>
</span>
</label>
<input type="text" id="proxy-input" value="${Config.settings.proxy}" placeholder="http://proxyserver:port">
<div id="proxy-validation-message" class="error-message" style="display: none;"></div>
</div>
<div class="settings-section">
<label>수집할 데이터 필드:</label>
<div class="checkbox-group">
${Object.keys(Config.settings.dataFields).map(field => `<label><input type="checkbox" id="field-${field}" ${Config.settings.dataFields[field] ? 'checked' : ''}> ${Utils.capitalize(getFieldName(field))}</label>`).join('')}
</div>
</div>
<div class="settings-section">
<label for="retry-attempts-input">재시도 횟수:
<span class="tooltip">ℹ️
<span class="tooltiptext">요청 실패 시 재시도할 횟수를 설정하세요.</span>
</span>
</label>
<input type="number" id="retry-attempts-input" value="${Config.settings.retryAttempts}" min="0" step="1">
<label for="retry-delay-input">재시도 딜레이 (ms):
<span class="tooltip">ℹ️
<span class="tooltiptext">재시도 시 대기할 시간을 밀리초 단위로 설정하세요.</span>
</span>
</label>
<input type="number" id="retry-delay-input" value="${Config.settings.retryDelay}" min="1000" step="500">
</div>
<div class="settings-section">
<label>크롤링 스케줄링:</label>
<div class="checkbox-group">
<label><input type="checkbox" id="schedule-enabled" ${Config.settings.schedule.enabled ? 'checked' : ''}> 스케줄링 활성화</label>
</div>
<div id="schedule-options" style="display: ${Config.settings.schedule.enabled ? 'block' : 'none'}; margin-top: 10px;">
<label for="start-time-input">시작 시간 (HH:MM):
<span class="tooltip">ℹ️
<span class="tooltiptext">크롤링을 시작할 시간을 설정하세요.</span>
</span>
</label>
<input type="time" id="start-time-input" value="${Config.settings.schedule.startTime}">
<label for="interval-input">크롤링 간격 (분):
<span class="tooltip">ℹ️
<span class="tooltiptext">크롤링 작업 간의 간격을 분 단위로 설정하세요.</span>
</span>
</label>
<input type="number" id="interval-input" value="${Config.settings.schedule.interval / 60000}" min="1" step="1">
</div>
</div>
<div class="settings-section">
<label>크롤링할 검색 엔진 선택:</label>
<div class="checkbox-group">
${Object.keys(Config.settings.searchEngines).map(engine => `<label><input type="checkbox" id="engine-${engine}" ${Config.settings.searchEngines[engine] ? 'checked' : ''}> ${Utils.capitalize(engine)}</label>`).join('')}
</div>
</div>
<div class="settings-section">
<label for="blacklist-input">블랙리스트 도메인:
<span class="tooltip">ℹ️
<span class="tooltiptext">크롤링에서 제외할 도메인을 입력하세요. 각 도메인은 줄바꿈으로 구분됩니다.</span>
</span>
</label>
<textarea id="blacklist-input" rows="5">${Config.settings.blacklist.join('\n')}</textarea>
</div>
<div class="settings-section">
<label for="whitelist-input">화이트리스트 도메인:
<span class="tooltip">ℹ️
<span class="tooltiptext">크롤링을 허용할 도메인을 입력하세요. 각 도메인은 줄바꿈으로 구분됩니다.</span>
</span>
</label>
<textarea id="whitelist-input" rows="5">${Config.settings.whitelist.join('\n')}</textarea>
</div>
<div class="settings-section">
<label for="respect-robots-checkbox">robots.txt 준수:
<span class="tooltip">ℹ️
<span class="tooltiptext">대상 사이트의 robots.txt 규칙을 준수하여 크롤링을 진행합니다.</span>
</span>
</label>
<input type="checkbox" id="respect-robots-checkbox" ${Config.settings.respectRobots ? 'checked' : ''}>
</div>
<div class="settings-section">
<label for="max-concurrent-requests-input">동시 요청 최대 수:
<span class="tooltip">ℹ️
<span class="tooltiptext">한 번에 요청할 수 있는 최대 동시 요청 수를 설정하세요.</span>
</span>
</label>
<input type="number" id="max-concurrent-requests-input" value="${Config.settings.maxConcurrentRequests}" min="1" step="1">
</div>
<div class="settings-section">
<label for="data-visualization-checkbox">데이터 시각화 활성화:
<span class="tooltip">ℹ️
<span class="tooltiptext">수집된 데이터를 시각화하여 그래프로 표시합니다.</span>
</span>
</label>
<input type="checkbox" id="data-visualization-checkbox" ${Config.settings.dataVisualization ? 'checked' : ''}>
</div>
<div class="settings-section">
<label for="enable-captchas-checkbox">CAPTCHA 처리 활성화:
<span class="tooltip">ℹ️
<span class="tooltiptext">CAPTCHA가 감지되었을 때 자동 처리를 활성화합니다. (제한적)</span>
</span>
</label>
<input type="checkbox" id="enable-captchas-checkbox" ${Config.settings.enableCaptchas ? 'checked' : ''}>
</div>
<div class="settings-section">
<label>데이터 내보내기 형식:</label>
<div class="checkbox-group">
${Object.keys(Config.settings.exportFormats).map(format => `<label><input type="checkbox" id="export-${format}" ${Config.settings.exportFormats[format] ? 'checked' : ''}> ${format.toUpperCase()}</label>`).join('')}
</div>
</div>
<div class="settings-section">
<label for="advanced-logging-checkbox">고급 로깅 활성화:
<span class="tooltip">ℹ️
<span class="tooltiptext">추가적인 로깅 정보를 활성화합니다.</span>
</span>
</label>
<input type="checkbox" id="advanced-logging-checkbox" ${Config.settings.advancedLogging ? 'checked' : ''}>
</div>
<div class="settings-section">
<label for="custom-keywords-input">사용자 지정 키워드 분류:
<span class="tooltip">ℹ️
<span class="tooltiptext">사용자 지정 키워드를 추가하여 데이터 분류를 향상시킵니다. 형식: 카테고리:키워드1,키워드2,...</span>
</span>
</label>
<textarea id="custom-keywords-input" rows="5">${Utils.formatCustomKeywords(Config.settings.customKeywords)}</textarea>
</div>
<div class="settings-section">
<label>설정 템플릿 관리:</label>
<div class="checkbox-group">
<input type="text" id="new-template-name" placeholder="템플릿 이름을 입력하세요" style="margin-bottom: 5px;">
<button id="save-template" style="padding: 8px 12px; background-color: #4caf50; color: white; border-radius: 4px; font-size: 14px; margin-right: 8px;">📁 템플릿 저장</button>
<button id="load-template" style="padding: 8px 12px; background-color: #2196F3; color: white; border-radius: 4px; font-size: 14px;">📂 템플릿 불러오기</button>
</div>
<select id="template-select" style="width: 100%; padding: 5px; margin-top: 10px; border: 1px solid #ccc; border-radius: 4px;">
<option value="" disabled selected>템플릿 선택</option>
${Object.keys(Config.settings.templates).map(template => `<option value="${template}">${template}</option>`).join('')}
</select>
<button id="create-template" style="margin-top: 10px; padding: 8px 12px; background-color: #4caf50; color: white; border-radius: 4px; font-size: 14px;">템플릿 생성</button>
<button id="delete-template" style="margin-top: 5px; padding: 8px 12px; background-color: #f44336; color: white; border-radius: 4px; font-size: 14px;">템플릿 삭제</button>
</div>
<div class="settings-section">
<button id="save-settings" style="margin-top: 10px; padding: 10px 20px; background-color: #4caf50; color: white; border-radius: 4px; font-size: 14px;">저장</button>
<button id="reset-settings" style="margin-top: 10px; padding: 10px 20px; background-color: #f44336; color: white; border-radius: 4px; font-size: 14px;">초기화</button>
</div>
</div>
<div id="tab-templates" style="display: none;">
<h3>설정 템플릿</h3>
<div class="settings-section">
<label for="new-template-name">새 템플릿 이름:</label>
<input type="text" id="new-template-name" placeholder="템플릿 이름을 입력하세요">
<button id="create-template" style="margin-top: 5px; padding: 8px 12px; background-color: #4caf50; color: white; border-radius: 4px; font-size: 14px;">템플릿 생성</button>
</div>
<div class="settings-section">
<label for="existing-templates">기존 템플릿:</label>
<select id="existing-templates" multiple size="10" style="width: 100%; padding: 5px; border: 1px solid #ccc; border-radius: 4px;">
${Object.keys(Config.settings.templates).map(template => `<option value="${template}">${template}</option>`).join('')}
</select>
<button id="delete-template" style="margin-top: 5px; padding: 8px 12px; background-color: #f44336; color: white; border-radius: 4px; font-size: 14px;">템플릿 삭제</button>
</div>
</div>
<div id="tab-previous-crawls" style="display: none;">
<h3>이전 크롤링 기록</h3>
<button id="clear-previous-crawls" style="padding: 8px 12px; background-color: #f44336; color: white; border-radius: 4px; font-size: 14px; margin-bottom: 10px;">기록 초기화</button>
<div style="overflow-x: auto;">
<table>
<thead>
<tr>
<th>날짜</th>
<th>크롤링 수</th>
<th>상태</th>
</tr>
</thead>
<tbody id="previous-crawls-body">
${renderPreviousCrawls()}
</tbody>
</table>
</div>
</div>
<div id="html-modal" class="modal">
<div class="modal-content">
<span class="close">×</span>
<pre id="modal-html-content"></pre>
</div>
</div>
</div>
</div>
`);
$('body').append(container);
$('#crawler-minimize').on('click', toggleMinimize);
$('.crawler-tab').on('click', switchTab);
$('#save-settings').on('click', saveSettingsHandler);
$('#reset-settings').on('click', resetSettingsHandler);
$('#add-user-agent').on('click', addUserAgent);
$('#remove-user-agent').on('click', removeUserAgent);
$('#download-csv').on('click', Download.downloadCSV);
$('#download-json').on('click', Download.downloadJSON);
$('#download-xml').on('click', Download.downloadXML);
$('#download-xls').on('click', Download.downloadXLS);
$('#search-filter').on('input', filterData);
$('#schedule-enabled').on('change', toggleScheduleOptions);
$('#data-visualization-checkbox').on('change', toggleDataVisualization);
$('#enable-captchas-checkbox').on('change', toggleCaptchaHandling);
$('#save-template').on('click', saveTemplateHandler);
$('#load-template').on('click', loadTemplateHandler);
$('#create-template').on('click', createTemplateHandler);
$('#delete-template').on('click', deleteTemplateHandler);
$('#template-select').on('change', selectTemplateHandler);
$('#existing-templates').on('change', existingTemplateSelectHandler);
$('#clear-previous-crawls').on('click', clearPreviousCrawls);
$('#reset-data').on('click', resetData);
$('#dark-mode-toggle').on('click', toggleDarkMode);
$('.close').on('click', () => $('#html-modal').hide());
$(window).on('click', function(e) { if (e.target.id === 'html-modal') $('#html-modal').hide(); });
$('#start-crawling').on('click', Crawling.startCrawling);
$('#stop-crawling').on('click', Crawling.stopCrawling);
$(document).on('keydown', handleShortcuts);
$('#proxy-input').on('input', validateProxyInput);
$('.settings-section input[type="checkbox"]').on('change', toggleDataField);
$('#crawler-container').on('mousedown', '#crawler-header', startDrag);
$(document).on('mouseup', stopDrag);
$(document).on('mousemove', drag);
updateUI();
makeResizable();
}
function toggleMinimize() {
$('#crawler-content').toggle();
const isHidden = $('#crawler-content').is(':hidden');
$('#crawler-minimize').text(isHidden ? '☐' : '─');
Config.saveSettings({ isMinimized: isHidden });
UI.updateLogs();
Logger.log(`크롤러 창이 ${isHidden ? '최소화' : '확대'}되었습니다.`, 'info');
}
function switchTab() {
const tab = $(this).data('tab');
$('.crawler-tab').removeClass('active');
$(this).addClass('active');
$('#crawler-content > div').hide();
$(`#tab-${tab}`).show();
Config.saveSettings({ currentTab: tab });
UI.updateLogs();
}
function updateUI() {
const currentTab = Config.settings.currentTab || 'logs';
$(`.crawler-tab[data-tab="${currentTab}"]`).addClass('active');
$('#crawler-content > div').hide();
$(`#tab-${currentTab}`).show();
UI.updateLogs();
UI.updateDataTable();
UI.updatePreviousCrawlsUI();
UI.updateTemplatesUI();
if (Config.settings.dataVisualization) DataVisualization.renderChart();
applyDarkMode();
}
function updateLogs() {
const logs = Logger.getLogs();
$('#log-messages').html(logs.map(l => `<div>[${l.timestamp}] [${l.type.toUpperCase()}] ${Utils.escape(l.message)}</div>`).join(''));
const progress = Config.settings.progress || 0;
const currentPage = Config.settings.currentPage || 0;
const totalPages = Config.settings.maxPages;
$('.progress-bar-fill').css('width', `${progress.toFixed(2)}%`);
$('.status-text').text(`진행률: ${progress.toFixed(2)}% (${currentPage}/${totalPages} 페이지)`);
const latestLog = logs[logs.length -1];
if (latestLog) {
if (latestLog.type === 'error') {
$('#error-message').text(latestLog.message).show();
setTimeout(() => { $('#error-message').fadeOut(); }, 5000);
}
if (latestLog.type === 'success') {
$('#success-message').text(latestLog.message).show();
setTimeout(() => { $('#success-message').fadeOut(); }, 5000);
}
if (latestLog.type === 'info') {
$('#success-message').text(latestLog.message).show();
setTimeout(() => { $('#success-message').fadeOut(); }, 5000);
}
if (latestLog.type === 'warning') {
$('#error-message').text(latestLog.message).show();
setTimeout(() => { $('#error-message').fadeOut(); }, 5000);
}
}
}
function updateDataTable() {
const data = JSON.parse(GM_getValue('data', '[]'));
const filter = $('#search-filter').val().toLowerCase();
const filtered = filter ? data.filter(item => Object.keys(Config.settings.dataFields).some(field => Config.settings.dataFields[field] && String(item[field]).toLowerCase().includes(filter))) : data;
const rows = filtered.map(item => `
<tr>
${Config.settings.dataFields.name ? `<td>${Utils.escape(item.name)}</td>` : ''}
${Config.settings.dataFields.url ? `<td><a href="${Utils.escape(item.url)}" target="_blank">${Utils.escape(item.url)}</a></td>` : ''}
${Config.settings.dataFields.description ? `<td>${Utils.escape(item.description)}</td>` : ''}
${Config.settings.dataFields.html ? `<td><button class="view-html" data-html="${encodeURIComponent(item.html)}">📄 보기</button></td>` : ''}
${Config.settings.dataFields.category ? `<td>${Utils.escape(item.category)}</td>` : ''}
${Config.settings.dataFields.quality ? `<td>${Utils.escape(item.quality.toString())}</td>` : ''}
${Config.settings.dataFields.language ? `<td>${Utils.escape(item.language)}</td>` : ''}
${Config.settings.dataFields.wordCount ? `<td>${Utils.escape(item.wordCount.toString())}</td>` : ''}
${Config.settings.dataFields.metaTags ? `<td>${Utils.escape(JSON.stringify(item.metaTags))}</td>` : ''}
${Config.settings.dataFields.headings ? `<td>${Utils.escape(item.headings.join(', '))}</td>` : ''}
</tr>
`).join('');
$('#data-table-body').html(rows);
$('.view-html').off('click').on('click', function() {
$('#modal-html-content').text(decodeURIComponent($(this).data('html')));
$('#html-modal').show();
});
if (Config.settings.dataVisualization) DataVisualization.renderChart();
}
function renderDataTable() {
const data = JSON.parse(GM_getValue('data', '[]'));
const filter = $('#search-filter').val().toLowerCase();
const filtered = filter ? data.filter(item => Object.keys(Config.settings.dataFields).some(field => Config.settings.dataFields[field] && String(item[field]).toLowerCase().includes(filter))) : data;
const rows = filtered.map(item => `
<tr>
${Config.settings.dataFields.name ? `<td>${Utils.escape(item.name)}</td>` : ''}
${Config.settings.dataFields.url ? `<td><a href="${Utils.escape(item.url)}" target="_blank">${Utils.escape(item.url)}</a></td>` : ''}
${Config.settings.dataFields.description ? `<td>${Utils.escape(item.description)}</td>` : ''}
${Config.settings.dataFields.html ? `<td><button class="view-html" data-html="${encodeURIComponent(item.html)}">📄 보기</button></td>` : ''}
${Config.settings.dataFields.category ? `<td>${Utils.escape(item.category)}</td>` : ''}
${Config.settings.dataFields.quality ? `<td>${Utils.escape(item.quality.toString())}</td>` : ''}
${Config.settings.dataFields.language ? `<td>${Utils.escape(item.language)}</td>` : ''}
${Config.settings.dataFields.wordCount ? `<td>${Utils.escape(item.wordCount.toString())}</td>` : ''}
${Config.settings.dataFields.metaTags ? `<td>${Utils.escape(JSON.stringify(item.metaTags))}</td>` : ''}
${Config.settings.dataFields.headings ? `<td>${Utils.escape(item.headings.join(', '))}</td>` : ''}
</tr>
`).join('');
return rows;
}
function renderPreviousCrawls() {
const previousCrawls = JSON.parse(GM_getValue('previousCrawls', '[]'));
if (!previousCrawls.length) return '<tr><td colspan="3">이전 크롤링 기록이 없습니다.</td></tr>';
return previousCrawls.map(crawl => `
<tr>
<td>${Utils.escape(crawl.date)}</td>
<td>${Utils.escape(crawl.count.toString())}</td>
<td>${Utils.escape(crawl.status)}</td>
</tr>
`).join('');
}
function updatePreviousCrawlsUI() {
const html = renderPreviousCrawls();
$('#previous-crawls-body').html(html);
}
function updateTemplatesUI() {
$('#template-select').empty().append(`<option value="" disabled selected>템플릿 선택</option>`);
Object.keys(Config.settings.templates).forEach(template => {
$('#template-select').append(`<option value="${template}">${template}</option>`);
});
$('#existing-templates').empty();
Object.keys(Config.settings.templates).forEach(template => {
$('#existing-templates').append(`<option value="${template}">${template}</option>`);
});
}
function addUserAgent() {
const ua = prompt("새 사용자 에이전트를 입력하세요:");
if (ua) {
Config.settings.userAgents.push(ua);
Config.saveSettings({ userAgents: Config.settings.userAgents });
updateUserAgentSelect();
Logger.log('새 사용자 에이전트가 추가되었습니다.', 'success');
}
}
function removeUserAgent() {
const selected = $('#user-agent-select').val();
if (selected && selected.length) {
selected.sort((a, b) => b - a).forEach(idx => { Config.settings.userAgents.splice(idx, 1); });
Config.settings.userAgentIndex = 0;
Config.saveSettings({
userAgents: Config.settings.userAgents,
userAgentIndex: Config.settings.userAgentIndex
});
updateUserAgentSelect();
Logger.log('선택한 사용자 에이전트가 제거되었습니다.', 'success');
} else {
alert('제거할 사용자 에이전트를 선택하세요.');
}
}
function updateUserAgentSelect() {
$('#user-agent-select').empty();
Config.settings.userAgents.forEach((ua, idx) => {
$('#user-agent-select').append(`<option value="${idx}" ${idx === Config.settings.userAgentIndex ? 'selected' : ''}>${ua}</option>`);
});
}
function saveSettingsHandler() {
const delay = parseInt($('#delay-input').val(), 10);
const delayRandom = parseInt($('#delay-random-input').val(), 10);
const maxPages = parseInt($('#max-pages-input').val(), 10);
const timeout = parseInt($('#timeout-input').val(), 10);
const minWordCount = parseInt($('#min-word-count-input').val(), 10);
const maxWordCount = parseInt($('#max-word-count-input').val(), 10);
const maxDepth = parseInt($('#max-depth-input').val(), 10);
const retryAttempts = parseInt($('#retry-attempts-input').val(), 10);
const retryDelay = parseInt($('#retry-delay-input').val(), 10);
const interval = parseInt($('#interval-input').val(), 10) * 60000;
const proxy = $('#proxy-input').val().trim();
const isValidProxy = proxy === '' || Config.validateProxy(proxy);
const isValidSettings = !isNaN(delay) && delay >= 500 &&
!isNaN(delayRandom) && delayRandom >= 0 &&
!isNaN(maxPages) && maxPages >= 1 &&
!isNaN(timeout) && timeout >= 1000 &&
!isNaN(minWordCount) && minWordCount >= 10 &&
!isNaN(maxWordCount) && maxWordCount >= 100 &&
!isNaN(maxDepth) && maxDepth >= 1 &&
!isNaN(retryAttempts) && retryAttempts >= 0 &&
!isNaN(retryDelay) && retryDelay >= 1000 &&
(!Config.settings.schedule.enabled || (!isNaN(interval) && interval >= 60000)) &&
isValidProxy;
if (!isValidSettings) {
$('#error-message').text('설정값이 유효하지 않습니다. 올바른 값을 입력하세요.').show();
setTimeout(() => { $('#error-message').fadeOut(); }, 5000);
return;
}
Config.settings.delay = delay;
Config.settings.delayRandom = delayRandom;
Config.settings.maxPages = maxPages;
const selectedUA = parseInt($('#user-agent-select').val(), 10);
Config.settings.userAgentIndex = isNaN(selectedUA) ? 0 : selectedUA;
Config.settings.contentTypes.include = $('#content-include-input').val().split(',').map(ct => ct.trim()).filter(ct => ct);
Config.settings.contentTypes.exclude = $('#content-exclude-input').val().split(',').map(ct => ct.trim()).filter(ct => ct);
Config.settings.timeout = timeout;
Config.settings.includeKeywords = $('#include-keywords-input').val().split('\n').map(kw => kw.trim()).filter(kw => kw);
Config.settings.excludeKeywords = $('#exclude-keywords-input').val().split('\n').map(kw => kw.trim()).filter(kw => kw);
Config.settings.regexInclude = $('#regex-include-input').val().split('\n').map(rx => rx.trim()).filter(rx => rx);
Config.settings.regexExclude = $('#regex-exclude-input').val().split('\n').map(rx => rx.trim()).filter(rx => rx);
Config.settings.language = $('#language-select').val() || [];
Config.settings.minWordCount = minWordCount;
Config.settings.maxWordCount = maxWordCount;
Config.settings.maxDepth = maxDepth;
Config.settings.proxy = proxy;
['name','url','description','html','category','quality','language','wordCount','metaTags','headings'].forEach(field => {
Config.settings.dataFields[field] = $(`#field-${field}`).is(':checked');
});
Config.settings.retryAttempts = retryAttempts;
Config.settings.retryDelay = retryDelay;
Config.settings.schedule.enabled = $('#schedule-enabled').is(':checked');
Config.settings.schedule.startTime = $('#start-time-input').val() || Config.DEFAULT_SETTINGS.schedule.startTime;
Config.settings.schedule.interval = Config.settings.schedule.enabled ? (isNaN(interval) ? Config.DEFAULT_SETTINGS.schedule.interval : interval) : Config.DEFAULT_SETTINGS.schedule.interval;
Object.keys(Config.settings.searchEngines).forEach(engine => { Config.settings.searchEngines[engine] = $(`#engine-${engine}`).is(':checked'); });
Config.settings.blacklist = $('#blacklist-input').val().split('\n').map(d => d.trim()).filter(d => d);
Config.settings.whitelist = $('#whitelist-input').val().split('\n').map(d => d.trim()).filter(d => d);
Config.settings.respectRobots = $('#respect-robots-checkbox').is(':checked');
Config.settings.maxConcurrentRequests = parseInt($('#max-concurrent-requests-input').val(), 10) || Config.DEFAULT_SETTINGS.maxConcurrentRequests;
Config.settings.dataVisualization = $('#data-visualization-checkbox').is(':checked');
Config.settings.exportFormats.csv = $('#export-csv').is(':checked');
Config.settings.exportFormats.json = $('#export-json').is(':checked');
Config.settings.exportFormats.xml = $('#export-xml').is(':checked');
Config.settings.exportFormats.xls = $('#export-xls').is(':checked');
Config.settings.advancedLogging = $('#advanced-logging-checkbox').is(':checked');
Config.settings.enableCaptchas = $('#enable-captchas-checkbox').is(':checked');
Config.settings.customKeywords = Utils.parseCustomKeywords($('#custom-keywords-input').val());
Config.saveSettings(Config.settings);
Crawling.updateCrawlSettings();
Logger.log('설정이 저장되었습니다.', 'success');
if (Config.settings.schedule.enabled) {
Scheduler.scheduleCrawling();
}
updateDataTable();
if (Config.settings.dataVisualization) DataVisualization.renderChart();
}
function resetSettingsHandler() {
if (confirm("기본 설정으로 초기화하시겠습니까?")) {
Config.resetSettings();
location.reload();
Logger.log('기본 설정으로 초기화되었습니다.', 'success');
}
}
function toggleScheduleOptions() {
$('#schedule-options').toggle($('#schedule-enabled').is(':checked'));
}
function toggleDataVisualization() {
Config.settings.dataVisualization = $('#data-visualization-checkbox').is(':checked');
Config.saveSettings({ dataVisualization: Config.settings.dataVisualization });
$('#data-visualization').toggle(Config.settings.dataVisualization);
Logger.log('데이터 시각화 설정이 변경되었습니다.', 'success');
if (Config.settings.dataVisualization) DataVisualization.renderChart();
}
function toggleCaptchaHandling() {
Config.settings.enableCaptchas = $('#enable-captchas-checkbox').is(':checked');
Config.saveSettings({ enableCaptchas: Config.settings.enableCaptchas });
Logger.log('CAPTCHA 처리 설정이 변경되었습니다.', 'success');
}
function filterData() {
updateDataTable();
}
function clearPreviousCrawls() {
if (confirm("이전 크롤링 기록을 모두 초기화하시겠습니까?")) {
GM_setValue('previousCrawls', JSON.stringify([]));
updatePreviousCrawlsUI();
Logger.log('이전 크롤링 기록이 초기화되었습니다.', 'success');
}
}
function resetData() {
if (confirm("모든 크롤링 데이터를 초기화하시겠습니까?")) {
GM_setValue('data', JSON.stringify([]));
GM_setValue('visitedUrls', JSON.stringify([]));
updateDataTable();
Logger.log('크롤링 데이터가 초기화되었습니다.', 'success');
}
}
function handleShortcuts(e) {
if (e.ctrlKey) {
const actions = { '1': 'logs', '2': 'data', '3': 'settings', '4': 'previous-crawls', '5': 'templates', 's': 'save', 'r': 'reset', 'p': 'start', 'q': 'stop', 'c': 'csv', 'j': 'json', 'x': 'xml', 'l': 'xls', 'd': 'dark-mode' };
const action = actions[e.key];
if (action) {
e.preventDefault();
switch(action) {
case 'logs': $('.crawler-tab[data-tab="logs"]').click(); break;
case 'data': $('.crawler-tab[data-tab="data"]').click(); break;
case 'settings': $('.crawler-tab[data-tab="settings"]').click(); break;
case 'previous-crawls': $('.crawler-tab[data-tab="previous-crawls"]').click(); break;
case 'templates': $('.crawler-tab[data-tab="templates"]').click(); break;
case 'save': saveSettingsHandler(); break;
case 'reset': resetSettingsHandler(); break;
case 'start': if (!Crawling.isCrawling()) Crawling.startCrawling(); break;
case 'stop': if (Crawling.isCrawling()) Crawling.stopCrawling(); break;
case 'csv': Config.settings.exportFormats.csv && Download.downloadCSV(); break;
case 'json': Config.settings.exportFormats.json && Download.downloadJSON(); break;
case 'xml': Config.settings.exportFormats.xml && Download.downloadXML(); break;
case 'xls': Config.settings.exportFormats.xls && Download.downloadXLS(); break;
case 'dark-mode': toggleDarkMode(); break;
}
}
}
}
function makeDraggable() {
// Handled via mousedown, mousemove, mouseup events
}
function startDrag(e) {
if (e.target.id === 'crawler-header' || $(e.target).closest('#crawler-header').length) {
isDragging = true;
const container = $('#crawler-container');
const offset = container.offset();
offsetX = e.pageX - offset.left;
offsetY = e.pageY - offset.top;
}
}
function drag(e) {
if (isDragging) {
let left = e.pageX - offsetX;
let top = e.pageY - offsetY;
left = Math.max(0, Math.min(left, $(window).width() - $('#crawler-container').outerWidth()));
top = Math.max(0, Math.min(top, $(window).height() - $('#crawler-container').outerHeight()));
$('#crawler-container').css({ left: `${left}px`, top: `${top}px` });
}
}
function stopDrag() {
if (isDragging) {
isDragging = false;
const position = $('#crawler-container').position();
Config.saveSettings({ uiPosition: { top: `${position.top}px`, left: `${position.left}px` } });
}
}
function makeResizable() {
// Already handled via CSS resize: both;
// Additional logic can be added if needed
}
function getFieldName(field) {
const names = {
name: '이름',
url: '주소',
description: '설명',
html: 'HTML 내용',
category: '분류',
quality: '품질점수',
language: '언어',
wordCount: '단어 수',
metaTags: '메타 태그',
headings: '헤딩'
};
return names[field] || field;
}
// Template Handlers
function saveTemplateHandler() {
const templateName = $('#new-template-name').val().trim();
if (!templateName) {
alert('템플릿 이름을 입력하세요.');
return;
}
Config.saveTemplate(templateName);
updateTemplatesUI();
$('#new-template-name').val('');
}
function loadTemplateHandler() {
const selected = $('#template-select').val();
if (!selected) {
alert('불러올 템플릿을 선택하세요.');
return;
}
Config.loadTemplate(selected);
updateUI();
$('.crawler-tab[data-tab="settings"]').click();
}
function createTemplateHandler() {
const templateName = $('#new-template-name').val().trim();
if (!templateName) {
alert('템플릿 이름을 입력하세요.');
return;
}
Config.saveTemplate(templateName);
updateTemplatesUI();
$('#new-template-name').val('');
}
function deleteTemplateHandler() {
const selectedTemplates = $('#existing-templates').val();
if (!selectedTemplates || !selectedTemplates.length) {
alert('삭제할 템플릿을 선택하세요.');
return;
}
selectedTemplates.forEach(template => {
delete Config.settings.templates[template];
Logger.log(`템플릿 '${template}'이(가) 삭제되었습니다.`, 'success');
});
Config.saveSettings({ templates: Config.settings.templates });
updateTemplatesUI();
}
function selectTemplateHandler() {
const selected = $(this).val();
if (selected) {
Config.loadTemplate(selected);
updateUI();
$('.crawler-tab[data-tab="settings"]').click();
}
}
function existingTemplateSelectHandler() {
// Placeholder for any additional actions when selecting existing templates
}
function toggleDarkMode() {
$('#crawler-container').toggleClass('dark');
const isDark = $('#crawler-container').hasClass('dark');
$('#dark-mode-toggle').text(isDark ? '☀️' : '🌙');
Config.saveSettings({ darkMode: isDark });
Logger.log(`다크 모드가 ${isDark ? '활성화' : '비활성화'}되었습니다.`, 'success');
}
function applyDarkMode() {
if (Config.settings.darkMode) {
$('#crawler-container').addClass('dark');
$('#dark-mode-toggle').text('☀️');
} else {
$('#crawler-container').removeClass('dark');
$('#dark-mode-toggle').text('🌙');
}
}
function toggleDataField() {
const field = $(this).attr('id').replace('field-', '');
Config.settings.dataFields[field] = $(this).is(':checked');
Config.saveSettings({ dataFields: Config.settings.dataFields });
Download.updateExportButtons();
updateDataTable();
}
function validateProxyInput() {
const proxy = $('#proxy-input').val().trim();
if (proxy && !Config.validateProxy(proxy)) {
$('#proxy-validation-message').text('유효하지 않은 프록시 URL입니다.').show();
} else {
$('#proxy-validation-message').hide();
}
}
return {
createUI,
updateUI,
updateLogs,
updateDataTable,
updatePreviousCrawlsUI,
updateTemplatesUI,
applyDarkMode
};
})();
/* ============================
Module: Download
============================ */
const Download = (function() {
function downloadCSV() {
const data = JSON.parse(GM_getValue('data', '[]'));
if (!data.length) { Logger.log('다운로드할 데이터가 없습니다.', 'warning'); return; }
const headers = Object.keys(Config.settings.dataFields).filter(f => Config.settings.dataFields[f]).map(f => Utils.capitalize(getFieldName(f)));
const csvContent = [headers.join(",")]
.concat(data.map(item => Object.keys(Config.settings.dataFields).filter(f => Config.settings.dataFields[f]).map(f => `"${Utils.escape(item[f], 'csv')}"`).join(",")))
.join("\n");
const blob = new Blob([csvContent], { type: "text/csv;charset=utf-8;" });
const link = document.createElement("a");
link.href = URL.createObjectURL(blob);
link.download = "data.csv";
document.body.appendChild(link);
link.click();
document.body.removeChild(link);
Logger.log('CSV 파일이 다운로드되었습니다.', 'success');
}
function downloadJSON() {
const data = JSON.parse(GM_getValue('data', '[]'));
if (!data.length) { Logger.log('다운로드할 데이터가 없습니다.', 'warning'); return; }
const jsonContent = JSON.stringify(data, null, 2);
const blob = new Blob([jsonContent], { type: "application/json" });
const link = document.createElement("a");
link.href = URL.createObjectURL(blob);
link.download = "data.json";
document.body.appendChild(link);
link.click();
document.body.removeChild(link);
Logger.log('JSON 파일이 다운로드되었습니다.', 'success');
}
function downloadXML() {
const data = JSON.parse(GM_getValue('data', '[]'));
if (!data.length) { Logger.log('다운로드할 데이터가 없습니다.', 'warning'); return; }
let xmlContent = '<?xml version="1.0" encoding="UTF-8"?><data>';
data.forEach(item => {
xmlContent += '<entry>';
if (Config.settings.dataFields.name) xmlContent += `<name>${Utils.escape(item.name || 'No Title', 'xml')}</name>`;
if (Config.settings.dataFields.url) xmlContent += `<url>${Utils.escape(item.url || '', 'xml')}</url>`;
if (Config.settings.dataFields.description) xmlContent += `<description>${Utils.escape(item.description || 'No Description', 'xml')}</description>`;
if (Config.settings.dataFields.html) xmlContent += `<html><![CDATA[${item.html || ''}]]></html>`;
if (Config.settings.dataFields.category) xmlContent += `<category>${Utils.escape(item.category || '', 'xml')}</category>`;
if (Config.settings.dataFields.quality) xmlContent += `<quality>${Utils.escape(item.quality)}</quality>`;
if (Config.settings.dataFields.language) xmlContent += `<language>${Utils.escape(item.language || 'unknown', 'xml')}</language>`;
if (Config.settings.dataFields.wordCount) xmlContent += `<wordCount>${Utils.escape(item.wordCount)}</wordCount>`;
if (Config.settings.dataFields.metaTags) xmlContent += `<metaTags>${Utils.escape(JSON.stringify(item.metaTags), 'xml')}</metaTags>`;
if (Config.settings.dataFields.headings) xmlContent += `<headings>${Utils.escape(item.headings.join('; '), 'xml')}</headings>`;
xmlContent += '</entry>';
});
xmlContent += '</data>';
const blob = new Blob([xmlContent], { type: "application/xml" });
const link = document.createElement("a");
link.href = URL.createObjectURL(blob);
link.download = "data.xml";
document.body.appendChild(link);
link.click();
document.body.removeChild(link);
Logger.log('XML 파일이 다운로드되었습니다.', 'success');
}
function downloadXLS() {
const data = JSON.parse(GM_getValue('data', '[]'));
if (!data.length) { Logger.log('다운로드할 데이터가 없습니다.', 'warning'); return; }
const worksheetData = data.map(item => {
const entry = {};
if (Config.settings.dataFields.name) entry['사이트 이름'] = item.name;
if (Config.settings.dataFields.url) entry['주소'] = item.url;
if (Config.settings.dataFields.description) entry['설명'] = item.description;
if (Config.settings.dataFields.html) entry['📄 HTML 내용'] = item.html;
if (Config.settings.dataFields.category) entry['분류'] = item.category;
if (Config.settings.dataFields.quality) entry['품질점수'] = item.quality;
if (Config.settings.dataFields.language) entry['언어'] = item.language;
if (Config.settings.dataFields.wordCount) entry['단어 수'] = item.wordCount;
if (Config.settings.dataFields.metaTags) entry['메타 태그'] = JSON.stringify(item.metaTags);
if (Config.settings.dataFields.headings) entry['헤딩'] = item.headings.join('; ');
return entry;
});
const worksheet = XLSX.utils.json_to_sheet(worksheetData);
const workbook = XLSX.utils.book_new();
XLSX.utils.book_append_sheet(workbook, worksheet, "Data");
XLSX.writeFile(workbook, "data.xlsx");
Logger.log('XLS 파일이 다운로드되었습니다.', 'success');
}
function getFieldName(field) {
const names = {
name: '이름',
url: '주소',
description: '설명',
html: 'HTML 내용',
category: '분류',
quality: '품질점수',
language: '언어',
wordCount: '단어 수',
metaTags: '메타 태그',
headings: '헤딩'
};
return names[field] || field;
}
function updateExportButtons() {
// Placeholder for dynamic enabling/disabling of export buttons based on data fields
}
return {
downloadCSV,
downloadJSON,
downloadXML,
downloadXLS,
updateExportButtons
};
})();
/* ============================
Module: DataVisualization
============================ */
const DataVisualization = (function() {
let chartInstance = null;
function renderChart() {
if (!document.getElementById('data-chart').getContext) return;
const ctx = document.getElementById('data-chart').getContext('2d');
if (chartInstance) chartInstance.destroy();
const data = JSON.parse(GM_getValue('data', '[]'));
const categories = {};
data.forEach(item => { const cat = item.category || '기타'; categories[cat] = (categories[cat] || 0) + 1; });
const chartData = {
labels: Object.keys(categories),
datasets: [{
label: '카테고리별 데이터 수',
data: Object.values(categories),
backgroundColor: Utils.generateColors(Object.keys(categories).length)
}]
};
chartInstance = new Chart(ctx, { type: 'pie', data: chartData, options: { responsive: true } });
}
return {
renderChart
};
})();
/* ============================
Module: Crawling
============================ */
const Crawling = (function() {
let isCrawlingFlag = Config.settings.isCrawling;
let currentPage = parseInt(GM_getValue('currentPage', 1), 10);
let totalPages = Config.settings.maxPages;
let progress = parseFloat(GM_getValue('progress', 0));
let currentDepth = parseInt(GM_getValue('currentDepth', 0), 10);
let activeRequests = 0;
let requestQueue = [];
let previousCrawls = JSON.parse(GM_getValue('previousCrawls', '[]'));
let visitedUrls = new Set(JSON.parse(GM_getValue('visitedUrls', '[]')));
let data = JSON.parse(GM_getValue('data', '[]'));
let navigationObserver = null;
const searchEngineHandlers = {
google: {
extractResults: function(doc) {
const results = [];
doc.querySelectorAll('div.g, div[data-hveid]').forEach(item => {
const name = item.querySelector('h3')?.innerText.trim() || 'No Title';
const url = item.querySelector('a')?.href || '';
const desc = item.querySelector('.aCOpRe, .VwiC3b')?.innerText.trim() || 'No Description';
if (Utils.isValidURL(url)) results.push({ name, url, description: desc });
});
return results;
},
getNextPageUrl: function(doc) {
return doc.querySelector('a#pnnext')?.href || null;
},
getContentContainer: function() {
return '#search';
}
},
bing: {
extractResults: function(doc) {
const results = [];
doc.querySelectorAll('.b_algo').forEach(item => {
const name = item.querySelector('h2')?.innerText.trim() || 'No Title';
const url = item.querySelector('h2 a')?.href || '';
const desc = item.querySelector('.b_caption p')?.innerText.trim() || 'No Description';
if (Utils.isValidURL(url)) results.push({ name, url, description: desc });
});
return results;
},
getNextPageUrl: function(doc) {
return doc.querySelector('a.sb_pagN')?.href || null;
},
getContentContainer: function() {
return '#b_content';
}
},
yahoo: {
extractResults: function(doc) {
const results = [];
doc.querySelectorAll('.dd.algo').forEach(item => {
const name = item.querySelector('h3')?.innerText.trim() || 'No Title';
const url = item.querySelector('a')?.href || '';
const desc = item.querySelector('.compText a')?.innerText.trim() || 'No Description';
if (Utils.isValidURL(url)) results.push({ name, url, description: desc });
});
return results;
},
getNextPageUrl: function(doc) {
return doc.querySelector('a.next')?.href || null;
},
getContentContainer: function() {
return '#web';
}
},
// Additional search engine handlers...
duckduckgo: {
extractResults: function(doc) {
const results = [];
doc.querySelectorAll('.result').forEach(item => {
const name = item.querySelector('h2')?.innerText.trim() || 'No Title';
const url = item.querySelector('a.result__a')?.href || '';
const desc = item.querySelector('.result__snippet')?.innerText.trim() || 'No Description';
if (Utils.isValidURL(url)) results.push({ name, url, description: desc });
});
return results;
},
getNextPageUrl: function(doc) {
return doc.querySelector('a.result--more__btn')?.href || null;
},
getContentContainer: function() {
return '#links';
}
},
bingimages: {
extractResults: function(doc) {
const results = [];
doc.querySelectorAll('.imgpt a.iusc').forEach(item => {
const url = item.getAttribute('m') ? JSON.parse(item.getAttribute('m')).murl : '';
const name = item.getAttribute('m') ? JSON.parse(item.getAttribute('m')).t : 'No Title';
const desc = item.getAttribute('m') ? JSON.parse(item.getAttribute('m')).p : 'No Description';
if (Utils.isValidURL(url)) results.push({ name, url, description: desc });
});
return results;
},
getNextPageUrl: function(doc) {
return doc.querySelector('a.sb_pagN')?.href || null;
},
getContentContainer: function() {
return '#image_results';
}
},
// Implement other search engines similarly...
};
function identifySearchEngine() {
const host = window.location.hostname;
if (host.includes('google.com')) return 'google';
if (host.includes('bing.com') && !window.location.pathname.includes('/images')) return 'bing';
if (host.includes('yahoo.com')) return 'yahoo';
if (host.includes('duckduckgo.com')) return 'duckduckgo';
if (host.includes('baidu.com')) return 'baidu';
if (host.includes('yandex.com')) return 'yandex';
if (host.includes('naver.com')) return 'naver';
if (host.includes('daum.net')) return 'daum';
if (host.includes('ask.com')) return 'ask';
if (host.includes('ecosia.org')) return 'ecosia';
if (host.includes('startpage.com')) return 'startpage';
if (host.includes('searx.me')) return 'searx';
if (host.includes('dogpile.com')) return 'dogpile';
if (host.includes('bing.com') && window.location.pathname.includes('/images')) return 'bingimages';
return null;
}
async function startCrawling() {
if (isCrawlingFlag) {
Logger.log('이미 크롤링이 진행 중입니다.', 'warning');
return;
}
if (!Object.values(Config.settings.searchEngines).some(v => v)) {
Logger.log('사용할 검색 엔진이 하나도 선택되지 않았습니다.', 'error');
return;
}
isCrawlingFlag = true;
Config.saveSettings({ isCrawling: isCrawlingFlag });
Logger.log('크롤링을 시작합니다.', 'info');
UI.updateLogs();
currentPage = Config.settings.currentPage || 1;
totalPages = Config.settings.maxPages;
currentDepth = Config.settings.currentDepth || 0;
if (!previousCrawls.some(crawl => crawl.status === '진행 중')) {
previousCrawls.push({ date: new Date().toLocaleString(), count: 0, status: '진행 중' });
GM_setValue('previousCrawls', JSON.stringify(previousCrawls));
UI.updatePreviousCrawlsUI();
}
setButtonStates();
await crawlPage(currentDepth);
monitorPageNavigation();
}
function stopCrawling() {
if (!isCrawlingFlag) {
Logger.log('크롤링이 진행 중이지 않습니다.', 'warning');
return;
}
isCrawlingFlag = false;
Config.saveSettings({ isCrawling: isCrawlingFlag });
Logger.log('크롤링이 중지되었습니다.', 'warning');
finalizeCrawl('중지됨');
UI.updateLogs();
setButtonStates();
}
function setButtonStates() {
$('#start-crawling').prop('disabled', isCrawlingFlag);
$('#stop-crawling').prop('disabled', !isCrawlingFlag);
$('#reset-data').prop('disabled', isCrawlingFlag);
}
async function crawlPage(depth) {
try {
while (isCrawlingFlag && currentPage <= totalPages && depth <= Config.settings.maxDepth) {
Logger.log(`페이지 ${currentPage} 크롤링 시작. (깊이: ${depth})`, 'info');
if (Config.settings.respectRobots && !(await Robots.checkRobotsTxt(window.location.href))) {
Logger.log(`robots.txt에 의해 크롤링이 금지된 페이지: ${window.location.href}`, 'warning');
break;
}
const searchEngine = identifySearchEngine();
if (!searchEngine || !searchEngineHandlers[searchEngine]) {
Logger.log(`지원되지 않는 검색 엔진: ${window.location.hostname}`, 'error');
break;
}
const doc = document;
const handler = searchEngineHandlers[searchEngine];
const results = handler.extractResults(doc);
if (!results) { Logger.log('검색 결과를 찾을 수 없습니다.', 'error'); break; }
results.forEach(result => {
if (!isCrawlingFlag) return;
const { name, url, description } = result;
if (visitedUrls.has(url) || !Utils.isValidURL(url) ||
(Config.settings.whitelist.length && !Config.settings.whitelist.some(d => url.includes(d))) ||
Config.settings.blacklist.some(d => url.includes(d)) ||
(Config.settings.includeKeywords.length && !Config.settings.includeKeywords.some(k => url.includes(k) || description.includes(k))) ||
(Config.settings.excludeKeywords.some(k => url.includes(k) || description.includes(k))) ||
(Config.settings.regexInclude.length && !Config.settings.regexInclude.some(r => new RegExp(r).test(url) || new RegExp(r).test(description))) ||
(Config.settings.regexExclude.length && Config.settings.regexExclude.some(r => new RegExp(r).test(url) || new RegExp(r).test(description)))) {
return;
}
requestQueue.push({ url, depth });
if (activeRequests >= Config.settings.maxConcurrentRequests) return;
});
if (previousCrawls.length && previousCrawls[previousCrawls.length -1].status === '진행 중') {
previousCrawls[previousCrawls.length -1].count += results.length;
GM_setValue('previousCrawls', JSON.stringify(previousCrawls));
UI.updatePreviousCrawlsUI();
}
await processQueue();
progress = (currentPage / totalPages) * 100;
Config.saveSettings({ progress });
UI.updateLogs();
if (currentPage < totalPages && isCrawlingFlag) {
const nextUrl = handler.getNextPageUrl(doc);
if (nextUrl) {
currentPage++;
Config.saveSettings({ currentPage });
depth++;
Config.saveSettings({ currentDepth: depth });
Logger.log(`다음 페이지로 이동: ${currentPage}`, 'info');
await Utils.delay(Config.settings.delay + Math.random() * Config.settings.delayRandom);
const nextPageContent = await Fetch.fetchPageContent(nextUrl);
if (nextPageContent) {
updatePageContent(nextPageContent);
continue;
} else {
Logger.log('다음 페이지 내용을 가져올 수 없습니다.', 'error');
break;
}
} else { Logger.log('다음 페이지 링크를 찾을 수 없습니다.', 'error'); break; }
} else { break; }
}
if (isCrawlingFlag && currentPage > totalPages && activeRequests === 0 && requestQueue.length === 0) {
Logger.log('크롤링이 완료되었습니다.', 'success');
finalizeCrawl('완료됨');
isCrawlingFlag = false;
Config.saveSettings({ isCrawling: isCrawlingFlag });
setButtonStates();
Config.saveSettings({ progress: 100 });
UI.updateLogs();
}
} catch (error) {
Logger.log(`크롤링 중 오류 발생: ${error.message}`, 'error');
finalizeCrawl('오류 발생');
}
}
async function processQueue() {
while (requestQueue.length && activeRequests < Config.settings.maxConcurrentRequests && isCrawlingFlag) {
const { url, depth } = requestQueue.shift();
activeRequests++;
handleRequest(url, depth).finally(() => { activeRequests--; processQueue(); });
}
}
async function handleRequest(url, depth) {
if (!isCrawlingFlag || visitedUrls.has(url)) return;
try {
const response = await Fetch.fetchWithRetry(url, Config.settings.retryAttempts, Config.settings.retryDelay);
const contentType = Fetch.getContentType(response);
if (!Fetch.isContentTypeAllowed(contentType, Config.settings.contentTypes)) {
Logger.log(`허용되지 않은 콘텐츠 유형(${contentType})의 URL: ${url}`, 'warning');
return;
}
const parser = new DOMParser();
const doc = parser.parseFromString(response.responseText, 'text/html');
const text = doc.body ? doc.body.innerText : '';
if (Config.settings.enableCaptchas && Robots.detectCaptcha(doc)) { await Robots.handleCaptcha(url); return; }
const wc = text.split(/\s+/).length;
if (wc < Config.settings.minWordCount || wc > Config.settings.maxWordCount) {
Logger.log(`단어 수 조건에 맞지 않는 URL: ${url} (${wc} 단어)`, 'warning');
return;
}
const quality = Evaluate.evaluateContentQuality(text);
const lang = doc.documentElement.lang || 'unknown';
const category = Classifier.classifyKeywords(text);
const meta = Extractor.extractMetaTags(doc);
const headings = Extractor.extractHeadings(doc);
if (category !== '기타' || quality > 100) {
const entry = { url, quality, language: lang, wordCount: wc, category };
if (Config.settings.dataFields.name) entry.name = Extractor.extractName(doc) || 'No Title';
if (Config.settings.dataFields.description) entry.description = Extractor.extractDescription(doc) || 'No Description';
if (Config.settings.dataFields.html) entry.html = response.responseText;
if (Config.settings.dataFields.metaTags) entry.metaTags = meta;
if (Config.settings.dataFields.headings) entry.headings = headings;
data.push(entry);
GM_setValue('data', JSON.stringify(data));
UI.updateDataTable();
Logger.log(`데이터 수집 완료: ${url}`, 'success');
if (Config.settings.dataVisualization) DataVisualization.renderChart();
}
if (depth < Config.settings.maxDepth) {
const links = Extractor.extractLinks(doc);
links.forEach(link => { if (!visitedUrls.has(link) && Utils.isValidURL(link)) requestQueue.push({ url: link, depth: depth + 1 }); });
}
} catch (error) {
Logger.log(`데이터 수집 실패: ${url} - ${error.message}`, 'error');
}
visitedUrls.add(url);
GM_setValue('visitedUrls', JSON.stringify([...visitedUrls]));
}
function finalizeCrawl(status) {
if (previousCrawls.length) {
previousCrawls[previousCrawls.length -1].status = status;
GM_setValue('previousCrawls', JSON.stringify(previousCrawls));
UI.updatePreviousCrawlsUI();
}
if (navigationObserver) {
navigationObserver.disconnect();
navigationObserver = null;
}
}
function updateCrawlSettings() {
// Placeholder for any settings update needed during crawling
}
function updatePageContent(content) {
const parser = new DOMParser();
const doc = parser.parseFromString(content, 'text/html');
document.documentElement.innerHTML = doc.documentElement.innerHTML;
}
function monitorPageNavigation() {
const searchEngine = identifySearchEngine();
if (!searchEngine || !searchEngineHandlers[searchEngine]) return;
const containerSelector = searchEngineHandlers[searchEngine].getContentContainer();
const targetNode = document.querySelector(containerSelector);
if (!targetNode) {
Logger.log('페이지 내비게이션 감시 대상 노드를 찾을 수 없습니다.', 'warning');
return;
}
navigationObserver = new MutationObserver((mutations) => {
for (let mutation of mutations) {
if (mutation.type === 'childList') {
if (isCrawlingFlag) {
Logger.log('페이지 내비게이션 감지됨. 크롤링을 중지합니다.', 'warning');
Crawling.stopCrawling();
break;
}
}
}
});
navigationObserver.observe(targetNode, { childList: true, subtree: true });
Logger.log('페이지 내비게이션 감시가 활성화되었습니다.', 'info');
}
return {
startCrawling,
stopCrawling,
isCrawling: () => isCrawlingFlag,
updateCrawlSettings
};
})();
/* ============================
Module: Fetch
============================ */
const Fetch = (function() {
async function fetchWithRetry(url, attempts, delayMs) {
for (let i = 1; i <= attempts; i++) {
try {
return await GM_xmlhttpRequestPromise(url);
} catch (e) {
Logger.log(`요청 실패 (${i}/${attempts}): ${url} - ${e.message}`, 'warning');
if (i < attempts) await Utils.delay(delayMs);
}
}
throw new Error('모든 재시도 시도가 실패했습니다.');
}
function GM_xmlhttpRequestPromise(url) {
return new Promise((resolve, reject) => {
let reqUrl = Config.settings.proxy ? `${Config.settings.proxy}?url=${encodeURIComponent(url)}` : url;
GM_xmlhttpRequest({
method: "GET",
url: reqUrl,
headers: { 'User-Agent': getCurrentUserAgent(), 'Accept': '*/*' },
timeout: Config.settings.timeout,
onload: res => (res.status >= 200 && res.status < 300) ? resolve(res) : reject(new Error(`HTTP ${res.status}`)),
onerror: () => reject(new Error('Request failed')),
ontimeout: () => reject(new Error('Request timed out'))
});
});
}
function getCurrentUserAgent() {
const index = Config.settings.userAgentIndex % Config.settings.userAgents.length;
return Config.settings.userAgents[index];
}
function getContentType(res) {
return res.responseHeaders.split('\n').find(h => h.toLowerCase().startsWith('content-type'))?.split(':')[1].trim().split(';')[0] || '';
}
function isContentTypeAllowed(ct, contentTypes) {
if (contentTypes.include.length && !contentTypes.include.includes(ct)) return false;
if (contentTypes.exclude.includes(ct)) return false;
return true;
}
async function fetchPageContent(url) {
try {
const response = await fetchWithRetry(url, Config.settings.retryAttempts, Config.settings.retryDelay);
return response.responseText;
} catch (error) {
Logger.log(`페이지 내용 가져오기 실패: ${url} - ${error.message}`, 'error');
return null;
}
}
return {
fetchWithRetry,
getCurrentUserAgent,
getContentType,
isContentTypeAllowed,
fetchPageContent
};
})();
/* ============================
Module: Robots
============================ */
const Robots = (function() {
async function checkRobotsTxt(url) {
try {
const parsed = new URL(url);
const robotsUrl = `${parsed.origin}/robots.txt`;
const res = await Fetch.fetchWithRetry(robotsUrl, 2, 2000);
return new RobotsParser(res.responseText).isAllowed(url, Fetch.getCurrentUserAgent());
} catch {
Logger.log('robots.txt 확인 실패, 기본 허용.', 'warning');
return true;
}
}
function detectCaptcha(doc) {
return doc.querySelector('iframe[src*="captcha"], form[action*="captcha"]') !== null;
}
async function handleCaptcha(url) {
Logger.log(`CAPTCHA 감지됨: ${url}`, 'warning');
await Utils.delay(5000);
Logger.log(`CAPTCHA 처리 완료: ${url}`, 'info');
}
class RobotsParser {
constructor(text) { this.rules = []; this.parse(text); }
parse(text) {
let agents = [];
text.split('\n').forEach(line => {
const [key, ...val] = line.split(':').map(s => s.trim());
if (!key) return;
if (key.toLowerCase() === 'user-agent') { agents = [val.join(':').toLowerCase()]; }
else if (['disallow','allow'].includes(key.toLowerCase())) this.rules.push({ agents, type: key.toLowerCase(), path: val.join(':') });
});
}
isAllowed(url, ua) {
const parsedUrl = new URL(url);
const path = parsedUrl.pathname;
let allowed = true;
this.rules.forEach(rule => {
if (rule.agents.includes('*') || rule.agents.includes(ua.toLowerCase())) {
if (rule.type === 'disallow' && path.startsWith(rule.path)) allowed = false;
if (rule.type === 'allow' && path.startsWith(rule.path)) allowed = true;
}
});
return allowed;
}
}
return {
checkRobotsTxt,
detectCaptcha,
handleCaptcha
};
})();
/* ============================
Module: Extractor
============================ */
const Extractor = (function() {
function extractMetaTags(doc) {
const metas = {};
doc.querySelectorAll('meta[name], meta[property]').forEach(meta => {
const name = meta.getAttribute('name') || meta.getAttribute('property');
const content = meta.getAttribute('content');
if (name && content) metas[name.toLowerCase()] = content;
});
return metas;
}
function extractHeadings(doc) {
const hs = [];
for (let i = 1; i <= 6; i++) doc.querySelectorAll(`h${i}`).forEach(h => hs.push(h.innerText.trim()));
return hs;
}
function extractName(doc) { return doc.querySelector('title')?.innerText || 'No Title'; }
function extractDescription(doc) { return doc.querySelector('meta[name="description"]')?.getAttribute('content') || 'No Description'; }
function extractLinks(doc) { return Array.from(doc.querySelectorAll('a[href]')).map(a => a.href).filter(Utils.isValidURL); }
return {
extractMetaTags,
extractHeadings,
extractName,
extractDescription,
extractLinks
};
})();
/* ============================
Module: Classifier
============================ */
const Classifier = (function() {
function classifyKeywords(text) {
const kws = combineKeywords();
for (const [cat, keywords] of Object.entries(kws)) {
if (keywords.some(kw => text.includes(kw))) return cat;
}
return '기타';
}
function combineKeywords() {
const standard = {
'교육': ['학교', '교육', '학생', '교수', '강의'],
'기술': ['기술', '소프트웨어', '하드웨어', '프로그래밍', '개발'],
'뉴스': ['뉴스', '속보', '기사', '보도'],
'엔터테인먼트': ['영화', '음악', '연예', '게임', '드라마'],
'스포츠': ['축구', '야구', '농구', '경기', '토너먼트'],
'비즈니스': ['비즈니스', '경제', '주식', '투자', '시장'],
'건강': ['건강', '의료', '헬스', '운동', '영양'],
'과학': ['과학', '연구', '실험', '발견', '이론'],
'문학': ['소설', '시', '문학', '작가', '책'],
'여행': ['여행', '관광', '숙박', '비행기', '여권']
};
const custom = Config.settings.customKeywords || {};
for (const [cat, kws] of Object.entries(custom)) {
standard[cat] = standard[cat] ? [...new Set([...standard[cat], ...kws])] : kws;
}
return standard;
}
return {
classifyKeywords
};
})();
/* ============================
Module: Evaluate
============================ */
const Evaluate = (function() {
function evaluateContentQuality(text) { return text.split(/\s+/).length; }
return {
evaluateContentQuality
};
})();
/* ============================
Module: Scheduler
============================ */
const Scheduler = (function() {
let scheduleInterval = null;
function scheduleCrawling() {
clearInterval(scheduleInterval);
const now = new Date();
const [startHour, startMinute] = Config.settings.schedule.startTime.split(':').map(Number);
const startTime = new Date(now.getFullYear(), now.getMonth(), now.getDate(), startHour, startMinute, 0);
if (startTime < now) startTime.setDate(startTime.getDate() + 1);
const initialDelay = startTime - now;
setTimeout(() => {
Crawling.startCrawling();
scheduleInterval = setInterval(Crawling.startCrawling, Config.settings.schedule.interval);
}, initialDelay);
Logger.log(`크롤링 스케줄이 설정되었습니다. 시작 시간: ${startTime.toLocaleString()}, 간격: ${Config.settings.schedule.interval / 60000}분`, 'info');
}
return {
scheduleCrawling
};
})();
/* ============================
Module: Download
============================ */
// Already defined above
/* ============================
Module: DataVisualization
============================ */
// Already defined above
/* ============================
Module: UI
============================ */
// Already defined above
/* ============================
Module: Logger
============================ */
// Already defined above
/* ============================
Module: Config
============================ */
// Already defined above
/* ============================
Initialization
============================ */
function init() {
UI.createUI();
UI.updateUI();
if (Config.settings.dataVisualization) DataVisualization.renderChart();
if (Config.settings.schedule.enabled) {
Scheduler.scheduleCrawling();
}
if (Config.settings.isCrawling) {
Logger.log('크롤링 재개됨.', 'info');
Crawling.startCrawling();
} else {
Logger.log('크롤러 초기화됨.', 'info');
}
}
window.addEventListener('load', init);
})();
```