USER
// ==UserScript==
// @name 종합 웹 크롤러 7.9.5
// @namespace http://tampermonkey.net/
// @version 7.9.5
// @description 고급 기능을 갖춘 웹 크롤러로 검색 결과를 실시간으로 수집하고 표시 (동시 크롤링, 번역 기능, UI 개선 등)
// @author YourName
// @match *://www.google.com/search*
// @match *://www.bing.com/search*
// @match *://search.daum.net/search*
// @match *://search.naver.com/search*
// @match *://search.yahoo.com/search*
// @match *://duckduckgo.com/*
// @match *://www.ecosia.org/search*
// @match *://*.onion/*
// @grant GM_xmlhttpRequest
// @grant GM_addStyle
// @grant GM_setValue
// @grant GM_getValue
// @grant GM_notification
// @connect *
// @require https://cdnjs.cloudflare.com/ajax/libs/xlsx/0.18.5/xlsx.full.min.js
// @require https://cdn.jsdelivr.net/npm/chart.js
// @require https://cdnjs.cloudflare.com/ajax/libs/Sortable/1.14.0/Sortable.min.js
// @require https://cdnjs.cloudflare.com/ajax/libs/i18next/21.6.15/i18next.min.js
// ==/UserScript==
(function() {
'use strict';
// CSS 스타일 추가 (UI 대폭 개선)
GM_addStyle(`
/* 컨테이너 스타일 */
#crawler-container {
position: fixed;
top: 20px;
right: 20px;
background-color: #2c3e50;
border: 2px solid #1abc9c;
padding: 20px;
z-index: 10000;
width: 800px;
max-height: 85vh;
overflow: hidden;
box-shadow: 0 8px 30px rgba(0,0,0,0.5);
border-radius: 12px;
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
color: #ecf0f1;
display: flex;
flex-direction: column;
transition: all 0.3s ease;
}
/* 헤더 스타일 */
#crawler-container header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 15px;
}
#crawler-container .title {
font-size: 20px;
font-weight: bold;
}
#crawler-container .close-btn {
cursor: pointer;
font-size: 24px;
color: #e74c3c;
transition: color 0.3s;
}
#crawler-container .close-btn:hover {
color: #c0392b;
}
/* 탭 스타일 */
#crawler-container .tabs {
display: flex;
border-bottom: 2px solid #1abc9c;
margin-bottom: 15px;
overflow-x: auto;
}
#crawler-container .tab {
flex: 1;
text-align: center;
padding: 10px 0;
cursor: pointer;
background-color: #34495e;
border-right: 1px solid #1abc9c;
transition: background-color 0.3s, color 0.3s;
white-space: nowrap;
}
#crawler-container .tab:last-child {
border-right: none;
}
#crawler-container .tab.active {
background-color: #1abc9c;
color: #2c3e50;
font-weight: bold;
}
/* 탭 콘텐츠 스타일 */
#crawler-container .tab-content {
display: none;
flex: 1;
overflow-y: auto;
}
#crawler-container .tab-content.active {
display: block;
}
/* 버튼 스타일 */
#crawler-container .controls button {
background-color: #1abc9c;
border: none;
color: #2c3e50;
padding: 10px 16px;
margin: 6px 3px;
border-radius: 5px;
cursor: pointer;
transition: background-color 0.3s, transform 0.2s;
font-size: 14px;
display: flex;
align-items: center;
}
#crawler-container .controls button:hover {
background-color: #16a085;
transform: translateY(-2px);
}
#crawler-container .controls button i {
margin-right: 6px;
}
/* 체크박스 및 라벨 스타일 */
#crawler-container .options label,
#crawler-container .filters label,
#crawler-container .exclude-domains label,
#crawler-container .advanced-options label {
display: flex;
align-items: center;
margin: 10px 0;
font-size: 14px;
}
#crawler-container .options input[type="checkbox"] {
margin-right: 8px;
transform: scale(1.2);
}
/* 스위치 스타일 */
.switch {
position: relative;
display: inline-block;
width: 50px;
height: 24px;
margin-left: 10px;
}
.switch input {
opacity: 0;
width: 0;
height: 0;
}
.switch .slider {
position: absolute;
cursor: pointer;
top: 0;
left: 0;
right: 0;
bottom: 0;
background-color: #ccc;
transition: 0.4s;
border-radius: 24px;
}
.switch .slider:before {
position: absolute;
content: "";
height: 18px;
width: 18px;
left: 3px;
bottom: 3px;
background-color: white;
transition: 0.4s;
border-radius: 50%;
}
.switch input:checked + .slider {
background-color: #1abc9c;
}
.switch input:checked + .slider:before {
transform: translateX(26px);
}
/* 입력 필드 스타일 */
#crawler-container input[type="text"],
#crawler-container input[type="number"] {
width: 100%;
padding: 8px;
margin: 6px 0;
border: 1px solid #bdc3c7;
border-radius: 4px;
font-size: 14px;
}
/* 테이블 스타일 */
#resultsTable {
width: 100%;
border-collapse: collapse;
margin-top: 15px;
}
#resultsTable th, #resultsTable td {
border: 1px solid #95a5a6;
padding: 10px;
text-align: left;
font-size: 13px;
}
#resultsTable th {
background-color: #1abc9c;
color: #2c3e50;
}
#resultsTable tbody tr:nth-child(even) {
background-color: #34495e;
}
/* 반응형 디자인 */
@media (max-width: 900px) {
#crawler-container {
width: 90%;
right: 5%;
padding: 15px;
}
#crawler-container .controls button {
padding: 8px 12px;
font-size: 13px;
}
#crawler-container .title {
font-size: 18px;
}
}
/* Papago iframe 스타일 */
#papagoIframe {
width: 100%;
height: 100%;
border: none;
}
/* 팝업 스타일 */
.search-engines-popup, .manage-darkweb-popup {
position: absolute;
background-color: #2c3e50;
border: 1px solid #1abc9c;
padding: 15px;
border-radius: 8px;
box-shadow: 0 4px 12px rgba(0,0,0,0.3);
max-height: 350px;
overflow-y: auto;
z-index: 10001;
}
.search-engines-popup button, .manage-darkweb-popup button {
width: 100%;
text-align: left;
margin: 6px 0;
background-color: #34495e;
color: #ecf0f1;
border: none;
padding: 10px;
border-radius: 5px;
cursor: pointer;
transition: background-color 0.3s;
font-size: 14px;
}
.search-engines-popup button:hover, .manage-darkweb-popup button:hover {
background-color: #1abc9c;
color: #2c3e50;
}
/* 로그 스타일 */
#log {
background-color: #34495e;
padding: 10px;
border-radius: 5px;
max-height: 200px;
overflow-y: auto;
font-size: 13px;
line-height: 1.4;
}
/* 차트 컨테이너 */
.chart-container {
margin-top: 15px;
height: 250px;
}
/* HTML 디스플레이 스타일 */
.html-display iframe {
width: 100%;
height: 500px;
border: none;
border-radius: 5px;
}
/* Papago 디스플레이 스타일 */
.papago-display {
width: 100%;
height: 500px;
position: relative;
}
/* 다크웹 엔진 목록 스타일 */
#darkwebEnginesList li {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 12px;
}
#darkwebEnginesList li span {
flex: 1;
margin-right: 10px;
word-break: break-all;
}
#darkwebEnginesList li button {
background-color: #e74c3c;
border: none;
color: #fff;
padding: 6px 10px;
border-radius: 4px;
cursor: pointer;
transition: background-color 0.3s;
}
#darkwebEnginesList li button:hover {
background-color: #c0392b;
}
/* 예외 도메인 리스트 스타일 */
.exclude-domains ul {
list-style: none;
padding: 0;
margin-top: 10px;
}
.exclude-domains ul li {
background-color: #34495e;
padding: 6px 10px;
border-radius: 4px;
margin-bottom: 6px;
display: flex;
justify-content: space-between;
align-items: center;
}
.exclude-domains ul li button {
background-color: #e74c3c;
border: none;
color: #fff;
padding: 4px 8px;
border-radius: 3px;
cursor: pointer;
transition: background-color 0.3s;
font-size: 12px;
}
.exclude-domains ul li button:hover {
background-color: #c0392b;
}
/* Search Engines Direct UI */
#search-engines-direct {
display: flex;
flex-wrap: wrap;
gap: 10px;
margin-top: 15px;
}
#search-engines-direct button {
flex: 1 1 45%;
background-color: #1abc9c;
border: none;
color: #2c3e50;
padding: 8px 12px;
border-radius: 5px;
cursor: pointer;
transition: background-color 0.3s;
font-size: 14px;
}
#search-engines-direct button:hover {
background-color: #16a085;
}
/* Preview Popup */
.preview-popup {
position: absolute;
background-color: #2c3e50;
border: 1px solid #1abc9c;
padding: 10px;
border-radius: 6px;
box-shadow: 0 4px 8px rgba(0,0,0,0.2);
width: 300px;
height: 150px;
z-index: 10002;
display: none;
}
`);
// UI 생성 (대폭 개선된 디자인)
const container = document.createElement('div');
container.id = 'crawler-container';
container.innerHTML = `
<header>
<span class="title">🔥 웹 크롤러 7.9.5</span>
<span class="close-btn" title="닫기">×</span>
</header>
<div class="tabs">
<div class="tab active" data-tab="settings">설정</div>
<div class="tab" data-tab="results">결과</div>
<div class="tab" data-tab="logs">로그</div>
<div class="tab" data-tab="site-filters">사이트 필터</div>
<div class="tab" data-tab="darkweb">다크웹</div>
<div class="tab" data-tab="html-display">HTML</div>
<div class="tab" data-tab="papago">Papago</div>
<div class="tab" data-tab="search-engines">검색 엔진</div>
</div>
<div class="tab-content active" id="settings">
<!-- 제어 버튼 -->
<div class="controls">
<button id="startBtn" title="크롤링 시작"><i>▶️</i> 시작</button>
<button id="pauseBtn" class="pause-btn" title="크롤링 일시정지/재개"><i>⏸️</i> 일시정지</button>
<button id="stopBtn" title="크롤링 중지"><i>⏹️</i> 중지</button>
<button id="resetBtn" title="크롤링 초기화"><i>🔄</i> 초기화</button>
<button id="exportBtn" class="export-btn" title="결과 내보내기"><i>💾</i> 내보내기</button>
<button id="searchEnginesBtn" class="search-engines-btn" title="검색 엔진 바로가기"><i>🔍</i> 검색 엔진</button>
</div>
<!-- 옵션 섹션 -->
<div class="options">
<label><input type="checkbox" id="toggleHTML" checked> HTML 가져오기</label>
<label><input type="checkbox" id="toggleImages" checked> 이미지 가져오기</label>
<label><input type="checkbox" id="toggleAds" checked> 광고 제외</label>
<label>엄격 사이트 필터링
<label class="switch">
<input type="checkbox" id="strictSiteFilter">
<span class="slider"></span>
</label>
</label>
<label>필터링 기능 활성화
<label class="switch">
<input type="checkbox" id="filteringToggle" checked>
<span class="slider"></span>
</label>
</label>
</div>
<!-- 필터 섹션 -->
<div class="filters">
<label for="filterInput">필터 단어 (정규식, 쉼표로 구분):</label>
<input type="text" id="filterInput" placeholder="예: 기부, 광고, /promo/i">
</div>
<!-- 도메인 제외 섹션 -->
<div class="exclude-domains">
<label for="excludeDomainsInput">제외할 도메인 (쉼표로 구분):</label>
<input type="text" id="excludeDomainsInput" placeholder="예: facebook.com, youtube.com">
<ul id="excludeDomainsList"></ul>
</div>
<!-- 고급 옵션 섹션 -->
<div class="advanced-options">
<label for="duplicateThreshold">중복 허용 횟수:</label>
<input type="number" id="duplicateThreshold" value="5" min="1" style="width: 80px;">
</div>
<!-- 제어 옵션 섹션 -->
<div class="controls">
<label for="retryAttempts">재시도 횟수:</label>
<input type="number" id="retryAttempts" value="10" min="1" style="width: 80px;">
<label for="retryInterval">재시도 간격 (ms):</label>
<input type="number" id="retryInterval" value="100" min="100" style="width: 80px;">
</div>
<!-- 다크웹 검색 엔진 토글 -->
<div class="controls">
<label>다크웹 검색 엔진 사용
<label class="switch">
<input type="checkbox" id="darkwebToggle" checked>
<span class="slider"></span>
</label>
</label>
</div>
<!-- 진행률 표시 섹션 -->
<div class="progress-section">
<progress id="progressBar" value="0" max="100" style="width: 100%; height: 20px;"></progress>
<span id="progressText">페이지 0</span>
</div>
<!-- 상태 정보 섹션 -->
<div class="status">
현재 페이지: <span id="currentPage">0</span>
<br>
방문된 사이트: <span id="visitedCount">0</span>
<br>
경과 시간: <span id="elapsedTime">00:00:00</span>
<br>
예상 남은 시간: <span id="remainingTime">00:00:00</span>
</div>
<!-- 차트 섹션 -->
<div class="chart-container">
<canvas id="crawlChart"></canvas>
</div>
</div>
<div class="tab-content" id="results">
<div style="margin-top:15px; overflow: auto;">
<table id="resultsTable">
<thead>
<tr>
<th>사이트 이름</th>
<th>주소</th>
<th>설명</th>
<th>이미지</th>
<th>HTML 링크</th>
<th>페이지 로딩 시간 (ms)</th>
</tr>
</thead>
<tbody id="resultsBody">
</tbody>
</table>
</div>
</div>
<div class="tab-content" id="logs">
<div class="log" id="log">
<strong>로그:</strong><br>
</div>
</div>
<div class="tab-content" id="site-filters">
<div class="site-filters">
<label for="siteFilterInput">사이트 필터링 (정규식, 쉼표로 구분):</label>
<input type="text" id="siteFilterInput" placeholder="예: .*example\\.com.*, /^https?:\\/\\/.*\\.example\\.org$/">
</div>
</div>
<div class="tab-content" id="darkweb">
<div class="darkweb-search">
<label>다크웹 검색 엔진 추가/관리</label>
<button class="add-darkweb-engine-btn" title="다크웹 검색 엔진 추가"><i>➕</i> 추가</button>
<ul id="darkwebEnginesList">
<!-- 다크웹 검색 엔진 목록 -->
</ul>
</div>
</div>
<div class="tab-content" id="html-display">
<div class="html-display"></div>
</div>
<div class="tab-content" id="papago">
<div class="papago-display">
<iframe id="papagoIframe" src="https://papago.naver.com/"></iframe>
</div>
</div>
<div class="tab-content" id="search-engines">
<div class="search-engines-direct" id="search-engines-direct">
<!-- 검색 엔진 바로가기 버튼 추가 -->
</div>
</div>
`;
document.body.appendChild(container);
// 드래그 가능하게 설정
(function makeDraggable(elmnt) {
const header = elmnt.querySelector('header');
const closeBtn = elmnt.querySelector('.close-btn');
let pos1 = 0, pos2 = 0, pos3 = 0, pos4 = 0;
header.onmousedown = dragMouseDown;
closeBtn.onclick = () => {
elmnt.style.display = 'none';
}
function dragMouseDown(e) {
e = e || window.event;
e.preventDefault();
pos3 = e.clientX;
pos4 = e.clientY;
document.onmouseup = closeDragElement;
document.onmousemove = elementDrag;
}
function elementDrag(e) {
e = e || window.event;
e.preventDefault();
pos1 = pos3 - e.clientX;
pos2 = pos4 - e.clientY;
pos3 = e.clientX;
pos4 = e.clientY;
elmnt.style.top = (elmnt.offsetTop - pos2) + "px";
elmnt.style.left = (elmnt.offsetLeft - pos1) + "px";
}
function closeDragElement() {
document.onmouseup = null;
document.onmousemove = null;
}
})(container);
// 탭 기능 구현
(function setupTabs() {
const tabs = container.querySelectorAll('.tab');
const tabContents = container.querySelectorAll('.tab-content');
tabs.forEach(tab => {
tab.addEventListener('click', () => {
tabs.forEach(t => t.classList.remove('active'));
tab.classList.add('active');
const target = tab.getAttribute('data-tab');
tabContents.forEach(content => {
if(content.id === target){
content.classList.add('active');
} else {
content.classList.remove('active');
}
});
});
});
})();
// 변수 초기화
let isRunning = false;
let isPaused = false;
let currentPage = 1;
let results = [];
let searchQuery = '';
let searchEngine = '';
let visitedLinks = new Map(); // 도메인별 방문 횟수 추적
let logMessages = [];
let filterPatterns = [];
let excludeDomains = [];
let siteFilterPatterns = [];
let duplicateThreshold = 5; // 중복 허용 횟수
let retryAttempts = 10; // 재시도 횟수
let retryInterval = 100; // 재시도 간격 100ms으로 설정
let startTime = null;
let elapsedInterval = null;
let crawlChart = null;
let crawlData = {
labels: [], // 도메인 이름
datasets: [{
label: '방문 횟수',
data: [],
backgroundColor: 'rgba(46, 204, 113, 0.6)',
borderColor: 'rgba(46, 204, 113, 1)',
borderWidth: 1
}]
};
let filteringEnabled = true; // 필터링 활성화 여부
let concurrentRequests = 40; // 동시 요청 수 40으로 설정
let currentRequests = 0; // 현재 요청 수
let requestQueue = []; // 요청 대기열
let translateLanguage = 'en'; // 기본 번역 언어
let darkwebEnabled = true; // 다크웹 검색 엔진 사용 여부
// 다크웹 검색 엔진 목록 (기본 엔진 추가 및 확장)
const defaultDarkwebEngines = [
{ name: 'Torch', url: 'http://xmh57jrzrnw6insl.onion/' },
{ name: 'Ahmia', url: 'http://msydqstlz2kzerdg.onion/' },
{ name: 'Haystak', url: 'http://haystakvxad7wbk5.onion/' },
{ name: 'Not Evil', url: 'http://hss3uro2hsxfogfq.onion/' },
{ name: 'Phobos', url: 'http://phobos6uj4abm4pf.onion/' },
{ name: 'Kilos', url: 'http://kilos7wbfn3ptgw.onion/' },
{ name: 'Candle', url: 'http://zg6ak7qqfhbkbppx.onion/' },
{ name: 'DarkSearch', url: 'http://darksearch3w3nrc.onion/' },
{ name: 'Dark Falcon', url: 'http://darkfalconphp5fh.onion/' },
{ name: 'SearXNG', url: 'http://searxngueyn5qzuv.onion/' },
{ name: 'DuckDuckGo (onion)', url: 'http://duckduckgogj7ay7.onion/' },
{ name: 'Phobos 2', url: 'http://phobos2uj4abm4pf.onion/' },
{ name: 'IntelligenceX', url: 'http://intelligencexmh5v.onion/' },
{ name: 'Silk Road 3.1', url: 'http://silkroad31lo5ya.onion/' },
{ name: 'Fresh Onions', url: 'http://rx7hmtf7k7xk6ked.onion/' },
// 추가 엔진 가능
{ name: 'Dark Web Link', url: 'http://darkweblink5wqk.onion/' },
{ name: 'Tor Tracker', url: 'http://tortracker5ujzra.onion/' },
{ name: 'Deep Web Search', url: 'http://deepwebsearch6b6x.onion/' },
{ name: 'Onion Monitor', url: 'http://onionmonitor3c7v.onion/' },
{ name: 'Hidden Wiki', url: 'http://hiddenwiki5z35.onion/' },
{ name: 'TorEX', url: 'http://torexk3wd2buz.onion/' },
{ name: 'BlackBook', url: 'http://blackbook51uv.onion/' },
{ name: 'Taboo Search', url: 'http://taboosearch37b.onion/' },
{ name: 'The Pirate Bay (onion)', url: 'http://uj3wazyk5u4hnvtk.onion/' },
// 더 많은 다크웹 검색 엔진을 추가할 수 있습니다.
];
// 버튼 및 요소 가져오기
const startBtn = document.getElementById('startBtn');
const pauseBtn = document.getElementById('pauseBtn');
const stopBtn = document.getElementById('stopBtn');
const resetBtn = document.getElementById('resetBtn');
const exportBtn = document.getElementById('exportBtn');
const searchEnginesBtn = document.getElementById('searchEnginesBtn');
const progressBar = document.getElementById('progressBar');
const progressText = document.getElementById('progressText');
const resultsBody = document.getElementById('resultsBody');
const logContainer = document.getElementById('log');
const currentPageElem = document.getElementById('currentPage');
const toggleHTML = document.getElementById('toggleHTML');
const toggleImages = document.getElementById('toggleImages');
const toggleAds = document.getElementById('toggleAds');
const filterInput = document.getElementById('filterInput');
const excludeDomainsInput = document.getElementById('excludeDomainsInput');
const excludeDomainsList = document.getElementById('excludeDomainsList');
const siteFilterInput = document.getElementById('siteFilterInput');
const duplicateThresholdInput = document.getElementById('duplicateThreshold');
const retryAttemptsInput = document.getElementById('retryAttempts');
const retryIntervalInput = document.getElementById('retryInterval');
const strictSiteFilter = document.getElementById('strictSiteFilter');
const visitedCountElem = document.getElementById('visitedCount');
const elapsedTimeElem = document.getElementById('elapsedTime');
const remainingTimeElem = document.getElementById('remainingTime');
const crawlChartCtx = document.getElementById('crawlChart').getContext('2d');
const filteringToggle = document.getElementById('filteringToggle');
const darkwebEnginesList = document.getElementById('darkwebEnginesList');
const addDarkwebEngineBtn = container.querySelector('.add-darkweb-engine-btn');
const darkwebToggle = document.getElementById('darkwebToggle');
const papagoIframe = document.getElementById('papagoIframe');
const searchEnginesDirect = document.getElementById('search-engines-direct');
// 이벤트 리스너
startBtn.addEventListener('click', startCrawling);
pauseBtn.addEventListener('click', togglePause);
stopBtn.addEventListener('click', stopCrawling);
resetBtn.addEventListener('click', resetCrawling);
exportBtn.addEventListener('click', exportResults);
searchEnginesBtn.addEventListener('click', openSearchEngines);
filterInput.addEventListener('input', updateFilterPatterns);
excludeDomainsInput.addEventListener('keydown', function(e) {
if (e.key === 'Enter') {
addExcludeDomain();
}
});
addDarkwebEngineBtn.addEventListener('click', addDarkwebEngine);
siteFilterInput.addEventListener('input', updateSiteFilterPatterns);
duplicateThresholdInput.addEventListener('change', updateDuplicateThreshold);
retryAttemptsInput.addEventListener('change', updateRetryAttempts);
retryIntervalInput.addEventListener('change', updateRetryInterval);
strictSiteFilter.addEventListener('change', updateStrictSiteFilter);
filteringToggle.addEventListener('change', toggleFiltering);
darkwebToggle.addEventListener('change', toggleDarkweb);
papagoIframe.addEventListener('load', () => {
log('🌐 Papago 번역기 로드 완료.');
});
// 페이지 새로 고침 시 크롤링 초기화
window.addEventListener('beforeunload', () => {
resetCrawling();
});
// 초기화 로드
loadLogs();
loadCrawlData();
loadDarkwebEngines();
loadExcludeDomains();
loadSearchEnginesDirect();
// 차트 초기화
function initializeChart() {
crawlChart = new Chart(crawlChartCtx, {
type: 'bar',
data: crawlData,
options: {
responsive: true,
maintainAspectRatio: false,
scales: {
y: { beginAtZero: true },
x: { ticks: { autoSkip: false } }
},
plugins: {
legend: {
display: false
},
tooltip: {
enabled: true
}
}
}
});
}
initializeChart();
// 로깅 함수
function log(message) {
const time = new Date().toLocaleTimeString();
const entry = `[${time}] ${message}`;
logMessages.push(entry);
logContainer.innerHTML += `${entry}<br>`;
logContainer.scrollTop = logContainer.scrollHeight;
// 저장
GM_setValue('crawlerLogs', JSON.stringify(logMessages));
}
function loadLogs() {
const storedLogs = GM_getValue('crawlerLogs', []);
if (storedLogs.length > 0) {
logMessages = JSON.parse(storedLogs);
logContainer.innerHTML = `<strong>로그:</strong><br>` + logMessages.join('<br>');
logContainer.scrollTop = logContainer.scrollHeight;
}
}
function updateFilterPatterns() {
if (!filteringEnabled) {
filterPatterns = [];
log(`🛠 필터링 기능 비활성화됨.`);
return;
}
const input = filterInput.value.trim();
filterPatterns = input ? input.split(',').map(word => {
word = word.trim();
if (word.startsWith('/') && word.endsWith('/')) {
try {
return new RegExp(word.slice(1, -1), 'i');
} catch (e) {
log(`⚠️ 유효하지 않은 정규식: ${word}`);
return null;
}
} else {
return new RegExp(word, 'i');
}
}).filter(regex => regex !== null) : [];
log(`🛠 필터 패턴 업데이트: [${filterPatterns.map(r => r.toString()).join(', ')}]`);
}
function updateExcludeDomains() {
// 이 함수는 제외 도메인이 추가될 때마다 호출됨
log(`🚫 제외 도메인 업데이트 완료.`);
}
function addExcludeDomain() {
const domain = excludeDomainsInput.value.trim().toLowerCase();
if (domain && !excludeDomains.includes(domain)) {
excludeDomains.push(domain);
renderExcludeDomains();
excludeDomainsInput.value = '';
log(`🚫 제외 도메인 추가: ${domain}`);
}
}
function renderExcludeDomains() {
excludeDomainsList.innerHTML = '';
excludeDomains.forEach((domain, index) => {
const li = document.createElement('li');
li.innerHTML = `
<span>${escapeHTML(domain)}</span>
<button data-index="${index}">✖️</button>
`;
excludeDomainsList.appendChild(li);
});
// 삭제 버튼 이벤트 리스너
excludeDomainsList.querySelectorAll('button').forEach(btn => {
btn.addEventListener('click', () => {
const index = parseInt(btn.getAttribute('data-index'));
if (!isNaN(index)) {
const removed = excludeDomains.splice(index, 1);
renderExcludeDomains();
log(`🚫 제외 도메인 제거: ${removed}`);
}
});
});
}
function loadExcludeDomains() {
const storedDomains = GM_getValue('excludeDomains', []);
excludeDomains = storedDomains.length > 0 ? storedDomains : [];
renderExcludeDomains();
}
function updateSiteFilterPatterns() {
const input = siteFilterInput.value.trim();
siteFilterPatterns = input ? input.split(',').map(pattern => {
pattern = pattern.trim();
if (pattern.startsWith('/') && pattern.endsWith('/')) {
try {
return new RegExp(pattern.slice(1, -1), 'i');
} catch (e) {
log(`⚠️ 유효하지 않은 사이트 필터 정규식: ${pattern}`);
return null;
}
} else {
return new RegExp(pattern, 'i');
}
}).filter(regex => regex !== null) : [];
log(`🛠 사이트 필터 패턴 업데이트: [${siteFilterPatterns.map(r => r.toString()).join(', ')}]`);
}
function updateDuplicateThreshold() {
const value = parseInt(duplicateThresholdInput.value);
duplicateThreshold = isNaN(value) || value < 1 ? 5 : value;
log(`🔄 중복 허용 횟수 설정: ${duplicateThreshold}`);
}
function updateRetryAttempts() {
const value = parseInt(retryAttemptsInput.value);
retryAttempts = isNaN(value) || value < 1 ? 10 : value;
log(`🔄 재시도 횟수 설정: ${retryAttempts}`);
}
function updateRetryInterval() {
const value = parseInt(retryIntervalInput.value);
retryInterval = isNaN(value) || value < 100 ? 100 : value; // 최소 값 100ms으로 설정
log(`🔄 재시도 간격 설정: ${retryInterval}ms`);
}
function updateStrictSiteFilter() {
const isStrict = strictSiteFilter.checked;
log(`🔄 엄격 사이트 필터링 설정: ${isStrict ? 'ON' : 'OFF'}`);
}
function toggleFiltering() {
filteringEnabled = filteringToggle.checked;
if (filteringEnabled) {
log('🔄 필터링 기능 활성화됨.');
updateFilterPatterns();
} else {
filterPatterns = [];
log('🔄 필터링 기능 비활성화됨.');
}
}
function toggleDarkweb() {
darkwebEnabled = darkwebToggle.checked;
log(`🔄 다크웹 검색 엔진 사용 설정: ${darkwebEnabled ? 'ON' : 'OFF'}`);
}
function getSearchQueryAndEngine() {
const url = window.location.href;
if (url.includes('google.com/search')) {
const params = new URLSearchParams(window.location.search);
searchQuery = params.get('q') || '';
searchEngine = 'google';
} else if (url.includes('bing.com/search')) {
const params = new URLSearchParams(window.location.search);
searchQuery = params.get('q') || '';
searchEngine = 'bing';
} else if (url.includes('daum.net/search')) {
const params = new URLSearchParams(window.location.search);
searchQuery = params.get('q') || '';
searchEngine = 'daum';
} else if (url.includes('search.naver.com/search')) {
const params = new URLSearchParams(window.location.search);
searchQuery = params.get('query') || '';
searchEngine = 'naver';
} else if (url.includes('search.yahoo.com/search')) {
const params = new URLSearchParams(window.location.search);
searchQuery = params.get('p') || '';
searchEngine = 'yahoo';
} else if (url.includes('duckduckgo.com/')) {
const params = new URLSearchParams(window.location.search);
searchQuery = params.get('q') || '';
searchEngine = 'duckduckgo';
} else if (url.includes('ecosia.org/search')) {
const params = new URLSearchParams(window.location.search);
searchQuery = params.get('q') || '';
searchEngine = 'ecosia';
} else if (url.includes('.onion')) {
// 다크웹 검색 엔진 중 하나로 설정
searchEngine = 'darkweb';
const params = new URLSearchParams(window.location.search);
searchQuery = params.get('q') || '';
if (!searchQuery) {
searchQuery = prompt('다크웹 검색어를 입력하세요:');
}
} else {
searchQuery = '';
searchEngine = '';
}
log(`🔍 검색 엔진: ${searchEngine}, 검색어: "${searchQuery}"`);
}
async function startCrawling() {
if (isRunning) {
alert('이미 크롤링이 진행 중입니다.');
return;
}
getSearchQueryAndEngine();
if (!searchQuery) {
alert('현재 페이지에서 검색어를 찾을 수 없습니다.');
log('⚠️ 검색어 미입력 또는 인식 실패.');
return;
}
isRunning = true;
isPaused = false;
currentPage = 1;
results = [];
visitedLinks.clear();
resultsBody.innerHTML = '';
visitedCountElem.innerText = '0';
elapsedTimeElem.innerText = '00:00:00';
remainingTimeElem.innerText = '00:00:00';
logMessages = [];
logContainer.innerHTML = `<strong>로그:</strong><br>`;
GM_setValue('crawlerLogs', JSON.stringify(logMessages));
log('✅ 크롤링을 시작합니다.');
updateDuplicateThreshold();
updateRetryAttempts();
updateRetryInterval();
await enqueueCrawlPages(concurrentRequests); // 동시 요청 수 40으로 설정
startTime = Date.now();
elapsedInterval = setInterval(updateElapsedTime, 1000);
}
function togglePause() {
if (!isRunning) {
alert('일시정지/재개할 크롤링이 없습니다.');
return;
}
isPaused = !isPaused;
if (isPaused) {
log('⏸️ 크롤링을 일시정지하였습니다.');
pauseBtn.innerHTML = '<i>▶️</i> 재개';
} else {
log('▶️ 크롤링을 재개하였습니다.');
pauseBtn.innerHTML = '<i>⏸️</i> 일시정지';
processQueue();
}
}
function stopCrawling() {
if (!isRunning) {
alert('중지할 크롤링이 없습니다.');
return;
}
isRunning = false;
isPaused = false;
pauseBtn.innerHTML = '<i>⏸️</i> 일시정지';
clearInterval(elapsedInterval);
updateProgress();
log('🚫 크롤링을 중지하였습니다.');
GM_notification({
text: '크롤링이 중지되었습니다.',
title: '웹 크롤러 7.9.5',
timeout: 3000
});
}
async function enqueueCrawlPages(limit) {
for (let i = 0; i < limit; i++) {
queueCrawlPage();
}
}
function queueCrawlPage() {
if (!isRunning || isPaused) return;
if (currentRequests >= concurrentRequests) return;
// Assign a unique page number before starting the crawl
const pageToCrawl = currentPage++;
currentPageElem.innerText = currentPage;
currentRequests++;
crawlPage(pageToCrawl).then(() => {
currentRequests--;
queueCrawlPage();
});
}
async function crawlPage(page) {
if (!isRunning || isPaused) {
return;
}
// 검색 엔진에 따른 URL 구성
let url = '';
if (searchEngine === 'google') {
url = `https://www.google.com/search?q=${encodeURIComponent(searchQuery)}&start=${(page-1)*10}`;
} else if (searchEngine === 'bing') {
url = `https://www.bing.com/search?q=${encodeURIComponent(searchQuery)}&first=${(page-1)*10 +1}`;
} else if (searchEngine === 'daum') {
url = `https://search.daum.net/search?q=${encodeURIComponent(searchQuery)}&page=${page}`;
} else if (searchEngine === 'naver') {
url = `https://search.naver.com/search.naver?query=${encodeURIComponent(searchQuery)}&start=${(page-1)*10 +1}`;
} else if (searchEngine === 'yahoo') {
url = `https://search.yahoo.com/search?p=${encodeURIComponent(searchQuery)}&b=${(page-1)*10 +1}`;
} else if (searchEngine === 'duckduckgo') {
url = `https://duckduckgo.com/html/?q=${encodeURIComponent(searchQuery)}&s=${(page-1)*10}`;
} else if (searchEngine === 'ecosia') {
url = `https://www.ecosia.org/search?q=${encodeURIComponent(searchQuery)}&page=${page}`;
} else if (searchEngine === 'darkweb') {
if (!darkwebEnabled) {
log('🛑 다크웹 검색 엔진이 비활성화되어 있습니다.');
isRunning = false;
return;
}
const darkwebEngines = getDarkwebEngines();
if (darkwebEngines.length === 0) {
log('⚠️ 다크웹 검색 엔진이 설정되지 않았습니다.');
isRunning = false;
return;
}
const randomEngine = darkwebEngines[Math.floor(Math.random() * darkwebEngines.length)];
url = `${randomEngine.url}/search?q=${encodeURIComponent(searchQuery)}&page=${page}`;
} else {
alert('지원하지 않는 검색 엔진입니다.');
log('❌ 지원하지 않는 검색 엔진.');
isRunning = false;
return;
}
log(`📄 페이지 ${page} 크롤링 중: ${url}`);
return new Promise((resolve) => {
GM_xmlhttpRequest({
method: "GET",
url: url,
headers: {
"User-Agent": navigator.userAgent,
"Accept": "text/html"
},
onload: async function(response) {
if (response.status !== 200) {
log(`❌ 페이지 ${page} 로드 실패: 상태 ${response.status}`);
await handleCrawlError(page, retryAttempts);
resolve();
return;
}
const parser = new DOMParser();
const doc = parser.parseFromString(response.responseText, 'text/html');
let items = [];
if (searchEngine === 'google') {
items = doc.querySelectorAll('div.g');
} else if (searchEngine === 'bing') {
items = doc.querySelectorAll('li.b_algo');
} else if (searchEngine === 'daum') {
items = doc.querySelectorAll('div.wrap_cont');
} else if (searchEngine === 'naver') {
items = doc.querySelectorAll('div.basic1');
} else if (searchEngine === 'yahoo') {
items = doc.querySelectorAll('div.compTitle');
} else if (searchEngine === 'duckduckgo') {
items = doc.querySelectorAll('div.result');
} else if (searchEngine === 'ecosia') {
items = doc.querySelectorAll('div.result');
} else if (searchEngine === 'darkweb') {
// 다크웹 검색 엔진 중 하나로 설정 (예: Torch 기준)
items = doc.querySelectorAll('.result'); // 실제 검색 엔진에 맞게 조정 필요
}
// 만약 검색 결과가 없다면 크롤링 종료
if (items.length === 0) {
log('🔍 더 이상 크롤링할 페이지가 없습니다. 크롤링을 종료합니다.');
isRunning = false;
isPaused = false;
pauseBtn.innerHTML = '<i>⏸️</i> 일시정지';
clearInterval(elapsedInterval);
updateProgress();
GM_notification({
text: '크롤링이 완료되었습니다.',
title: '웹 크롤러 7.9.5',
timeout: 3000
});
resolve();
return;
}
log(`📊 페이지 ${page}에서 ${items.length}개 항목 발견.`);
// 병렬 처리: Promise.all을 이용하여 비동기적으로 처리
const itemPromises = Array.from(items).map(async (item) => {
let title = 'N/A';
let link = 'N/A';
let description = 'N/A';
let image = 'N/A';
let htmlLink = '비활성화';
let pageLoadTime = 'N/A';
// 각 검색 엔진에 따른 데이터 추출
if (searchEngine === 'google') {
const titleElem = item.querySelector('h3');
const linkElem = item.querySelector('a');
const descElem = item.querySelector('.IsZvec') || item.querySelector('.VwiC3b');
if (titleElem) title = titleElem.innerText;
if (linkElem) link = linkElem.href;
if (descElem) description = descElem.innerText;
// 이미지 가져오기 (가능한 경우)
const imgElem = item.querySelector('img');
if (imgElem && imgElem.src) {
image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 80px; max-height: 80px;">`;
}
// HTML 보기 링크
if (toggleHTML.checked && link !== 'N/A') {
htmlLink = `<button class="html-btn" data-url="${encodeURIComponent(link)}">보기</button>`;
}
} else if (searchEngine === 'bing') {
const titleElem = item.querySelector('h2 a');
const descElem = item.querySelector('p');
if (titleElem) {
title = titleElem.innerText;
link = titleElem.href;
}
if (descElem) description = descElem.innerText;
// 이미지 가져오기 (가능한 경우)
const imgElem = item.querySelector('img');
if (imgElem && imgElem.src) {
image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 80px; max-height: 80px;">`;
}
// HTML 보기 링크
if (toggleHTML.checked && link !== 'N/A') {
htmlLink = `<button class="html-btn" data-url="${encodeURIComponent(link)}">보기</button>`;
}
} else if (searchEngine === 'daum') {
const titleElem = item.querySelector('a.link_fax');
const descElem = item.querySelector('div.cont_inner');
if (titleElem) {
title = titleElem.innerText.trim();
link = titleElem.href;
}
if (descElem) description = descElem.innerText.trim();
// 이미지 가져오기 (가능한 경우)
const imgElem = item.querySelector('img');
if (imgElem && imgElem.src) {
image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 80px; max-height: 80px;">`;
}
// HTML 보기 링크
if (toggleHTML.checked && link !== 'N/A') {
htmlLink = `<button class="html-btn" data-url="${encodeURIComponent(link)}">보기</button>`;
}
} else if (searchEngine === 'naver') {
const titleElem = item.querySelector('a.tit');
const descElem = item.querySelector('div.desc');
if (titleElem) {
title = titleElem.innerText;
link = titleElem.href;
}
if (descElem) description = descElem.innerText;
// 이미지 가져오기 (가능한 경우)
const imgElem = item.querySelector('img.thumb');
if (imgElem && imgElem.src) {
image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 80px; max-height: 80px;">`;
}
// HTML 보기 링크
if (toggleHTML.checked && link !== 'N/A') {
htmlLink = `<button class="html-btn" data-url="${encodeURIComponent(link)}">보기</button>`;
}
} else if (searchEngine === 'yahoo') {
const titleElem = item.querySelector('h3');
const linkElem = item.querySelector('a');
const descElem = item.querySelector('div.compText');
if (titleElem) title = titleElem.innerText;
if (linkElem) link = linkElem.href;
if (descElem) description = descElem.innerText;
// 이미지 가져오기 (가능한 경우)
const imgElem = item.querySelector('img');
if (imgElem && imgElem.src) {
image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 80px; max-height: 80px;">`;
}
// HTML 보기 링크
if (toggleHTML.checked && link !== 'N/A') {
htmlLink = `<button class="html-btn" data-url="${encodeURIComponent(link)}">보기</button>`;
}
} else if (searchEngine === 'duckduckgo') {
const titleElem = item.querySelector('a.result__a');
const descElem = item.querySelector('a.result__snippet');
if (titleElem) {
title = titleElem.innerText;
link = titleElem.href;
}
if (descElem) description = descElem.innerText;
// 이미지 가져오기 (가능한 경우)
const imgElem = item.querySelector('img.result__thumbnail');
if (imgElem && imgElem.src) {
image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 80px; max-height: 80px;">`;
}
// HTML 보기 링크
if (toggleHTML.checked && link !== 'N/A') {
htmlLink = `<button class="html-btn" data-url="${encodeURIComponent(link)}">보기</button>`;
}
} else if (searchEngine === 'ecosia') {
const titleElem = item.querySelector('a.result-title');
const descElem = item.querySelector('div.result-description');
if (titleElem) {
title = titleElem.innerText;
link = titleElem.href;
}
if (descElem) description = descElem.innerText;
// 이미지 가져오기 (가능한 경우)
const imgElem = item.querySelector('img.result-image');
if (imgElem && imgElem.src) {
image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 80px; max-height: 80px;">`;
}
// HTML 보기 링크
if (toggleHTML.checked && link !== 'N/A') {
htmlLink = `<button class="html-btn" data-url="${encodeURIComponent(link)}">보기</button>`;
}
} else if (searchEngine === 'darkweb') {
// 다크웹 검색 결과 추출 (예: Torch 기준)
const titleElem = item.querySelector('a');
const descElem = item.querySelector('p');
if (titleElem) {
title = titleElem.innerText;
link = titleElem.href;
}
if (descElem) description = descElem.innerText;
// 이미지 가져오기 (가능한 경우)
const imgElem = item.querySelector('img');
if (imgElem && imgElem.src) {
image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 80px; max-height: 80px;">`;
}
// HTML 보기 링크
if (toggleHTML.checked && link !== 'N/A') {
htmlLink = `<button class="html-btn" data-url="${encodeURIComponent(link)}">보기</button>`;
}
}
// 광고 제외 기능 적용
if (toggleAds.checked) {
const combinedText = `${title} ${description}`.toLowerCase();
const isAd = combinedText.includes('advertisement') || combinedText.includes('광고');
if (isAd) {
log(`🚫 광고 항목 제외: "${title}"`);
return null;
}
}
// 필터 단어 적용
if (filterPatterns.length > 0) {
const combinedText = `${title} ${description}`.toLowerCase();
const isFiltered = filterPatterns.some(regex => regex.test(combinedText));
if (isFiltered) {
log(`❌ 필터 단어 포함 항목 제외: "${title}"`);
return null;
}
}
// 사이트 필터링 적용 (사이트 제목 또는 HTML 내용 기반)
if (siteFilterPatterns.length > 0) {
const isSiteFiltered = siteFilterPatterns.some(regex => regex.test(link) || regex.test(title));
if (isSiteFiltered) {
log(`🚫 사이트 필터링에 의해 제외된 항목: "${link}"`);
return null;
}
}
// 도메인 추출
const domain = extractDomain(link);
if (!domain) return null;
// 제외 도메인 검사
if (excludeDomains.includes(domain.toLowerCase())) {
log(`🚫 제외 도메인 항목 제외: "${link}"`);
return null;
}
// 엄격 사이트 필터링 적용
if (strictSiteFilter.checked) {
if (!siteFilterPatterns.some(regex => regex.test(link) || regex.test(title))) {
log(`🚫 엄격 사이트 필터링에 의해 제외된 항목: "${link}"`);
return null;
}
}
// 중복 검사 (도메인별 방문 횟수)
let visitCount = visitedLinks.get(domain) || 0;
if (visitCount >= duplicateThreshold) {
log(`🔄 중복 방문 제한: "${domain}"`);
return null;
}
// 방문 횟수 업데이트
visitedLinks.set(domain, visitCount + 1);
visitedCountElem.innerText = visitedLinks.size.toString();
// 페이지 로딩 시간 측정
const loadStart = performance.now();
try {
// mode: 'cors'로 변경하여 실제 응답을 받을 수 있도록 함
const fetchResponse = await fetch(link, { mode: 'cors' });
if (fetchResponse.ok) {
const loadEnd = performance.now();
pageLoadTime = Math.round(loadEnd - loadStart);
} else {
pageLoadTime = '응답 불량';
}
} catch (err) {
pageLoadTime = '계산 불가';
}
// HTML 옵션
if (!toggleHTML.checked) {
htmlLink = '비활성화';
}
// 이미지 옵션
if (!toggleImages.checked) {
image = '비활성화';
}
const result = { title, link, description, image, htmlLink, pageLoadTime };
results.push(result);
// 결과 테이블에 추가
const row = document.createElement('tr');
row.innerHTML = `
<td>${escapeHTML(title)}</td>
<td><a href="${escapeHTML(link)}" target="_blank">${escapeHTML(link)}</a></td>
<td>${escapeHTML(description)}</td>
<td>${image}</td>
<td>${htmlLink}</td>
<td>${pageLoadTime}</td>
`;
resultsBody.appendChild(row);
return result;
});
// Promise.all을 통해 모든 항목 처리 완료 대기
await Promise.all(itemPromises);
// 데이터 시각화 업데이트
updateChart();
// 진행률 업데이트
updateProgress();
// 대기 시간 추가하여 서버에 부담 주지 않기
setTimeout(() => {
queueCrawlPage();
}, isPaused ? 0 : 1000); // 1초 대기
resolve();
},
onerror: function() {
log(`❌ 페이지 ${page} 요청 중 오류 발생.`);
handleCrawlError(page, retryAttempts).then(() => {
resolve();
});
}
});
});
}
async function handleCrawlError(page, retriesLeft) {
if (retriesLeft > 0 && isRunning) {
log(`🔄 페이지 ${page} 재시도 (${retriesLeft}회 남음).`);
return new Promise((resolve) => {
setTimeout(() => {
const url = constructURL(page);
if (!url) {
log(`❌ 페이지 ${page} 재시도 실패: URL이 유효하지 않습니다.`);
resolve();
return;
}
GM_xmlhttpRequest({
method: "GET",
url: url,
headers: {
"User-Agent": navigator.userAgent,
"Accept": "text/html"
},
onload: async function(response) {
if (response.status === 200) {
log(`✅ 페이지 ${page} 재시도 성공.`);
await crawlPage(page);
} else {
await handleCrawlError(page, retriesLeft - 1);
}
resolve();
},
onerror: async function() {
await handleCrawlError(page, retriesLeft - 1);
resolve();
}
});
}, retryInterval); // retryInterval이 100ms으로 설정됨
});
} else {
log(`❌ 페이지 ${page} 로드 실패: 최대 재시도 횟수 초과.`);
// 크롤링을 중지하지 않고 다음 페이지로 넘어감
currentPage++;
currentPageElem.innerText = currentPage;
setTimeout(() => {
queueCrawlPage();
}, 1000);
return Promise.resolve();
}
}
function constructURL(page) {
if (searchEngine === 'google') {
return `https://www.google.com/search?q=${encodeURIComponent(searchQuery)}&start=${(page-1)*10}`;
} else if (searchEngine === 'bing') {
return `https://www.bing.com/search?q=${encodeURIComponent(searchQuery)}&first=${(page-1)*10 +1}`;
} else if (searchEngine === 'daum') {
return `https://search.daum.net/search?q=${encodeURIComponent(searchQuery)}&page=${page}`;
} else if (searchEngine === 'naver') {
return `https://search.naver.com/search.naver?query=${encodeURIComponent(searchQuery)}&start=${(page-1)*10 +1}`;
} else if (searchEngine === 'yahoo') {
return `https://search.yahoo.com/search?p=${encodeURIComponent(searchQuery)}&b=${(page-1)*10 +1}`;
} else if (searchEngine === 'duckduckgo') {
return `https://duckduckgo.com/html/?q=${encodeURIComponent(searchQuery)}&s=${(page-1)*10}`;
} else if (searchEngine === 'ecosia') {
return `https://www.ecosia.org/search?q=${encodeURIComponent(searchQuery)}&page=${page}`;
} else if (searchEngine === 'darkweb') {
const darkwebEngines = getDarkwebEngines();
if (darkwebEngines.length === 0) {
return '';
}
const randomEngine = darkwebEngines[Math.floor(Math.random() * darkwebEngines.length)];
return `${randomEngine.url}/search?q=${encodeURIComponent(searchQuery)}&page=${page}`;
} else {
return '';
}
}
function updateProgress() {
// 무제한이므로 진행률을 페이지 수에 기반하여 표시
const progressPercent = (currentPage % 100) * 1; // 100페이지마다 100%로 리셋
progressBar.value = progressPercent > 100 ? 100 : progressPercent;
progressText.innerText = `페이지 ${currentPage}`;
if (progressPercent >= 100) {
progressBar.value = 0;
}
}
function resetCrawling() {
isRunning = false;
isPaused = false;
currentPage = 1;
results = [];
visitedLinks.clear();
resultsBody.innerHTML = '';
progressBar.value = 0;
progressText.innerText = '페이지 0';
currentPageElem.innerText = '0';
visitedCountElem.innerText = '0';
elapsedTimeElem.innerText = '00:00:00';
remainingTimeElem.innerText = '00:00:00';
logMessages = [];
logContainer.innerHTML = `<strong>로그:</strong><br>`;
GM_setValue('crawlerLogs', JSON.stringify(logMessages));
// 차트 초기화
crawlData.labels = [];
crawlData.datasets[0].data = [];
if (crawlChart) {
crawlChart.update();
}
clearInterval(elapsedInterval);
log('🔄 크롤링을 초기화하였습니다.');
}
function exportResults() {
if (results.length === 0) {
alert('내보낼 결과가 없습니다.');
return;
}
const format = prompt('내보낼 형식을 선택하세요 (xlsx, csv, json):', 'xlsx');
if (!format) return;
if (format.toLowerCase() === 'xlsx') {
const ws = XLSX.utils.json_to_sheet(results);
const wb = XLSX.utils.book_new();
XLSX.utils.book_append_sheet(wb, ws, "결과");
XLSX.writeFile(wb, "크롤링_결과.xlsx");
log('💾 결과를 "크롤링_결과.xlsx" 파일로 내보냈습니다.');
GM_notification({
text: '결과를 "크롤링_결과.xlsx" 파일로 내보냈습니다.',
title: '웹 크롤러 7.9.5',
timeout: 3000
});
} else if (format.toLowerCase() === 'csv') {
const csv = results.map(row => `"${row.title.replace(/"/g, '""')}","${row.link.replace(/"/g, '""')}","${row.description.replace(/"/g, '""')}","${row.image.replace(/"/g, '""')}","${row.htmlLink.replace(/"/g, '""')}","${row.pageLoadTime}"`).join('\n');
const blob = new Blob([csv], { type: 'text/csv;charset=utf-8;' });
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = '크롤링_결과.csv';
a.click();
log('💾 결과를 "크롤링_결과.csv" 파일로 내보냈습니다.');
GM_notification({
text: '결과를 "크롤링_결과.csv" 파일로 내보냈습니다.',
title: '웹 크롤러 7.9.5',
timeout: 3000
});
} else if (format.toLowerCase() === 'json') {
const json = JSON.stringify(results, null, 2);
const blob = new Blob([json], { type: 'application/json;charset=utf-8;' });
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = '크롤링_결과.json';
a.click();
log('💾 결과를 "크롤링_결과.json" 파일로 내보냈습니다.');
GM_notification({
text: '결과를 "크롤링_결과.json" 파일로 내보냈습니다.',
title: '웹 크롤러 7.9.5',
timeout: 3000
});
} else {
alert('지원하지 않는 형식입니다.');
}
}
function escapeHTML(str) {
if (!str) return '';
return str.replace(/&/g, "&")
.replace(/</g, "<")
.replace(/>/g, ">")
.replace(/"/g, """)
.replace(/'/g, "'");
}
function extractDomain(url) {
try {
const urlObj = new URL(url);
return urlObj.hostname.replace('www.', '');
} catch (e) {
return null;
}
}
function updateElapsedTime() {
if (!startTime) return;
const now = Date.now();
const elapsed = Math.floor((now - startTime) / 1000);
elapsedTimeElem.innerText = formatTime(elapsed);
const estimatedTotal = currentPage * 1; // 예측: 각 페이지 1초
const remaining = Math.max(estimatedTotal - elapsed, 0);
remainingTimeElem.innerText = formatTime(remaining);
}
function formatTime(seconds) {
const hrs = String(Math.floor(seconds / 3600)).padStart(2, '0');
const mins = String(Math.floor((seconds % 3600) / 60)).padStart(2, '0');
const secs = String(seconds % 60).padStart(2, '0');
return `${hrs}:${mins}:${secs}`;
}
function updateChart() {
crawlData.labels = Array.from(visitedLinks.keys());
crawlData.datasets[0].data = Array.from(visitedLinks.values());
if (crawlChart) {
crawlChart.update();
}
// 저장
GM_setValue('crawlerData', JSON.stringify(crawlData));
}
function loadCrawlData() {
const storedData = GM_getValue('crawlerData', null);
if (storedData) {
const parsedData = JSON.parse(storedData);
crawlData = parsedData;
if (crawlChart) {
crawlChart.data = crawlData;
crawlChart.update();
}
}
}
function saveCrawlData() {
GM_setValue('crawlerData', JSON.stringify(crawlData));
}
// 주기적으로 크롤링 데이터를 저장
setInterval(saveCrawlData, 5000);
// 검색 엔진 바로 가기 버튼 기능
function openSearchEngines() {
const engineUrls = {
'Google': 'https://www.google.com/',
'Bing': 'https://www.bing.com/',
'Daum': 'https://www.daum.net/',
'Naver': 'https://www.naver.com/',
'Yahoo': 'https://www.yahoo.com/',
'DuckDuckGo': 'https://duckduckgo.com/',
'Ecosia': 'https://www.ecosia.org/',
'Yahoo Japan': 'https://www.yahoo.co.jp/',
'Baidu': 'https://www.baidu.com/',
'Yandex': 'https://www.yandex.com/',
'Ask': 'https://www.ask.com/',
'Startpage': 'https://www.startpage.com/',
'Qwant': 'https://www.qwant.com/',
'Searx': 'https://searx.me/',
'Swisscows': 'https://swisscows.com/',
'Gibiru': 'https://www.gibiru.com/',
// 다크웹 검색 엔진도 포함
'TOR Search': 'http://torsearch7uu5ond.onion/',
'Ahmia': 'http://msydqstlz2kzerdg.onion/',
'Haystak': 'http://haystakvxad7wbk5.onion/',
'Dark Search': 'http://darksearch3w3nrc.onion/',
'Phobos': 'http://phobos6uj4abm4pf.onion/',
'Not Evil': 'http://hss3uro2hsxfogfq.onion/',
'IntelligenceX': 'http://intelligencexmh5v.onion/',
'Torch': 'http://xmh57jrzrnw6insl.onion/',
'Silk Road 3.1': 'http://silkroad31lo5ya.onion/',
'Fresh Onions': 'http://rx7hmtf7k7xk6ked.onion/',
// 추가 엔진 가능
'Dark Web Link': 'http://darkweblink5wqk.onion/',
'Tor Tracker': 'http://tortracker5ujzra.onion/',
'Deep Web Search': 'http://deepwebsearch6b6x.onion/',
'Onion Monitor': 'http://onionmonitor3c7v.onion/',
'Hidden Wiki': 'http://hiddenwiki5z35.onion/',
'TorEX': 'http://torexk3wd2buz.onion/',
'BlackBook': 'http://blackbook51uv.onion/',
'Taboo Search': 'http://taboosearch37b.onion/',
'The Pirate Bay (onion)': 'http://uj3wazyk5u4hnvtk.onion/',
// 더 많은 검색 엔진을 추가할 수 있습니다.
};
let html = '📡 <strong>검색 엔진:</strong><br>';
for (const [name, url] of Object.entries(engineUrls)) {
html += `<button class="search-engine-btn" data-url="${url}">${name}</button><br>`;
}
// 팝업 생성
const popup = document.createElement('div');
popup.className = 'search-engines-popup';
popup.innerHTML = html;
container.appendChild(popup);
popup.style.display = 'block';
// 버튼 이벤트 리스너
popup.querySelectorAll('.search-engine-btn').forEach(btn => {
btn.addEventListener('click', () => {
window.open(btn.getAttribute('data-url'), '_blank');
popup.style.display = 'none';
});
});
// 외부 클릭 시 팝업 닫기
document.addEventListener('click', function(event) {
if (!popup.contains(event.target) && event.target !== searchEnginesBtn) {
popup.style.display = 'none';
}
}, { once: true });
}
// HTML 탭에 콘텐츠 표시 기능
container.addEventListener('click', function(event) {
if (event.target.classList.contains('html-btn')) {
const url = decodeURIComponent(event.target.getAttribute('data-url'));
showHTMLInUI(url);
}
});
container.addEventListener('mouseover', function(event) {
if (event.target.classList.contains('html-btn')) {
const url = decodeURIComponent(event.target.getAttribute('data-url'));
showPreviewPopup(event.target, url);
}
});
container.addEventListener('mouseout', function(event) {
if (event.target.classList.contains('html-btn')) {
const popup = container.querySelector('.preview-popup');
if (popup) {
popup.style.display = 'none';
}
}
});
function showHTMLInUI(url) {
// UI 내에 iframe을 사용하여 사이트 내용 표시
let iframe = container.querySelector('.html-display iframe');
if (!iframe) {
iframe = document.createElement('iframe');
container.querySelector('.html-display').appendChild(iframe);
}
iframe.src = url;
// 탭을 'HTML'로 전환
switchTab('html-display');
log(`🔍 HTML 보기: ${url}`);
}
function showPreviewPopup(target, url) {
// 팝업 생성
let popup = container.querySelector('.preview-popup');
if (!popup) {
popup = document.createElement('div');
popup.className = 'preview-popup';
popup.innerHTML = `<iframe src="${url}" style="width:100%; height:100%; border:none;"></iframe>`;
container.appendChild(popup);
}
popup.style.display = 'block';
// 위치 조정
const rect = target.getBoundingClientRect();
const containerRect = container.getBoundingClientRect();
popup.style.left = `${rect.left - containerRect.left + 20}px`;
popup.style.top = `${rect.bottom - containerRect.top + 5}px`;
}
function switchTab(tabId) {
const tabs = container.querySelectorAll('.tab');
const tabContents = container.querySelectorAll('.tab-content');
tabs.forEach(t => {
if(t.getAttribute('data-tab') === tabId){
t.classList.add('active');
} else {
t.classList.remove('active');
}
});
tabContents.forEach(content => {
if(content.id === tabId){
content.classList.add('active');
} else {
content.classList.remove('active');
}
});
}
// 다크웹 검색 엔진 관리
function loadDarkwebEngines() {
const storedEngines = GM_getValue('darkwebEngines', defaultDarkwebEngines);
darkwebEnginesList.innerHTML = '';
storedEngines.forEach(engine => {
const li = document.createElement('li');
li.innerHTML = `
<span>${escapeHTML(engine.name)}</span>
<button class="remove-darkweb-engine-btn" data-url="${engine.url}">🗑️</button>
`;
darkwebEnginesList.appendChild(li);
});
// 삭제 버튼 이벤트 리스너
darkwebEnginesList.querySelectorAll('.remove-darkweb-engine-btn').forEach(btn => {
btn.addEventListener('click', () => {
const url = btn.getAttribute('data-url');
removeDarkwebEngine(url);
});
});
}
function addDarkwebEngine() {
const name = prompt('다크웹 검색 엔진 이름을 입력하세요:');
if (!name) {
alert('이름을 입력하지 않았습니다.');
return;
}
const url = prompt('다크웹 검색 엔진 URL을 입력하세요 (예: http://exampleonion.onion/):');
if (!url) {
alert('URL을 입력하지 않았습니다.');
return;
}
const newEngine = { name, url };
const storedEngines = GM_getValue('darkwebEngines', defaultDarkwebEngines);
storedEngines.push(newEngine);
GM_setValue('darkwebEngines', storedEngines);
loadDarkwebEngines();
log(`🔄 다크웹 검색 엔진 추가: ${name} (${url})`);
}
function removeDarkwebEngine(url) {
let storedEngines = GM_getValue('darkwebEngines', defaultDarkwebEngines);
storedEngines = storedEngines.filter(engine => engine.url !== url);
GM_setValue('darkwebEngines', storedEngines);
loadDarkwebEngines();
log(`🔄 다크웹 검색 엔진 제거: ${url}`);
}
function getDarkwebEngines() {
return GM_getValue('darkwebEngines', defaultDarkwebEngines);
}
// 번역기 기능 추가
async function translateContent() {
if (results.length === 0) {
alert('번역할 결과가 없습니다.');
return;
}
translatedContent.innerHTML = '번역 중...';
try {
// i18next를 이용한 번역
const texts = results.map(r => r.description).join('. ');
const translatedText = await translateText(texts, translateLanguage);
translatedContent.innerText = translatedText;
log('🔄 번역이 완료되었습니다.');
} catch (error) {
translatedContent.innerText = '번역 중 오류가 발생했습니다.';
log(`❌ 번역 오류: ${error}`);
}
}
async function translateText(text, targetLang) {
// Papago 번역 API 사용 (API 키 필요)
const apiUrl = 'https://openapi.naver.com/v1/papago/n2mt';
const clientId = 'YOUR_NAVER_CLIENT_ID'; // [필수] 실제 클라이언트 ID로 교체하세요.
const clientSecret = 'YOUR_NAVER_CLIENT_SECRET'; // [필수] 실제 클라이언트 시크릿으로 교체하세요.
const data = new URLSearchParams();
data.append('source', 'ko');
data.append('target', targetLang);
data.append('text', text);
const response = await fetch(apiUrl, {
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'X-Naver-Client-Id': clientId,
'X-Naver-Client-Secret': clientSecret
},
body: data
});
const result = await response.json();
if (result.errorMessage) {
throw new Error(result.errorMessage);
}
return result.message.result.translatedText;
}
// 번역기 섹션 버튼 클릭 처리
// 번역 기능은 위에서 정의되었습니다.
// 로깅, 필터링, 크롤링 함수들은 이미 위에서 상세히 구현되었습니다.
// 검색 엔진 바로가기 직접 UI에 표시
function loadSearchEnginesDirect() {
const engineUrls = {
'Google': 'https://www.google.com/',
'Bing': 'https://www.bing.com/',
'Daum': 'https://www.daum.net/',
'Naver': 'https://www.naver.com/',
'Yahoo': 'https://www.yahoo.com/',
'DuckDuckGo': 'https://duckduckgo.com/',
'Ecosia': 'https://www.ecosia.org/',
'Yahoo Japan': 'https://www.yahoo.co.jp/',
'Baidu': 'https://www.baidu.com/',
'Yandex': 'https://www.yandex.com/',
'Ask': 'https://www.ask.com/',
'Startpage': 'https://www.startpage.com/',
'Qwant': 'https://www.qwant.com/',
'Searx': 'https://searx.me/',
'Swisscows': 'https://swisscows.com/',
'Gibiru': 'https://www.gibiru.com/',
'DuckDuckGo (Privacy)': 'https://duckduckgo.com/?t=hk&kat=privacy',
'Brave Search': 'https://search.brave.com/',
'Kiddle': 'https://www.kiddle.co/',
'Gigablast': 'https://www.gigablast.com/',
'Mojeek': 'https://www.mojeek.com/',
'Yippy': 'https://yippy.com/',
'Infinity Search': 'https://www.infinitysearch.co/',
'WolframAlpha': 'https://www.wolframalpha.com/',
'Shodan': 'https://www.shodan.io/',
// 다크웹 검색 엔진도 포함
'TOR Search': 'http://torsearch7uu5ond.onion/',
'Ahmia': 'http://msydqstlz2kzerdg.onion/',
'Haystak': 'http://haystakvxad7wbk5.onion/',
'Dark Search': 'http://darksearch3w3nrc.onion/',
'Phobos': 'http://phobos6uj4abm4pf.onion/',
'Not Evil': 'http://hss3uro2hsxfogfq.onion/',
'IntelligenceX': 'http://intelligencexmh5v.onion/',
'Torch': 'http://xmh57jrzrnw6insl.onion/',
'Silk Road 3.1': 'http://silkroad31lo5ya.onion/',
'Fresh Onions': 'http://rx7hmtf7k7xk6ked.onion/',
'Dark Web Link': 'http://darkweblink5wqk.onion/',
'Tor Tracker': 'http://tortracker5ujzra.onion/',
'Deep Web Search': 'http://deepwebsearch6b6x.onion/',
'Onion Monitor': 'http://onionmonitor3c7v.onion/',
'Hidden Wiki': 'http://hiddenwiki5z35.onion/',
'TorEX': 'http://torexk3wd2buz.onion/',
'BlackBook': 'http://blackbook51uv.onion/',
'Taboo Search': 'http://taboosearch37b.onion/',
'The Pirate Bay (onion)': 'http://uj3wazyk5u4hnvtk.onion/',
// 더 많은 검색 엔진을 추가할 수 있습니다.
};
let html = '';
for (const [name, url] of Object.entries(engineUrls)) {
html += `<button class="search-engine-direct-btn" data-url="${url}">${name}</button>`;
}
searchEnginesDirect.innerHTML = html;
// 버튼 이벤트 리스너
searchEnginesDirect.querySelectorAll('.search-engine-direct-btn').forEach(btn => {
btn.addEventListener('click', () => {
window.open(btn.getAttribute('data-url'), '_blank');
});
});
}
})(); Google
URL: https://www.google.com/
Bing
URL: https://www.bing.com/
Yahoo
URL: https://www.yahoo.com/
DuckDuckGo
URL: https://duckduckgo.com/
Ecosia
URL: https://www.ecosia.org/
Ask.com
URL: https://www.ask.com/
AOL Search
URL: https://search.aol.com/
Baidu
URL: https://www.baidu.com/
Yandex
URL: https://www.yandex.com/
Startpage
URL: https://www.startpage.com/
🔒 프라이버시 중심 검색 엔진
Qwant
URL: https://www.qwant.com/
Swisscows
URL: https://swisscows.com/
Mojeek
URL: https://www.mojeek.com/
Gibiru
URL: https://www.gibiru.com/
MetaGer
URL: https://metager.org/
Peekier
URL: https://peekier.com/
Disconnect Search
URL: https://search.disconnect.me/
Ekoru
URL: https://www.ekoru.org/
Yippy
URL: https://yippy.com/
Infinity Search
URL: https://www.infinitysearch.co/
🌍 지역별 검색 엔진
Naver (한국)
URL: https://www.naver.com/
Daum (한국)
URL: https://www.daum.net/
Sogou (중국)
URL: https://www.sogou.com/
Nate (한국)
URL: https://www.nate.com/
Seznam (체코)
URL: https://www.seznam.cz/
Yandex.ru (러시아)
URL: https://www.yandex.ru/
Qwant.fr (프랑스)
URL: https://www.qwant.com/
Zyxc (독일)
URL: https://www.zyxc.search.de/
Exalead (프랑스)
URL: https://www.exalead.com/search/
Biglobe (일본)
URL: https://www.biglobe.ne.jp/
🛠️ 전문 검색 엔진
WolframAlpha (지식 엔진)
URL: https://www.wolframalpha.com/
Shodan (사물인터넷 검색)
URL: https://www.shodan.io/
Archive.org (인터넷 아카이브)
URL: https://archive.org/
Wayback Machine (웹사이트 아카이브)
URL: https://web.archive.org/
PubMed (의학 논문)
URL: https://pubmed.ncbi.nlm.nih.gov/
IEEE Xplore (공학 논문)
URL: https://ieeexplore.ieee.org/
Google Scholar (학술 자료)
URL: https://scholar.google.com/
LexisNexis (법률 자료)
URL: https://www.lexisnexis.com/
Google Books (도서 검색)
URL: https://books.google.com/
Creative Commons Search (저작권 무료 자료)
URL: https://search.creativecommons.org/
🌐 다크웹 검색 엔진
Torch
URL: http://xmh57jrzrnw6insl.onion/
Ahmia
URL: http://msydqstlz2kzerdg.onion/
Haystak
URL: http://haystakvxad7wbk5.onion/
Not Evil
URL: http://hss3uro2hsxfogfq.onion/
Phobos
URL: http://phobos6uj4abm4pf.onion/
DarkSearch
URL: http://darksearch3w3nrc.onion/
IntelligenceX
URL: http://intelligencexmh5v.onion/
The Pirate Bay (onion)
URL: http://uj3wazyk5u4hnvtk.onion/
Silk Road 3.1
URL: http://silkroad31lo5ya.onion/
Deep Web Search
URL: http://deepwebsearch6b6x.onion/
🔍 특화된 기능의 검색 엔진
Dogpile (메타 검색 엔진)
URL: https://www.dogpile.com/
Startpage (프라이버시 중심)
URL: https://www.startpage.com/
Exalead (비즈니스 중심)
URL: https://www.exalead.com/
Gigablast (오픈소스 검색 엔진)
URL: https://www.gigablast.com/
Searx (오픈소스 메타 검색 엔진)
URL: https://searx.me/
FindSounds (음악 검색)
URL: https://www.findsounds.com/
TinEye (이미지 역검색)
URL: https://tineye.com/
Boardreader (포럼 검색)
URL: https://www.boardreader.com/
PubCrawl (학술 논문 연관 검색)
URL: https://pubcrawl.org/
BeautyFetch (사진 및 이미지 검색)
URL: http://beautyfetch.com/
🌟 추가 가능한 기타 검색 엔진
Search Encrypt
URL: https://www.searchencrypt.com/
Oscobo
URL: https://www.oscobo.com/
HotBot
URL: https://www.hotbot.com/
Peekier
URL: https://peekier.com/
Lukol
URL: https://www.lukol.com/
Disconnect Search
URL: https://search.disconnect.me/
Privatelee
URL: https://privatelee.com/
Infinity Search
URL: https://www.infinitysearch.co/
DuckDuckGo (Tor)
URL: http://3g2upl4pq6kufc4m.onion/
TorEX
URL: http://torexk3wd2buz.onion/
BlackBook
URL: http://blackbook51uv.onion/
Taboo Search
URL: http://taboosearch37b.onion/
Kilos
URL: http://kilos7wbfn3ptgw.onion/
Candle
URL: http://zg6ak7qqfhbkbppx.onion/
Dark Falcon
URL: http://darkfalconphp5fh.onion/
Hidden Wiki
URL: http://hiddenwiki5z35.onion/
Onion Monitor
URL: http://onionmonitor3c7v.onion/
Phobos 2
URL: http://phobos2uj4abm4pf.onion/
Deep Web Search
URL: http://deepwebsearch6b6x.onion/
Dark Web Link
URL: http://darkweblink5wqk.onion/
Tor Tracker
URL: http://tortracker5ujzra.onion/
IntelligenceX
URL: http://intelligencexmh5v.onion/
Fresh Onions
URL: http://rx7hmtf7k7xk6ked.onion/
Phobos
URL: http://phobos6uj4abm4pf.onion/
Not Evil
URL: http://hss3uro2hsxfogfq.onion/
The Pirate Bay (onion)
URL: http://uj3wazyk5u4hnvtk.onion/
Shinobi Search
URL: http://shinobisearch6airie.onion/
Royal Dark Web Search
URL: http://royaldarkwebsearch57af.onion/
DarkSearchBox
URL: http://darksearchbox5a3x.onion/
Black Search
URL: http://blacksearch7xm7.onion/
📚 기타 유용한 검색 엔진
Google Images
URL: https://images.google.com/
Bing Images
URL: https://www.bing.com/images
DuckDuckGo Images
URL: https://duckduckgo.com/?ia=images
Pinterest
URL: https://www.pinterest.com/
Flickr
URL: https://www.flickr.com/
Shutterstock
URL: https://www.shutterstock.com/
Unsplash
URL: https://unsplash.com/
Pixabay
URL: https://pixabay.com/
500px
URL: https://500px.com/
Goodreads (도서 검색)
URL: https://www.goodreads.com/ 이것들을 검색 엔진 탭에 추가하셈 + 코드만 쓰셈 코드 생략하지 마셈 코드 삭제 하지 마셈 ASSISTANT
```javascript
// ==UserScript==
// @name 종합 웹 크롤러 7.9.5
// @namespace http://tampermonkey.net/
// @version 7.9.5
// @description 고급 기능을 갖춘 웹 크롤러로 검색 결과를 실시간으로 수집하고 표시 (동시 크롤링, 번역 기능, UI 개선 등)
// @author YourName
// @match *://www.google.com/search*
// @match *://www.bing.com/search*
// @match *://search.daum.net/search*
// @match *://search.naver.com/search*
// @match *://search.yahoo.com/search*
// @match *://duckduckgo.com/*
// @match *://www.ecosia.org/search*
// @match *://*.onion/*
// @grant GM_xmlhttpRequest
// @grant GM_addStyle
// @grant GM_setValue
// @grant GM_getValue
// @grant GM_notification
// @connect *
// @require https://cdnjs.cloudflare.com/ajax/libs/xlsx/0.18.5/xlsx.full.min.js
// @require https://cdn.jsdelivr.net/npm/chart.js
// @require https://cdnjs.cloudflare.com/ajax/libs/Sortable/1.14.0/Sortable.min.js
// @require https://cdnjs.cloudflare.com/ajax/libs/i18next/21.6.15/i18next.min.js
// ==/UserScript==
(function() {
'use strict';
// CSS 스타일 추가 (UI 대폭 개선)
GM_addStyle(`
/* 컨테이너 스타일 */
#crawler-container {
position: fixed;
top: 20px;
right: 20px;
background-color: #2c3e50;
border: 2px solid #1abc9c;
padding: 20px;
z-index: 10000;
width: 800px;
max-height: 85vh;
overflow: hidden;
box-shadow: 0 8px 30px rgba(0,0,0,0.5);
border-radius: 12px;
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
color: #ecf0f1;
display: flex;
flex-direction: column;
transition: all 0.3s ease;
}
/* 헤더 스타일 */
#crawler-container header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 15px;
}
#crawler-container .title {
font-size: 20px;
font-weight: bold;
}
#crawler-container .close-btn {
cursor: pointer;
font-size: 24px;
color: #e74c3c;
transition: color 0.3s;
}
#crawler-container .close-btn:hover {
color: #c0392b;
}
/* 탭 스타일 */
#crawler-container .tabs {
display: flex;
border-bottom: 2px solid #1abc9c;
margin-bottom: 15px;
overflow-x: auto;
}
#crawler-container .tab {
flex: 1;
text-align: center;
padding: 10px 0;
cursor: pointer;
background-color: #34495e;
border-right: 1px solid #1abc9c;
transition: background-color 0.3s, color 0.3s;
white-space: nowrap;
}
#crawler-container .tab:last-child {
border-right: none;
}
#crawler-container .tab.active {
background-color: #1abc9c;
color: #2c3e50;
font-weight: bold;
}
/* 탭 콘텐츠 스타일 */
#crawler-container .tab-content {
display: none;
flex: 1;
overflow-y: auto;
}
#crawler-container .tab-content.active {
display: block;
}
/* 버튼 스타일 */
#crawler-container .controls button {
background-color: #1abc9c;
border: none;
color: #2c3e50;
padding: 10px 16px;
margin: 6px 3px;
border-radius: 5px;
cursor: pointer;
transition: background-color 0.3s, transform 0.2s;
font-size: 14px;
display: flex;
align-items: center;
}
#crawler-container .controls button:hover {
background-color: #16a085;
transform: translateY(-2px);
}
#crawler-container .controls button i {
margin-right: 6px;
}
/* 체크박스 및 라벨 스타일 */
#crawler-container .options label,
#crawler-container .filters label,
#crawler-container .exclude-domains label,
#crawler-container .advanced-options label {
display: flex;
align-items: center;
margin: 10px 0;
font-size: 14px;
}
#crawler-container .options input[type="checkbox"] {
margin-right: 8px;
transform: scale(1.2);
}
/* 스위치 스타일 */
.switch {
position: relative;
display: inline-block;
width: 50px;
height: 24px;
margin-left: 10px;
}
.switch input {
opacity: 0;
width: 0;
height: 0;
}
.switch .slider {
position: absolute;
cursor: pointer;
top: 0;
left: 0;
right: 0;
bottom: 0;
background-color: #ccc;
transition: 0.4s;
border-radius: 24px;
}
.switch .slider:before {
position: absolute;
content: "";
height: 18px;
width: 18px;
left: 3px;
bottom: 3px;
background-color: white;
transition: 0.4s;
border-radius: 50%;
}
.switch input:checked + .slider {
background-color: #1abc9c;
}
.switch input:checked + .slider:before {
transform: translateX(26px);
}
/* 입력 필드 스타일 */
#crawler-container input[type="text"],
#crawler-container input[type="number"] {
width: 100%;
padding: 8px;
margin: 6px 0;
border: 1px solid #bdc3c7;
border-radius: 4px;
font-size: 14px;
}
/* 테이블 스타일 */
#resultsTable {
width: 100%;
border-collapse: collapse;
margin-top: 15px;
}
#resultsTable th, #resultsTable td {
border: 1px solid #95a5a6;
padding: 10px;
text-align: left;
font-size: 13px;
}
#resultsTable th {
background-color: #1abc9c;
color: #2c3e50;
}
#resultsTable tbody tr:nth-child(even) {
background-color: #34495e;
}
/* 반응형 디자인 */
@media (max-width: 900px) {
#crawler-container {
width: 90%;
right: 5%;
padding: 15px;
}
#crawler-container .controls button {
padding: 8px 12px;
font-size: 13px;
}
#crawler-container .title {
font-size: 18px;
}
}
/* Papago iframe 스타일 */
#papagoIframe {
width: 100%;
height: 100%;
border: none;
}
/* 팝업 스타일 */
.search-engines-popup, .manage-darkweb-popup {
position: absolute;
background-color: #2c3e50;
border: 1px solid #1abc9c;
padding: 15px;
border-radius: 8px;
box-shadow: 0 4px 12px rgba(0,0,0,0.3);
max-height: 350px;
overflow-y: auto;
z-index: 10001;
}
.search-engines-popup button, .manage-darkweb-popup button {
width: 100%;
text-align: left;
margin: 6px 0;
background-color: #34495e;
color: #ecf0f1;
border: none;
padding: 10px;
border-radius: 5px;
cursor: pointer;
transition: background-color 0.3s;
font-size: 14px;
}
.search-engines-popup button:hover, .manage-darkweb-popup button:hover {
background-color: #1abc9c;
color: #2c3e50;
}
/* 로그 스타일 */
#log {
background-color: #34495e;
padding: 10px;
border-radius: 5px;
max-height: 200px;
overflow-y: auto;
font-size: 13px;
line-height: 1.4;
}
/* 차트 컨테이너 */
.chart-container {
margin-top: 15px;
height: 250px;
}
/* HTML 디스플레이 스타일 */
.html-display iframe {
width: 100%;
height: 500px;
border: none;
border-radius: 5px;
}
/* Papago 디스플레이 스타일 */
.papago-display {
width: 100%;
height: 500px;
position: relative;
}
/* 다크웹 엔진 목록 스타일 */
#darkwebEnginesList li {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 12px;
}
#darkwebEnginesList li span {
flex: 1;
margin-right: 10px;
word-break: break-all;
}
#darkwebEnginesList li button {
background-color: #e74c3c;
border: none;
color: #fff;
padding: 6px 10px;
border-radius: 4px;
cursor: pointer;
transition: background-color 0.3s;
}
#darkwebEnginesList li button:hover {
background-color: #c0392b;
}
/* 예외 도메인 리스트 스타일 */
.exclude-domains ul {
list-style: none;
padding: 0;
margin-top: 10px;
}
.exclude-domains ul li {
background-color: #34495e;
padding: 6px 10px;
border-radius: 4px;
margin-bottom: 6px;
display: flex;
justify-content: space-between;
align-items: center;
}
.exclude-domains ul li button {
background-color: #e74c3c;
border: none;
color: #fff;
padding: 4px 8px;
border-radius: 3px;
cursor: pointer;
transition: background-color 0.3s;
font-size: 12px;
}
.exclude-domains ul li button:hover {
background-color: #c0392b;
}
/* Search Engines Direct UI */
#search-engines-direct {
display: flex;
flex-wrap: wrap;
gap: 10px;
margin-top: 15px;
}
#search-engines-direct button {
flex: 1 1 45%;
background-color: #1abc9c;
border: none;
color: #2c3e50;
padding: 8px 12px;
border-radius: 5px;
cursor: pointer;
transition: background-color 0.3s;
font-size: 14px;
}
#search-engines-direct button:hover {
background-color: #16a085;
}
/* Preview Popup */
.preview-popup {
position: absolute;
background-color: #2c3e50;
border: 1px solid #1abc9c;
padding: 10px;
border-radius: 6px;
box-shadow: 0 4px 8px rgba(0,0,0,0.2);
width: 300px;
height: 150px;
z-index: 10002;
display: none;
}
`);
// UI 생성 (대폭 개선된 디자인)
const container = document.createElement('div');
container.id = 'crawler-container';
container.innerHTML = `
<header>
<span class="title">🔥 웹 크롤러 7.9.5</span>
<span class="close-btn" title="닫기">×</span>
</header>
<div class="tabs">
<div class="tab active" data-tab="settings">설정</div>
<div class="tab" data-tab="results">결과</div>
<div class="tab" data-tab="logs">로그</div>
<div class="tab" data-tab="site-filters">사이트 필터</div>
<div class="tab" data-tab="darkweb">다크웹</div>
<div class="tab" data-tab="html-display">HTML</div>
<div class="tab" data-tab="papago">Papago</div>
<div class="tab" data-tab="search-engines">검색 엔진</div>
</div>
<div class="tab-content active" id="settings">
<!-- 제어 버튼 -->
<div class="controls">
<button id="startBtn" title="크롤링 시작"><i>▶️</i> 시작</button>
<button id="pauseBtn" class="pause-btn" title="크롤링 일시정지/재개"><i>⏸️</i> 일시정지</button>
<button id="stopBtn" title="크롤링 중지"><i>⏹️</i> 중지</button>
<button id="resetBtn" title="크롤링 초기화"><i>🔄</i> 초기화</button>
<button id="exportBtn" class="export-btn" title="결과 내보내기"><i>💾</i> 내보내기</button>
<button id="searchEnginesBtn" class="search-engines-btn" title="검색 엔진 바로가기"><i>🔍</i> 검색 엔진</button>
</div>
<!-- 옵션 섹션 -->
<div class="options">
<label><input type="checkbox" id="toggleHTML" checked> HTML 가져오기</label>
<label><input type="checkbox" id="toggleImages" checked> 이미지 가져오기</label>
<label><input type="checkbox" id="toggleAds" checked> 광고 제외</label>
<label>엄격 사이트 필터링
<label class="switch">
<input type="checkbox" id="strictSiteFilter">
<span class="slider"></span>
</label>
</label>
<label>필터링 기능 활성화
<label class="switch">
<input type="checkbox" id="filteringToggle" checked>
<span class="slider"></span>
</label>
</label>
</div>
<!-- 필터 섹션 -->
<div class="filters">
<label for="filterInput">필터 단어 (정규식, 쉼표로 구분):</label>
<input type="text" id="filterInput" placeholder="예: 기부, 광고, /promo/i">
</div>
<!-- 도메인 제외 섹션 -->
<div class="exclude-domains">
<label for="excludeDomainsInput">제외할 도메인 (쉼표로 구분):</label>
<input type="text" id="excludeDomainsInput" placeholder="예: facebook.com, youtube.com">
<ul id="excludeDomainsList"></ul>
</div>
<!-- 고급 옵션 섹션 -->
<div class="advanced-options">
<label for="duplicateThreshold">중복 허용 횟수:</label>
<input type="number" id="duplicateThreshold" value="5" min="1" style="width: 80px;">
</div>
<!-- 제어 옵션 섹션 -->
<div class="controls">
<label for="retryAttempts">재시도 횟수:</label>
<input type="number" id="retryAttempts" value="10" min="1" style="width: 80px;">
<label for="retryInterval">재시도 간격 (ms):</label>
<input type="number" id="retryInterval" value="100" min="100" style="width: 80px;">
</div>
<!-- 다크웹 검색 엔진 토글 -->
<div class="controls">
<label>다크웹 검색 엔진 사용
<label class="switch">
<input type="checkbox" id="darkwebToggle" checked>
<span class="slider"></span>
</label>
</label>
</div>
<!-- 진행률 표시 섹션 -->
<div class="progress-section">
<progress id="progressBar" value="0" max="100" style="width: 100%; height: 20px;"></progress>
<span id="progressText">페이지 0</span>
</div>
<!-- 상태 정보 섹션 -->
<div class="status">
현재 페이지: <span id="currentPage">0</span>
<br>
방문된 사이트: <span id="visitedCount">0</span>
<br>
경과 시간: <span id="elapsedTime">00:00:00</span>
<br>
예상 남은 시간: <span id="remainingTime">00:00:00</span>
</div>
<!-- 차트 섹션 -->
<div class="chart-container">
<canvas id="crawlChart"></canvas>
</div>
</div>
<div class="tab-content" id="results">
<div style="margin-top:15px; overflow: auto;">
<table id="resultsTable">
<thead>
<tr>
<th>사이트 이름</th>
<th>주소</th>
<th>설명</th>
<th>이미지</th>
<th>HTML 링크</th>
<th>페이지 로딩 시간 (ms)</th>
</tr>
</thead>
<tbody id="resultsBody">
</tbody>
</table>
</div>
</div>
<div class="tab-content" id="logs">
<div class="log" id="log">
<strong>로그:</strong><br>
</div>
</div>
<div class="tab-content" id="site-filters">
<div class="site-filters">
<label for="siteFilterInput">사이트 필터링 (정규식, 쉼표로 구분):</label>
<input type="text" id="siteFilterInput" placeholder="예: .*example\\.com.*, /^https?:\\/\\/.*\\.example\\.org$/">
</div>
</div>
<div class="tab-content" id="darkweb">
<div class="darkweb-search">
<label>다크웹 검색 엔진 추가/관리</label>
<button class="add-darkweb-engine-btn" title="다크웹 검색 엔진 추가"><i>➕</i> 추가</button>
<ul id="darkwebEnginesList">
<!-- 다크웹 검색 엔진 목록 -->
</ul>
</div>
</div>
<div class="tab-content" id="html-display">
<div class="html-display"></div>
</div>
<div class="tab-content" id="papago">
<div class="papago-display">
<iframe id="papagoIframe" src="https://papago.naver.com/"></iframe>
</div>
</div>
<div class="tab-content" id="search-engines">
<div class="search-engines-direct" id="search-engines-direct">
<!-- 검색 엔진 바로가기 버튼 추가 -->
</div>
</div>
`;
document.body.appendChild(container);
// 드래그 가능하게 설정
(function makeDraggable(elmnt) {
const header = elmnt.querySelector('header');
const closeBtn = elmnt.querySelector('.close-btn');
let pos1 = 0, pos2 = 0, pos3 = 0, pos4 = 0;
header.onmousedown = dragMouseDown;
closeBtn.onclick = () => {
elmnt.style.display = 'none';
}
function dragMouseDown(e) {
e = e || window.event;
e.preventDefault();
pos3 = e.clientX;
pos4 = e.clientY;
document.onmouseup = closeDragElement;
document.onmousemove = elementDrag;
}
function elementDrag(e) {
e = e || window.event;
e.preventDefault();
pos1 = pos3 - e.clientX;
pos2 = pos4 - e.clientY;
pos3 = e.clientX;
pos4 = e.clientY;
elmnt.style.top = (elmnt.offsetTop - pos2) + "px";
elmnt.style.left = (elmnt.offsetLeft - pos1) + "px";
}
function closeDragElement() {
document.onmouseup = null;
document.onmousemove = null;
}
})(container);
// 탭 기능 구현
(function setupTabs() {
const tabs = container.querySelectorAll('.tab');
const tabContents = container.querySelectorAll('.tab-content');
tabs.forEach(tab => {
tab.addEventListener('click', () => {
tabs.forEach(t => t.classList.remove('active'));
tab.classList.add('active');
const target = tab.getAttribute('data-tab');
tabContents.forEach(content => {
if(content.id === target){
content.classList.add('active');
} else {
content.classList.remove('active');
}
});
});
});
})();
// 변수 초기화
let isRunning = false;
let isPaused = false;
let currentPage = 1;
let results = [];
let searchQuery = '';
let searchEngine = '';
let visitedLinks = new Map(); // 도메인별 방문 횟수 추적
let logMessages = [];
let filterPatterns = [];
let excludeDomains = [];
let siteFilterPatterns = [];
let duplicateThreshold = 5; // 중복 허용 횟수
let retryAttempts = 10; // 재시도 횟수
let retryInterval = 100; // 재시도 간격 100ms으로 설정
let startTime = null;
let elapsedInterval = null;
let crawlChart = null;
let crawlData = {
labels: [], // 도메인 이름
datasets: [{
label: '방문 횟수',
data: [],
backgroundColor: 'rgba(46, 204, 113, 0.6)',
borderColor: 'rgba(46, 204, 113, 1)',
borderWidth: 1
}]
};
let filteringEnabled = true; // 필터링 활성화 여부
let concurrentRequests = 40; // 동시 요청 수 40으로 설정
let currentRequests = 0; // 현재 요청 수
let requestQueue = []; // 요청 대기열
let translateLanguage = 'en'; // 기본 번역 언어
let darkwebEnabled = true; // 다크웹 검색 엔진 사용 여부
// 다크웹 검색 엔진 목록 (기본 엔진 추가 및 확장)
const defaultDarkwebEngines = [
{ name: 'Torch', url: 'http://xmh57jrzrnw6insl.onion/' },
{ name: 'Ahmia', url: 'http://msydqstlz2kzerdg.onion/' },
{ name: 'Haystak', url: 'http://haystakvxad7wbk5.onion/' },
{ name: 'Not Evil', url: 'http://hss3uro2hsxfogfq.onion/' },
{ name: 'Phobos', url: 'http://phobos6uj4abm4pf.onion/' },
{ name: 'DarkSearch', url: 'http://darksearch3w3nrc.onion/' },
{ name: 'IntelligenceX', url: 'http://intelligencexmh5v.onion/' },
{ name: 'Silk Road 3.1', url: 'http://silkroad31lo5ya.onion/' },
{ name: 'Fresh Onions', url: 'http://rx7hmtf7k7xk6ked.onion/' },
{ name: 'Dark Web Link', url: 'http://darkweblink5wqk.onion/' },
{ name: 'Tor Tracker', url: 'http://tortracker5ujzra.onion/' },
{ name: 'Deep Web Search', url: 'http://deepwebsearch6b6x.onion/' },
{ name: 'Onion Monitor', url: 'http://onionmonitor3c7v.onion/' },
{ name: 'Hidden Wiki', url: 'http://hiddenwiki5z35.onion/' },
{ name: 'TorEX', url: 'http://torexk3wd2buz.onion/' },
{ name: 'BlackBook', url: 'http://blackbook51uv.onion/' },
{ name: 'Taboo Search', url: 'http://taboosearch37b.onion/' },
{ name: 'The Pirate Bay (onion)', url: 'http://uj3wazyk5u4hnvtk.onion/' },
// 더 많은 다크웹 검색 엔진을 추가할 수 있습니다.
];
// 버튼 및 요소 가져오기
const startBtn = document.getElementById('startBtn');
const pauseBtn = document.getElementById('pauseBtn');
const stopBtn = document.getElementById('stopBtn');
const resetBtn = document.getElementById('resetBtn');
const exportBtn = document.getElementById('exportBtn');
const searchEnginesBtn = document.getElementById('searchEnginesBtn');
const progressBar = document.getElementById('progressBar');
const progressText = document.getElementById('progressText');
const resultsBody = document.getElementById('resultsBody');
const logContainer = document.getElementById('log');
const currentPageElem = document.getElementById('currentPage');
const toggleHTML = document.getElementById('toggleHTML');
const toggleImages = document.getElementById('toggleImages');
const toggleAds = document.getElementById('toggleAds');
const filterInput = document.getElementById('filterInput');
const excludeDomainsInput = document.getElementById('excludeDomainsInput');
const excludeDomainsList = document.getElementById('excludeDomainsList');
const siteFilterInput = document.getElementById('siteFilterInput');
const duplicateThresholdInput = document.getElementById('duplicateThreshold');
const retryAttemptsInput = document.getElementById('retryAttempts');
const retryIntervalInput = document.getElementById('retryInterval');
const strictSiteFilter = document.getElementById('strictSiteFilter');
const visitedCountElem = document.getElementById('visitedCount');
const elapsedTimeElem = document.getElementById('elapsedTime');
const remainingTimeElem = document.getElementById('remainingTime');
const crawlChartCtx = document.getElementById('crawlChart').getContext('2d');
const filteringToggle = document.getElementById('filteringToggle');
const darkwebEnginesList = document.getElementById('darkwebEnginesList');
const addDarkwebEngineBtn = container.querySelector('.add-darkweb-engine-btn');
const darkwebToggle = document.getElementById('darkwebToggle');
const papagoIframe = document.getElementById('papagoIframe');
const searchEnginesDirect = document.getElementById('search-engines-direct');
// 이벤트 리스너
startBtn.addEventListener('click', startCrawling);
pauseBtn.addEventListener('click', togglePause);
stopBtn.addEventListener('click', stopCrawling);
resetBtn.addEventListener('click', resetCrawling);
exportBtn.addEventListener('click', exportResults);
searchEnginesBtn.addEventListener('click', openSearchEngines);
filterInput.addEventListener('input', updateFilterPatterns);
excludeDomainsInput.addEventListener('keydown', function(e) {
if (e.key === 'Enter') {
addExcludeDomain();
}
});
addDarkwebEngineBtn.addEventListener('click', addDarkwebEngine);
siteFilterInput.addEventListener('input', updateSiteFilterPatterns);
duplicateThresholdInput.addEventListener('change', updateDuplicateThreshold);
retryAttemptsInput.addEventListener('change', updateRetryAttempts);
retryIntervalInput.addEventListener('change', updateRetryInterval);
strictSiteFilter.addEventListener('change', updateStrictSiteFilter);
filteringToggle.addEventListener('change', toggleFiltering);
darkwebToggle.addEventListener('change', toggleDarkweb);
papagoIframe.addEventListener('load', () => {
log('🌐 Papago 번역기 로드 완료.');
});
// 페이지 새로 고침 시 크롤링 초기화
window.addEventListener('beforeunload', () => {
resetCrawling();
});
// 초기화 로드
loadLogs();
loadCrawlData();
loadDarkwebEngines();
loadExcludeDomains();
loadSearchEnginesDirect();
// 차트 초기화
function initializeChart() {
crawlChart = new Chart(crawlChartCtx, {
type: 'bar',
data: crawlData,
options: {
responsive: true,
maintainAspectRatio: false,
scales: {
y: { beginAtZero: true },
x: { ticks: { autoSkip: false } }
},
plugins: {
legend: {
display: false
},
tooltip: {
enabled: true
}
}
}
});
}
initializeChart();
// 로깅 함수
function log(message) {
const time = new Date().toLocaleTimeString();
const entry = `[${time}] ${message}`;
logMessages.push(entry);
logContainer.innerHTML += `${entry}<br>`;
logContainer.scrollTop = logContainer.scrollHeight;
// 저장
GM_setValue('crawlerLogs', JSON.stringify(logMessages));
}
function loadLogs() {
const storedLogs = GM_getValue('crawlerLogs', []);
if (storedLogs.length > 0) {
logMessages = JSON.parse(storedLogs);
logContainer.innerHTML = `<strong>로그:</strong><br>` + logMessages.join('<br>');
logContainer.scrollTop = logContainer.scrollHeight;
}
}
function updateFilterPatterns() {
if (!filteringEnabled) {
filterPatterns = [];
log(`🛠 필터링 기능 비활성화됨.`);
return;
}
const input = filterInput.value.trim();
filterPatterns = input ? input.split(',').map(word => {
word = word.trim();
if (word.startsWith('/') && word.endsWith('/')) {
try {
return new RegExp(word.slice(1, -1), 'i');
} catch (e) {
log(`⚠️ 유효하지 않은 정규식: ${word}`);
return null;
}
} else {
return new RegExp(word, 'i');
}
}).filter(regex => regex !== null) : [];
log(`🛠 필터 패턴 업데이트: [${filterPatterns.map(r => r.toString()).join(', ')}]`);
}
function updateExcludeDomains() {
// 이 함수는 제외 도메인이 추가될 때마다 호출됨
log(`🚫 제외 도메인 업데이트 완료.`);
}
function addExcludeDomain() {
const domain = excludeDomainsInput.value.trim().toLowerCase();
if (domain && !excludeDomains.includes(domain)) {
excludeDomains.push(domain);
renderExcludeDomains();
excludeDomainsInput.value = '';
log(`🚫 제외 도메인 추가: ${domain}`);
}
}
function renderExcludeDomains() {
excludeDomainsList.innerHTML = '';
excludeDomains.forEach((domain, index) => {
const li = document.createElement('li');
li.innerHTML = `
<span>${escapeHTML(domain)}</span>
<button data-index="${index}">✖️</button>
`;
excludeDomainsList.appendChild(li);
});
// 삭제 버튼 이벤트 리스너
excludeDomainsList.querySelectorAll('button').forEach(btn => {
btn.addEventListener('click', () => {
const index = parseInt(btn.getAttribute('data-index'));
if (!isNaN(index)) {
const removed = excludeDomains.splice(index, 1);
renderExcludeDomains();
log(`🚫 제외 도메인 제거: ${removed}`);
}
});
});
}
function loadExcludeDomains() {
const storedDomains = GM_getValue('excludeDomains', []);
excludeDomains = storedDomains.length > 0 ? storedDomains : [];
renderExcludeDomains();
}
function updateSiteFilterPatterns() {
const input = siteFilterInput.value.trim();
siteFilterPatterns = input ? input.split(',').map(pattern => {
pattern = pattern.trim();
if (pattern.startsWith('/') && pattern.endsWith('/')) {
try {
return new RegExp(pattern.slice(1, -1), 'i');
} catch (e) {
log(`⚠️ 유효하지 않은 사이트 필터 정규식: ${pattern}`);
return null;
}
} else {
return new RegExp(pattern, 'i');
}
}).filter(regex => regex !== null) : [];
log(`🛠 사이트 필터 패턴 업데이트: [${siteFilterPatterns.map(r => r.toString()).join(', ')}]`);
}
function updateDuplicateThreshold() {
const value = parseInt(duplicateThresholdInput.value);
duplicateThreshold = isNaN(value) || value < 1 ? 5 : value;
log(`🔄 중복 허용 횟수 설정: ${duplicateThreshold}`);
}
function updateRetryAttempts() {
const value = parseInt(retryAttemptsInput.value);
retryAttempts = isNaN(value) || value < 1 ? 10 : value;
log(`🔄 재시도 횟수 설정: ${retryAttempts}`);
}
function updateRetryInterval() {
const value = parseInt(retryIntervalInput.value);
retryInterval = isNaN(value) || value < 100 ? 100 : value; // 최소 값 100ms으로 설정
log(`🔄 재시도 간격 설정: ${retryInterval}ms`);
}
function updateStrictSiteFilter() {
const isStrict = strictSiteFilter.checked;
log(`🔄 엄격 사이트 필터링 설정: ${isStrict ? 'ON' : 'OFF'}`);
}
function toggleFiltering() {
filteringEnabled = filteringToggle.checked;
if (filteringEnabled) {
log('🔄 필터링 기능 활성화됨.');
updateFilterPatterns();
} else {
filterPatterns = [];
log('🔄 필터링 기능 비활성화됨.');
}
}
function toggleDarkweb() {
darkwebEnabled = darkwebToggle.checked;
log(`🔄 다크웹 검색 엔진 사용 설정: ${darkwebEnabled ? 'ON' : 'OFF'}`);
}
function getSearchQueryAndEngine() {
const url = window.location.href;
if (url.includes('google.com/search')) {
const params = new URLSearchParams(window.location.search);
searchQuery = params.get('q') || '';
searchEngine = 'google';
} else if (url.includes('bing.com/search')) {
const params = new URLSearchParams(window.location.search);
searchQuery = params.get('q') || '';
searchEngine = 'bing';
} else if (url.includes('daum.net/search')) {
const params = new URLSearchParams(window.location.search);
searchQuery = params.get('q') || '';
searchEngine = 'daum';
} else if (url.includes('search.naver.com/search')) {
const params = new URLSearchParams(window.location.search);
searchQuery = params.get('query') || '';
searchEngine = 'naver';
} else if (url.includes('search.yahoo.com/search')) {
const params = new URLSearchParams(window.location.search);
searchQuery = params.get('p') || '';
searchEngine = 'yahoo';
} else if (url.includes('duckduckgo.com/')) {
const params = new URLSearchParams(window.location.search);
searchQuery = params.get('q') || '';
searchEngine = 'duckduckgo';
} else if (url.includes('ecosia.org/search')) {
const params = new URLSearchParams(window.location.search);
searchQuery = params.get('q') || '';
searchEngine = 'ecosia';
} else if (url.includes('.onion')) {
// 다크웹 검색 엔진 중 하나로 설정
searchEngine = 'darkweb';
const params = new URLSearchParams(window.location.search);
searchQuery = params.get('q') || '';
if (!searchQuery) {
searchQuery = prompt('다크웹 검색어를 입력하세요:');
}
} else {
searchQuery = '';
searchEngine = '';
}
log(`🔍 검색 엔진: ${searchEngine}, 검색어: "${searchQuery}"`);
}
async function startCrawling() {
if (isRunning) {
alert('이미 크롤링이 진행 중입니다.');
return;
}
getSearchQueryAndEngine();
if (!searchQuery) {
alert('현재 페이지에서 검색어를 찾을 수 없습니다.');
log('⚠️ 검색어 미입력 또는 인식 실패.');
return;
}
isRunning = true;
isPaused = false;
currentPage = 1;
results = [];
visitedLinks.clear();
resultsBody.innerHTML = '';
visitedCountElem.innerText = '0';
elapsedTimeElem.innerText = '00:00:00';
remainingTimeElem.innerText = '00:00:00';
logMessages = [];
logContainer.innerHTML = `<strong>로그:</strong><br>`;
GM_setValue('crawlerLogs', JSON.stringify(logMessages));
log('✅ 크롤링을 시작합니다.');
updateDuplicateThreshold();
updateRetryAttempts();
updateRetryInterval();
await enqueueCrawlPages(concurrentRequests); // 동시 요청 수 40으로 설정
startTime = Date.now();
elapsedInterval = setInterval(updateElapsedTime, 1000);
}
function togglePause() {
if (!isRunning) {
alert('일시정지/재개할 크롤링이 없습니다.');
return;
}
isPaused = !isPaused;
if (isPaused) {
log('⏸️ 크롤링을 일시정지하였습니다.');
pauseBtn.innerHTML = '<i>▶️</i> 재개';
} else {
log('▶️ 크롤링을 재개하였습니다.');
pauseBtn.innerHTML = '<i>⏸️</i> 일시정지';
processQueue();
}
}
function stopCrawling() {
if (!isRunning) {
alert('중지할 크롤링이 없습니다.');
return;
}
isRunning = false;
isPaused = false;
pauseBtn.innerHTML = '<i>⏸️</i> 일시정지';
clearInterval(elapsedInterval);
updateProgress();
log('🚫 크롤링을 중지하였습니다.');
GM_notification({
text: '크롤링이 중지되었습니다.',
title: '웹 크롤러 7.9.5',
timeout: 3000
});
}
async function enqueueCrawlPages(limit) {
for (let i = 0; i < limit; i++) {
queueCrawlPage();
}
}
function queueCrawlPage() {
if (!isRunning || isPaused) return;
if (currentRequests >= concurrentRequests) return;
// Assign a unique page number before starting the crawl
const pageToCrawl = currentPage++;
currentPageElem.innerText = currentPage;
currentRequests++;
crawlPage(pageToCrawl).then(() => {
currentRequests--;
queueCrawlPage();
});
}
async function crawlPage(page) {
if (!isRunning || isPaused) {
return;
}
// 검색 엔진에 따른 URL 구성
let url = '';
if (searchEngine === 'google') {
url = `https://www.google.com/search?q=${encodeURIComponent(searchQuery)}&start=${(page-1)*10}`;
} else if (searchEngine === 'bing') {
url = `https://www.bing.com/search?q=${encodeURIComponent(searchQuery)}&first=${(page-1)*10 +1}`;
} else if (searchEngine === 'daum') {
url = `https://search.daum.net/search?q=${encodeURIComponent(searchQuery)}&page=${page}`;
} else if (searchEngine === 'naver') {
url = `https://search.naver.com/search.naver?query=${encodeURIComponent(searchQuery)}&start=${(page-1)*10 +1}`;
} else if (searchEngine === 'yahoo') {
url = `https://search.yahoo.com/search?p=${encodeURIComponent(searchQuery)}&b=${(page-1)*10 +1}`;
} else if (searchEngine === 'duckduckgo') {
url = `https://duckduckgo.com/html/?q=${encodeURIComponent(searchQuery)}&s=${(page-1)*10}`;
} else if (searchEngine === 'ecosia') {
url = `https://www.ecosia.org/search?q=${encodeURIComponent(searchQuery)}&page=${page}`;
} else if (searchEngine === 'darkweb') {
if (!darkwebEnabled) {
log('🛑 다크웹 검색 엔진이 비활성화되어 있습니다.');
isRunning = false;
return;
}
const darkwebEngines = getDarkwebEngines();
if (darkwebEngines.length === 0) {
log('⚠️ 다크웹 검색 엔진이 설정되지 않았습니다.');
isRunning = false;
return;
}
const randomEngine = darkwebEngines[Math.floor(Math.random() * darkwebEngines.length)];
url = `${randomEngine.url}/search?q=${encodeURIComponent(searchQuery)}&page=${page}`;
} else {
alert('지원하지 않는 검색 엔진입니다.');
log('❌ 지원하지 않는 검색 엔진.');
isRunning = false;
return;
}
log(`📄 페이지 ${page} 크롤링 중: ${url}`);
return new Promise((resolve) => {
GM_xmlhttpRequest({
method: "GET",
url: url,
headers: {
"User-Agent": navigator.userAgent,
"Accept": "text/html"
},
onload: async function(response) {
if (response.status !== 200) {
log(`❌ 페이지 ${page} 로드 실패: 상태 ${response.status}`);
await handleCrawlError(page, retryAttempts);
resolve();
return;
}
const parser = new DOMParser();
const doc = parser.parseFromString(response.responseText, 'text/html');
let items = [];
if (searchEngine === 'google') {
items = doc.querySelectorAll('div.g');
} else if (searchEngine === 'bing') {
items = doc.querySelectorAll('li.b_algo');
} else if (searchEngine === 'daum') {
items = doc.querySelectorAll('div.wrap_cont');
} else if (searchEngine === 'naver') {
items = doc.querySelectorAll('div.basic1');
} else if (searchEngine === 'yahoo') {
items = doc.querySelectorAll('div.compTitle');
} else if (searchEngine === 'duckduckgo') {
items = doc.querySelectorAll('div.result');
} else if (searchEngine === 'ecosia') {
items = doc.querySelectorAll('div.result');
} else if (searchEngine === 'darkweb') {
// 다크웹 검색 엔진 중 하나로 설정 (예: Torch 기준)
items = doc.querySelectorAll('.result'); // 실제 검색 엔진에 맞게 조정 필요
}
// 만약 검색 결과가 없다면 크롤링 종료
if (items.length === 0) {
log('🔍 더 이상 크롤링할 페이지가 없습니다. 크롤링을 종료합니다.');
isRunning = false;
isPaused = false;
pauseBtn.innerHTML = '<i>⏸️</i> 일시정지';
clearInterval(elapsedInterval);
updateProgress();
GM_notification({
text: '크롤링이 완료되었습니다.',
title: '웹 크롤러 7.9.5',
timeout: 3000
});
resolve();
return;
}
log(`📊 페이지 ${page}에서 ${items.length}개 항목 발견.`);
// 병렬 처리: Promise.all을 이용하여 비동기적으로 처리
const itemPromises = Array.from(items).map(async (item) => {
let title = 'N/A';
let link = 'N/A';
let description = 'N/A';
let image = 'N/A';
let htmlLink = '비활성화';
let pageLoadTime = 'N/A';
// 각 검색 엔진에 따른 데이터 추출
if (searchEngine === 'google') {
const titleElem = item.querySelector('h3');
const linkElem = item.querySelector('a');
const descElem = item.querySelector('.IsZvec') || item.querySelector('.VwiC3b');
if (titleElem) title = titleElem.innerText;
if (linkElem) link = linkElem.href;
if (descElem) description = descElem.innerText;
// 이미지 가져오기 (가능한 경우)
const imgElem = item.querySelector('img');
if (imgElem && imgElem.src) {
image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 80px; max-height: 80px;">`;
}
// HTML 보기 링크
if (toggleHTML.checked && link !== 'N/A') {
htmlLink = `<button class="html-btn" data-url="${encodeURIComponent(link)}">보기</button>`;
}
} else if (searchEngine === 'bing') {
const titleElem = item.querySelector('h2 a');
const descElem = item.querySelector('p');
if (titleElem) {
title = titleElem.innerText;
link = titleElem.href;
}
if (descElem) description = descElem.innerText;
// 이미지 가져오기 (가능한 경우)
const imgElem = item.querySelector('img');
if (imgElem && imgElem.src) {
image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 80px; max-height: 80px;">`;
}
// HTML 보기 링크
if (toggleHTML.checked && link !== 'N/A') {
htmlLink = `<button class="html-btn" data-url="${encodeURIComponent(link)}">보기</button>`;
}
} else if (searchEngine === 'daum') {
const titleElem = item.querySelector('a.link_fax');
const descElem = item.querySelector('div.cont_inner');
if (titleElem) {
title = titleElem.innerText.trim();
link = titleElem.href;
}
if (descElem) description = descElem.innerText.trim();
// 이미지 가져오기 (가능한 경우)
const imgElem = item.querySelector('img');
if (imgElem && imgElem.src) {
image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 80px; max-height: 80px;">`;
}
// HTML 보기 링크
if (toggleHTML.checked && link !== 'N/A') {
htmlLink = `<button class="html-btn" data-url="${encodeURIComponent(link)}">보기</button>`;
}
} else if (searchEngine === 'naver') {
const titleElem = item.querySelector('a.tit');
const descElem = item.querySelector('div.desc');
if (titleElem) {
title = titleElem.innerText;
link = titleElem.href;
}
if (descElem) description = descElem.innerText;
// 이미지 가져오기 (가능한 경우)
const imgElem = item.querySelector('img.thumb');
if (imgElem && imgElem.src) {
image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 80px; max-height: 80px;">`;
}
// HTML 보기 링크
if (toggleHTML.checked && link !== 'N/A') {
htmlLink = `<button class="html-btn" data-url="${encodeURIComponent(link)}">보기</button>`;
}
} else if (searchEngine === 'yahoo') {
const titleElem = item.querySelector('h3');
const linkElem = item.querySelector('a');
const descElem = item.querySelector('div.compText');
if (titleElem) title = titleElem.innerText;
if (linkElem) link = linkElem.href;
if (descElem) description = descElem.innerText;
// 이미지 가져오기 (가능한 경우)
const imgElem = item.querySelector('img');
if (imgElem && imgElem.src) {
image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 80px; max-height: 80px;">`;
}
// HTML 보기 링크
if (toggleHTML.checked && link !== 'N/A') {
htmlLink = `<button class="html-btn" data-url="${encodeURIComponent(link)}">보기</button>`;
}
} else if (searchEngine === 'duckduckgo') {
const titleElem = item.querySelector('a.result__a');
const descElem = item.querySelector('a.result__snippet');
if (titleElem) {
title = titleElem.innerText;
link = titleElem.href;
}
if (descElem) description = descElem.innerText;
// 이미지 가져오기 (가능한 경우)
const imgElem = item.querySelector('img.result__thumbnail');
if (imgElem && imgElem.src) {
image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 80px; max-height: 80px;">`;
}
// HTML 보기 링크
if (toggleHTML.checked && link !== 'N/A') {
htmlLink = `<button class="html-btn" data-url="${encodeURIComponent(link)}">보기</button>`;
}
} else if (searchEngine === 'ecosia') {
const titleElem = item.querySelector('a.result-title');
const descElem = item.querySelector('div.result-description');
if (titleElem) {
title = titleElem.innerText;
link = titleElem.href;
}
if (descElem) description = descElem.innerText;
// 이미지 가져오기 (가능한 경우)
const imgElem = item.querySelector('img.result-image');
if (imgElem && imgElem.src) {
image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 80px; max-height: 80px;">`;
}
// HTML 보기 링크
if (toggleHTML.checked && link !== 'N/A') {
htmlLink = `<button class="html-btn" data-url="${encodeURIComponent(link)}">보기</button>`;
}
} else if (searchEngine === 'darkweb') {
// 다크웹 검색 결과 추출 (예: Torch 기준)
const titleElem = item.querySelector('a');
const descElem = item.querySelector('p');
if (titleElem) {
title = titleElem.innerText;
link = titleElem.href;
}
if (descElem) description = descElem.innerText;
// 이미지 가져오기 (가능한 경우)
const imgElem = item.querySelector('img');
if (imgElem && imgElem.src) {
image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 80px; max-height: 80px;">`;
}
// HTML 보기 링크
if (toggleHTML.checked && link !== 'N/A') {
htmlLink = `<button class="html-btn" data-url="${encodeURIComponent(link)}">보기</button>`;
}
}
// 광고 제외 기능 적용
if (toggleAds.checked) {
const combinedText = `${title} ${description}`.toLowerCase();
const isAd = combinedText.includes('advertisement') || combinedText.includes('광고');
if (isAd) {
log(`🚫 광고 항목 제외: "${title}"`);
return null;
}
}
// 필터 단어 적용
if (filterPatterns.length > 0) {
const combinedText = `${title} ${description}`.toLowerCase();
const isFiltered = filterPatterns.some(regex => regex.test(combinedText));
if (isFiltered) {
log(`❌ 필터 단어 포함 항목 제외: "${title}"`);
return null;
}
}
// 사이트 필터링 적용 (사이트 제목 또는 HTML 내용 기반)
if (siteFilterPatterns.length > 0) {
const isSiteFiltered = siteFilterPatterns.some(regex => regex.test(link) || regex.test(title));
if (isSiteFiltered) {
log(`🚫 사이트 필터링에 의해 제외된 항목: "${link}"`);
return null;
}
}
// 도메인 추출
const domain = extractDomain(link);
if (!domain) return null;
// 제외 도메인 검사
if (excludeDomains.includes(domain.toLowerCase())) {
log(`🚫 제외 도메인 항목 제외: "${link}"`);
return null;
}
// 엄격 사이트 필터링 적용
if (strictSiteFilter.checked) {
if (!siteFilterPatterns.some(regex => regex.test(link) || regex.test(title))) {
log(`🚫 엄격 사이트 필터링에 의해 제외된 항목: "${link}"`);
return null;
}
}
// 중복 검사 (도메인별 방문 횟수)
let visitCount = visitedLinks.get(domain) || 0;
if (visitCount >= duplicateThreshold) {
log(`🔄 중복 방문 제한: "${domain}"`);
return null;
}
// 방문 횟수 업데이트
visitedLinks.set(domain, visitCount + 1);
visitedCountElem.innerText = visitedLinks.size.toString();
// 페이지 로딩 시간 측정
const loadStart = performance.now();
try {
// mode: 'cors'로 변경하여 실제 응답을 받을 수 있도록 함
const fetchResponse = await fetch(link, { mode: 'cors' });
if (fetchResponse.ok) {
const loadEnd = performance.now();
pageLoadTime = Math.round(loadEnd - loadStart);
} else {
pageLoadTime = '응답 불량';
}
} catch (err) {
pageLoadTime = '계산 불가';
}
// HTML 옵션
if (!toggleHTML.checked) {
htmlLink = '비활성화';
}
// 이미지 옵션
if (!toggleImages.checked) {
image = '비활성화';
}
const result = { title, link, description, image, htmlLink, pageLoadTime };
results.push(result);
// 결과 테이블에 추가
const row = document.createElement('tr');
row.innerHTML = `
<td>${escapeHTML(title)}</td>
<td><a href="${escapeHTML(link)}" target="_blank">${escapeHTML(link)}</a></td>
<td>${escapeHTML(description)}</td>
<td>${image}</td>
<td>${htmlLink}</td>
<td>${pageLoadTime}</td>
`;
resultsBody.appendChild(row);
return result;
});
// Promise.all을 통해 모든 항목 처리 완료 대기
await Promise.all(itemPromises);
// 데이터 시각화 업데이트
updateChart();
// 진행률 업데이트
updateProgress();
// 대기 시간 추가하여 서버에 부담 주지 않기
setTimeout(() => {
queueCrawlPage();
}, isPaused ? 0 : 1000); // 1초 대기
resolve();
},
onerror: function() {
log(`❌ 페이지 ${page} 요청 중 오류 발생.`);
handleCrawlError(page, retryAttempts).then(() => {
resolve();
});
}
});
});
}
async function handleCrawlError(page, retriesLeft) {
if (retriesLeft > 0 && isRunning) {
log(`🔄 페이지 ${page} 재시도 (${retriesLeft}회 남음).`);
return new Promise((resolve) => {
setTimeout(() => {
const url = constructURL(page);
if (!url) {
log(`❌ 페이지 ${page} 재시도 실패: URL이 유효하지 않습니다.`);
resolve();
return;
}
GM_xmlhttpRequest({
method: "GET",
url: url,
headers: {
"User-Agent": navigator.userAgent,
"Accept": "text/html"
},
onload: async function(response) {
if (response.status === 200) {
log(`✅ 페이지 ${page} 재시도 성공.`);
await crawlPage(page);
} else {
await handleCrawlError(page, retriesLeft - 1);
}
resolve();
},
onerror: async function() {
await handleCrawlError(page, retriesLeft - 1);
resolve();
}
});
}, retryInterval); // retryInterval이 100ms으로 설정됨
});
} else {
log(`❌ 페이지 ${page} 로드 실패: 최대 재시도 횟수 초과.`);
// 크롤링을 중지하지 않고 다음 페이지로 넘어감
currentPage++;
currentPageElem.innerText = currentPage;
setTimeout(() => {
queueCrawlPage();
}, 1000);
return Promise.resolve();
}
}
function constructURL(page) {
if (searchEngine === 'google') {
return `https://www.google.com/search?q=${encodeURIComponent(searchQuery)}&start=${(page-1)*10}`;
} else if (searchEngine === 'bing') {
return `https://www.bing.com/search?q=${encodeURIComponent(searchQuery)}&first=${(page-1)*10 +1}`;
} else if (searchEngine === 'daum') {
return `https://search.daum.net/search?q=${encodeURIComponent(searchQuery)}&page=${page}`;
} else if (searchEngine === 'naver') {
return `https://search.naver.com/search.naver?query=${encodeURIComponent(searchQuery)}&start=${(page-1)*10 +1}`;
} else if (searchEngine === 'yahoo') {
return `https://search.yahoo.com/search?p=${encodeURIComponent(searchQuery)}&b=${(page-1)*10 +1}`;
} else if (searchEngine === 'duckduckgo') {
return `https://duckduckgo.com/html/?q=${encodeURIComponent(searchQuery)}&s=${(page-1)*10}`;
} else if (searchEngine === 'ecosia') {
return `https://www.ecosia.org/search?q=${encodeURIComponent(searchQuery)}&page=${page}`;
} else if (searchEngine === 'darkweb') {
const darkwebEngines = getDarkwebEngines();
if (darkwebEngines.length === 0) {
return '';
}
const randomEngine = darkwebEngines[Math.floor(Math.random() * darkwebEngines.length)];
return `${randomEngine.url}/search?q=${encodeURIComponent(searchQuery)}&page=${page}`;
} else {
return '';
}
}
function updateProgress() {
// 무제한이므로 진행률을 페이지 수에 기반하여 표시
const progressPercent = (currentPage % 100) * 1; // 100페이지마다 100%로 리셋
progressBar.value = progressPercent > 100 ? 100 : progressPercent;
progressText.innerText = `페이지 ${currentPage}`;
if (progressPercent >= 100) {
progressBar.value = 0;
}
}
function resetCrawling() {
isRunning = false;
isPaused = false;
currentPage = 1;
results = [];
visitedLinks.clear();
resultsBody.innerHTML = '';
progressBar.value = 0;
progressText.innerText = '페이지 0';
currentPageElem.innerText = '0';
visitedCountElem.innerText = '0';
elapsedTimeElem.innerText = '00:00:00';
remainingTimeElem.innerText = '00:00:00';
logMessages = [];
logContainer.innerHTML = `<strong>로그:</strong><br>`;
GM_setValue('crawlerLogs', JSON.stringify(logMessages));
// 차트 초기화
crawlData.labels = [];
crawlData.datasets[0].data = [];
if (crawlChart) {
crawlChart.update();
}
clearInterval(elapsedInterval);
log('🔄 크롤링을 초기화하였습니다.');
}
function exportResults() {
if (results.length === 0) {
alert('내보낼 결과가 없습니다.');
return;
}
const format = prompt('내보낼 형식을 선택하세요 (xlsx, csv, json):', 'xlsx');
if (!format) return;
if (format.toLowerCase() === 'xlsx') {
const ws = XLSX.utils.json_to_sheet(results);
const wb = XLSX.utils.book_new();
XLSX.utils.book_append_sheet(wb, ws, "결과");
XLSX.writeFile(wb, "크롤링_결과.xlsx");
log('💾 결과를 "크롤링_결과.xlsx" 파일로 내보냈습니다.');
GM_notification({
text: '결과를 "크롤링_결과.xlsx" 파일로 내보냈습니다.',
title: '웹 크롤러 7.9.5',
timeout: 3000
});
} else if (format.toLowerCase() === 'csv') {
const csv = results.map(row => `"${row.title.replace(/"/g, '""')}","${row.link.replace(/"/g, '""')}","${row.description.replace(/"/g, '""')}","${row.image.replace(/"/g, '""')}","${row.htmlLink.replace(/"/g, '""')}","${row.pageLoadTime}"`).join('\n');
const blob = new Blob([csv], { type: 'text/csv;charset=utf-8;' });
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = '크롤링_결과.csv';
a.click();
log('💾 결과를 "크롤링_결과.csv" 파일로 내보냈습니다.');
GM_notification({
text: '결과를 "크롤링_결과.csv" 파일로 내보냈습니다.',
title: '웹 크롤러 7.9.5',
timeout: 3000
});
} else if (format.toLowerCase() === 'json') {
const json = JSON.stringify(results, null, 2);
const blob = new Blob([json], { type: 'application/json;charset=utf-8;' });
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = '크롤링_결과.json';
a.click();
log('💾 결과를 "크롤링_결과.json" 파일로 내보냈습니다.');
GM_notification({
text: '결과를 "크롤링_결과.json" 파일로 내보냈습니다.',
title: '웹 크롤러 7.9.5',
timeout: 3000
});
} else {
alert('지원하지 않는 형식입니다.');
}
}
function escapeHTML(str) {
if (!str) return '';
return str.replace(/&/g, "&")
.replace(/</g, "<")
.replace(/>/g, ">")
.replace(/"/g, """)
.replace(/'/g, "'");
}
function extractDomain(url) {
try {
const urlObj = new URL(url);
return urlObj.hostname.replace('www.', '');
} catch (e) {
return null;
}
}
function updateElapsedTime() {
if (!startTime) return;
const now = Date.now();
const elapsed = Math.floor((now - startTime) / 1000);
elapsedTimeElem.innerText = formatTime(elapsed);
const estimatedTotal = currentPage * 1; // 예측: 각 페이지 1초
const remaining = Math.max(estimatedTotal - elapsed, 0);
remainingTimeElem.innerText = formatTime(remaining);
}
function formatTime(seconds) {
const hrs = String(Math.floor(seconds / 3600)).padStart(2, '0');
const mins = String(Math.floor((seconds % 3600) / 60)).padStart(2, '0');
const secs = String(seconds % 60).padStart(2, '0');
return `${hrs}:${mins}:${secs}`;
}
function updateChart() {
crawlData.labels = Array.from(visitedLinks.keys());
crawlData.datasets[0].data = Array.from(visitedLinks.values());
if (crawlChart) {
crawlChart.update();
}
// 저장
GM_setValue('crawlerData', JSON.stringify(crawlData));
}
function loadCrawlData() {
const storedData = GM_getValue('crawlerData', null);
if (storedData) {
const parsedData = JSON.parse(storedData);
crawlData = parsedData;
if (crawlChart) {
crawlChart.data = crawlData;
crawlChart.update();
}
}
}
function saveCrawlData() {
GM_setValue('crawlerData', JSON.stringify(crawlData));
}
// 주기적으로 크롤링 데이터를 저장
setInterval(saveCrawlData, 5000);
// 검색 엔진 바로 가기 버튼 기능
function openSearchEngines() {
const engineUrls = {
'Google': 'https://www.google.com/',
'Bing': 'https://www.bing.com/',
'Daum': 'https://www.daum.net/',
'Naver': 'https://www.naver.com/',
'Yahoo': 'https://www.yahoo.com/',
'DuckDuckGo': 'https://duckduckgo.com/',
'Ecosia': 'https://www.ecosia.org/',
'Yahoo Japan': 'https://www.yahoo.co.jp/',
'Baidu': 'https://www.baidu.com/',
'Yandex': 'https://www.yandex.com/',
'Ask': 'https://www.ask.com/',
'Startpage': 'https://www.startpage.com/',
'Qwant': 'https://www.qwant.com/',
'Swisscows': 'https://swisscows.com/',
'Mojeek': 'https://www.mojeek.com/',
'Gibiru': 'https://www.gibiru.com/',
'MetaGer': 'https://metager.org/',
'Peekier': 'https://peekier.com/',
'Disconnect Search': 'https://search.disconnect.me/',
'Ekoru': 'https://www.ekoru.org/',
'Yippy': 'https://yippy.com/',
'Infinity Search': 'https://www.infinitysearch.co/',
'WolframAlpha': 'https://www.wolframalpha.com/',
'Shodan': 'https://www.shodan.io/',
'Archive.org': 'https://archive.org/',
'Wayback Machine': 'https://web.archive.org/',
'PubMed': 'https://pubmed.ncbi.nlm.nih.gov/',
'IEEE Xplore': 'https://ieeexplore.ieee.org/',
'Google Scholar': 'https://scholar.google.com/',
'LexisNexis': 'https://www.lexisnexis.com/',
'Google Books': 'https://books.google.com/',
'Creative Commons Search': 'https://search.creativecommons.org/',
'TOR Search': 'http://torsearch7uu5ond.onion/',
'Ahmia': 'http://msydqstlz2kzerdg.onion/',
'Haystak': 'http://haystakvxad7wbk5.onion/',
'Dark Search': 'http://darksearch3w3nrc.onion/',
'Phobos': 'http://phobos6uj4abm4pf.onion/',
'Not Evil': 'http://hss3uro2hsxfogfq.onion/',
'IntelligenceX': 'http://intelligencexmh5v.onion/',
'Torch': 'http://xmh57jrzrnw6insl.onion/',
'Silk Road 3.1': 'http://silkroad31lo5ya.onion/',
'Fresh Onions': 'http://rx7hmtf7k7xk6ked.onion/',
'Dark Web Link': 'http://darkweblink5wqk.onion/',
'Tor Tracker': 'http://tortracker5ujzra.onion/',
'Deep Web Search': 'http://deepwebsearch6b6x.onion/',
'Onion Monitor': 'http://onionmonitor3c7v.onion/',
'Hidden Wiki': 'http://hiddenwiki5z35.onion/',
'TorEX': 'http://torexk3wd2buz.onion/',
'BlackBook': 'http://blackbook51uv.onion/',
'Taboo Search': 'http://taboosearch37b.onion/',
'The Pirate Bay (onion)': 'http://uj3wazyk5u4hnvtk.onion/',
'Dogpile': 'https://www.dogpile.com/',
'Oscobo': 'https://www.oscobo.com/',
'HotBot': 'https://www.hotbot.com/',
'Peekier': 'https://peekier.com/',
'Lukol': 'https://www.lukol.com/',
'Privatelee': 'https://privatelee.com/',
'DuckDuckGo (Tor)': 'http://3g2upl4pq6kufc4m.onion/',
'TorEX': 'http://torexk3wd2buz.onion/',
'BlackBook': 'http://blackbook51uv.onion/',
'Taboo Search': 'http://taboosearch37b.onion/',
'Kilos': 'http://kilos7wbfn3ptgw.onion/',
'Candle': 'http://zg6ak7qqfhbkbppx.onion/',
'Dark Falcon': 'http://darkfalconphp5fh.onion/',
'Hidden Wiki': 'http://hiddenwiki5z35.onion/',
'Onion Monitor': 'http://onionmonitor3c7v.onion/',
'Phobos 2': 'http://phobos2uj4abm4pf.onion/',
'Deep Web Search': 'http://deepwebsearch6b6x.onion/',
'DarkSearchBox': 'http://darksearchbox5a3x.onion/',
'Black Search': 'http://blacksearch7xm7.onion/',
'Search Encrypt': 'https://www.searchencrypt.com/',
'Peekier': 'https://peekier.com/',
'Lukol': 'https://www.lukol.com/',
'Disconnect Search': 'https://search.disconnect.me/',
'Privatelee': 'https://privatelee.com/',
'Infinity Search': 'https://www.infinitysearch.co/',
'Dogpile': 'https://www.dogpile.com/',
'Startpage': 'https://www.startpage.com/',
'Exalead': 'https://www.exalead.com/',
'Gigablast': 'https://www.gigablast.com/',
'Searx': 'https://searx.me/',
'FindSounds': 'https://www.findsounds.com/',
'TinEye': 'https://tineye.com/',
'Boardreader': 'https://www.boardreader.com/',
'PubCrawl': 'https://pubcrawl.org/',
'BeautyFetch': 'http://beautyfetch.com/',
'WolframAlpha': 'https://www.wolframalpha.com/',
'Shodan': 'https://www.shodan.io/',
'Archive.org': 'https://archive.org/',
'Wayback Machine': 'https://web.archive.org/',
'PubMed': 'https://pubmed.ncbi.nlm.nih.gov/',
'IEEE Xplore': 'https://ieeexplore.ieee.org/',
'Google Scholar': 'https://scholar.google.com/',
'LexisNexis': 'https://www.lexisnexis.com/',
'Google Books': 'https://books.google.com/',
'Creative Commons Search': 'https://search.creativecommons.org/',
// 추가 엔진 가능
};
let html = '📡 <strong>검색 엔진:</strong><br>';
for (const [name, url] of Object.entries(engineUrls)) {
html += `<button class="search-engine-btn" data-url="${url}">${name}</button><br>`;
}
// 팝업 생성
const popup = document.createElement('div');
popup.className = 'search-engines-popup';
popup.innerHTML = html;
container.appendChild(popup);
popup.style.display = 'block';
// 버튼 이벤트 리스너
popup.querySelectorAll('.search-engine-btn').forEach(btn => {
btn.addEventListener('click', () => {
window.open(btn.getAttribute('data-url'), '_blank');
popup.style.display = 'none';
});
});
// 외부 클릭 시 팝업 닫기
document.addEventListener('click', function(event) {
if (!popup.contains(event.target) && event.target !== searchEnginesBtn) {
popup.style.display = 'none';
}
}, { once: true });
}
// HTML 탭에 콘텐츠 표시 기능
container.addEventListener('click', function(event) {
if (event.target.classList.contains('html-btn')) {
const url = decodeURIComponent(event.target.getAttribute('data-url'));
showHTMLInUI(url);
}
});
container.addEventListener('mouseover', function(event) {
if (event.target.classList.contains('html-btn')) {
const url = decodeURIComponent(event.target.getAttribute('data-url'));
showPreviewPopup(event.target, url);
}
});
container.addEventListener('mouseout', function(event) {
if (event.target.classList.contains('html-btn')) {
const popup = container.querySelector('.preview-popup');
if (popup) {
popup.style.display = 'none';
}
}
});
function showHTMLInUI(url) {
// UI 내에 iframe을 사용하여 사이트 내용 표시
let iframe = container.querySelector('.html-display iframe');
if (!iframe) {
iframe = document.createElement('iframe');
container.querySelector('.html-display').appendChild(iframe);
}
iframe.src = url;
// 탭을 'HTML'로 전환
switchTab('html-display');
log(`🔍 HTML 보기: ${url}`);
}
function showPreviewPopup(target, url) {
// 팝업 생성
let popup = container.querySelector('.preview-popup');
if (!popup) {
popup = document.createElement('div');
popup.className = 'preview-popup';
popup.innerHTML = `<iframe src="${url}" style="width:100%; height:100%; border:none;"></iframe>`;
container.appendChild(popup);
}
popup.style.display = 'block';
// 위치 조정
const rect = target.getBoundingClientRect();
const containerRect = container.getBoundingClientRect();
popup.style.left = `${rect.left - containerRect.left + 20}px`;
popup.style.top = `${rect.bottom - containerRect.top + 5}px`;
}
function switchTab(tabId) {
const tabs = container.querySelectorAll('.tab');
const tabContents = container.querySelectorAll('.tab-content');
tabs.forEach(t => {
if(t.getAttribute('data-tab') === tabId){
t.classList.add('active');
} else {
t.classList.remove('active');
}
});
tabContents.forEach(content => {
if(content.id === tabId){
content.classList.add('active');
} else {
content.classList.remove('active');
}
});
}
// 다크웹 검색 엔진 관리
function loadDarkwebEngines() {
const storedEngines = GM_getValue('darkwebEngines', defaultDarkwebEngines);
darkwebEnginesList.innerHTML = '';
storedEngines.forEach(engine => {
const li = document.createElement('li');
li.innerHTML = `
<span>${escapeHTML(engine.name)}</span>
<button class="remove-darkweb-engine-btn" data-url="${engine.url}">🗑️</button>
`;
darkwebEnginesList.appendChild(li);
});
// 삭제 버튼 이벤트 리스너
darkwebEnginesList.querySelectorAll('.remove-darkweb-engine-btn').forEach(btn => {
btn.addEventListener('click', () => {
const url = btn.getAttribute('data-url');
removeDarkwebEngine(url);
});
});
}
function addDarkwebEngine() {
const name = prompt('다크웹 검색 엔진 이름을 입력하세요:');
if (!name) {
alert('이름을 입력하지 않았습니다.');
return;
}
const url = prompt('다크웹 검색 엔진 URL을 입력하세요 (예: http://exampleonion.onion/):');
if (!url) {
alert('URL을 입력하지 않았습니다.');
return;
}
const newEngine = { name, url };
const storedEngines = GM_getValue('darkwebEngines', defaultDarkwebEngines);
storedEngines.push(newEngine);
GM_setValue('darkwebEngines', storedEngines);
loadDarkwebEngines();
log(`🔄 다크웹 검색 엔진 추가: ${name} (${url})`);
}
function removeDarkwebEngine(url) {
let storedEngines = GM_getValue('darkwebEngines', defaultDarkwebEngines);
storedEngines = storedEngines.filter(engine => engine.url !== url);
GM_setValue('darkwebEngines', storedEngines);
loadDarkwebEngines();
log(`🔄 다크웹 검색 엔진 제거: ${url}`);
}
function getDarkwebEngines() {
return GM_getValue('darkwebEngines', defaultDarkwebEngines);
}
// 번역기 기능 추가
async function translateContent() {
if (results.length === 0) {
alert('번역할 결과가 없습니다.');
return;
}
translatedContent.innerHTML = '번역 중...';
try {
// i18next를 이용한 번역
const texts = results.map(r => r.description).join('. ');
const translatedText = await translateText(texts, translateLanguage);
translatedContent.innerText = translatedText;
log('🔄 번역이 완료되었습니다.');
} catch (error) {
translatedContent.innerText = '번역 중 오류가 발생했습니다.';
log(`❌ 번역 오류: ${error}`);
}
}
async function translateText(text, targetLang) {
// Papago 번역 API 사용 (API 키 필요)
const apiUrl = 'https://openapi.naver.com/v1/papago/n2mt';
const clientId = 'YOUR_NAVER_CLIENT_ID'; // [필수] 실제 클라이언트 ID로 교체하세요.
const clientSecret = 'YOUR_NAVER_CLIENT_SECRET'; // [필수] 실제 클라이언트 시크릿으로 교체하세요.
const data = new URLSearchParams();
data.append('source', 'ko');
data.append('target', targetLang);
data.append('text', text);
const response = await fetch(apiUrl, {
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'X-Naver-Client-Id': clientId,
'X-Naver-Client-Secret': clientSecret
},
body: data
});
const result = await response.json();
if (result.errorMessage) {
throw new Error(result.errorMessage);
}
return result.message.result.translatedText;
}
// 번역기 섹션 버튼 클릭 처리
// 번역 기능은 위에서 정의되었습니다.
// 로깅, 필터링, 크롤링 함수들은 이미 위에서 상세히 구현되었습니다.
// 검색 엔진 바로가기 직접 UI에 표시
function loadSearchEnginesDirect() {
const engineUrls = {
'Google': 'https://www.google.com/',
'Bing': 'https://www.bing.com/',
'Daum': 'https://www.daum.net/',
'Naver': 'https://www.naver.com/',
'Yahoo': 'https://www.yahoo.com/',
'DuckDuckGo': 'https://duckduckgo.com/',
'Ecosia': 'https://www.ecosia.org/',
'Yahoo Japan': 'https://www.yahoo.co.jp/',
'Baidu': 'https://www.baidu.com/',
'Yandex': 'https://www.yandex.com/',
'Ask': 'https://www.ask.com/',
'Startpage': 'https://www.startpage.com/',
'Qwant': 'https://www.qwant.com/',
'Swisscows': 'https://swisscows.com/',
'Mojeek': 'https://www.mojeek.com/',
'Gibiru': 'https://www.gibiru.com/',
'MetaGer': 'https://metager.org/',
'Peekier': 'https://peekier.com/',
'Disconnect Search': 'https://search.disconnect.me/',
'Ekoru': 'https://www.ekoru.org/',
'Yippy': 'https://yippy.com/',
'Infinity Search': 'https://www.infinitysearch.co/',
'WolframAlpha': 'https://www.wolframalpha.com/',
'Shodan': 'https://www.shodan.io/',
'Archive.org': 'https://archive.org/',
'Wayback Machine': 'https://web.archive.org/',
'PubMed': 'https://pubmed.ncbi.nlm.nih.gov/',
'IEEE Xplore': 'https://ieeexplore.ieee.org/',
'Google Scholar': 'https://scholar.google.com/',
'LexisNexis': 'https://www.lexisnexis.com/',
'Google Books': 'https://books.google.com/',
'Creative Commons Search': 'https://search.creativecommons.org/',
'TOR Search': 'http://torsearch7uu5ond.onion/',
'Ahmia': 'http://msydqstlz2kzerdg.onion/',
'Haystak': 'http://haystakvxad7wbk5.onion/',
'Dark Search': 'http://darksearch3w3nrc.onion/',
'Phobos': 'http://phobos6uj4abm4pf.onion/',
'Not Evil': 'http://hss3uro2hsxfogfq.onion/',
'IntelligenceX': 'http://intelligencexmh5v.onion/',
'Torch': 'http://xmh57jrzrnw6insl.onion/',
'Silk Road 3.1': 'http://silkroad31lo5ya.onion/',
'Fresh Onions': 'http://rx7hmtf7k7xk6ked.onion/',
'Dark Web Link': 'http://darkweblink5wqk.onion/',
'Tor Tracker': 'http://tortracker5ujzra.onion/',
'Deep Web Search': 'http://deepwebsearch6b6x.onion/',
'Onion Monitor': 'http://onionmonitor3c7v.onion/',
'Hidden Wiki': 'http://hiddenwiki5z35.onion/',
'TorEX': 'http://torexk3wd2buz.onion/',
'BlackBook': 'http://blackbook51uv.onion/',
'Taboo Search': 'http://taboosearch37b.onion/',
'The Pirate Bay (onion)': 'http://uj3wazyk5u4hnvtk.onion/',
'Dogpile': 'https://www.dogpile.com/',
'Oscobo': 'https://www.oscobo.com/',
'HotBot': 'https://www.hotbot.com/',
'Peekier': 'https://peekier.com/',
'Lukol': 'https://www.lukol.com/',
'Privatelee': 'https://privatelee.com/',
'Infinity Search': 'https://www.infinitysearch.co/',
'Dogpile': 'https://www.dogpile.com/',
'Startpage': 'https://www.startpage.com/',
'Exalead': 'https://www.exalead.com/',
'Gigablast': 'https://www.gigablast.com/',
'Searx': 'https://searx.me/',
'FindSounds': 'https://www.findsounds.com/',
'TinEye': 'https://tineye.com/',
'Boardreader': 'https://www.boardreader.com/',
'PubCrawl': 'https://pubcrawl.org/',
'BeautyFetch': 'http://beautyfetch.com/',
'WolframAlpha': 'https://www.wolframalpha.com/',
'Shodan': 'https://www.shodan.io/',
'Archive.org': 'https://archive.org/',
'Wayback Machine': 'https://web.archive.org/',
'PubMed': 'https://pubmed.ncbi.nlm.nih.gov/',
'IEEE Xplore': 'https://ieeexplore.ieee.org/',
'Google Scholar': 'https://scholar.google.com/',
'LexisNexis': 'https://www.lexisnexis.com/',
'Google Books': 'https://books.google.com/',
'Creative Commons Search': 'https://search.creativecommons.org/',
// 추가 엔진 가능
};
let html = '';
for (const [name, url] of Object.entries(engineUrls)) {
html += `<button class="search-engine-direct-btn" data-url="${url}">${name}</button>`;
}
searchEnginesDirect.innerHTML = html;
// 버튼 이벤트 리스너
searchEnginesDirect.querySelectorAll('.search-engine-direct-btn').forEach(btn => {
btn.addEventListener('click', () => {
window.open(btn.getAttribute('data-url'), '_blank');
});
});
}
})();
```