TheHarvester: Email Enumeration e Subdomain Discovery Rapida per Initial Reconnaissance

# Verifica installazione
theharvester --version
# Output: theHarvester 4.8.2

# Update se necessario
sudo apt update && sudo apt install theharvester

git clone https://github.com/laramies/theHarvester.git
cd theHarvester
python3 -m pip install -r requirements/base.txt
python3 theHarvester.py --help

# Via pip (richiede Python 3.10+)
pip install theHarvester

# Oppure da source
git clone https://github.com/laramies/theHarvester.git
cd theHarvester
pip install -r requirements/base.txt
python theHarvester.py --help

brew install theharvester

theharvester -d example.com -l 50 -b google

# Screenshot functionality (playwright)
pip3 install playwright
playwright install chromium

# DNS bruteforce (nessuna extra dependency richiesta)

theharvester -d targetcorp.com -b google

-b baidu          # Baidu (Cina)
-b bing           # Microsoft Bing
-b brave          # Brave Search
-b duckduckgo     # DuckDuckGo
-b google         # Google Search
-b yahoo          # Yahoo Search

-b certspotter    # SSLMate CertSpotter
-b crtsh          # crt.sh database

-b dnsdumpster    # DNSdumpster.com
-b hackertarget   # HackerTarget API
-b rapiddns       # RapidDNS
-b sublist3r      # Sublist3r wrapper
-b threatminer    # ThreatMiner

-b hunter         # Hunter.io (richiede API key)
-b linkedin       # LinkedIn (rate limited)

-b pgp            # MIT PGP keyserver

-b otx            # AlienVault OTX
-b securityTrails # SecurityTrails
-b shodan         # Shodan
-b virustotal     # VirusTotal

-b all            # Query tutti i source disponibili

theharvester -d example.com -b google -l 100
# Ferma dopo 100 risultati da Google

theharvester -d example.com -b all -f results
# Genera: results.html + results.xml

theharvester -d targetcorp.com -b google,bing -l 200

theharvester -d targetcorp.com -b all -l 500

theharvester -d example.com -b shodan -l 100

theharvester -d example.com -b all -f report
# Genera: report.html + report.xml

theharvester -d example.com -b all -f results.json

{
  "emails": ["admin@example.com", "info@example.com"],
  "hosts": ["www.example.com:93.184.216.34", "mail.example.com:93.184.216.35"],
  "ips": ["93.184.216.34", "93.184.216.35"],
  "asns": ["AS15169"],
  "interesting_urls": [...]
}

# Step 1: Harvest da multiple sources
theharvester -d targetcorp.com -b google,bing,linkedin,pgp -l 500 -f emails_raw

# Step 2: Extract solo email da output
grep -oE '[a-zA-Z0-9._%+-]+@targetcorp\.com' emails_raw.html | sort -u > emails_clean.txt

# Step 3: Validate email format, rimuovi generici
grep -v 'noreply\|no-reply\|postmaster\|abuse' emails_clean.txt > emails_valid.txt

# Step 4: Count risultati
wc -l emails_valid.txt
# Output: 147 emails_valid.txt

# Osserva pattern da email trovate
cat emails_valid.txt | head -10
# Output pattern: john.doe@targetcorp.com, jane.smith@targetcorp.com
# Pattern identificato: firstname.lastname@domain

# Genera email candidate da LinkedIn employee list
# (raccolto separatamente via scraping)
cat linkedin_employees.txt
# Output:
# John Doe
# Jane Smith  
# Michael Johnson

# Generate email con awk
cat linkedin_employees.txt | awk '{print tolower($1"."$2"@targetcorp.com")}' > guessed_emails.txt

# Merge con harvested
cat emails_valid.txt guessed_emails.txt | sort -u > final_email_list.txt

smtp-user-enum -M VRFY -U final_email_list.txt -t mail.targetcorp.com

# Passive discovery da multiple sources
theharvester -d example.com -b all -l 1000 -f subdomains_all

# Extract hosts da output HTML
grep -oP '(?<=<td>)[a-z0-9.-]+\.example\.com' subdomains_all.html | sort -u > subs_harvester.txt

# DNS bruteforce con wordlist (opzionale, più noisy)
theharvester -d example.com -b dnsdumpster,crtsh,certspotter -l 2000

# Merge con other tools per coverage massimo
# Esempio: combinare con Amass, Sublist3r
cat subs_harvester.txt subs_amass.txt subs_sublist3r.txt | sort -u > all_subdomains.txt

# Resolve to IPs
cat all_subdomains.txt | while read sub; do 
    host $sub | grep "has address" | awk '{print $1","$NF}'
done > resolved_subs.csv

# Count unique IPs
cut -d',' -f2 resolved_subs.csv | sort -u | wc -l
# Output: 89 unique IPs

# Extract solo IPs
cut -d',' -f2 resolved_subs.csv | sort -u > target_ips.txt

# Nmap full port scan
nmap -iL target_ips.txt -p- -sV -sC -oA nmap_full_scan

# LinkedIn harvesting (richiede setup specifico, spesso rate-limited)
theharvester -d targetcorp.com -b linkedin -l 500

# PGP key server search
theharvester -d targetcorp.com -b pgp -l 200

# Twitter mentions
theharvester -d targetcorp.com -b twitter -l 300

# Da output HTML/text
grep -oP 'Name: \K[A-Z][a-z]+ [A-Z][a-z]+' results.html | sort -u > employee_names.txt

# Associate con job titles (se disponibili in LinkedIn output)
# Esempio output manuale:
# John Doe - Senior DevOps Engineer
# Jane Smith - Security Analyst
# Michael Johnson - CTO

#!/usr/bin/env python3
# correlate_emails_names.py

emails = open('emails_valid.txt').read().splitlines()
names = open('employee_names.txt').read().splitlines()

for name in names:
    parts = name.lower().split()
    if len(parts) >= 2:
        fname, lname = parts[0], parts[-1]
        
        # Check various email formats
        patterns = [
            f"{fname}.{lname}@",
            f"{fname[0]}{lname}@",
            f"{fname}@",
            f"{fname}_{lname}@"
        ]
        
        for pattern in patterns:
            matches = [e for e in emails if pattern in e]
            if matches:
                print(f"[CONFIRMED] {name} → {matches[0]}")
                break
        else:
            print(f"[GUESSED] {name} → {fname}.{lname}@targetcorp.com")

# Enable DNS bruteforce con wordlist default (builtin)
theharvester -d example.com -b google,bing --dns-brute

# Con wordlist custom
theharvester -d example.com -b google --dns-brute -w /usr/share/seclists/Discovery/DNS/subdomains-top1million-5000.txt

theharvester -d example.com -b bing,google --virtual-host

# Setup Playwright (one-time)
pip3 install playwright
playwright install chromium

# Run TheHarvester con screenshot
theharvester -d example.com -b all --screenshot /tmp/screenshots/

# ~/.theHarvester/api-keys.yaml
apikeys:
  bevigil:
    key: YOUR_BEVIGIL_KEY
  binaryedge:
    key: YOUR_BINARYEDGE_KEY
  censys:
    id: YOUR_CENSYS_ID
    secret: YOUR_CENSYS_SECRET
  criminalip:
    key: YOUR_CRIMINALIP_KEY
  fullhunt:
    key: YOUR_FULLHUNT_KEY
  github:
    key: YOUR_GITHUB_TOKEN
  hunter:
    key: YOUR_HUNTER_KEY
  intelx:
    key: YOUR_INTELX_KEY
  netlas:
    key: YOUR_NETLAS_KEY
  pentesttools:
    key: YOUR_PENTESTTOOLS_KEY
  projectdiscovery:
    key: YOUR_CHAOS_KEY
  rocketreach:
    key: YOUR_ROCKETREACH_KEY
  securityTrails:
    key: YOUR_SECURITYTRAILS_KEY
  shodan:
    key: YOUR_SHODAN_KEY
  tomba:
    key: YOUR_TOMBA_KEY
  virustotal:
    key: YOUR_VIRUSTOTAL_KEY
  zoomeye:
    key: YOUR_ZOOMEYE_KEY

# Shodan
# Register: https://account.shodan.io/register
# API Key: Account → API Key section

# Hunter.io  
# Register: https://hunter.io/users/sign_up
# Free tier: 25 searches/month
# API Key: Dashboard → API section

# GitHub Personal Access Token
# Settings → Developer settings → Personal access tokens → Generate
# Scopes needed: public_repo (minimum)

# VirusTotal
# Register: https://www.virustotal.com/gui/join-us
# API Key: Profile → API Key
# Free: 4 requests/minute limit

#!/bin/bash
# batch_harvest.sh

DOMAINS_FILE="domains.txt"
OUTPUT_DIR="./results"

mkdir -p $OUTPUT_DIR

while read domain; do
    echo "[*] Scanning $domain..."
    
    # Quick harvest
    theharvester -d $domain -b google,bing,hunter -l 200 \
        -f "$OUTPUT_DIR/${domain}_quick"
    
    # Sleep per rate limiting
    sleep 5
    
done < $DOMAINS_FILE

echo "[+] Batch scan complete. Results in $OUTPUT_DIR/"

chmod +x batch_harvest.sh
./batch_harvest.sh

# 1. Harvest emails con TheHarvester
theharvester -d target.com -b all -f harvest.json

# 2. Parse JSON e import in Recon-ng database
python3 << 'PYEOF'
import json
import sqlite3

with open('harvest.json') as f:
    data = json.load(f)

# Connect to Recon-ng workspace DB
conn = sqlite3.connect(os.path.expanduser('~/.recon-ng/workspaces/target/data.db'))
cursor = conn.cursor()

# Insert emails
for email in data.get('emails', []):
    cursor.execute("INSERT OR IGNORE INTO contacts (email) VALUES (?)", (email,))

# Insert hosts
for host in data.get('hosts', []):
    hostname = host.split(':')[0]
    cursor.execute("INSERT OR IGNORE INTO hosts (host) VALUES (?)", (hostname,))

conn.commit()
conn.close()
print("[+] Data imported to Recon-ng")
PYEOF

# Extract unique hosts
cat harvest.html | grep -oE '[a-z0-9.-]+\.target\.com' | sort -u > hosts.txt

# Feed each to SpiderFoot
cat hosts.txt | while read host; do
    python3 /opt/spiderfoot/sf.py -s $host -u investigate -o csv -f "spider_${host}.csv"
done

# Complete subdomain discovery pipeline
theharvester -d target.com -b all --dns-brute -f harvest_subs.txt
amass enum -passive -d target.com -o amass_subs.txt

# Merge results
cat harvest_subs.txt amass_subs.txt | sort -u > merged_subs.txt

# Visual reconnaissance
cat merged_subs.txt | aquatone -out aquatone_results/

#!/bin/bash
# daily_harvest.sh - Automated OSINT collection

DOMAIN_LIST="targets.txt"
OUTPUT_BASE="/var/osint"
DATE=$(date +%Y%m%d)
OUTPUT_DIR="$OUTPUT_BASE/$DATE"

mkdir -p $OUTPUT_DIR

while read domain; do
    echo "[*] Processing $domain at $(date)"
    
    # Quick harvest
    theharvester -d $domain -b google,bing,hunter -l 200 \
        -f "$OUTPUT_DIR/${domain}_quick.json"
    
    # Email extraction
    cat "$OUTPUT_DIR/${domain}_quick.json" | jq -r '.emails[]' 2>/dev/null \
        > "$OUTPUT_DIR/${domain}_emails.txt"
    
    # Subdomain extraction  
    cat "$OUTPUT_DIR/${domain}_quick.json" | jq -r '.hosts[]' 2>/dev/null | cut -d: -f1 \
        > "$OUTPUT_DIR/${domain}_subs.txt"
    
    # Alert on new findings (compare con previous run)
    PREV_EMAILS="$OUTPUT_BASE/previous/${domain}_emails.txt"
    if [ -f "$PREV_EMAILS" ]; then
        NEW_COUNT=$(comm -13 <(sort "$PREV_EMAILS") <(sort "$OUTPUT_DIR/${domain}_emails.txt") | wc -l)
        if [ $NEW_COUNT -gt 0 ]; then
            echo "[ALERT] $NEW_COUNT new emails for $domain" | \
                mail -s "TheHarvester Alert: $domain" security@company.com
        fi
    fi
    
    # Backup for next diff
    mkdir -p "$OUTPUT_BASE/previous"
    cp "$OUTPUT_DIR/${domain}_emails.txt" "$OUTPUT_BASE/previous/"
    
    # Rate limiting delay
    sleep $((RANDOM % 300 + 60))  # 1-5 min random
    
done < $DOMAIN_LIST

echo "[+] Daily harvest complete: $OUTPUT_DIR"

crontab -e

# Add line (run every Sunday at 3 AM):
0 3 * * 0 /opt/scripts/daily_harvest.sh >> /var/log/harvest.log 2>&1

#!/usr/bin/env python3
"""
harvest_and_validate.py - TheHarvester wrapper with validation
"""

import subprocess
import json
import re
import sys

def harvest_domain(domain, sources='all', limit=500):
    """Run TheHarvester and return parsed JSON output"""
    
    output_file = f"harvest_{domain}.json"
    cmd = [
        'theharvester',
        '-d', domain,
        '-b', sources,
        '-l', str(limit),
        '-f', output_file
    ]
    
    print(f"[*] Running: {' '.join(cmd)}")
    result = subprocess.run(cmd, capture_output=True, text=True)
    
    if result.returncode != 0:
        print(f"[-] Error: {result.stderr}")
        return None
    
    try:
        with open(output_file) as f:
            return json.load(f)
    except:
        print(f"[-] Failed to parse JSON output")
        return None

def validate_emails(emails):
    """Validate email format and filter generics"""
    
    valid_emails = []
    email_pattern = re.compile(r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$')
    
    generic_patterns = ['noreply', 'no-reply', 'postmaster', 'abuse', 'info', 'support']
    
    for email in emails:
        # Format validation
        if not email_pattern.match(email):
            continue
        
        # Filter generic emails
        if any(pattern in email.lower() for pattern in generic_patterns):
            continue
        
        valid_emails.append(email)
    
    return valid_emails

def detect_email_pattern(emails):
    """Detect naming convention from email samples"""
    
    patterns = {
        'firstname.lastname': 0,
        'firstinitial.lastname': 0,
        'firstname': 0,
        'firstname_lastname': 0,
        'unknown': 0
    }
    
    for email in emails:
        local = email.split('@')[0]
        
        if '.' in local and not any(char.isdigit() for char in local):
            patterns['firstname.lastname'] += 1
        elif local.count('.') == 1 and len(local.split('.')[0]) == 1:
            patterns['firstinitial.lastname'] += 1
        elif '_' in local:
            patterns['firstname_lastname'] += 1
        elif not any(char in local for char in ['.', '_', '-']) and not any(char.isdigit() for char in local):
            patterns['firstname'] += 1
        else:
            patterns['unknown'] += 1
    
    dominant = max(patterns, key=patterns.get) if patterns else 'unknown'
    confidence = patterns[dominant] / len(emails) if emails else 0
    
    return dominant, confidence

def main():
    if len(sys.argv) < 2:
        print("Usage: python3 harvest_and_validate.py <domain>")
        sys.exit(1)
    
    domain = sys.argv[1]
    
    # Harvest data
    data = harvest_domain(domain)
    if not data:
        sys.exit(1)
    
    # Process emails
    raw_emails = data.get('emails', [])
    valid_emails = validate_emails(raw_emails)
    
    # Detect pattern
    pattern, confidence = detect_email_pattern(valid_emails)
    
    # Results
    results = {
        'domain': domain,
        'total_emails_found': len(raw_emails),
        'valid_emails': len(valid_emails),
        'email_pattern': pattern,
        'pattern_confidence': f"{confidence:.2%}",
        'emails': valid_emails,
        'subdomains': [h.split(':')[0] for h in data.get('hosts', [])],
        'total_subdomains': len(data.get('hosts', []))
    }
    
    print(json.dumps(results, indent=2))
    
    # Save validated results
    with open(f'{domain}_validated.json', 'w') as f:
        json.dump(results, f, indent=2)
    
    print(f"\n[+] Validated results saved to {domain}_validated.json")

if __name__ == "__main__":
    main()

python3 harvest_and_validate.py targetcorp.com

import random

user_agents = [
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36',
    'Mozilla/5.0 (X11; Ubuntu; Linux x86_64) AppleWebKit/537.36'
]

headers = {
    'User-Agent': random.choice(user_agents)
}

# Throttle requests con sleep between sources
theharvester -d target.com -b google -l 50
sleep 300  # 5 min
theharvester -d target.com -b bing -l 50
sleep 300
# etc.

# Via proxychains
proxychains4 theharvester -d target.com -b all

# Via SOCKS proxy
export ALL_PROXY=socks5://127.0.0.1:9050
theharvester -d target.com -b all

# Via Tor
torsocks theharvester -d target.com -b google,bing

# VPS 1: Google + Bing
ssh vps1.provider.com "theharvester -d target.com -b google,bing -f vps1.json"

# VPS 2: Shodan + VirusTotal
ssh vps2.provider.com "theharvester -d target.com -b shodan,virustotal -f vps2.json"

# VPS 3: DNS brute (se necessario)
ssh vps3.provider.com "theharvester -d target.com --dns-brute -f vps3.json"

# Merge results offline
scp vps1:vps1.json vps2:vps2.json vps3:vps3.json ./
python3 merge_results.py vps*.json > merged_final.json

# Stick to passive sources only
theharvester -d target.com -b google,bing,certspotter,crtsh,hackertarget

# NO --dns-brute flag

# Scan 50 domains in parallel (10 concurrent)
cat domains.txt | parallel -j 10 'theharvester -d {} -b all -f results_{}.json'

# .gitlab-ci.yml
recon_stage:
  stage: reconnaissance
  script:
    - theharvester -d $CI_PROJECT_DOMAIN -b all -f recon.json
    - python3 parse_and_alert.py recon.json
  artifacts:
    paths:
      - recon.json
  only:
    - schedules  # Weekly cron job

# k8s-harvester-job.yaml
apiVersion: batch/v1
kind: Job
metadata:
  name: theharvester-scan
spec:
  parallelism: 20
  template:
    spec:
      containers:
      - name: harvester
        image: theharvester:latest
        command: ["theharvester"]
        args: ["-d", "$(TARGET_DOMAIN)", "-b", "all", "-f", "/output/$(TARGET_DOMAIN).json"]
        volumeMounts:
        - name: output
          mountPath: /output
      volumes:
      - name: output
        persistentVolumeClaim:
          claimName: harvester-pvc

# Verify domain exists
dig target.com ANY
nslookup target.com

# Try specific source
theharvester -d target.com -b google -l 100

# Increase limit
theharvester -d target.com -b all -l 2000

# Check rate limiting
curl -I "https://www.google.com/search?q=site:target.com"
# Se 429 Too Many Requests → wait e retry

# Verify API key file exists
cat ~/.theHarvester/api-keys.yaml

# Test key manually
# Shodan example:
curl "https://api.shodan.io/api-info?key=YOUR_KEY"

# Hunter.io example:
curl "https://api.hunter.io/v2/domain-search?domain=example.com&api_key=YOUR_KEY"

# Re-register se necessario
# Revoke old key, generate new

# Limit sources (skip slow ones)
theharvester -d largedomain.com -b google,bing -l 500

# Split execution by source
theharvester -d largedomain.com -b google &
theharvester -d largedomain.com -b bing &
theharvester -d largedomain.com -b shodan &
wait
# Merge dopo completion

# Force UTF-8 encoding
export PYTHONIOENCODING=utf-8
theharvester -d target.com -b all

# Or filter output
theharvester -d target.com -b all | iconv -f UTF-8 -t ASCII//TRANSLIT

# Use larger wordlist
theharvester -d target.com --dns-brute \
    -w /usr/share/seclists/Discovery/DNS/subdomains-top1million-110000.txt

# Custom DNS resolver
echo "8.8.8.8" > resolvers.txt
echo "1.1.1.1" >> resolvers.txt
theharvester -d target.com --dns-brute --dns-server resolvers.txt

# Check for wildcard DNS
dig random-nonexistent-subdomain-12345.target.com
# Se returns IP → wildcard enabled (brute ineffective)

# SMTP VRFY command
smtp-user-enum -M VRFY -U emails.txt -t mail.target.com

# Email verification services (API)
# Hunter.io, NeverBounce, ZeroBounce

# Manual test (OPSEC risk!)
echo "test" | mail -s "test" target@domain.com
# Check bounce message

cd /opt/theHarvester
git pull

# Export emails
theharvester -d target.com -b all > emails.txt

# Import in Metasploit auxiliary modules
# Or use directly for phishing payloads

# INSTALLATION
pip3 install theHarvester                     # Via pip
git clone https://github.com/laramies/theHarvester.git  # From source
sudo apt install theharvester                 # Kali/Debian

# BASIC USAGE
theharvester -d domain.com -b google          # Single source
theharvester -d domain.com -b google,bing     # Multiple sources
theharvester -d domain.com -b all             # All sources
theharvester -d domain.com -b all -l 500      # Limit results

# OUTPUT FORMATS
-f filename.html                              # HTML + XML output
-f filename.json                              # JSON output

# DNS OPERATIONS
--dns-brute                                   # Enable DNS bruteforce
-w wordlist.txt                               # Custom wordlist
--dns-server resolvers.txt                    # Custom DNS servers

# ADVANCED
--virtual-host                                # Virtual host discovery
--screenshot /path/                           # Auto-screenshot subdomains

# COMMON SOURCES (no API key)
google,bing,yahoo,duckduckgo                  # Search engines
crtsh,certspotter                             # Certificate Transparency
dnsdumpster,hackertarget,rapiddns            # DNS enumeration
pgp                                           # PGP keyservers

# API SOURCES (require keys)
shodan,virustotal,securityTrails             # Threat intelligence
hunter                                        # Professional email discovery
github-code                                   # GitHub code search

# API KEY CONFIG
~/.theHarvester/api-keys.yaml                # Config file location

# BATCH OPERATIONS
for d in $(cat domains.txt); do \
  theharvester -d $d -b all -f ${d}.html; \
done

# OUTPUT PARSING
grep -oE '[a-z0-9._-]+@[a-z0-9.-]+' results.html      # Extract emails
grep -oE '[a-z0-9.-]+\.domain\.com' results.html      # Extract subdomains
cat results.json | jq -r '.emails[]'                   # JSON email extraction

# PIPELINE INTEGRATION
theharvester -d target.com -b all | \
  grep "Host:" | awk '{print $2}' | \
  sort -u > subdomains.txt

# PROXY/VPN
proxychains4 theharvester -d target.com -b all
torsocks theharvester -d target.com -b google
export ALL_PROXY=socks5://127.0.0.1:9050

# OPSEC
- Use VPN/proxy for all operations
- Throttle requests (sleep between scans)
- Randomize User-Agent (code modification)
- Avoid --dns-brute unless authorized
- Rotate API keys per engagement

# Local artifacts
rm -f *.html *.xml *.json
history -c  # Clear bash history

# No artifacts su target system (passive tool)