Web Application Survey Tutorial

%pylab inline
import binascii
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import networkx as nx
import datetime as dt
import time
import ipy_table
import dnslib
import pythonwhois
import urlparse
import tldextract
import json
import os
import sys
import urllib2


from yurl import URL
from urlparse import urlparse
from IPython.display import display_pretty, display_html, display_jpeg, display_png, display_json, display_latex, display_svg

# Ipython settings
pd.set_option('display.height', 1000)
pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.max_colwidth', 100)
pd.set_option('display.width', 3000)
pd.set_option('display.column_space', 1000)

# Change working directory
os.chdir("/root/work/appsurvey")

# Join root domain + suffix
extract_root_domain =  lambda x: '.'.join(tldextract.extract(x)[1:3])

target_columns = ['scheme', 'userinfo', 'host', 'port', 'path', 'query', 'fragment', 'decoded']
target_component = [list(URL(t)) for t in targets['Target']]

df_targets = pd.DataFrame(target_component, columns=target_columns)
empty_hosts = df_targets[df_targets['host'] == '']

# Copy path information to host
for index,row in empty_hosts.iterrows():
    df_targets.ix[index:index]['host'] = df_targets.ix[index:index]['path']
    df_targets.ix[index:index]['path'] = ''
    
# Extract root tld
df_targets['root_domain'] = df_targets['host'].apply(extract_root_domain)

# Drop unnecessary columns
df_targets.drop(['query', 'fragment', 'decoded'], axis=1, inplace=True)

# Write df to file (for later use)
df_targets.to_csv("targets_df.csv", sep="\t")

print("First 20 Entries")
df_targets[:20]

def date_handler(obj):
    return obj.isoformat() if hasattr(obj, 'isoformat') else obj

target_whois = {}

def fetch_whois(domains):
    """ Fetch WHOIS information for specified domains (list) """
    for d in domains:
        print("Get WHOIS for\t %s ..." % d)

        # Check if file already exists
        if os.path.isfile("WHOIS/%s.json" % d):
            print("File exists already. Aborting.")
            continue

        try:
            # Get whois information
            whois_data = pythonwhois.get_whois(d)

            # Convert to JSON$
            json_data = json.dumps(whois_data, default=date_handler)

            # Write contents to file
            with open('WHOIS/%s.json' % d, 'w') as outfile:
              json.dump(json_data, outfile)

            # Sleep for 20s    
            time.sleep(20)
        except:
            print("[ERROR] Couldn't retrieve WHOIS for\t %s" % d)
            
# I'll only fetch the root domains and only the first 20. Feel free to uncomment this
# and adapt it to your needs.
#fetch_whois(uniq_subdomains)
fetch_whois(uniq_roots[:20])

def fetch_dns(domains):
    """ Fetch all DNS records for specified domains (list) """
    for d in domains:
        print("Dig DNS records for\t %s ..." % d)

        # Check if file already exists
        if os.path.isfile("DNS/%s.txt" % d):
            print("File exists already. Aborting.")
            continue
            
        # Get DNS info
        dig_data = !dig +nocmd $d any +multiline +noall +answer
        dig_output = "\n".join(dig_data)
        
        # Write contents to file
        with open('DNS/%s.txt' % d, 'w') as outfile:
            outfile.write(dig_output)
            outfile.close()
        
        time.sleep(5)
        
# I'll only fetch the root domains and only the first 20. Feel free to uncomment this
# and adapt it to your needs.
#fetch_dns(uniq_subdomains)
fetch_dns(uniq_roots[:20])

from pprint import pprint

# Global DF frames
frames = []

def read_whois(domains):
    for d in domains:
        print("Reading WHOIS for\t %s" % d)
        
        try:
            with open('WHOIS/%s.json' % d, 'r') as inputfile:
                whois = json.loads(json.load(inputfile))

                # Delete raw record
                whois.pop('raw', None)

                data = []
                
                # Iterate contacts -> tech
                if whois['contacts']['tech']:
                    for i in whois['contacts']['tech']:
                        data.append([d, 'contacts', 'tech', i, whois['contacts']['tech'][i]])

                # Iterate contacts -> admin
                if whois['contacts']['admin']:
                    for i in whois['contacts']['admin']:
                        data.append([d, 'contacts', 'admin', i, whois['contacts']['admin'][i]])

                # Nameservers
                if "nameservers" in whois:
                    for i in whois['nameservers']:
                        data.append([d, 'nameservers', '', '', i])

                # Create DF only if data is not empty
                if data:
                    df = pd.DataFrame(data, columns=['domain', 'element', 'type', 'field', 'value'])
                    frames.append(df)

                # Close file
                inputfile.close()
        except:
            print("[ERROR] Couldn't read WHOIS for\t %s" % d)

#read_whois(uniq_subdomains)
read_whois(uniq_roots[:20])

from pprint import pprint
import re
import traceback

# Global DF frames
frames = []

def read_dns(domains):
    for d in domains:
        print("Reading WHOIS for\t %s" % d)
        data = []
        try:
            with open('DNS/%s.txt' % d, 'r') as inputfile:
                dns = inputfile.read()
                
                for l in dns.splitlines():
                    records = l.split()
                    
                    # Check only for NS, MX, A, CNAME, TXT
                    a = re.compile("^(NS|MX|A|CNAME|TXT)$")
                    if len(records) >= 4:
                        if a.match(records[3]):
                            data.append([d, records[3], records[4]])
                
                # Create DF only if data is not empty
                if data:
                    df = pd.DataFrame(data, columns=['domain', 'dns_record', 'value'])
                    frames.append(df)      
                    
                # Close file
                inputfile.close()
                
        except Exception, err:
            print("[ERROR] Couldn't read WHOIS for\t %s" % d)
            traceback.print_exc()

#read_dns(uniq_subdomains)            
read_dns(uniq_roots[:20])

import urllib2
import httplib


c_targets = [t for t in targets['Target'][:20]]
frames = []

# Collect here all URLs failed to connect to
error_urls = []

def send_request(target, data):
    """ Sends a single request to the target """            
    
    # Set own headers
    headers = {'User-Agent' : 'Mozilla 5.10'}

    # Create request
    request = urllib2.Request(target, None, headers)
    
    # Default response
    response = None
        
    try:
        # Send request
        response = urllib2.urlopen(request, timeout=5)
        
        # Add headers
        for h in response.info():
            data.append([target, response.code, h, response.info()[h]])
        
    except urllib2.HTTPError, e:
        print('[ERROR] HTTPError = ' + str(e.code))
        data.append([target, e.code, '', ''])
            
    except urllib2.URLError, e:
        print('[ERROR] URLError = ' + str(e.reason))
        data.append([target, e.reason, '', ''])
            
    except ValueError, e:
        # Most probably the target didn't have any schema
        # So send the request again with HTTP
        error_urls.append(target)
        print('[ERROR] ValueError = ' + e.message)
            
    except httplib.HTTPException, e:
        print('[ERROR] HTTPException')
            
    except Exception:
        import traceback
        print('[ERROR] Exception: ' + traceback.format_exc())
        
    finally:
        return response
        

    
    
def open_connection(targets):
    """ Iterate through targets and send requests """
    data = []
    for t in targets:
        print("Connecting to\t %s" % t)
        
        response = send_request(t, data)
        
    # Create DF only if data is not empty
    if data:
        df = pd.DataFrame(data, columns=['url', 'response', 'header', 'value'])
        frames.append(df)    
        

# Open connection to targets and collect information
open_connection(c_targets)

# If there are any urls not having been tested, then
# prepend http:// to <target> and run again
new_targets =  ["http://"+u for u in error_urls]
open_connection(new_targets)