update
This commit is contained in:
Autumn.home
parent
aa4717ef97
commit
94f59ad438
|
|
@ -47,7 +47,7 @@ Scope Sentry是一款具有资产测绘、子域名枚举、信息泄露检测
|
|||
|
||||
Discord:
|
||||
|
||||
[https://discord.gg/agsYdAyN](https://discord.gg/agsYdAyN)
|
||||
[https://discord.gg/GWVwSBBm48](https://discord.gg/GWVwSBBm48)
|
||||
|
||||
QQ:
|
||||
|
||||
|
|
|
|||
|
|
@ -43,14 +43,14 @@ async def asset_data(request_data: dict, db=Depends(get_mongo_db), _: dict = Dep
|
|||
try:
|
||||
if len(APP) == 0:
|
||||
collection = db["FingerprintRules"]
|
||||
cursor = collection.find({}, {"_id": 1, "name": 1})
|
||||
cursor = await collection.find({}, {"_id": 1, "name": 1})
|
||||
async for document in cursor:
|
||||
document['id'] = str(document['_id'])
|
||||
del document['_id']
|
||||
APP[document['id']] = document['name']
|
||||
if len(SensitiveRuleList) == 0:
|
||||
collection = db["SensitiveRule"]
|
||||
cursor = collection.find({}, {"_id": 1, "name": 1})
|
||||
cursor = await collection.find({}, {"_id": 1, "name": 1})
|
||||
async for document in cursor:
|
||||
document['id'] = str(document['_id'])
|
||||
del document['_id']
|
||||
|
|
|
|||
|
|
@ -118,3 +118,20 @@ async def save_system_data(data: dict, db=Depends(get_mongo_db), _: dict = Depen
|
|||
return {"message": "error", "code": 500}
|
||||
|
||||
|
||||
@router.get("/system/deduplication/config")
|
||||
async def get_deduplication_config(_: dict = Depends(verify_token), db=Depends(get_mongo_db)):
|
||||
try:
|
||||
# 查询所有 type 为 "system" 的文档
|
||||
cursor = await db.config.find_one({"name": "deduplication"})
|
||||
deduplication_data = {}
|
||||
|
||||
async for document in cursor:
|
||||
deduplication_data[document["name"]] = document["value"]
|
||||
return {
|
||||
"code": 200,
|
||||
"data": deduplication_data
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(str(e))
|
||||
# 根据需要处理异常
|
||||
return {"message": "error", "code": 500}
|
||||
17
api/node.py
17
api/node.py
|
|
@ -150,3 +150,20 @@ async def get_redis_online_data(redis_con):
|
|||
else:
|
||||
result.append(name)
|
||||
return result
|
||||
|
||||
|
||||
async def get_node_all(redis_con):
|
||||
try:
|
||||
result = []
|
||||
async with redis_con as redis:
|
||||
# 获取所有以 node: 开头的键
|
||||
keys = await redis.keys("node:*")
|
||||
for key in keys:
|
||||
name = key.split(":")[1]
|
||||
hash_data = await redis.hgetall(key)
|
||||
if hash_data.get('state') != '2':
|
||||
result.append(name)
|
||||
|
||||
return result
|
||||
except:
|
||||
return []
|
||||
473
api/project.py
473
api/project.py
|
|
@ -170,7 +170,7 @@ async def add_project_rule(request_data: dict, db=Depends(get_mongo_db), _: dict
|
|||
{"id": str(result.inserted_id), "name": name, 'hour': hour, 'type': 'Project', 'state': True,
|
||||
'lastTime': get_now_time(), 'nextTime': formatted_time, 'runner_id': str(result.inserted_id)})
|
||||
await scheduler_project(str(result.inserted_id))
|
||||
background_tasks.add_task(update_project, tmsg, str(result.inserted_id))
|
||||
background_tasks.add_task(update_project, root_domains, str(result.inserted_id))
|
||||
await refresh_config('all', 'project')
|
||||
Project_List[name] = str(result.inserted_id)
|
||||
return {"code": 200, "message": "Project added successfully"}
|
||||
|
|
@ -239,7 +239,9 @@ async def update_project_data(request_data: dict, db=Depends(get_mongo_db), _: d
|
|||
id=str(pro_id), jobstore='mongo')
|
||||
await db.ScheduledTasks.update_one({"id": pro_id}, {"$set": {'state': True}})
|
||||
else:
|
||||
scheduler.remove_job(pro_id)
|
||||
job = scheduler.get_job(pro_id)
|
||||
if job is not None:
|
||||
scheduler.remove_job(pro_id)
|
||||
await db.ScheduledTasks.update_one({"id": pro_id}, {"$set": {'state': False}})
|
||||
else:
|
||||
if newScheduledTasks:
|
||||
|
|
@ -278,7 +280,7 @@ async def update_project_data(request_data: dict, db=Depends(get_mongo_db), _: d
|
|||
new_root_domain.append(t_root_domain)
|
||||
request_data["root_domains"] = new_root_domain
|
||||
await db.ProjectTargetData.update_one({"id": pro_id}, update_document)
|
||||
background_tasks.add_task(change_update_project, new_targets.strip().strip('\n'), pro_id)
|
||||
background_tasks.add_task(update_project, new_root_domain, pro_id, True)
|
||||
if old_name != new_name:
|
||||
del Project_List[old_name]
|
||||
Project_List[new_name] = pro_id
|
||||
|
|
@ -300,415 +302,72 @@ async def update_project_data(request_data: dict, db=Depends(get_mongo_db), _: d
|
|||
# Handle exceptions as needed
|
||||
return {"message": "error", "code": 500}
|
||||
|
||||
|
||||
async def change_update_project(domain, project_id):
|
||||
async def update_project(root_domain, project_id, change=False):
|
||||
asset_collection_list = {
|
||||
'asset': ["url", "host", "ip"],
|
||||
'subdomain': ["host", "ip"],
|
||||
'DirScanResult': ["url"],
|
||||
'vulnerability': ["url"],
|
||||
'SubdoaminTakerResult': ["input"],
|
||||
'PageMonitoring': ["url"],
|
||||
'SensitiveResult': ["url"],
|
||||
'UrlScan': ["input"],
|
||||
'crawler': ["url"]}
|
||||
async for db in get_mongo_db():
|
||||
await add_asset_project(db, domain, project_id, True)
|
||||
await add_subdomain_project(db, domain, project_id, True)
|
||||
await add_dir_project(db, domain, project_id, True)
|
||||
await add_vul_project(db, domain, project_id, True)
|
||||
await add_SubTaker_project(db, domain, project_id, True)
|
||||
await add_PageMonitoring_project(db, domain, project_id, True)
|
||||
await add_sensitive_project(db, domain, project_id, True)
|
||||
await add_url_project(db, domain, project_id, True)
|
||||
await add_crawler_project(db, domain, project_id, True)
|
||||
for a in asset_collection_list:
|
||||
if change:
|
||||
await asset_update_project(root_domain, asset_collection_list[a], a, db, project_id)
|
||||
else:
|
||||
await asset_add_project(root_domain, asset_collection_list[a], a, db, project_id)
|
||||
|
||||
|
||||
async def add_asset_project(db, domain, project_id, updata=False):
|
||||
try:
|
||||
if updata:
|
||||
query = {"$or": [{"project": ""}, {"project": project_id}]}
|
||||
else:
|
||||
query = {"project": {"$eq": ""}}
|
||||
cursor: AsyncIOMotorCursor = ((db['asset'].find(query, {
|
||||
"_id": 0, "id": {"$toString": "$_id"},
|
||||
"url": 1,
|
||||
"host": 1,
|
||||
"project": 1,
|
||||
})))
|
||||
result = await cursor.to_list(length=None)
|
||||
logger.debug(f"asset project null number is {len(result)}")
|
||||
if len(result) != 0:
|
||||
domain_root_list = []
|
||||
for d in domain.split("\n"):
|
||||
u = get_root_domain(d)
|
||||
if u not in domain_root_list:
|
||||
domain_root_list.append(u)
|
||||
for r in result:
|
||||
url = ""
|
||||
if "url" in r:
|
||||
url = r['url']
|
||||
else:
|
||||
url = r['host']
|
||||
if url != "":
|
||||
targer_url = get_root_domain(url)
|
||||
if targer_url in domain_root_list:
|
||||
update_document = {
|
||||
"$set": {
|
||||
"project": project_id,
|
||||
}
|
||||
}
|
||||
await db['asset'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
||||
else:
|
||||
if r["project"] != "":
|
||||
update_document = {
|
||||
"$set": {
|
||||
"project": "",
|
||||
}
|
||||
}
|
||||
await db['asset'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
||||
except Exception as e:
|
||||
logger.error(f"add_asset_project error:{e}")
|
||||
async def asset_add_project(root_domain, db_key, doc_name, db, project_id):
|
||||
regex_patterns = [f".*{domain}.*" for domain in root_domain]
|
||||
pattern = "|".join(regex_patterns)
|
||||
# 构建查询条件
|
||||
query = {
|
||||
"$and": [
|
||||
{
|
||||
"$or": [
|
||||
{key: {"$regex": pattern, "$options": "i"}} for key in db_key
|
||||
]
|
||||
},
|
||||
{"project": {"$exists": True, "$eq": ""}}
|
||||
]
|
||||
}
|
||||
update_query = {
|
||||
"$set": {
|
||||
"project": project_id
|
||||
}
|
||||
}
|
||||
result = await db[doc_name].update_many(query, update_query)
|
||||
# 打印更新的文档数量
|
||||
logger.info(f"Updated {doc_name} {result.modified_count} documents")
|
||||
|
||||
|
||||
async def add_subdomain_project(db, domain, project_id, updata=False):
|
||||
try:
|
||||
if updata:
|
||||
query = {"$or": [{"project": ""}, {"project": project_id}]}
|
||||
else:
|
||||
query = {"project": {"$eq": ""}}
|
||||
cursor: AsyncIOMotorCursor = ((db['subdomain'].find(query, {
|
||||
"_id": 0, "id": {"$toString": "$_id"},
|
||||
"host": 1,
|
||||
"project": 1
|
||||
})))
|
||||
result = await cursor.to_list(length=None)
|
||||
logger.debug(f"subdomain project null number is {len(result)}")
|
||||
if len(result) != 0:
|
||||
domain_root_list = []
|
||||
for d in domain.split("\n"):
|
||||
u = get_root_domain(d)
|
||||
if u not in domain_root_list:
|
||||
domain_root_list.append(u)
|
||||
for r in result:
|
||||
url = r['host']
|
||||
if url != "":
|
||||
targer_url = get_root_domain(url)
|
||||
if targer_url in domain_root_list:
|
||||
update_document = {
|
||||
"$set": {
|
||||
"project": project_id,
|
||||
}
|
||||
}
|
||||
await db['subdomain'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
||||
else:
|
||||
if r["project"] != "":
|
||||
update_document = {
|
||||
"$set": {
|
||||
"project": "",
|
||||
}
|
||||
}
|
||||
await db['subdomain'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
||||
except Exception as e:
|
||||
logger.error(f"add_subdomain_project error:{e}")
|
||||
|
||||
|
||||
async def add_url_project(db, domain, project_id, updata=False):
|
||||
try:
|
||||
if updata:
|
||||
query = {"$or": [{"project": ""}, {"project": project_id}]}
|
||||
else:
|
||||
query = {"project": {"$eq": ""}}
|
||||
cursor: AsyncIOMotorCursor = ((db['UrlScan'].find(query, {
|
||||
"_id": 0, "id": {"$toString": "$_id"},
|
||||
"input": 1,
|
||||
"project": 1
|
||||
})))
|
||||
result = await cursor.to_list(length=None)
|
||||
logger.debug(f"url project null number is {len(result)}")
|
||||
if len(result) != 0:
|
||||
domain_root_list = []
|
||||
for d in domain.split("\n"):
|
||||
u = get_root_domain(d)
|
||||
if u not in domain_root_list:
|
||||
domain_root_list.append(u)
|
||||
for r in result:
|
||||
url = r['input']
|
||||
if url != "":
|
||||
targer_url = get_root_domain(url)
|
||||
if targer_url in domain_root_list:
|
||||
update_document = {
|
||||
"$set": {
|
||||
"project": project_id,
|
||||
}
|
||||
}
|
||||
await db['UrlScan'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
||||
else:
|
||||
if r["project"] != "":
|
||||
update_document = {
|
||||
"$set": {
|
||||
"project": "",
|
||||
}
|
||||
}
|
||||
await db['UrlScan'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
||||
except Exception as e:
|
||||
logger.error(f"add_url_project error:{e}")
|
||||
|
||||
|
||||
async def add_crawler_project(db, domain, project_id, updata=False):
|
||||
try:
|
||||
if updata:
|
||||
query = {"$or": [{"project": ""}, {"project": project_id}]}
|
||||
else:
|
||||
query = {"project": {"$eq": ""}}
|
||||
cursor: AsyncIOMotorCursor = ((db['crawler'].find(query, {
|
||||
"_id": 0, "id": {"$toString": "$_id"},
|
||||
"url": 1,
|
||||
"project": 1
|
||||
})))
|
||||
result = await cursor.to_list(length=None)
|
||||
logger.debug(f"crawler project null number is {len(result)}")
|
||||
if len(result) != 0:
|
||||
domain_root_list = []
|
||||
for d in domain.split("\n"):
|
||||
u = get_root_domain(d)
|
||||
if u not in domain_root_list:
|
||||
domain_root_list.append(u)
|
||||
for r in result:
|
||||
url = r['url']
|
||||
if url != "":
|
||||
targer_url = get_root_domain(url)
|
||||
if targer_url in domain_root_list:
|
||||
update_document = {
|
||||
"$set": {
|
||||
"project": project_id,
|
||||
}
|
||||
}
|
||||
await db['crawler'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
||||
else:
|
||||
if r["project"] != "":
|
||||
update_document = {
|
||||
"$set": {
|
||||
"project": "",
|
||||
}
|
||||
}
|
||||
await db['crawler'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
||||
except Exception as e:
|
||||
logger.error(f"add_crawler_project error:{e}")
|
||||
|
||||
|
||||
async def add_sensitive_project(db, domain, project_id, updata=False):
|
||||
try:
|
||||
if updata:
|
||||
query = {"$or": [{"project": ""}, {"project": project_id}]}
|
||||
else:
|
||||
query = {"project": {"$eq": ""}}
|
||||
cursor: AsyncIOMotorCursor = ((db['SensitiveResult'].find(query, {
|
||||
"_id": 0, "id": {"$toString": "$_id"},
|
||||
"url": 1,
|
||||
"project": 1
|
||||
})))
|
||||
result = await cursor.to_list(length=None)
|
||||
logger.debug(f"sensitive project null number is {len(result)}")
|
||||
if len(result) != 0:
|
||||
domain_root_list = []
|
||||
for d in domain.split("\n"):
|
||||
u = get_root_domain(d)
|
||||
if u not in domain_root_list:
|
||||
domain_root_list.append(u)
|
||||
for r in result:
|
||||
url = r['url']
|
||||
if url != "":
|
||||
targer_url = get_root_domain(url)
|
||||
if targer_url in domain_root_list:
|
||||
update_document = {
|
||||
"$set": {
|
||||
"project": project_id,
|
||||
}
|
||||
}
|
||||
await db['SensitiveResult'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
||||
else:
|
||||
if r["project"] != "":
|
||||
update_document = {
|
||||
"$set": {
|
||||
"project": "",
|
||||
}
|
||||
}
|
||||
await db['SensitiveResult'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
||||
except Exception as e:
|
||||
logger.error(f"add_sensitive_project error:{e}")
|
||||
|
||||
|
||||
async def add_dir_project(db, domain, project_id, updata=False):
|
||||
try:
|
||||
if updata:
|
||||
query = {"$or": [{"project": ""}, {"project": project_id}]}
|
||||
else:
|
||||
query = {"project": {"$eq": ""}}
|
||||
cursor: AsyncIOMotorCursor = ((db['DirScanResult'].find(query, {
|
||||
"_id": 0, "id": {"$toString": "$_id"},
|
||||
"url": 1,
|
||||
"project": 1
|
||||
})))
|
||||
result = await cursor.to_list(length=None)
|
||||
logger.debug(f"dir project null number is {len(result)}")
|
||||
if len(result) != 0:
|
||||
domain_root_list = []
|
||||
for d in domain.split("\n"):
|
||||
u = get_root_domain(d)
|
||||
if u not in domain_root_list:
|
||||
domain_root_list.append(u)
|
||||
for r in result:
|
||||
url = r['url']
|
||||
if url != "":
|
||||
targer_url = get_root_domain(url)
|
||||
if targer_url in domain_root_list:
|
||||
update_document = {
|
||||
"$set": {
|
||||
"project": project_id,
|
||||
}
|
||||
}
|
||||
await db['DirScanResult'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
||||
else:
|
||||
if r["project"] != "":
|
||||
update_document = {
|
||||
"$set": {
|
||||
"project": "",
|
||||
}
|
||||
}
|
||||
await db['DirScanResult'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
||||
except Exception as e:
|
||||
logger.error(f"add_dir_project error:{e}")
|
||||
|
||||
|
||||
async def add_vul_project(db, domain, project_id, updata=False):
|
||||
try:
|
||||
if updata:
|
||||
query = {"$or": [{"project": ""}, {"project": project_id}]}
|
||||
else:
|
||||
query = {"project": {"$eq": ""}}
|
||||
cursor: AsyncIOMotorCursor = ((db['vulnerability'].find(query, {
|
||||
"_id": 0, "id": {"$toString": "$_id"},
|
||||
"url": 1,
|
||||
"project": 1
|
||||
})))
|
||||
result = await cursor.to_list(length=None)
|
||||
logger.debug(f"vul project null number is {len(result)}")
|
||||
if len(result) != 0:
|
||||
domain_root_list = []
|
||||
for d in domain.split("\n"):
|
||||
u = get_root_domain(d)
|
||||
if u not in domain_root_list:
|
||||
domain_root_list.append(u)
|
||||
for r in result:
|
||||
url = r['url']
|
||||
if url != "":
|
||||
targer_url = get_root_domain(url)
|
||||
if targer_url in domain_root_list:
|
||||
update_document = {
|
||||
"$set": {
|
||||
"project": project_id,
|
||||
}
|
||||
}
|
||||
await db['vulnerability'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
||||
else:
|
||||
if r["project"] != "":
|
||||
update_document = {
|
||||
"$set": {
|
||||
"project": "",
|
||||
}
|
||||
}
|
||||
await db['vulnerability'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"add_vul_project error:{e}")
|
||||
|
||||
|
||||
async def add_PageMonitoring_project(db, domain, project_id, updata=False):
|
||||
try:
|
||||
if updata:
|
||||
query = {"$or": [{"project": ""}, {"project": project_id}]}
|
||||
else:
|
||||
query = {"project": {"$eq": ""}}
|
||||
cursor: AsyncIOMotorCursor = ((db['PageMonitoring'].find(query, {
|
||||
"_id": 0, "id": {"$toString": "$_id"},
|
||||
"url": 1,
|
||||
"project": 1
|
||||
})))
|
||||
result = await cursor.to_list(length=None)
|
||||
logger.debug(f"PageMonitoring project null number is {len(result)}")
|
||||
if len(result) != 0:
|
||||
domain_root_list = []
|
||||
for d in domain.split("\n"):
|
||||
u = get_root_domain(d)
|
||||
if u not in domain_root_list:
|
||||
domain_root_list.append(u)
|
||||
for r in result:
|
||||
url = r['url']
|
||||
if url != "":
|
||||
targer_url = get_root_domain(url)
|
||||
if targer_url in domain_root_list:
|
||||
update_document = {
|
||||
"$set": {
|
||||
"project": project_id,
|
||||
}
|
||||
}
|
||||
await db['PageMonitoring'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
||||
else:
|
||||
if r["project"] != "":
|
||||
update_document = {
|
||||
"$set": {
|
||||
"project": "",
|
||||
}
|
||||
}
|
||||
await db['PageMonitoring'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
||||
except Exception as e:
|
||||
logger.error(f"add_PageMonitoring_project error:{e}")
|
||||
|
||||
|
||||
async def add_SubTaker_project(db, domain, project_id, updata=False):
|
||||
try:
|
||||
if updata:
|
||||
query = {"$or": [{"project": ""}, {"project": project_id}]}
|
||||
else:
|
||||
query = {"project": {"$eq": ""}}
|
||||
cursor: AsyncIOMotorCursor = ((db['SubdoaminTakerResult'].find(query, {
|
||||
"_id": 0, "id": {"$toString": "$_id"},
|
||||
"Input": 1,
|
||||
"project": 1
|
||||
})))
|
||||
result = await cursor.to_list(length=None)
|
||||
logger.debug(f"SubTaker project null number is {len(result)}")
|
||||
if len(result) != 0:
|
||||
domain_root_list = []
|
||||
for d in domain.split("\n"):
|
||||
u = get_root_domain(d)
|
||||
if u not in domain_root_list:
|
||||
domain_root_list.append(u)
|
||||
for r in result:
|
||||
url = r['input']
|
||||
if url != "":
|
||||
targer_url = get_root_domain(url)
|
||||
if targer_url in domain_root_list:
|
||||
update_document = {
|
||||
"$set": {
|
||||
"project": project_id,
|
||||
}
|
||||
}
|
||||
await db['SubdoaminTakerResult'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
||||
else:
|
||||
if r["project"] != "":
|
||||
update_document = {
|
||||
"$set": {
|
||||
"project": "",
|
||||
}
|
||||
}
|
||||
await db['SubdoaminTakerResult'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
||||
except Exception as e:
|
||||
logger.error(f"add_SubTaker_project error:{e}")
|
||||
|
||||
|
||||
async def update_project(domain, project_id):
|
||||
async for db in get_mongo_db():
|
||||
await add_asset_project(db, domain, project_id)
|
||||
await add_subdomain_project(db, domain, project_id)
|
||||
await add_dir_project(db, domain, project_id)
|
||||
await add_vul_project(db, domain, project_id)
|
||||
await add_SubTaker_project(db, domain, project_id)
|
||||
await add_PageMonitoring_project(db, domain, project_id)
|
||||
await add_sensitive_project(db, domain, project_id)
|
||||
await add_url_project(db, domain, project_id)
|
||||
await add_crawler_project(db, domain, project_id)
|
||||
async def asset_update_project(root_domain, db_key, doc_name, db, project_id):
|
||||
regex_patterns = [f".*{domain}.*" for domain in root_domain]
|
||||
pattern = "|".join(regex_patterns)
|
||||
# 构建查询条件
|
||||
query = {
|
||||
"$and": [
|
||||
{"project": project_id},
|
||||
{
|
||||
"$nor": [
|
||||
{key: {"$regex": pattern, "$options": "i"}} for key in db_key
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
update_query = {
|
||||
"$set": {
|
||||
"project": ""
|
||||
}
|
||||
}
|
||||
result = await db[doc_name].update_many(query, update_query)
|
||||
# 打印更新的文档数量
|
||||
logger.info(f"Updated {doc_name} {result.modified_count} documents to null ")
|
||||
await asset_add_project(root_domain, db_key, doc_name, db, project_id)
|
||||
|
||||
|
||||
async def delete_asset_project(db, collection, project_id):
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ from core.apscheduler_handler import scheduler
|
|||
from core.db import get_mongo_db
|
||||
from core.redis_handler import get_redis_pool, check_redis_task_target_is_null
|
||||
from core.util import *
|
||||
from api.node import get_redis_online_data
|
||||
from api.node import get_redis_online_data, get_node_all
|
||||
from api.page_monitoring import get_page_monitoring_data
|
||||
router = APIRouter()
|
||||
|
||||
|
|
@ -254,7 +254,7 @@ async def create_scan_task(request_data, id, targetList, redis_con):
|
|||
try:
|
||||
request_data["id"] = str(id)
|
||||
if request_data['allNode']:
|
||||
request_data["node"] = await get_redis_online_data(redis_con)
|
||||
request_data["node"] = await get_node_all(redis_con)
|
||||
|
||||
keys_to_delete = [
|
||||
f"TaskInfo:tmp:{id}",
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
# @version:
|
||||
import json
|
||||
import os
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from bson import ObjectId
|
||||
|
||||
|
|
@ -58,6 +59,7 @@ def get_finger():
|
|||
d.pop('_id', None)
|
||||
return data
|
||||
|
||||
|
||||
def get_project_data():
|
||||
project_path = os.path.join(combined_directory, "ScopeSentry.project.json")
|
||||
data = read_json_file(project_path)
|
||||
|
|
@ -68,8 +70,9 @@ def get_project_data():
|
|||
tmp = []
|
||||
for t in d['target'].split('\n'):
|
||||
root_domain = get_root_domain(t)
|
||||
if root_domain not in tmp:
|
||||
tmp.append(root_domain)
|
||||
if root_domain is not None and root_domain != "":
|
||||
if root_domain not in tmp:
|
||||
tmp.append(root_domain)
|
||||
d["root_domains"] = tmp
|
||||
d['_id'] = ObjectId(project_id)
|
||||
target_data.append({"id": project_id, "target": d['target']})
|
||||
|
|
@ -503,3 +506,50 @@ def get_fingerprint_data():
|
|||
except FileNotFoundError:
|
||||
logger.error("文件不存在")
|
||||
return json.loads(fingerprint)
|
||||
|
||||
|
||||
def get_root_domain(url):
|
||||
# 如果URL不带协议，添加一个默认的http协议
|
||||
global root_domain
|
||||
if not url.startswith(('http://', 'https://')):
|
||||
url = 'http://' + url
|
||||
|
||||
parsed_url = urlparse(url)
|
||||
|
||||
# 检查是否为IP地址
|
||||
try:
|
||||
# 使用ip_address来检查
|
||||
from ipaddress import ip_address
|
||||
ip_address(parsed_url.netloc)
|
||||
return parsed_url.netloc # 如果是IP地址，直接返回
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
domain_parts = parsed_url.netloc.split('.')
|
||||
|
||||
# 复合域名列表
|
||||
compound_domains = [
|
||||
'com.cn', 'net.cn', 'org.cn', 'gov.cn', 'edu.cn', 'ac.cn', 'mil.cn',
|
||||
'co.uk', 'org.uk', 'net.uk', 'gov.uk', 'ac.uk', 'sch.uk',
|
||||
'co.jp', 'ne.jp', 'or.jp', 'go.jp', 'ac.jp', 'ad.jp',
|
||||
'com.de', 'org.de', 'net.de', 'gov.de',
|
||||
'com.ca', 'net.ca', 'org.ca', 'gov.ca',
|
||||
'com.au', 'net.au', 'org.au', 'gov.au', 'edu.au',
|
||||
'com.fr', 'net.fr', 'org.fr', 'gov.fr',
|
||||
'com.br', 'com.mx', 'com.ar', 'com.ru',
|
||||
'co.in', 'co.za',
|
||||
'co.kr', 'com.tw'
|
||||
]
|
||||
|
||||
# 检查是否为复合域名
|
||||
is_compound_domain = False
|
||||
for compound_domain in compound_domains:
|
||||
if domain_parts[-2:] == compound_domain.split('.'):
|
||||
is_compound_domain = True
|
||||
root_domain = '.'.join(domain_parts[-3:])
|
||||
break
|
||||
|
||||
if not is_compound_domain:
|
||||
root_domain = '.'.join(domain_parts[-2:])
|
||||
|
||||
return root_domain
|
||||
Loading…
Reference in New Issue