update
This commit is contained in:
parent
aa4717ef97
commit
94f59ad438
|
@ -47,7 +47,7 @@ Scope Sentry是一款具有资产测绘、子域名枚举、信息泄露检测
|
||||||
|
|
||||||
Discord:
|
Discord:
|
||||||
|
|
||||||
[https://discord.gg/agsYdAyN](https://discord.gg/agsYdAyN)
|
[https://discord.gg/GWVwSBBm48](https://discord.gg/GWVwSBBm48)
|
||||||
|
|
||||||
QQ:
|
QQ:
|
||||||
|
|
||||||
|
|
|
@ -43,14 +43,14 @@ async def asset_data(request_data: dict, db=Depends(get_mongo_db), _: dict = Dep
|
||||||
try:
|
try:
|
||||||
if len(APP) == 0:
|
if len(APP) == 0:
|
||||||
collection = db["FingerprintRules"]
|
collection = db["FingerprintRules"]
|
||||||
cursor = collection.find({}, {"_id": 1, "name": 1})
|
cursor = await collection.find({}, {"_id": 1, "name": 1})
|
||||||
async for document in cursor:
|
async for document in cursor:
|
||||||
document['id'] = str(document['_id'])
|
document['id'] = str(document['_id'])
|
||||||
del document['_id']
|
del document['_id']
|
||||||
APP[document['id']] = document['name']
|
APP[document['id']] = document['name']
|
||||||
if len(SensitiveRuleList) == 0:
|
if len(SensitiveRuleList) == 0:
|
||||||
collection = db["SensitiveRule"]
|
collection = db["SensitiveRule"]
|
||||||
cursor = collection.find({}, {"_id": 1, "name": 1})
|
cursor = await collection.find({}, {"_id": 1, "name": 1})
|
||||||
async for document in cursor:
|
async for document in cursor:
|
||||||
document['id'] = str(document['_id'])
|
document['id'] = str(document['_id'])
|
||||||
del document['_id']
|
del document['_id']
|
||||||
|
|
|
@ -118,3 +118,20 @@ async def save_system_data(data: dict, db=Depends(get_mongo_db), _: dict = Depen
|
||||||
return {"message": "error", "code": 500}
|
return {"message": "error", "code": 500}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/system/deduplication/config")
|
||||||
|
async def get_deduplication_config(_: dict = Depends(verify_token), db=Depends(get_mongo_db)):
|
||||||
|
try:
|
||||||
|
# 查询所有 type 为 "system" 的文档
|
||||||
|
cursor = await db.config.find_one({"name": "deduplication"})
|
||||||
|
deduplication_data = {}
|
||||||
|
|
||||||
|
async for document in cursor:
|
||||||
|
deduplication_data[document["name"]] = document["value"]
|
||||||
|
return {
|
||||||
|
"code": 200,
|
||||||
|
"data": deduplication_data
|
||||||
|
}
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(str(e))
|
||||||
|
# 根据需要处理异常
|
||||||
|
return {"message": "error", "code": 500}
|
17
api/node.py
17
api/node.py
|
@ -150,3 +150,20 @@ async def get_redis_online_data(redis_con):
|
||||||
else:
|
else:
|
||||||
result.append(name)
|
result.append(name)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
async def get_node_all(redis_con):
|
||||||
|
try:
|
||||||
|
result = []
|
||||||
|
async with redis_con as redis:
|
||||||
|
# 获取所有以 node: 开头的键
|
||||||
|
keys = await redis.keys("node:*")
|
||||||
|
for key in keys:
|
||||||
|
name = key.split(":")[1]
|
||||||
|
hash_data = await redis.hgetall(key)
|
||||||
|
if hash_data.get('state') != '2':
|
||||||
|
result.append(name)
|
||||||
|
|
||||||
|
return result
|
||||||
|
except:
|
||||||
|
return []
|
463
api/project.py
463
api/project.py
|
@ -170,7 +170,7 @@ async def add_project_rule(request_data: dict, db=Depends(get_mongo_db), _: dict
|
||||||
{"id": str(result.inserted_id), "name": name, 'hour': hour, 'type': 'Project', 'state': True,
|
{"id": str(result.inserted_id), "name": name, 'hour': hour, 'type': 'Project', 'state': True,
|
||||||
'lastTime': get_now_time(), 'nextTime': formatted_time, 'runner_id': str(result.inserted_id)})
|
'lastTime': get_now_time(), 'nextTime': formatted_time, 'runner_id': str(result.inserted_id)})
|
||||||
await scheduler_project(str(result.inserted_id))
|
await scheduler_project(str(result.inserted_id))
|
||||||
background_tasks.add_task(update_project, tmsg, str(result.inserted_id))
|
background_tasks.add_task(update_project, root_domains, str(result.inserted_id))
|
||||||
await refresh_config('all', 'project')
|
await refresh_config('all', 'project')
|
||||||
Project_List[name] = str(result.inserted_id)
|
Project_List[name] = str(result.inserted_id)
|
||||||
return {"code": 200, "message": "Project added successfully"}
|
return {"code": 200, "message": "Project added successfully"}
|
||||||
|
@ -239,6 +239,8 @@ async def update_project_data(request_data: dict, db=Depends(get_mongo_db), _: d
|
||||||
id=str(pro_id), jobstore='mongo')
|
id=str(pro_id), jobstore='mongo')
|
||||||
await db.ScheduledTasks.update_one({"id": pro_id}, {"$set": {'state': True}})
|
await db.ScheduledTasks.update_one({"id": pro_id}, {"$set": {'state': True}})
|
||||||
else:
|
else:
|
||||||
|
job = scheduler.get_job(pro_id)
|
||||||
|
if job is not None:
|
||||||
scheduler.remove_job(pro_id)
|
scheduler.remove_job(pro_id)
|
||||||
await db.ScheduledTasks.update_one({"id": pro_id}, {"$set": {'state': False}})
|
await db.ScheduledTasks.update_one({"id": pro_id}, {"$set": {'state': False}})
|
||||||
else:
|
else:
|
||||||
|
@ -278,7 +280,7 @@ async def update_project_data(request_data: dict, db=Depends(get_mongo_db), _: d
|
||||||
new_root_domain.append(t_root_domain)
|
new_root_domain.append(t_root_domain)
|
||||||
request_data["root_domains"] = new_root_domain
|
request_data["root_domains"] = new_root_domain
|
||||||
await db.ProjectTargetData.update_one({"id": pro_id}, update_document)
|
await db.ProjectTargetData.update_one({"id": pro_id}, update_document)
|
||||||
background_tasks.add_task(change_update_project, new_targets.strip().strip('\n'), pro_id)
|
background_tasks.add_task(update_project, new_root_domain, pro_id, True)
|
||||||
if old_name != new_name:
|
if old_name != new_name:
|
||||||
del Project_List[old_name]
|
del Project_List[old_name]
|
||||||
Project_List[new_name] = pro_id
|
Project_List[new_name] = pro_id
|
||||||
|
@ -300,415 +302,72 @@ async def update_project_data(request_data: dict, db=Depends(get_mongo_db), _: d
|
||||||
# Handle exceptions as needed
|
# Handle exceptions as needed
|
||||||
return {"message": "error", "code": 500}
|
return {"message": "error", "code": 500}
|
||||||
|
|
||||||
|
async def update_project(root_domain, project_id, change=False):
|
||||||
async def change_update_project(domain, project_id):
|
asset_collection_list = {
|
||||||
|
'asset': ["url", "host", "ip"],
|
||||||
|
'subdomain': ["host", "ip"],
|
||||||
|
'DirScanResult': ["url"],
|
||||||
|
'vulnerability': ["url"],
|
||||||
|
'SubdoaminTakerResult': ["input"],
|
||||||
|
'PageMonitoring': ["url"],
|
||||||
|
'SensitiveResult': ["url"],
|
||||||
|
'UrlScan': ["input"],
|
||||||
|
'crawler': ["url"]}
|
||||||
async for db in get_mongo_db():
|
async for db in get_mongo_db():
|
||||||
await add_asset_project(db, domain, project_id, True)
|
for a in asset_collection_list:
|
||||||
await add_subdomain_project(db, domain, project_id, True)
|
if change:
|
||||||
await add_dir_project(db, domain, project_id, True)
|
await asset_update_project(root_domain, asset_collection_list[a], a, db, project_id)
|
||||||
await add_vul_project(db, domain, project_id, True)
|
else:
|
||||||
await add_SubTaker_project(db, domain, project_id, True)
|
await asset_add_project(root_domain, asset_collection_list[a], a, db, project_id)
|
||||||
await add_PageMonitoring_project(db, domain, project_id, True)
|
|
||||||
await add_sensitive_project(db, domain, project_id, True)
|
|
||||||
await add_url_project(db, domain, project_id, True)
|
|
||||||
await add_crawler_project(db, domain, project_id, True)
|
|
||||||
|
|
||||||
|
|
||||||
async def add_asset_project(db, domain, project_id, updata=False):
|
async def asset_add_project(root_domain, db_key, doc_name, db, project_id):
|
||||||
try:
|
regex_patterns = [f".*{domain}.*" for domain in root_domain]
|
||||||
if updata:
|
pattern = "|".join(regex_patterns)
|
||||||
query = {"$or": [{"project": ""}, {"project": project_id}]}
|
# 构建查询条件
|
||||||
else:
|
query = {
|
||||||
query = {"project": {"$eq": ""}}
|
"$and": [
|
||||||
cursor: AsyncIOMotorCursor = ((db['asset'].find(query, {
|
{
|
||||||
"_id": 0, "id": {"$toString": "$_id"},
|
"$or": [
|
||||||
"url": 1,
|
{key: {"$regex": pattern, "$options": "i"}} for key in db_key
|
||||||
"host": 1,
|
]
|
||||||
"project": 1,
|
},
|
||||||
})))
|
{"project": {"$exists": True, "$eq": ""}}
|
||||||
result = await cursor.to_list(length=None)
|
]
|
||||||
logger.debug(f"asset project null number is {len(result)}")
|
}
|
||||||
if len(result) != 0:
|
update_query = {
|
||||||
domain_root_list = []
|
|
||||||
for d in domain.split("\n"):
|
|
||||||
u = get_root_domain(d)
|
|
||||||
if u not in domain_root_list:
|
|
||||||
domain_root_list.append(u)
|
|
||||||
for r in result:
|
|
||||||
url = ""
|
|
||||||
if "url" in r:
|
|
||||||
url = r['url']
|
|
||||||
else:
|
|
||||||
url = r['host']
|
|
||||||
if url != "":
|
|
||||||
targer_url = get_root_domain(url)
|
|
||||||
if targer_url in domain_root_list:
|
|
||||||
update_document = {
|
|
||||||
"$set": {
|
"$set": {
|
||||||
"project": project_id,
|
"project": project_id
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
await db['asset'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
result = await db[doc_name].update_many(query, update_query)
|
||||||
else:
|
# 打印更新的文档数量
|
||||||
if r["project"] != "":
|
logger.info(f"Updated {doc_name} {result.modified_count} documents")
|
||||||
update_document = {
|
|
||||||
|
|
||||||
|
async def asset_update_project(root_domain, db_key, doc_name, db, project_id):
|
||||||
|
regex_patterns = [f".*{domain}.*" for domain in root_domain]
|
||||||
|
pattern = "|".join(regex_patterns)
|
||||||
|
# 构建查询条件
|
||||||
|
query = {
|
||||||
|
"$and": [
|
||||||
|
{"project": project_id},
|
||||||
|
{
|
||||||
|
"$nor": [
|
||||||
|
{key: {"$regex": pattern, "$options": "i"}} for key in db_key
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
update_query = {
|
||||||
"$set": {
|
"$set": {
|
||||||
"project": "",
|
"project": ""
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
await db['asset'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
result = await db[doc_name].update_many(query, update_query)
|
||||||
except Exception as e:
|
# 打印更新的文档数量
|
||||||
logger.error(f"add_asset_project error:{e}")
|
logger.info(f"Updated {doc_name} {result.modified_count} documents to null ")
|
||||||
|
await asset_add_project(root_domain, db_key, doc_name, db, project_id)
|
||||||
|
|
||||||
async def add_subdomain_project(db, domain, project_id, updata=False):
|
|
||||||
try:
|
|
||||||
if updata:
|
|
||||||
query = {"$or": [{"project": ""}, {"project": project_id}]}
|
|
||||||
else:
|
|
||||||
query = {"project": {"$eq": ""}}
|
|
||||||
cursor: AsyncIOMotorCursor = ((db['subdomain'].find(query, {
|
|
||||||
"_id": 0, "id": {"$toString": "$_id"},
|
|
||||||
"host": 1,
|
|
||||||
"project": 1
|
|
||||||
})))
|
|
||||||
result = await cursor.to_list(length=None)
|
|
||||||
logger.debug(f"subdomain project null number is {len(result)}")
|
|
||||||
if len(result) != 0:
|
|
||||||
domain_root_list = []
|
|
||||||
for d in domain.split("\n"):
|
|
||||||
u = get_root_domain(d)
|
|
||||||
if u not in domain_root_list:
|
|
||||||
domain_root_list.append(u)
|
|
||||||
for r in result:
|
|
||||||
url = r['host']
|
|
||||||
if url != "":
|
|
||||||
targer_url = get_root_domain(url)
|
|
||||||
if targer_url in domain_root_list:
|
|
||||||
update_document = {
|
|
||||||
"$set": {
|
|
||||||
"project": project_id,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await db['subdomain'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
|
||||||
else:
|
|
||||||
if r["project"] != "":
|
|
||||||
update_document = {
|
|
||||||
"$set": {
|
|
||||||
"project": "",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await db['subdomain'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"add_subdomain_project error:{e}")
|
|
||||||
|
|
||||||
|
|
||||||
async def add_url_project(db, domain, project_id, updata=False):
|
|
||||||
try:
|
|
||||||
if updata:
|
|
||||||
query = {"$or": [{"project": ""}, {"project": project_id}]}
|
|
||||||
else:
|
|
||||||
query = {"project": {"$eq": ""}}
|
|
||||||
cursor: AsyncIOMotorCursor = ((db['UrlScan'].find(query, {
|
|
||||||
"_id": 0, "id": {"$toString": "$_id"},
|
|
||||||
"input": 1,
|
|
||||||
"project": 1
|
|
||||||
})))
|
|
||||||
result = await cursor.to_list(length=None)
|
|
||||||
logger.debug(f"url project null number is {len(result)}")
|
|
||||||
if len(result) != 0:
|
|
||||||
domain_root_list = []
|
|
||||||
for d in domain.split("\n"):
|
|
||||||
u = get_root_domain(d)
|
|
||||||
if u not in domain_root_list:
|
|
||||||
domain_root_list.append(u)
|
|
||||||
for r in result:
|
|
||||||
url = r['input']
|
|
||||||
if url != "":
|
|
||||||
targer_url = get_root_domain(url)
|
|
||||||
if targer_url in domain_root_list:
|
|
||||||
update_document = {
|
|
||||||
"$set": {
|
|
||||||
"project": project_id,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await db['UrlScan'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
|
||||||
else:
|
|
||||||
if r["project"] != "":
|
|
||||||
update_document = {
|
|
||||||
"$set": {
|
|
||||||
"project": "",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await db['UrlScan'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"add_url_project error:{e}")
|
|
||||||
|
|
||||||
|
|
||||||
async def add_crawler_project(db, domain, project_id, updata=False):
|
|
||||||
try:
|
|
||||||
if updata:
|
|
||||||
query = {"$or": [{"project": ""}, {"project": project_id}]}
|
|
||||||
else:
|
|
||||||
query = {"project": {"$eq": ""}}
|
|
||||||
cursor: AsyncIOMotorCursor = ((db['crawler'].find(query, {
|
|
||||||
"_id": 0, "id": {"$toString": "$_id"},
|
|
||||||
"url": 1,
|
|
||||||
"project": 1
|
|
||||||
})))
|
|
||||||
result = await cursor.to_list(length=None)
|
|
||||||
logger.debug(f"crawler project null number is {len(result)}")
|
|
||||||
if len(result) != 0:
|
|
||||||
domain_root_list = []
|
|
||||||
for d in domain.split("\n"):
|
|
||||||
u = get_root_domain(d)
|
|
||||||
if u not in domain_root_list:
|
|
||||||
domain_root_list.append(u)
|
|
||||||
for r in result:
|
|
||||||
url = r['url']
|
|
||||||
if url != "":
|
|
||||||
targer_url = get_root_domain(url)
|
|
||||||
if targer_url in domain_root_list:
|
|
||||||
update_document = {
|
|
||||||
"$set": {
|
|
||||||
"project": project_id,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await db['crawler'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
|
||||||
else:
|
|
||||||
if r["project"] != "":
|
|
||||||
update_document = {
|
|
||||||
"$set": {
|
|
||||||
"project": "",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await db['crawler'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"add_crawler_project error:{e}")
|
|
||||||
|
|
||||||
|
|
||||||
async def add_sensitive_project(db, domain, project_id, updata=False):
|
|
||||||
try:
|
|
||||||
if updata:
|
|
||||||
query = {"$or": [{"project": ""}, {"project": project_id}]}
|
|
||||||
else:
|
|
||||||
query = {"project": {"$eq": ""}}
|
|
||||||
cursor: AsyncIOMotorCursor = ((db['SensitiveResult'].find(query, {
|
|
||||||
"_id": 0, "id": {"$toString": "$_id"},
|
|
||||||
"url": 1,
|
|
||||||
"project": 1
|
|
||||||
})))
|
|
||||||
result = await cursor.to_list(length=None)
|
|
||||||
logger.debug(f"sensitive project null number is {len(result)}")
|
|
||||||
if len(result) != 0:
|
|
||||||
domain_root_list = []
|
|
||||||
for d in domain.split("\n"):
|
|
||||||
u = get_root_domain(d)
|
|
||||||
if u not in domain_root_list:
|
|
||||||
domain_root_list.append(u)
|
|
||||||
for r in result:
|
|
||||||
url = r['url']
|
|
||||||
if url != "":
|
|
||||||
targer_url = get_root_domain(url)
|
|
||||||
if targer_url in domain_root_list:
|
|
||||||
update_document = {
|
|
||||||
"$set": {
|
|
||||||
"project": project_id,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await db['SensitiveResult'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
|
||||||
else:
|
|
||||||
if r["project"] != "":
|
|
||||||
update_document = {
|
|
||||||
"$set": {
|
|
||||||
"project": "",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await db['SensitiveResult'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"add_sensitive_project error:{e}")
|
|
||||||
|
|
||||||
|
|
||||||
async def add_dir_project(db, domain, project_id, updata=False):
|
|
||||||
try:
|
|
||||||
if updata:
|
|
||||||
query = {"$or": [{"project": ""}, {"project": project_id}]}
|
|
||||||
else:
|
|
||||||
query = {"project": {"$eq": ""}}
|
|
||||||
cursor: AsyncIOMotorCursor = ((db['DirScanResult'].find(query, {
|
|
||||||
"_id": 0, "id": {"$toString": "$_id"},
|
|
||||||
"url": 1,
|
|
||||||
"project": 1
|
|
||||||
})))
|
|
||||||
result = await cursor.to_list(length=None)
|
|
||||||
logger.debug(f"dir project null number is {len(result)}")
|
|
||||||
if len(result) != 0:
|
|
||||||
domain_root_list = []
|
|
||||||
for d in domain.split("\n"):
|
|
||||||
u = get_root_domain(d)
|
|
||||||
if u not in domain_root_list:
|
|
||||||
domain_root_list.append(u)
|
|
||||||
for r in result:
|
|
||||||
url = r['url']
|
|
||||||
if url != "":
|
|
||||||
targer_url = get_root_domain(url)
|
|
||||||
if targer_url in domain_root_list:
|
|
||||||
update_document = {
|
|
||||||
"$set": {
|
|
||||||
"project": project_id,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await db['DirScanResult'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
|
||||||
else:
|
|
||||||
if r["project"] != "":
|
|
||||||
update_document = {
|
|
||||||
"$set": {
|
|
||||||
"project": "",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await db['DirScanResult'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"add_dir_project error:{e}")
|
|
||||||
|
|
||||||
|
|
||||||
async def add_vul_project(db, domain, project_id, updata=False):
|
|
||||||
try:
|
|
||||||
if updata:
|
|
||||||
query = {"$or": [{"project": ""}, {"project": project_id}]}
|
|
||||||
else:
|
|
||||||
query = {"project": {"$eq": ""}}
|
|
||||||
cursor: AsyncIOMotorCursor = ((db['vulnerability'].find(query, {
|
|
||||||
"_id": 0, "id": {"$toString": "$_id"},
|
|
||||||
"url": 1,
|
|
||||||
"project": 1
|
|
||||||
})))
|
|
||||||
result = await cursor.to_list(length=None)
|
|
||||||
logger.debug(f"vul project null number is {len(result)}")
|
|
||||||
if len(result) != 0:
|
|
||||||
domain_root_list = []
|
|
||||||
for d in domain.split("\n"):
|
|
||||||
u = get_root_domain(d)
|
|
||||||
if u not in domain_root_list:
|
|
||||||
domain_root_list.append(u)
|
|
||||||
for r in result:
|
|
||||||
url = r['url']
|
|
||||||
if url != "":
|
|
||||||
targer_url = get_root_domain(url)
|
|
||||||
if targer_url in domain_root_list:
|
|
||||||
update_document = {
|
|
||||||
"$set": {
|
|
||||||
"project": project_id,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await db['vulnerability'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
|
||||||
else:
|
|
||||||
if r["project"] != "":
|
|
||||||
update_document = {
|
|
||||||
"$set": {
|
|
||||||
"project": "",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await db['vulnerability'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"add_vul_project error:{e}")
|
|
||||||
|
|
||||||
|
|
||||||
async def add_PageMonitoring_project(db, domain, project_id, updata=False):
|
|
||||||
try:
|
|
||||||
if updata:
|
|
||||||
query = {"$or": [{"project": ""}, {"project": project_id}]}
|
|
||||||
else:
|
|
||||||
query = {"project": {"$eq": ""}}
|
|
||||||
cursor: AsyncIOMotorCursor = ((db['PageMonitoring'].find(query, {
|
|
||||||
"_id": 0, "id": {"$toString": "$_id"},
|
|
||||||
"url": 1,
|
|
||||||
"project": 1
|
|
||||||
})))
|
|
||||||
result = await cursor.to_list(length=None)
|
|
||||||
logger.debug(f"PageMonitoring project null number is {len(result)}")
|
|
||||||
if len(result) != 0:
|
|
||||||
domain_root_list = []
|
|
||||||
for d in domain.split("\n"):
|
|
||||||
u = get_root_domain(d)
|
|
||||||
if u not in domain_root_list:
|
|
||||||
domain_root_list.append(u)
|
|
||||||
for r in result:
|
|
||||||
url = r['url']
|
|
||||||
if url != "":
|
|
||||||
targer_url = get_root_domain(url)
|
|
||||||
if targer_url in domain_root_list:
|
|
||||||
update_document = {
|
|
||||||
"$set": {
|
|
||||||
"project": project_id,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await db['PageMonitoring'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
|
||||||
else:
|
|
||||||
if r["project"] != "":
|
|
||||||
update_document = {
|
|
||||||
"$set": {
|
|
||||||
"project": "",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await db['PageMonitoring'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"add_PageMonitoring_project error:{e}")
|
|
||||||
|
|
||||||
|
|
||||||
async def add_SubTaker_project(db, domain, project_id, updata=False):
|
|
||||||
try:
|
|
||||||
if updata:
|
|
||||||
query = {"$or": [{"project": ""}, {"project": project_id}]}
|
|
||||||
else:
|
|
||||||
query = {"project": {"$eq": ""}}
|
|
||||||
cursor: AsyncIOMotorCursor = ((db['SubdoaminTakerResult'].find(query, {
|
|
||||||
"_id": 0, "id": {"$toString": "$_id"},
|
|
||||||
"Input": 1,
|
|
||||||
"project": 1
|
|
||||||
})))
|
|
||||||
result = await cursor.to_list(length=None)
|
|
||||||
logger.debug(f"SubTaker project null number is {len(result)}")
|
|
||||||
if len(result) != 0:
|
|
||||||
domain_root_list = []
|
|
||||||
for d in domain.split("\n"):
|
|
||||||
u = get_root_domain(d)
|
|
||||||
if u not in domain_root_list:
|
|
||||||
domain_root_list.append(u)
|
|
||||||
for r in result:
|
|
||||||
url = r['input']
|
|
||||||
if url != "":
|
|
||||||
targer_url = get_root_domain(url)
|
|
||||||
if targer_url in domain_root_list:
|
|
||||||
update_document = {
|
|
||||||
"$set": {
|
|
||||||
"project": project_id,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await db['SubdoaminTakerResult'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
|
||||||
else:
|
|
||||||
if r["project"] != "":
|
|
||||||
update_document = {
|
|
||||||
"$set": {
|
|
||||||
"project": "",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await db['SubdoaminTakerResult'].update_one({"_id": ObjectId(r['id'])}, update_document)
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"add_SubTaker_project error:{e}")
|
|
||||||
|
|
||||||
|
|
||||||
async def update_project(domain, project_id):
|
|
||||||
async for db in get_mongo_db():
|
|
||||||
await add_asset_project(db, domain, project_id)
|
|
||||||
await add_subdomain_project(db, domain, project_id)
|
|
||||||
await add_dir_project(db, domain, project_id)
|
|
||||||
await add_vul_project(db, domain, project_id)
|
|
||||||
await add_SubTaker_project(db, domain, project_id)
|
|
||||||
await add_PageMonitoring_project(db, domain, project_id)
|
|
||||||
await add_sensitive_project(db, domain, project_id)
|
|
||||||
await add_url_project(db, domain, project_id)
|
|
||||||
await add_crawler_project(db, domain, project_id)
|
|
||||||
|
|
||||||
|
|
||||||
async def delete_asset_project(db, collection, project_id):
|
async def delete_asset_project(db, collection, project_id):
|
||||||
|
|
|
@ -16,7 +16,7 @@ from core.apscheduler_handler import scheduler
|
||||||
from core.db import get_mongo_db
|
from core.db import get_mongo_db
|
||||||
from core.redis_handler import get_redis_pool, check_redis_task_target_is_null
|
from core.redis_handler import get_redis_pool, check_redis_task_target_is_null
|
||||||
from core.util import *
|
from core.util import *
|
||||||
from api.node import get_redis_online_data
|
from api.node import get_redis_online_data, get_node_all
|
||||||
from api.page_monitoring import get_page_monitoring_data
|
from api.page_monitoring import get_page_monitoring_data
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
|
|
||||||
|
@ -254,7 +254,7 @@ async def create_scan_task(request_data, id, targetList, redis_con):
|
||||||
try:
|
try:
|
||||||
request_data["id"] = str(id)
|
request_data["id"] = str(id)
|
||||||
if request_data['allNode']:
|
if request_data['allNode']:
|
||||||
request_data["node"] = await get_redis_online_data(redis_con)
|
request_data["node"] = await get_node_all(redis_con)
|
||||||
|
|
||||||
keys_to_delete = [
|
keys_to_delete = [
|
||||||
f"TaskInfo:tmp:{id}",
|
f"TaskInfo:tmp:{id}",
|
||||||
|
|
|
@ -4,6 +4,7 @@
|
||||||
# @version:
|
# @version:
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
from bson import ObjectId
|
from bson import ObjectId
|
||||||
|
|
||||||
|
@ -58,6 +59,7 @@ def get_finger():
|
||||||
d.pop('_id', None)
|
d.pop('_id', None)
|
||||||
return data
|
return data
|
||||||
|
|
||||||
|
|
||||||
def get_project_data():
|
def get_project_data():
|
||||||
project_path = os.path.join(combined_directory, "ScopeSentry.project.json")
|
project_path = os.path.join(combined_directory, "ScopeSentry.project.json")
|
||||||
data = read_json_file(project_path)
|
data = read_json_file(project_path)
|
||||||
|
@ -68,6 +70,7 @@ def get_project_data():
|
||||||
tmp = []
|
tmp = []
|
||||||
for t in d['target'].split('\n'):
|
for t in d['target'].split('\n'):
|
||||||
root_domain = get_root_domain(t)
|
root_domain = get_root_domain(t)
|
||||||
|
if root_domain is not None and root_domain != "":
|
||||||
if root_domain not in tmp:
|
if root_domain not in tmp:
|
||||||
tmp.append(root_domain)
|
tmp.append(root_domain)
|
||||||
d["root_domains"] = tmp
|
d["root_domains"] = tmp
|
||||||
|
@ -503,3 +506,50 @@ def get_fingerprint_data():
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
logger.error("文件不存在")
|
logger.error("文件不存在")
|
||||||
return json.loads(fingerprint)
|
return json.loads(fingerprint)
|
||||||
|
|
||||||
|
|
||||||
|
def get_root_domain(url):
|
||||||
|
# 如果URL不带协议,添加一个默认的http协议
|
||||||
|
global root_domain
|
||||||
|
if not url.startswith(('http://', 'https://')):
|
||||||
|
url = 'http://' + url
|
||||||
|
|
||||||
|
parsed_url = urlparse(url)
|
||||||
|
|
||||||
|
# 检查是否为IP地址
|
||||||
|
try:
|
||||||
|
# 使用ip_address来检查
|
||||||
|
from ipaddress import ip_address
|
||||||
|
ip_address(parsed_url.netloc)
|
||||||
|
return parsed_url.netloc # 如果是IP地址,直接返回
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
domain_parts = parsed_url.netloc.split('.')
|
||||||
|
|
||||||
|
# 复合域名列表
|
||||||
|
compound_domains = [
|
||||||
|
'com.cn', 'net.cn', 'org.cn', 'gov.cn', 'edu.cn', 'ac.cn', 'mil.cn',
|
||||||
|
'co.uk', 'org.uk', 'net.uk', 'gov.uk', 'ac.uk', 'sch.uk',
|
||||||
|
'co.jp', 'ne.jp', 'or.jp', 'go.jp', 'ac.jp', 'ad.jp',
|
||||||
|
'com.de', 'org.de', 'net.de', 'gov.de',
|
||||||
|
'com.ca', 'net.ca', 'org.ca', 'gov.ca',
|
||||||
|
'com.au', 'net.au', 'org.au', 'gov.au', 'edu.au',
|
||||||
|
'com.fr', 'net.fr', 'org.fr', 'gov.fr',
|
||||||
|
'com.br', 'com.mx', 'com.ar', 'com.ru',
|
||||||
|
'co.in', 'co.za',
|
||||||
|
'co.kr', 'com.tw'
|
||||||
|
]
|
||||||
|
|
||||||
|
# 检查是否为复合域名
|
||||||
|
is_compound_domain = False
|
||||||
|
for compound_domain in compound_domains:
|
||||||
|
if domain_parts[-2:] == compound_domain.split('.'):
|
||||||
|
is_compound_domain = True
|
||||||
|
root_domain = '.'.join(domain_parts[-3:])
|
||||||
|
break
|
||||||
|
|
||||||
|
if not is_compound_domain:
|
||||||
|
root_domain = '.'.join(domain_parts[-2:])
|
||||||
|
|
||||||
|
return root_domain
|
Loading…
Reference in New Issue