add
This commit is contained in:
parent
0eb0f122a2
commit
0618ef2dc7
|
@ -6,9 +6,7 @@
|
|||
# -------------------------------------------
|
||||
from fastapi import APIRouter, Depends
|
||||
from motor.motor_asyncio import AsyncIOMotorCursor
|
||||
from pymongo import DESCENDING
|
||||
from api.users import verify_token
|
||||
from core.config import POC_LIST
|
||||
from core.db import get_mongo_db
|
||||
from core.util import search_to_mongodb
|
||||
from loguru import logger
|
||||
|
|
|
@ -177,14 +177,14 @@ async def do_asset_deduplication():
|
|||
"filters": [],
|
||||
"groups": ["url", "status", "msg"]
|
||||
},
|
||||
"PageMonitoring": {
|
||||
"filters": [],
|
||||
"groups": ["url"]
|
||||
},
|
||||
"SensitiveResult": {
|
||||
"filters": [],
|
||||
"groups": ["url"]
|
||||
},#############
|
||||
# "PageMonitoring": {
|
||||
# "filters": [],
|
||||
# "groups": ["url"]
|
||||
# },
|
||||
# "SensitiveResult": {
|
||||
# "filters": [],
|
||||
# "groups": ["url"]
|
||||
# },
|
||||
"SubdoaminTakerResult": {
|
||||
"filters": [],
|
||||
"groups": ["input", "value"]
|
||||
|
@ -196,7 +196,7 @@ async def do_asset_deduplication():
|
|||
"asset": {
|
||||
"filters": [],
|
||||
"groups": [""]
|
||||
},################
|
||||
},
|
||||
"crawler": {
|
||||
"filters": [],
|
||||
"groups": ["url", "body"]
|
||||
|
@ -205,14 +205,24 @@ async def do_asset_deduplication():
|
|||
"filters": [],
|
||||
"groups": ["host", "type", "ip"]
|
||||
},
|
||||
"vulnerability": {
|
||||
"filters": [],
|
||||
"groups": ["url", "vulnid", "matched"]
|
||||
}
|
||||
# "vulnerability": {
|
||||
# "filters": [],
|
||||
# "groups": ["url", "vulnid", "matched"]
|
||||
# }
|
||||
}
|
||||
for r in result:
|
||||
if result[r]:
|
||||
await asset_data_dedup(db, r, )
|
||||
if r in f_g_k:
|
||||
if r == "asset":
|
||||
# http资产去重
|
||||
http_filter = [{"type": {"$ne": "other"}}]
|
||||
http_group = ["url", "statuscode", "hashes.body_mmh3"]
|
||||
await asset_data_dedup(db, r, http_filter, http_group)
|
||||
other_filter = [{"type":"other"}]
|
||||
other_group = ["host", "ip", "protocol"]
|
||||
await asset_data_dedup(db, r, other_filter, other_group)
|
||||
else:
|
||||
await asset_data_dedup(db, r, f_g_k[r]['filters'], f_g_k[r]['groups'])
|
||||
|
||||
|
||||
async def asset_data_dedup(db, collection_name, filters, groups):
|
||||
|
|
|
@ -10,31 +10,35 @@ from fastapi import APIRouter, Depends
|
|||
from motor.motor_asyncio import AsyncIOMotorCursor
|
||||
from api.users import verify_token
|
||||
from core.db import get_mongo_db
|
||||
from core.util import search_to_mongodb
|
||||
from core.util import search_to_mongodb, get_search_query
|
||||
from loguru import logger
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post("/dirscan/result/data")
|
||||
async def dirscan_data(request_data: dict, db=Depends(get_mongo_db), _: dict = Depends(verify_token)):
|
||||
try:
|
||||
search_query = request_data.get("search", "")
|
||||
page_index = request_data.get("pageIndex", 1)
|
||||
page_size = request_data.get("pageSize", 10)
|
||||
keyword = {
|
||||
'project': 'project',
|
||||
'statuscode': 'status',
|
||||
'url': 'url',
|
||||
'redirect': 'msg',
|
||||
'length': 'length'
|
||||
}
|
||||
query = await search_to_mongodb(search_query, keyword)
|
||||
if query == "" or query is None:
|
||||
query = await get_search_query("dir", request_data)
|
||||
if query == "":
|
||||
return {"message": "Search condition parsing error", "code": 500}
|
||||
query = query[0]
|
||||
total_count = await db['DirScanResult'].count_documents(query)
|
||||
cursor: AsyncIOMotorCursor = ((db['DirScanResult'].find(query, {"_id": 0, "id": {"$toString": "$_id"}, "url": 1, "status": 1, "msg":1, "length": 1})
|
||||
.sort([('_id', -1)])
|
||||
sort = request_data.get("sort", {})
|
||||
sort_by = [('_id', -1)]
|
||||
if sort != {}:
|
||||
if 'length' in sort:
|
||||
sort_value = sort['length']
|
||||
if sort_value is not None:
|
||||
if sort_value == "ascending":
|
||||
sort_value = 1
|
||||
else:
|
||||
sort_value = -1
|
||||
sort_by = [('length', sort_value)]
|
||||
cursor: AsyncIOMotorCursor = ((db['DirScanResult'].find(query, {"_id": 0, "id": {"$toString": "$_id"}, "url": 1,
|
||||
"status": 1, "msg": 1, "length": 1})
|
||||
.sort(sort_by)
|
||||
.skip((page_index - 1) * page_size)
|
||||
.limit(page_size)))
|
||||
result = await cursor.to_list(length=None)
|
||||
|
|
253
api/export.py
253
api/export.py
|
@ -124,7 +124,7 @@ async def export_data(request_data: dict, db=Depends(get_mongo_db), _: dict = De
|
|||
"file_size": ""
|
||||
})
|
||||
if result.inserted_id:
|
||||
background_tasks.add_task(export_data_from_mongodb, quantity, query, file_name, index, db)
|
||||
background_tasks.add_task(export_data_from_mongodb, quantity, query, file_name, index)
|
||||
return {"message": "Successfully added data export task", "code": 200}
|
||||
else:
|
||||
return {"message": "Failed to export data", "code": 500}
|
||||
|
@ -154,133 +154,134 @@ async def fetch_data(db, collection, query, quantity, project_list):
|
|||
return cursor
|
||||
|
||||
|
||||
async def export_data_from_mongodb(quantity, query, file_name, index, db):
|
||||
try:
|
||||
cursor = await fetch_data(db, index, query, quantity, Project_List)
|
||||
result = await cursor.to_list(length=None)
|
||||
relative_path = f'file/{file_name}.xlsx'
|
||||
file_path = os.path.join(os.getcwd(), relative_path)
|
||||
if index == "asset":
|
||||
http_columns = {
|
||||
"timestamp": "时间",
|
||||
"tlsdata": "TLS_Data",
|
||||
"hashes": "Hash",
|
||||
"cdnname": "Cdn_Name",
|
||||
"port": "端口",
|
||||
"url": "url",
|
||||
"title": "标题",
|
||||
"type": "类型",
|
||||
"error": "错误",
|
||||
"responsebody": "响应体",
|
||||
"host": "IP",
|
||||
"faviconmmh3": "图标Hash",
|
||||
"faviconpath": "faviconpath",
|
||||
"rawheaders": "响应头",
|
||||
"jarm": "jarm",
|
||||
"technologies": "technologies",
|
||||
"statuscode": "响应码",
|
||||
"contentlength": "contentlength",
|
||||
"cdn": "cdn",
|
||||
"webcheck": "webcheck",
|
||||
"project": "项目",
|
||||
"webfinger": "指纹",
|
||||
"iconcontent": "图标",
|
||||
"domain": "域名"
|
||||
async def export_data_from_mongodb(quantity, query, file_name, index):
|
||||
async for db in get_mongo_db():
|
||||
try:
|
||||
cursor = await fetch_data(db, index, query, quantity, Project_List)
|
||||
result = await cursor.to_list(length=None)
|
||||
relative_path = f'file/{file_name}.xlsx'
|
||||
file_path = os.path.join(os.getcwd(), relative_path)
|
||||
if index == "asset":
|
||||
http_columns = {
|
||||
"timestamp": "时间",
|
||||
"tlsdata": "TLS_Data",
|
||||
"hashes": "Hash",
|
||||
"cdnname": "Cdn_Name",
|
||||
"port": "端口",
|
||||
"url": "url",
|
||||
"title": "标题",
|
||||
"type": "类型",
|
||||
"error": "错误",
|
||||
"responsebody": "响应体",
|
||||
"host": "IP",
|
||||
"faviconmmh3": "图标Hash",
|
||||
"faviconpath": "faviconpath",
|
||||
"rawheaders": "响应头",
|
||||
"jarm": "jarm",
|
||||
"technologies": "technologies",
|
||||
"statuscode": "响应码",
|
||||
"contentlength": "contentlength",
|
||||
"cdn": "cdn",
|
||||
"webcheck": "webcheck",
|
||||
"project": "项目",
|
||||
"webfinger": "指纹",
|
||||
"iconcontent": "图标",
|
||||
"domain": "域名"
|
||||
}
|
||||
other_columns = {
|
||||
"timestamp": "时间",
|
||||
"host": "域名",
|
||||
"ip": "IP",
|
||||
"port": "端口",
|
||||
"protocol": "协议",
|
||||
"tls": "TLS",
|
||||
"transport": "transport",
|
||||
"version": "版本",
|
||||
"raw": "banner",
|
||||
"project": "项目",
|
||||
"type": "类型"
|
||||
}
|
||||
other_df = pd.DataFrame()
|
||||
http_df = pd.DataFrame()
|
||||
for doc in result:
|
||||
if doc["type"] == "other":
|
||||
other_df = pd.concat([other_df, pd.DataFrame([doc])], ignore_index=True)
|
||||
else:
|
||||
if doc['webfinger'] is not None:
|
||||
webfinger = []
|
||||
for webfinger_id in doc['webfinger']:
|
||||
webfinger.append(APP[webfinger_id])
|
||||
doc['webfinger'] = webfinger
|
||||
http_df = pd.concat([http_df, pd.DataFrame([doc])], ignore_index=True)
|
||||
try:
|
||||
excel_writer = pd.ExcelWriter(file_path, engine='xlsxwriter')
|
||||
http_df.rename(columns=http_columns, inplace=True)
|
||||
http_df.to_excel(excel_writer, sheet_name='HTTP Data', index=False)
|
||||
other_df.rename(columns=other_columns, inplace=True)
|
||||
other_df.to_excel(excel_writer, sheet_name='Other Data', index=False)
|
||||
excel_writer.close()
|
||||
except IllegalCharacterError as e:
|
||||
logger.error("导出内容有不可见字符,忽略此错误")
|
||||
else:
|
||||
columns = {}
|
||||
if index == "subdomain":
|
||||
columns = {'host': '域名', 'type': '解析类型', 'value': '解析值', 'ip': '解析IP', 'project': '项目',
|
||||
'time': '时间'}
|
||||
if index == "SubdoaminTakerResult":
|
||||
columns = {
|
||||
'input': '源域名', 'value': '解析值', 'cname': '接管类型', 'response': '响应体', 'project': '项目'
|
||||
}
|
||||
if index == "UrlScan":
|
||||
columns = {
|
||||
'input': '输入', 'source': '来源', 'outputtype': '输出类型', 'output': '输出',
|
||||
'statuscode': 'statuscode', 'length': 'length', 'time': '时间', 'project': '项目'
|
||||
}
|
||||
if index == "crawler":
|
||||
columns = {
|
||||
'url': 'URL', 'method': 'Method', 'body': 'Body', 'project': '项目'
|
||||
}
|
||||
if index == "SensitiveResult":
|
||||
columns = {
|
||||
'url': 'URL', 'sid': '规则名称', 'match': '匹配内容', 'project': '项目', 'body': '响应体',
|
||||
'color': '等级', 'time': '时间', 'md5': '响应体MD5'
|
||||
}
|
||||
if index == "DirScanResult":
|
||||
columns = {
|
||||
'url': 'URL', 'status': '响应码', 'msg': '跳转', 'project': '项目'
|
||||
}
|
||||
if index == "vulnerability":
|
||||
columns = {
|
||||
'url': 'URL', 'vulname': '漏洞', 'matched': '匹配', 'project': '项目', 'level': '危害等级',
|
||||
'time': '时间', 'request': '请求', 'response': '响应'
|
||||
}
|
||||
if index == "PageMonitoring":
|
||||
columns = {
|
||||
'url': 'URL', 'content': '响应体', 'hash': '响应体Hash', 'diff': 'Diff',
|
||||
'state': '状态', 'project': '项目', 'time': '时间'
|
||||
}
|
||||
try:
|
||||
df = pd.DataFrame(result)
|
||||
df.rename(columns=columns, inplace=True)
|
||||
df.to_excel(file_path, index=False)
|
||||
except IllegalCharacterError as e:
|
||||
logger.error("导出内容有不可见字符,忽略此错误")
|
||||
file_size = os.path.getsize(file_path) / (1024 * 1024) # kb
|
||||
update_document = {
|
||||
"$set": {
|
||||
"state": 1,
|
||||
"end_time": get_now_time(),
|
||||
"file_size": str(round(file_size, 2))
|
||||
}
|
||||
}
|
||||
other_columns = {
|
||||
"timestamp": "时间",
|
||||
"host": "域名",
|
||||
"ip": "IP",
|
||||
"port": "端口",
|
||||
"protocol": "协议",
|
||||
"tls": "TLS",
|
||||
"transport": "transport",
|
||||
"version": "版本",
|
||||
"raw": "banner",
|
||||
"project": "项目",
|
||||
"type": "类型"
|
||||
await db.export.update_one({"file_name": file_name}, update_document)
|
||||
except Exception as e:
|
||||
logger.error(str(e))
|
||||
update_document = {
|
||||
"$set": {
|
||||
"state": 2,
|
||||
}
|
||||
}
|
||||
other_df = pd.DataFrame()
|
||||
http_df = pd.DataFrame()
|
||||
for doc in result:
|
||||
if doc["type"] == "other":
|
||||
other_df = pd.concat([other_df, pd.DataFrame([doc])], ignore_index=True)
|
||||
else:
|
||||
if doc['webfinger'] is not None:
|
||||
webfinger = []
|
||||
for webfinger_id in doc['webfinger']:
|
||||
webfinger.append(APP[webfinger_id])
|
||||
doc['webfinger'] = webfinger
|
||||
http_df = pd.concat([http_df, pd.DataFrame([doc])], ignore_index=True)
|
||||
try:
|
||||
excel_writer = pd.ExcelWriter(file_path, engine='xlsxwriter')
|
||||
http_df.rename(columns=http_columns, inplace=True)
|
||||
http_df.to_excel(excel_writer, sheet_name='HTTP Data', index=False)
|
||||
other_df.rename(columns=other_columns, inplace=True)
|
||||
other_df.to_excel(excel_writer, sheet_name='Other Data', index=False)
|
||||
excel_writer.close()
|
||||
except IllegalCharacterError as e:
|
||||
logger.error("导出内容有不可见字符,忽略此错误")
|
||||
else:
|
||||
columns = {}
|
||||
if index == "subdomain":
|
||||
columns = {'host': '域名', 'type': '解析类型', 'value': '解析值', 'ip': '解析IP', 'project': '项目',
|
||||
'time': '时间'}
|
||||
if index == "SubdoaminTakerResult":
|
||||
columns = {
|
||||
'input': '源域名', 'value': '解析值', 'cname': '接管类型', 'response': '响应体', 'project': '项目'
|
||||
}
|
||||
if index == "UrlScan":
|
||||
columns = {
|
||||
'input': '输入', 'source': '来源', 'outputtype': '输出类型', 'output': '输出',
|
||||
'statuscode': 'statuscode', 'length': 'length', 'time': '时间', 'project': '项目'
|
||||
}
|
||||
if index == "crawler":
|
||||
columns = {
|
||||
'url': 'URL', 'method': 'Method', 'body': 'Body', 'project': '项目'
|
||||
}
|
||||
if index == "SensitiveResult":
|
||||
columns = {
|
||||
'url': 'URL', 'sid': '规则名称', 'match': '匹配内容', 'project': '项目', 'body': '响应体',
|
||||
'color': '等级', 'time': '时间', 'md5': '响应体MD5'
|
||||
}
|
||||
if index == "DirScanResult":
|
||||
columns = {
|
||||
'url': 'URL', 'status': '响应码', 'msg': '跳转', 'project': '项目'
|
||||
}
|
||||
if index == "vulnerability":
|
||||
columns = {
|
||||
'url': 'URL', 'vulname': '漏洞', 'matched': '匹配', 'project': '项目', 'level': '危害等级',
|
||||
'time': '时间', 'request': '请求', 'response': '响应'
|
||||
}
|
||||
if index == "PageMonitoring":
|
||||
columns = {
|
||||
'url': 'URL', 'content': '响应体', 'hash': '响应体Hash', 'diff': 'Diff',
|
||||
'state': '状态', 'project': '项目', 'time': '时间'
|
||||
}
|
||||
try:
|
||||
df = pd.DataFrame(result)
|
||||
df.rename(columns=columns, inplace=True)
|
||||
df.to_excel(file_path, index=False)
|
||||
except IllegalCharacterError as e:
|
||||
logger.error("导出内容有不可见字符,忽略此错误")
|
||||
file_size = os.path.getsize(file_path) / (1024 * 1024) # kb
|
||||
update_document = {
|
||||
"$set": {
|
||||
"state": 1,
|
||||
"end_time": get_now_time(),
|
||||
"file_size": str(round(file_size, 2))
|
||||
}
|
||||
}
|
||||
await db.export.update_one({"file_name": file_name}, update_document)
|
||||
except Exception as e:
|
||||
logger.error(str(e))
|
||||
update_document = {
|
||||
"$set": {
|
||||
"state": 2,
|
||||
}
|
||||
}
|
||||
await db.export.update_one({"file_name": file_name}, update_document)
|
||||
await db.export.update_one({"file_name": file_name}, update_document)
|
||||
|
||||
|
||||
@router.get("/export/record")
|
||||
|
|
|
@ -11,7 +11,6 @@ from pymongo import ASCENDING, DESCENDING
|
|||
from loguru import logger
|
||||
from core.redis_handler import refresh_config
|
||||
from core.util import *
|
||||
from core.config import POC_LIST
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
|
@ -124,7 +123,6 @@ async def update_poc_data(request_data: dict, db=Depends(get_mongo_db), _: dict
|
|||
result = await db.PocList.update_one({"_id": ObjectId(poc_id)}, update_document)
|
||||
# Check if the update was successful
|
||||
if result:
|
||||
POC_LIST[poc_id] = level
|
||||
await refresh_config('all', 'poc')
|
||||
return {"message": "Data updated successfully", "code": 200}
|
||||
else:
|
||||
|
@ -160,7 +158,6 @@ async def add_poc_data(request_data: dict, db=Depends(get_mongo_db), _: dict = D
|
|||
|
||||
# Check if the insertion was successful
|
||||
if result.inserted_id:
|
||||
POC_LIST[str(result.inserted_id)] = level
|
||||
await refresh_config('all', 'poc')
|
||||
return {"message": "Data added successfully", "code": 200}
|
||||
else:
|
||||
|
@ -186,9 +183,6 @@ async def delete_poc_rules(request_data: dict, db=Depends(get_mongo_db), _: dict
|
|||
|
||||
# Check if the deletion was successful
|
||||
if result.deleted_count > 0:
|
||||
for pid in poc_ids:
|
||||
if pid in POC_LIST:
|
||||
del POC_LIST[pid]
|
||||
return {"code": 200, "message": "Poc deleted successfully"}
|
||||
else:
|
||||
return {"code": 404, "message": "Poc not found"}
|
||||
|
|
|
@ -68,7 +68,7 @@ async def get_projects_data(request_data: dict, db=Depends(get_mongo_db), _: dic
|
|||
for result in results:
|
||||
result["AssetCount"] = result.get("AssetCount", 0)
|
||||
result_list[tag].append(result)
|
||||
background_tasks.add_task(update_project_count, db=db, id=result["id"])
|
||||
background_tasks.add_task(update_project_count, id=result["id"])
|
||||
|
||||
return {
|
||||
"code": 200,
|
||||
|
@ -79,15 +79,16 @@ async def get_projects_data(request_data: dict, db=Depends(get_mongo_db), _: dic
|
|||
}
|
||||
|
||||
|
||||
async def update_project_count(db, id):
|
||||
query = {"project": {"$eq": id}}
|
||||
total_count = await db['asset'].count_documents(query)
|
||||
update_document = {
|
||||
"$set": {
|
||||
"AssetCount": total_count
|
||||
async def update_project_count(id):
|
||||
async for db in get_mongo_db():
|
||||
query = {"project": {"$eq": id}}
|
||||
total_count = await db['asset'].count_documents(query)
|
||||
update_document = {
|
||||
"$set": {
|
||||
"AssetCount": total_count
|
||||
}
|
||||
}
|
||||
}
|
||||
await db.project.update_one({"_id": ObjectId(id)}, update_document)
|
||||
await db.project.update_one({"_id": ObjectId(id)}, update_document)
|
||||
|
||||
|
||||
@router.post("/project/content")
|
||||
|
@ -190,7 +191,7 @@ async def delete_project_rules(request_data: dict, db=Depends(get_mongo_db), _:
|
|||
pro_ids = request_data.get("ids", [])
|
||||
delA = request_data.get("delA", False)
|
||||
if delA:
|
||||
background_tasks.add_task(delete_asset, pro_ids, db, True)
|
||||
background_tasks.add_task(delete_asset, pro_ids, True)
|
||||
obj_ids = [ObjectId(poc_id) for poc_id in pro_ids]
|
||||
result = await db.project.delete_many({"_id": {"$in": obj_ids}})
|
||||
await db.ProjectTargetData.delete_many({"id": {"$in": pro_ids}})
|
||||
|
|
|
@ -16,7 +16,7 @@ from core.db import get_mongo_db
|
|||
from core.redis_handler import refresh_config
|
||||
from loguru import logger
|
||||
|
||||
from core.util import search_to_mongodb
|
||||
from core.util import search_to_mongodb, get_search_query
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
@ -223,23 +223,13 @@ async def get_sensitive_result_rules(request_data: dict, db=Depends(get_mongo_db
|
|||
|
||||
|
||||
@router.post("/sensitive/result/data2")
|
||||
async def get_sensitive_result_rules(request_data: dict, db=Depends(get_mongo_db), _: dict = Depends(verify_token)):
|
||||
async def get_sensitive_result_data2(request_data: dict, db=Depends(get_mongo_db), _: dict = Depends(verify_token)):
|
||||
try:
|
||||
search_query = request_data.get("search", "")
|
||||
page_index = request_data.get("pageIndex", 1)
|
||||
page_size = request_data.get("pageSize", 10)
|
||||
keyword = {
|
||||
'url': 'url',
|
||||
'sname': 'sid',
|
||||
"body": "body",
|
||||
"info": "match",
|
||||
'project': 'project',
|
||||
'md5': 'md5'
|
||||
}
|
||||
query = await search_to_mongodb(search_query, keyword)
|
||||
if query == "" or query is None:
|
||||
query = await get_search_query("sens", request_data)
|
||||
if query == "":
|
||||
return {"message": "Search condition parsing error", "code": 500}
|
||||
query = query[0]
|
||||
total_count = await db['SensitiveResult'].count_documents(query)
|
||||
pipeline = [
|
||||
{
|
||||
|
|
35
api/task.py
35
api/task.py
|
@ -181,7 +181,7 @@ async def delete_task(request_data: dict, db=Depends(get_mongo_db), _: dict = De
|
|||
scheduler.remove_job(task_id)
|
||||
await db.ScheduledTasks.delete_many({"id": {"$in": task_ids}})
|
||||
if delA:
|
||||
background_tasks.add_task(delete_asset, task_ids, db)
|
||||
background_tasks.add_task(delete_asset, task_ids)
|
||||
await redis_con.delete(*redis_key)
|
||||
# Delete the SensitiveRule documents based on the provided IDs
|
||||
result = await db.task.delete_many({"_id": {"$in": obj_ids}})
|
||||
|
@ -558,19 +558,20 @@ async def scheduler_scan_task(id):
|
|||
await create_scan_task(doc, task_id, targetList, redis)
|
||||
|
||||
|
||||
async def delete_asset(task_ids, db, is_project = False):
|
||||
key = ["asset", "subdomain", "SubdoaminTakerResult", "UrlScan", "crawler", "SensitiveResult", "DirScanResult", "vulnerability", "PageMonitoring"]
|
||||
del_query = {"taskId": {"$in": task_ids}}
|
||||
if is_project:
|
||||
del_query = {
|
||||
"$or": [
|
||||
{"taskId": {"$in": task_ids}},
|
||||
{"project": {"$in": task_ids}}
|
||||
]
|
||||
}
|
||||
for k in key:
|
||||
result = await db[k].delete_many(del_query)
|
||||
if result.deleted_count > 0:
|
||||
logger.info("Deleted {} {} documents".format(k, result.deleted_count))
|
||||
else:
|
||||
logger.info("Deleted {} None documents".format(k))
|
||||
async def delete_asset(task_ids, is_project = False):
|
||||
async for db in get_mongo_db():
|
||||
key = ["asset", "subdomain", "SubdoaminTakerResult", "UrlScan", "crawler", "SensitiveResult", "DirScanResult", "vulnerability", "PageMonitoring"]
|
||||
del_query = {"taskId": {"$in": task_ids}}
|
||||
if is_project:
|
||||
del_query = {
|
||||
"$or": [
|
||||
{"taskId": {"$in": task_ids}},
|
||||
{"project": {"$in": task_ids}}
|
||||
]
|
||||
}
|
||||
for k in key:
|
||||
result = await db[k].delete_many(del_query)
|
||||
if result.deleted_count > 0:
|
||||
logger.info("Deleted {} {} documents".format(k, result.deleted_count))
|
||||
else:
|
||||
logger.info("Deleted {} None documents".format(k))
|
|
@ -9,33 +9,20 @@ from fastapi import APIRouter, Depends
|
|||
from motor.motor_asyncio import AsyncIOMotorCursor
|
||||
from pymongo import DESCENDING
|
||||
from api.users import verify_token
|
||||
from core.config import POC_LIST
|
||||
from core.db import get_mongo_db
|
||||
from core.util import search_to_mongodb
|
||||
from core.util import search_to_mongodb, get_search_query
|
||||
from loguru import logger
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post("/vul/data")
|
||||
async def get_vul_data(request_data: dict, db=Depends(get_mongo_db), _: dict = Depends(verify_token)):
|
||||
try:
|
||||
search_query = request_data.get("search", "")
|
||||
page_index = request_data.get("pageIndex", 1)
|
||||
page_size = request_data.get("pageSize", 10)
|
||||
# MongoDB collection for SensitiveRule
|
||||
# Fuzzy search based on the name field
|
||||
keyword = {
|
||||
'url': 'url',
|
||||
'vulname': 'vulname',
|
||||
'project': 'project',
|
||||
'matched': 'matched',
|
||||
'request': 'request',
|
||||
'response': 'response',
|
||||
'level': 'level'
|
||||
}
|
||||
query = await search_to_mongodb(search_query, keyword)
|
||||
if query == "" or query is None:
|
||||
query = await get_search_query("vul", request_data)
|
||||
if query == "":
|
||||
return {"message": "Search condition parsing error", "code": 500}
|
||||
query = query[0]
|
||||
# Get the total count of documents matching the search criteria
|
||||
total_count = await db.vulnerability.count_documents(query)
|
||||
if total_count == 0:
|
||||
|
@ -60,9 +47,8 @@ async def get_vul_data(request_data: dict, db=Depends(get_mongo_db), _: dict = D
|
|||
"time": doc["time"],
|
||||
"request": doc["request"],
|
||||
"response": doc["response"],
|
||||
"level": doc['level']
|
||||
}
|
||||
if doc["vulnid"] in POC_LIST:
|
||||
data["level"] = POC_LIST[doc["vulnid"]]
|
||||
response_data.append(data)
|
||||
return {
|
||||
"code": 200,
|
||||
|
|
|
@ -27,7 +27,6 @@ NODE_TIMEOUT = 50
|
|||
TOTAL_LOGS = 1000
|
||||
APP = {}
|
||||
SensitiveRuleList = {}
|
||||
POC_LIST = {}
|
||||
Project_List = {}
|
||||
def set_timezone(t):
|
||||
global TIMEZONE
|
||||
|
|
|
@ -133,7 +133,6 @@ async def create_database():
|
|||
{"id": "page_monitoring", "name": "Page Monitoring", 'hour': 24, 'type': 'Page Monitoring', 'state': True})
|
||||
await get_fingerprint(client[DATABASE_NAME])
|
||||
await get_sens_rule(client[DATABASE_NAME])
|
||||
await get_pocList(client[DATABASE_NAME])
|
||||
await get_project(client[DATABASE_NAME])
|
||||
except Exception as e:
|
||||
# 处理异常
|
||||
|
@ -166,14 +165,6 @@ async def get_sens_rule(client):
|
|||
}
|
||||
|
||||
|
||||
async def get_pocList(client):
|
||||
collection = client["PocList"]
|
||||
cursor = collection.find({}, {"_id": 1, "level": 1})
|
||||
async for document in cursor:
|
||||
document['id'] = str(document['_id'])
|
||||
POC_LIST[document['id']] = document['level']
|
||||
|
||||
|
||||
async def get_project(client):
|
||||
collection = client["project"]
|
||||
cursor = collection.find({}, {"_id": 1, "name": 1})
|
||||
|
|
49
core/util.py
49
core/util.py
|
@ -321,6 +321,55 @@ async def search_to_mongodb(expression_raw, keyword):
|
|||
logger.error(e)
|
||||
return ""
|
||||
|
||||
async def get_search_query(name, request_data):
|
||||
search_query = request_data.get("search", "")
|
||||
search_key_v = {
|
||||
'sens':{
|
||||
'url': 'url',
|
||||
'sname': 'sid',
|
||||
"body": "body",
|
||||
"info": "match",
|
||||
'project': 'project',
|
||||
'md5': 'md5'
|
||||
},
|
||||
'dir': {
|
||||
'project': 'project',
|
||||
'statuscode': 'status',
|
||||
'url': 'url',
|
||||
'redirect': 'msg',
|
||||
'length': 'length'
|
||||
},
|
||||
'vul': {
|
||||
'url': 'url',
|
||||
'vulname': 'vulname',
|
||||
'project': 'project',
|
||||
'matched': 'matched',
|
||||
'request': 'request',
|
||||
'response': 'response',
|
||||
'level': 'level'
|
||||
}
|
||||
}
|
||||
keyword = search_key_v[name]
|
||||
query = await search_to_mongodb(search_query, keyword)
|
||||
if query == "" or query is None:
|
||||
return ""
|
||||
query = query[0]
|
||||
filter_key = ['color', 'status', 'level']
|
||||
filter = request_data.get("filter", {})
|
||||
if filter:
|
||||
query["$and"] = []
|
||||
for f in filter:
|
||||
if f in filter_key:
|
||||
tmp_or = []
|
||||
for v in filter[f]:
|
||||
tmp_or.append({f: v})
|
||||
if len(tmp_or) != 0:
|
||||
query["$and"].append({"$or": tmp_or})
|
||||
if "$and" in query:
|
||||
if len(query["$and"]) == 0:
|
||||
query.pop("$and")
|
||||
return query
|
||||
|
||||
|
||||
def get_root_domain(url):
|
||||
# 如果URL不带协议,添加一个默认的http协议
|
||||
|
|
Loading…
Reference in New Issue