add
This commit is contained in:
parent
0eb0f122a2
commit
0618ef2dc7
|
@ -6,9 +6,7 @@
|
||||||
# -------------------------------------------
|
# -------------------------------------------
|
||||||
from fastapi import APIRouter, Depends
|
from fastapi import APIRouter, Depends
|
||||||
from motor.motor_asyncio import AsyncIOMotorCursor
|
from motor.motor_asyncio import AsyncIOMotorCursor
|
||||||
from pymongo import DESCENDING
|
|
||||||
from api.users import verify_token
|
from api.users import verify_token
|
||||||
from core.config import POC_LIST
|
|
||||||
from core.db import get_mongo_db
|
from core.db import get_mongo_db
|
||||||
from core.util import search_to_mongodb
|
from core.util import search_to_mongodb
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
|
|
|
@ -177,14 +177,14 @@ async def do_asset_deduplication():
|
||||||
"filters": [],
|
"filters": [],
|
||||||
"groups": ["url", "status", "msg"]
|
"groups": ["url", "status", "msg"]
|
||||||
},
|
},
|
||||||
"PageMonitoring": {
|
# "PageMonitoring": {
|
||||||
"filters": [],
|
# "filters": [],
|
||||||
"groups": ["url"]
|
# "groups": ["url"]
|
||||||
},
|
# },
|
||||||
"SensitiveResult": {
|
# "SensitiveResult": {
|
||||||
"filters": [],
|
# "filters": [],
|
||||||
"groups": ["url"]
|
# "groups": ["url"]
|
||||||
},#############
|
# },
|
||||||
"SubdoaminTakerResult": {
|
"SubdoaminTakerResult": {
|
||||||
"filters": [],
|
"filters": [],
|
||||||
"groups": ["input", "value"]
|
"groups": ["input", "value"]
|
||||||
|
@ -196,7 +196,7 @@ async def do_asset_deduplication():
|
||||||
"asset": {
|
"asset": {
|
||||||
"filters": [],
|
"filters": [],
|
||||||
"groups": [""]
|
"groups": [""]
|
||||||
},################
|
},
|
||||||
"crawler": {
|
"crawler": {
|
||||||
"filters": [],
|
"filters": [],
|
||||||
"groups": ["url", "body"]
|
"groups": ["url", "body"]
|
||||||
|
@ -205,14 +205,24 @@ async def do_asset_deduplication():
|
||||||
"filters": [],
|
"filters": [],
|
||||||
"groups": ["host", "type", "ip"]
|
"groups": ["host", "type", "ip"]
|
||||||
},
|
},
|
||||||
"vulnerability": {
|
# "vulnerability": {
|
||||||
"filters": [],
|
# "filters": [],
|
||||||
"groups": ["url", "vulnid", "matched"]
|
# "groups": ["url", "vulnid", "matched"]
|
||||||
}
|
# }
|
||||||
}
|
}
|
||||||
for r in result:
|
for r in result:
|
||||||
if result[r]:
|
if result[r]:
|
||||||
await asset_data_dedup(db, r, )
|
if r in f_g_k:
|
||||||
|
if r == "asset":
|
||||||
|
# http资产去重
|
||||||
|
http_filter = [{"type": {"$ne": "other"}}]
|
||||||
|
http_group = ["url", "statuscode", "hashes.body_mmh3"]
|
||||||
|
await asset_data_dedup(db, r, http_filter, http_group)
|
||||||
|
other_filter = [{"type":"other"}]
|
||||||
|
other_group = ["host", "ip", "protocol"]
|
||||||
|
await asset_data_dedup(db, r, other_filter, other_group)
|
||||||
|
else:
|
||||||
|
await asset_data_dedup(db, r, f_g_k[r]['filters'], f_g_k[r]['groups'])
|
||||||
|
|
||||||
|
|
||||||
async def asset_data_dedup(db, collection_name, filters, groups):
|
async def asset_data_dedup(db, collection_name, filters, groups):
|
||||||
|
|
|
@ -10,31 +10,35 @@ from fastapi import APIRouter, Depends
|
||||||
from motor.motor_asyncio import AsyncIOMotorCursor
|
from motor.motor_asyncio import AsyncIOMotorCursor
|
||||||
from api.users import verify_token
|
from api.users import verify_token
|
||||||
from core.db import get_mongo_db
|
from core.db import get_mongo_db
|
||||||
from core.util import search_to_mongodb
|
from core.util import search_to_mongodb, get_search_query
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
|
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
|
|
||||||
|
|
||||||
@router.post("/dirscan/result/data")
|
@router.post("/dirscan/result/data")
|
||||||
async def dirscan_data(request_data: dict, db=Depends(get_mongo_db), _: dict = Depends(verify_token)):
|
async def dirscan_data(request_data: dict, db=Depends(get_mongo_db), _: dict = Depends(verify_token)):
|
||||||
try:
|
try:
|
||||||
search_query = request_data.get("search", "")
|
|
||||||
page_index = request_data.get("pageIndex", 1)
|
page_index = request_data.get("pageIndex", 1)
|
||||||
page_size = request_data.get("pageSize", 10)
|
page_size = request_data.get("pageSize", 10)
|
||||||
keyword = {
|
query = await get_search_query("dir", request_data)
|
||||||
'project': 'project',
|
if query == "":
|
||||||
'statuscode': 'status',
|
|
||||||
'url': 'url',
|
|
||||||
'redirect': 'msg',
|
|
||||||
'length': 'length'
|
|
||||||
}
|
|
||||||
query = await search_to_mongodb(search_query, keyword)
|
|
||||||
if query == "" or query is None:
|
|
||||||
return {"message": "Search condition parsing error", "code": 500}
|
return {"message": "Search condition parsing error", "code": 500}
|
||||||
query = query[0]
|
|
||||||
total_count = await db['DirScanResult'].count_documents(query)
|
total_count = await db['DirScanResult'].count_documents(query)
|
||||||
cursor: AsyncIOMotorCursor = ((db['DirScanResult'].find(query, {"_id": 0, "id": {"$toString": "$_id"}, "url": 1, "status": 1, "msg":1, "length": 1})
|
sort = request_data.get("sort", {})
|
||||||
.sort([('_id', -1)])
|
sort_by = [('_id', -1)]
|
||||||
|
if sort != {}:
|
||||||
|
if 'length' in sort:
|
||||||
|
sort_value = sort['length']
|
||||||
|
if sort_value is not None:
|
||||||
|
if sort_value == "ascending":
|
||||||
|
sort_value = 1
|
||||||
|
else:
|
||||||
|
sort_value = -1
|
||||||
|
sort_by = [('length', sort_value)]
|
||||||
|
cursor: AsyncIOMotorCursor = ((db['DirScanResult'].find(query, {"_id": 0, "id": {"$toString": "$_id"}, "url": 1,
|
||||||
|
"status": 1, "msg": 1, "length": 1})
|
||||||
|
.sort(sort_by)
|
||||||
.skip((page_index - 1) * page_size)
|
.skip((page_index - 1) * page_size)
|
||||||
.limit(page_size)))
|
.limit(page_size)))
|
||||||
result = await cursor.to_list(length=None)
|
result = await cursor.to_list(length=None)
|
||||||
|
@ -48,4 +52,4 @@ async def dirscan_data(request_data: dict, db=Depends(get_mongo_db), _: dict = D
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(str(e))
|
logger.error(str(e))
|
||||||
# Handle exceptions as needed
|
# Handle exceptions as needed
|
||||||
return {"message": "error", "code": 500}
|
return {"message": "error", "code": 500}
|
||||||
|
|
253
api/export.py
253
api/export.py
|
@ -124,7 +124,7 @@ async def export_data(request_data: dict, db=Depends(get_mongo_db), _: dict = De
|
||||||
"file_size": ""
|
"file_size": ""
|
||||||
})
|
})
|
||||||
if result.inserted_id:
|
if result.inserted_id:
|
||||||
background_tasks.add_task(export_data_from_mongodb, quantity, query, file_name, index, db)
|
background_tasks.add_task(export_data_from_mongodb, quantity, query, file_name, index)
|
||||||
return {"message": "Successfully added data export task", "code": 200}
|
return {"message": "Successfully added data export task", "code": 200}
|
||||||
else:
|
else:
|
||||||
return {"message": "Failed to export data", "code": 500}
|
return {"message": "Failed to export data", "code": 500}
|
||||||
|
@ -154,133 +154,134 @@ async def fetch_data(db, collection, query, quantity, project_list):
|
||||||
return cursor
|
return cursor
|
||||||
|
|
||||||
|
|
||||||
async def export_data_from_mongodb(quantity, query, file_name, index, db):
|
async def export_data_from_mongodb(quantity, query, file_name, index):
|
||||||
try:
|
async for db in get_mongo_db():
|
||||||
cursor = await fetch_data(db, index, query, quantity, Project_List)
|
try:
|
||||||
result = await cursor.to_list(length=None)
|
cursor = await fetch_data(db, index, query, quantity, Project_List)
|
||||||
relative_path = f'file/{file_name}.xlsx'
|
result = await cursor.to_list(length=None)
|
||||||
file_path = os.path.join(os.getcwd(), relative_path)
|
relative_path = f'file/{file_name}.xlsx'
|
||||||
if index == "asset":
|
file_path = os.path.join(os.getcwd(), relative_path)
|
||||||
http_columns = {
|
if index == "asset":
|
||||||
"timestamp": "时间",
|
http_columns = {
|
||||||
"tlsdata": "TLS_Data",
|
"timestamp": "时间",
|
||||||
"hashes": "Hash",
|
"tlsdata": "TLS_Data",
|
||||||
"cdnname": "Cdn_Name",
|
"hashes": "Hash",
|
||||||
"port": "端口",
|
"cdnname": "Cdn_Name",
|
||||||
"url": "url",
|
"port": "端口",
|
||||||
"title": "标题",
|
"url": "url",
|
||||||
"type": "类型",
|
"title": "标题",
|
||||||
"error": "错误",
|
"type": "类型",
|
||||||
"responsebody": "响应体",
|
"error": "错误",
|
||||||
"host": "IP",
|
"responsebody": "响应体",
|
||||||
"faviconmmh3": "图标Hash",
|
"host": "IP",
|
||||||
"faviconpath": "faviconpath",
|
"faviconmmh3": "图标Hash",
|
||||||
"rawheaders": "响应头",
|
"faviconpath": "faviconpath",
|
||||||
"jarm": "jarm",
|
"rawheaders": "响应头",
|
||||||
"technologies": "technologies",
|
"jarm": "jarm",
|
||||||
"statuscode": "响应码",
|
"technologies": "technologies",
|
||||||
"contentlength": "contentlength",
|
"statuscode": "响应码",
|
||||||
"cdn": "cdn",
|
"contentlength": "contentlength",
|
||||||
"webcheck": "webcheck",
|
"cdn": "cdn",
|
||||||
"project": "项目",
|
"webcheck": "webcheck",
|
||||||
"webfinger": "指纹",
|
"project": "项目",
|
||||||
"iconcontent": "图标",
|
"webfinger": "指纹",
|
||||||
"domain": "域名"
|
"iconcontent": "图标",
|
||||||
|
"domain": "域名"
|
||||||
|
}
|
||||||
|
other_columns = {
|
||||||
|
"timestamp": "时间",
|
||||||
|
"host": "域名",
|
||||||
|
"ip": "IP",
|
||||||
|
"port": "端口",
|
||||||
|
"protocol": "协议",
|
||||||
|
"tls": "TLS",
|
||||||
|
"transport": "transport",
|
||||||
|
"version": "版本",
|
||||||
|
"raw": "banner",
|
||||||
|
"project": "项目",
|
||||||
|
"type": "类型"
|
||||||
|
}
|
||||||
|
other_df = pd.DataFrame()
|
||||||
|
http_df = pd.DataFrame()
|
||||||
|
for doc in result:
|
||||||
|
if doc["type"] == "other":
|
||||||
|
other_df = pd.concat([other_df, pd.DataFrame([doc])], ignore_index=True)
|
||||||
|
else:
|
||||||
|
if doc['webfinger'] is not None:
|
||||||
|
webfinger = []
|
||||||
|
for webfinger_id in doc['webfinger']:
|
||||||
|
webfinger.append(APP[webfinger_id])
|
||||||
|
doc['webfinger'] = webfinger
|
||||||
|
http_df = pd.concat([http_df, pd.DataFrame([doc])], ignore_index=True)
|
||||||
|
try:
|
||||||
|
excel_writer = pd.ExcelWriter(file_path, engine='xlsxwriter')
|
||||||
|
http_df.rename(columns=http_columns, inplace=True)
|
||||||
|
http_df.to_excel(excel_writer, sheet_name='HTTP Data', index=False)
|
||||||
|
other_df.rename(columns=other_columns, inplace=True)
|
||||||
|
other_df.to_excel(excel_writer, sheet_name='Other Data', index=False)
|
||||||
|
excel_writer.close()
|
||||||
|
except IllegalCharacterError as e:
|
||||||
|
logger.error("导出内容有不可见字符,忽略此错误")
|
||||||
|
else:
|
||||||
|
columns = {}
|
||||||
|
if index == "subdomain":
|
||||||
|
columns = {'host': '域名', 'type': '解析类型', 'value': '解析值', 'ip': '解析IP', 'project': '项目',
|
||||||
|
'time': '时间'}
|
||||||
|
if index == "SubdoaminTakerResult":
|
||||||
|
columns = {
|
||||||
|
'input': '源域名', 'value': '解析值', 'cname': '接管类型', 'response': '响应体', 'project': '项目'
|
||||||
|
}
|
||||||
|
if index == "UrlScan":
|
||||||
|
columns = {
|
||||||
|
'input': '输入', 'source': '来源', 'outputtype': '输出类型', 'output': '输出',
|
||||||
|
'statuscode': 'statuscode', 'length': 'length', 'time': '时间', 'project': '项目'
|
||||||
|
}
|
||||||
|
if index == "crawler":
|
||||||
|
columns = {
|
||||||
|
'url': 'URL', 'method': 'Method', 'body': 'Body', 'project': '项目'
|
||||||
|
}
|
||||||
|
if index == "SensitiveResult":
|
||||||
|
columns = {
|
||||||
|
'url': 'URL', 'sid': '规则名称', 'match': '匹配内容', 'project': '项目', 'body': '响应体',
|
||||||
|
'color': '等级', 'time': '时间', 'md5': '响应体MD5'
|
||||||
|
}
|
||||||
|
if index == "DirScanResult":
|
||||||
|
columns = {
|
||||||
|
'url': 'URL', 'status': '响应码', 'msg': '跳转', 'project': '项目'
|
||||||
|
}
|
||||||
|
if index == "vulnerability":
|
||||||
|
columns = {
|
||||||
|
'url': 'URL', 'vulname': '漏洞', 'matched': '匹配', 'project': '项目', 'level': '危害等级',
|
||||||
|
'time': '时间', 'request': '请求', 'response': '响应'
|
||||||
|
}
|
||||||
|
if index == "PageMonitoring":
|
||||||
|
columns = {
|
||||||
|
'url': 'URL', 'content': '响应体', 'hash': '响应体Hash', 'diff': 'Diff',
|
||||||
|
'state': '状态', 'project': '项目', 'time': '时间'
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
df = pd.DataFrame(result)
|
||||||
|
df.rename(columns=columns, inplace=True)
|
||||||
|
df.to_excel(file_path, index=False)
|
||||||
|
except IllegalCharacterError as e:
|
||||||
|
logger.error("导出内容有不可见字符,忽略此错误")
|
||||||
|
file_size = os.path.getsize(file_path) / (1024 * 1024) # kb
|
||||||
|
update_document = {
|
||||||
|
"$set": {
|
||||||
|
"state": 1,
|
||||||
|
"end_time": get_now_time(),
|
||||||
|
"file_size": str(round(file_size, 2))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
other_columns = {
|
await db.export.update_one({"file_name": file_name}, update_document)
|
||||||
"timestamp": "时间",
|
except Exception as e:
|
||||||
"host": "域名",
|
logger.error(str(e))
|
||||||
"ip": "IP",
|
update_document = {
|
||||||
"port": "端口",
|
"$set": {
|
||||||
"protocol": "协议",
|
"state": 2,
|
||||||
"tls": "TLS",
|
}
|
||||||
"transport": "transport",
|
|
||||||
"version": "版本",
|
|
||||||
"raw": "banner",
|
|
||||||
"project": "项目",
|
|
||||||
"type": "类型"
|
|
||||||
}
|
}
|
||||||
other_df = pd.DataFrame()
|
await db.export.update_one({"file_name": file_name}, update_document)
|
||||||
http_df = pd.DataFrame()
|
|
||||||
for doc in result:
|
|
||||||
if doc["type"] == "other":
|
|
||||||
other_df = pd.concat([other_df, pd.DataFrame([doc])], ignore_index=True)
|
|
||||||
else:
|
|
||||||
if doc['webfinger'] is not None:
|
|
||||||
webfinger = []
|
|
||||||
for webfinger_id in doc['webfinger']:
|
|
||||||
webfinger.append(APP[webfinger_id])
|
|
||||||
doc['webfinger'] = webfinger
|
|
||||||
http_df = pd.concat([http_df, pd.DataFrame([doc])], ignore_index=True)
|
|
||||||
try:
|
|
||||||
excel_writer = pd.ExcelWriter(file_path, engine='xlsxwriter')
|
|
||||||
http_df.rename(columns=http_columns, inplace=True)
|
|
||||||
http_df.to_excel(excel_writer, sheet_name='HTTP Data', index=False)
|
|
||||||
other_df.rename(columns=other_columns, inplace=True)
|
|
||||||
other_df.to_excel(excel_writer, sheet_name='Other Data', index=False)
|
|
||||||
excel_writer.close()
|
|
||||||
except IllegalCharacterError as e:
|
|
||||||
logger.error("导出内容有不可见字符,忽略此错误")
|
|
||||||
else:
|
|
||||||
columns = {}
|
|
||||||
if index == "subdomain":
|
|
||||||
columns = {'host': '域名', 'type': '解析类型', 'value': '解析值', 'ip': '解析IP', 'project': '项目',
|
|
||||||
'time': '时间'}
|
|
||||||
if index == "SubdoaminTakerResult":
|
|
||||||
columns = {
|
|
||||||
'input': '源域名', 'value': '解析值', 'cname': '接管类型', 'response': '响应体', 'project': '项目'
|
|
||||||
}
|
|
||||||
if index == "UrlScan":
|
|
||||||
columns = {
|
|
||||||
'input': '输入', 'source': '来源', 'outputtype': '输出类型', 'output': '输出',
|
|
||||||
'statuscode': 'statuscode', 'length': 'length', 'time': '时间', 'project': '项目'
|
|
||||||
}
|
|
||||||
if index == "crawler":
|
|
||||||
columns = {
|
|
||||||
'url': 'URL', 'method': 'Method', 'body': 'Body', 'project': '项目'
|
|
||||||
}
|
|
||||||
if index == "SensitiveResult":
|
|
||||||
columns = {
|
|
||||||
'url': 'URL', 'sid': '规则名称', 'match': '匹配内容', 'project': '项目', 'body': '响应体',
|
|
||||||
'color': '等级', 'time': '时间', 'md5': '响应体MD5'
|
|
||||||
}
|
|
||||||
if index == "DirScanResult":
|
|
||||||
columns = {
|
|
||||||
'url': 'URL', 'status': '响应码', 'msg': '跳转', 'project': '项目'
|
|
||||||
}
|
|
||||||
if index == "vulnerability":
|
|
||||||
columns = {
|
|
||||||
'url': 'URL', 'vulname': '漏洞', 'matched': '匹配', 'project': '项目', 'level': '危害等级',
|
|
||||||
'time': '时间', 'request': '请求', 'response': '响应'
|
|
||||||
}
|
|
||||||
if index == "PageMonitoring":
|
|
||||||
columns = {
|
|
||||||
'url': 'URL', 'content': '响应体', 'hash': '响应体Hash', 'diff': 'Diff',
|
|
||||||
'state': '状态', 'project': '项目', 'time': '时间'
|
|
||||||
}
|
|
||||||
try:
|
|
||||||
df = pd.DataFrame(result)
|
|
||||||
df.rename(columns=columns, inplace=True)
|
|
||||||
df.to_excel(file_path, index=False)
|
|
||||||
except IllegalCharacterError as e:
|
|
||||||
logger.error("导出内容有不可见字符,忽略此错误")
|
|
||||||
file_size = os.path.getsize(file_path) / (1024 * 1024) # kb
|
|
||||||
update_document = {
|
|
||||||
"$set": {
|
|
||||||
"state": 1,
|
|
||||||
"end_time": get_now_time(),
|
|
||||||
"file_size": str(round(file_size, 2))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await db.export.update_one({"file_name": file_name}, update_document)
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(str(e))
|
|
||||||
update_document = {
|
|
||||||
"$set": {
|
|
||||||
"state": 2,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await db.export.update_one({"file_name": file_name}, update_document)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/export/record")
|
@router.get("/export/record")
|
||||||
|
|
|
@ -11,7 +11,6 @@ from pymongo import ASCENDING, DESCENDING
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from core.redis_handler import refresh_config
|
from core.redis_handler import refresh_config
|
||||||
from core.util import *
|
from core.util import *
|
||||||
from core.config import POC_LIST
|
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
|
|
||||||
|
|
||||||
|
@ -124,7 +123,6 @@ async def update_poc_data(request_data: dict, db=Depends(get_mongo_db), _: dict
|
||||||
result = await db.PocList.update_one({"_id": ObjectId(poc_id)}, update_document)
|
result = await db.PocList.update_one({"_id": ObjectId(poc_id)}, update_document)
|
||||||
# Check if the update was successful
|
# Check if the update was successful
|
||||||
if result:
|
if result:
|
||||||
POC_LIST[poc_id] = level
|
|
||||||
await refresh_config('all', 'poc')
|
await refresh_config('all', 'poc')
|
||||||
return {"message": "Data updated successfully", "code": 200}
|
return {"message": "Data updated successfully", "code": 200}
|
||||||
else:
|
else:
|
||||||
|
@ -160,7 +158,6 @@ async def add_poc_data(request_data: dict, db=Depends(get_mongo_db), _: dict = D
|
||||||
|
|
||||||
# Check if the insertion was successful
|
# Check if the insertion was successful
|
||||||
if result.inserted_id:
|
if result.inserted_id:
|
||||||
POC_LIST[str(result.inserted_id)] = level
|
|
||||||
await refresh_config('all', 'poc')
|
await refresh_config('all', 'poc')
|
||||||
return {"message": "Data added successfully", "code": 200}
|
return {"message": "Data added successfully", "code": 200}
|
||||||
else:
|
else:
|
||||||
|
@ -186,9 +183,6 @@ async def delete_poc_rules(request_data: dict, db=Depends(get_mongo_db), _: dict
|
||||||
|
|
||||||
# Check if the deletion was successful
|
# Check if the deletion was successful
|
||||||
if result.deleted_count > 0:
|
if result.deleted_count > 0:
|
||||||
for pid in poc_ids:
|
|
||||||
if pid in POC_LIST:
|
|
||||||
del POC_LIST[pid]
|
|
||||||
return {"code": 200, "message": "Poc deleted successfully"}
|
return {"code": 200, "message": "Poc deleted successfully"}
|
||||||
else:
|
else:
|
||||||
return {"code": 404, "message": "Poc not found"}
|
return {"code": 404, "message": "Poc not found"}
|
||||||
|
|
|
@ -68,7 +68,7 @@ async def get_projects_data(request_data: dict, db=Depends(get_mongo_db), _: dic
|
||||||
for result in results:
|
for result in results:
|
||||||
result["AssetCount"] = result.get("AssetCount", 0)
|
result["AssetCount"] = result.get("AssetCount", 0)
|
||||||
result_list[tag].append(result)
|
result_list[tag].append(result)
|
||||||
background_tasks.add_task(update_project_count, db=db, id=result["id"])
|
background_tasks.add_task(update_project_count, id=result["id"])
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"code": 200,
|
"code": 200,
|
||||||
|
@ -79,15 +79,16 @@ async def get_projects_data(request_data: dict, db=Depends(get_mongo_db), _: dic
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async def update_project_count(db, id):
|
async def update_project_count(id):
|
||||||
query = {"project": {"$eq": id}}
|
async for db in get_mongo_db():
|
||||||
total_count = await db['asset'].count_documents(query)
|
query = {"project": {"$eq": id}}
|
||||||
update_document = {
|
total_count = await db['asset'].count_documents(query)
|
||||||
"$set": {
|
update_document = {
|
||||||
"AssetCount": total_count
|
"$set": {
|
||||||
|
"AssetCount": total_count
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
await db.project.update_one({"_id": ObjectId(id)}, update_document)
|
||||||
await db.project.update_one({"_id": ObjectId(id)}, update_document)
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/project/content")
|
@router.post("/project/content")
|
||||||
|
@ -190,7 +191,7 @@ async def delete_project_rules(request_data: dict, db=Depends(get_mongo_db), _:
|
||||||
pro_ids = request_data.get("ids", [])
|
pro_ids = request_data.get("ids", [])
|
||||||
delA = request_data.get("delA", False)
|
delA = request_data.get("delA", False)
|
||||||
if delA:
|
if delA:
|
||||||
background_tasks.add_task(delete_asset, pro_ids, db, True)
|
background_tasks.add_task(delete_asset, pro_ids, True)
|
||||||
obj_ids = [ObjectId(poc_id) for poc_id in pro_ids]
|
obj_ids = [ObjectId(poc_id) for poc_id in pro_ids]
|
||||||
result = await db.project.delete_many({"_id": {"$in": obj_ids}})
|
result = await db.project.delete_many({"_id": {"$in": obj_ids}})
|
||||||
await db.ProjectTargetData.delete_many({"id": {"$in": pro_ids}})
|
await db.ProjectTargetData.delete_many({"id": {"$in": pro_ids}})
|
||||||
|
|
|
@ -16,7 +16,7 @@ from core.db import get_mongo_db
|
||||||
from core.redis_handler import refresh_config
|
from core.redis_handler import refresh_config
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
|
|
||||||
from core.util import search_to_mongodb
|
from core.util import search_to_mongodb, get_search_query
|
||||||
|
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
|
|
||||||
|
@ -223,23 +223,13 @@ async def get_sensitive_result_rules(request_data: dict, db=Depends(get_mongo_db
|
||||||
|
|
||||||
|
|
||||||
@router.post("/sensitive/result/data2")
|
@router.post("/sensitive/result/data2")
|
||||||
async def get_sensitive_result_rules(request_data: dict, db=Depends(get_mongo_db), _: dict = Depends(verify_token)):
|
async def get_sensitive_result_data2(request_data: dict, db=Depends(get_mongo_db), _: dict = Depends(verify_token)):
|
||||||
try:
|
try:
|
||||||
search_query = request_data.get("search", "")
|
|
||||||
page_index = request_data.get("pageIndex", 1)
|
page_index = request_data.get("pageIndex", 1)
|
||||||
page_size = request_data.get("pageSize", 10)
|
page_size = request_data.get("pageSize", 10)
|
||||||
keyword = {
|
query = await get_search_query("sens", request_data)
|
||||||
'url': 'url',
|
if query == "":
|
||||||
'sname': 'sid',
|
|
||||||
"body": "body",
|
|
||||||
"info": "match",
|
|
||||||
'project': 'project',
|
|
||||||
'md5': 'md5'
|
|
||||||
}
|
|
||||||
query = await search_to_mongodb(search_query, keyword)
|
|
||||||
if query == "" or query is None:
|
|
||||||
return {"message": "Search condition parsing error", "code": 500}
|
return {"message": "Search condition parsing error", "code": 500}
|
||||||
query = query[0]
|
|
||||||
total_count = await db['SensitiveResult'].count_documents(query)
|
total_count = await db['SensitiveResult'].count_documents(query)
|
||||||
pipeline = [
|
pipeline = [
|
||||||
{
|
{
|
||||||
|
|
35
api/task.py
35
api/task.py
|
@ -181,7 +181,7 @@ async def delete_task(request_data: dict, db=Depends(get_mongo_db), _: dict = De
|
||||||
scheduler.remove_job(task_id)
|
scheduler.remove_job(task_id)
|
||||||
await db.ScheduledTasks.delete_many({"id": {"$in": task_ids}})
|
await db.ScheduledTasks.delete_many({"id": {"$in": task_ids}})
|
||||||
if delA:
|
if delA:
|
||||||
background_tasks.add_task(delete_asset, task_ids, db)
|
background_tasks.add_task(delete_asset, task_ids)
|
||||||
await redis_con.delete(*redis_key)
|
await redis_con.delete(*redis_key)
|
||||||
# Delete the SensitiveRule documents based on the provided IDs
|
# Delete the SensitiveRule documents based on the provided IDs
|
||||||
result = await db.task.delete_many({"_id": {"$in": obj_ids}})
|
result = await db.task.delete_many({"_id": {"$in": obj_ids}})
|
||||||
|
@ -558,19 +558,20 @@ async def scheduler_scan_task(id):
|
||||||
await create_scan_task(doc, task_id, targetList, redis)
|
await create_scan_task(doc, task_id, targetList, redis)
|
||||||
|
|
||||||
|
|
||||||
async def delete_asset(task_ids, db, is_project = False):
|
async def delete_asset(task_ids, is_project = False):
|
||||||
key = ["asset", "subdomain", "SubdoaminTakerResult", "UrlScan", "crawler", "SensitiveResult", "DirScanResult", "vulnerability", "PageMonitoring"]
|
async for db in get_mongo_db():
|
||||||
del_query = {"taskId": {"$in": task_ids}}
|
key = ["asset", "subdomain", "SubdoaminTakerResult", "UrlScan", "crawler", "SensitiveResult", "DirScanResult", "vulnerability", "PageMonitoring"]
|
||||||
if is_project:
|
del_query = {"taskId": {"$in": task_ids}}
|
||||||
del_query = {
|
if is_project:
|
||||||
"$or": [
|
del_query = {
|
||||||
{"taskId": {"$in": task_ids}},
|
"$or": [
|
||||||
{"project": {"$in": task_ids}}
|
{"taskId": {"$in": task_ids}},
|
||||||
]
|
{"project": {"$in": task_ids}}
|
||||||
}
|
]
|
||||||
for k in key:
|
}
|
||||||
result = await db[k].delete_many(del_query)
|
for k in key:
|
||||||
if result.deleted_count > 0:
|
result = await db[k].delete_many(del_query)
|
||||||
logger.info("Deleted {} {} documents".format(k, result.deleted_count))
|
if result.deleted_count > 0:
|
||||||
else:
|
logger.info("Deleted {} {} documents".format(k, result.deleted_count))
|
||||||
logger.info("Deleted {} None documents".format(k))
|
else:
|
||||||
|
logger.info("Deleted {} None documents".format(k))
|
|
@ -9,33 +9,20 @@ from fastapi import APIRouter, Depends
|
||||||
from motor.motor_asyncio import AsyncIOMotorCursor
|
from motor.motor_asyncio import AsyncIOMotorCursor
|
||||||
from pymongo import DESCENDING
|
from pymongo import DESCENDING
|
||||||
from api.users import verify_token
|
from api.users import verify_token
|
||||||
from core.config import POC_LIST
|
|
||||||
from core.db import get_mongo_db
|
from core.db import get_mongo_db
|
||||||
from core.util import search_to_mongodb
|
from core.util import search_to_mongodb, get_search_query
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
|
|
||||||
|
|
||||||
@router.post("/vul/data")
|
@router.post("/vul/data")
|
||||||
async def get_vul_data(request_data: dict, db=Depends(get_mongo_db), _: dict = Depends(verify_token)):
|
async def get_vul_data(request_data: dict, db=Depends(get_mongo_db), _: dict = Depends(verify_token)):
|
||||||
try:
|
try:
|
||||||
search_query = request_data.get("search", "")
|
|
||||||
page_index = request_data.get("pageIndex", 1)
|
page_index = request_data.get("pageIndex", 1)
|
||||||
page_size = request_data.get("pageSize", 10)
|
page_size = request_data.get("pageSize", 10)
|
||||||
# MongoDB collection for SensitiveRule
|
query = await get_search_query("vul", request_data)
|
||||||
# Fuzzy search based on the name field
|
if query == "":
|
||||||
keyword = {
|
|
||||||
'url': 'url',
|
|
||||||
'vulname': 'vulname',
|
|
||||||
'project': 'project',
|
|
||||||
'matched': 'matched',
|
|
||||||
'request': 'request',
|
|
||||||
'response': 'response',
|
|
||||||
'level': 'level'
|
|
||||||
}
|
|
||||||
query = await search_to_mongodb(search_query, keyword)
|
|
||||||
if query == "" or query is None:
|
|
||||||
return {"message": "Search condition parsing error", "code": 500}
|
return {"message": "Search condition parsing error", "code": 500}
|
||||||
query = query[0]
|
|
||||||
# Get the total count of documents matching the search criteria
|
# Get the total count of documents matching the search criteria
|
||||||
total_count = await db.vulnerability.count_documents(query)
|
total_count = await db.vulnerability.count_documents(query)
|
||||||
if total_count == 0:
|
if total_count == 0:
|
||||||
|
@ -60,9 +47,8 @@ async def get_vul_data(request_data: dict, db=Depends(get_mongo_db), _: dict = D
|
||||||
"time": doc["time"],
|
"time": doc["time"],
|
||||||
"request": doc["request"],
|
"request": doc["request"],
|
||||||
"response": doc["response"],
|
"response": doc["response"],
|
||||||
|
"level": doc['level']
|
||||||
}
|
}
|
||||||
if doc["vulnid"] in POC_LIST:
|
|
||||||
data["level"] = POC_LIST[doc["vulnid"]]
|
|
||||||
response_data.append(data)
|
response_data.append(data)
|
||||||
return {
|
return {
|
||||||
"code": 200,
|
"code": 200,
|
||||||
|
|
|
@ -27,7 +27,6 @@ NODE_TIMEOUT = 50
|
||||||
TOTAL_LOGS = 1000
|
TOTAL_LOGS = 1000
|
||||||
APP = {}
|
APP = {}
|
||||||
SensitiveRuleList = {}
|
SensitiveRuleList = {}
|
||||||
POC_LIST = {}
|
|
||||||
Project_List = {}
|
Project_List = {}
|
||||||
def set_timezone(t):
|
def set_timezone(t):
|
||||||
global TIMEZONE
|
global TIMEZONE
|
||||||
|
|
|
@ -133,7 +133,6 @@ async def create_database():
|
||||||
{"id": "page_monitoring", "name": "Page Monitoring", 'hour': 24, 'type': 'Page Monitoring', 'state': True})
|
{"id": "page_monitoring", "name": "Page Monitoring", 'hour': 24, 'type': 'Page Monitoring', 'state': True})
|
||||||
await get_fingerprint(client[DATABASE_NAME])
|
await get_fingerprint(client[DATABASE_NAME])
|
||||||
await get_sens_rule(client[DATABASE_NAME])
|
await get_sens_rule(client[DATABASE_NAME])
|
||||||
await get_pocList(client[DATABASE_NAME])
|
|
||||||
await get_project(client[DATABASE_NAME])
|
await get_project(client[DATABASE_NAME])
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
# 处理异常
|
# 处理异常
|
||||||
|
@ -166,14 +165,6 @@ async def get_sens_rule(client):
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async def get_pocList(client):
|
|
||||||
collection = client["PocList"]
|
|
||||||
cursor = collection.find({}, {"_id": 1, "level": 1})
|
|
||||||
async for document in cursor:
|
|
||||||
document['id'] = str(document['_id'])
|
|
||||||
POC_LIST[document['id']] = document['level']
|
|
||||||
|
|
||||||
|
|
||||||
async def get_project(client):
|
async def get_project(client):
|
||||||
collection = client["project"]
|
collection = client["project"]
|
||||||
cursor = collection.find({}, {"_id": 1, "name": 1})
|
cursor = collection.find({}, {"_id": 1, "name": 1})
|
||||||
|
|
49
core/util.py
49
core/util.py
|
@ -321,6 +321,55 @@ async def search_to_mongodb(expression_raw, keyword):
|
||||||
logger.error(e)
|
logger.error(e)
|
||||||
return ""
|
return ""
|
||||||
|
|
||||||
|
async def get_search_query(name, request_data):
|
||||||
|
search_query = request_data.get("search", "")
|
||||||
|
search_key_v = {
|
||||||
|
'sens':{
|
||||||
|
'url': 'url',
|
||||||
|
'sname': 'sid',
|
||||||
|
"body": "body",
|
||||||
|
"info": "match",
|
||||||
|
'project': 'project',
|
||||||
|
'md5': 'md5'
|
||||||
|
},
|
||||||
|
'dir': {
|
||||||
|
'project': 'project',
|
||||||
|
'statuscode': 'status',
|
||||||
|
'url': 'url',
|
||||||
|
'redirect': 'msg',
|
||||||
|
'length': 'length'
|
||||||
|
},
|
||||||
|
'vul': {
|
||||||
|
'url': 'url',
|
||||||
|
'vulname': 'vulname',
|
||||||
|
'project': 'project',
|
||||||
|
'matched': 'matched',
|
||||||
|
'request': 'request',
|
||||||
|
'response': 'response',
|
||||||
|
'level': 'level'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
keyword = search_key_v[name]
|
||||||
|
query = await search_to_mongodb(search_query, keyword)
|
||||||
|
if query == "" or query is None:
|
||||||
|
return ""
|
||||||
|
query = query[0]
|
||||||
|
filter_key = ['color', 'status', 'level']
|
||||||
|
filter = request_data.get("filter", {})
|
||||||
|
if filter:
|
||||||
|
query["$and"] = []
|
||||||
|
for f in filter:
|
||||||
|
if f in filter_key:
|
||||||
|
tmp_or = []
|
||||||
|
for v in filter[f]:
|
||||||
|
tmp_or.append({f: v})
|
||||||
|
if len(tmp_or) != 0:
|
||||||
|
query["$and"].append({"$or": tmp_or})
|
||||||
|
if "$and" in query:
|
||||||
|
if len(query["$and"]) == 0:
|
||||||
|
query.pop("$and")
|
||||||
|
return query
|
||||||
|
|
||||||
|
|
||||||
def get_root_domain(url):
|
def get_root_domain(url):
|
||||||
# 如果URL不带协议,添加一个默认的http协议
|
# 如果URL不带协议,添加一个默认的http协议
|
||||||
|
|
Loading…
Reference in New Issue