liangsheng8708
/
ScopeSentry
mirror of https://mirror.ghproxy.com/https://github.com/Autumn-27/ScopeSentry.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add

This commit is contained in:
Autumn.home 2024-07-04 23:28:11 +08:00
parent 0eb0f122a2
commit 0618ef2dc7
12 changed files with 257 additions and 233 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
api/SubdoaminTaker.py
View File

@ -6,9 +6,7 @@
# -------------------------------------------
from fastapi import APIRouter, Depends
from motor.motor_asyncio import AsyncIOMotorCursor
from pymongo import DESCENDING
from api.users import verify_token
from core.config import POC_LIST
from core.db import get_mongo_db
from core.util import search_to_mongodb
from loguru import logger

38
api/configuration.py
View File

@ -177,14 +177,14 @@ async def do_asset_deduplication():
"filters": [],
"groups": ["url", "status", "msg"]
},
"PageMonitoring": {
"filters": [],
"groups": ["url"]
},
"SensitiveResult": {
"filters": [],
"groups": ["url"]
},#############
# "PageMonitoring": {
# "filters": [],
# "groups": ["url"]
# },
# "SensitiveResult": {
# "filters": [],
# "groups": ["url"]
# },
"SubdoaminTakerResult": {
"filters": [],
"groups": ["input", "value"]
@ -196,7 +196,7 @@ async def do_asset_deduplication():
"asset": {
"filters": [],
"groups": [""]
},################
},
"crawler": {
"filters": [],
"groups": ["url", "body"]
@ -205,14 +205,24 @@ async def do_asset_deduplication():
"filters": [],
"groups": ["host", "type", "ip"]
},
"vulnerability": {
"filters": [],
"groups": ["url", "vulnid", "matched"]
}
# "vulnerability": {
# "filters": [],
# "groups": ["url", "vulnid", "matched"]
# }
}
for r in result:
if result[r]:
await asset_data_dedup(db, r, )
if r in f_g_k:
if r == "asset":
# http资产去重
http_filter = [{"type": {"$ne": "other"}}]
http_group = ["url", "statuscode", "hashes.body_mmh3"]
await asset_data_dedup(db, r, http_filter, http_group)
other_filter = [{"type":"other"}]
other_group = ["host", "ip", "protocol"]
await asset_data_dedup(db, r, other_filter, other_group)
else:
await asset_data_dedup(db, r, f_g_k[r]['filters'], f_g_k[r]['groups'])
async def asset_data_dedup(db, collection_name, filters, groups):

34
api/dirscan.py
View File

@ -10,31 +10,35 @@ from fastapi import APIRouter, Depends
from motor.motor_asyncio import AsyncIOMotorCursor
from api.users import verify_token
from core.db import get_mongo_db
from core.util import search_to_mongodb
from core.util import search_to_mongodb, get_search_query
from loguru import logger
router = APIRouter()
@router.post("/dirscan/result/data")
async def dirscan_data(request_data: dict, db=Depends(get_mongo_db), _: dict = Depends(verify_token)):
try:
search_query = request_data.get("search", "")
page_index = request_data.get("pageIndex", 1)
page_size = request_data.get("pageSize", 10)
keyword = {
'project': 'project',
'statuscode': 'status',
'url': 'url',
'redirect': 'msg',
'length': 'length'
}
query = await search_to_mongodb(search_query, keyword)
if query == "" or query is None:
query = await get_search_query("dir", request_data)
if query == "":
return {"message": "Search condition parsing error", "code": 500}
query = query[0]
total_count = await db['DirScanResult'].count_documents(query)
cursor: AsyncIOMotorCursor = ((db['DirScanResult'].find(query, {"_id": 0, "id": {"$toString": "$_id"}, "url": 1, "status": 1, "msg":1, "length": 1})
.sort([('_id', -1)])
sort = request_data.get("sort", {})
sort_by = [('_id', -1)]
if sort != {}:
if 'length' in sort:
sort_value = sort['length']
if sort_value is not None:
if sort_value == "ascending":
sort_value = 1
else:
sort_value = -1
sort_by = [('length', sort_value)]
cursor: AsyncIOMotorCursor = ((db['DirScanResult'].find(query, {"_id": 0, "id": {"$toString": "$_id"}, "url": 1,
"status": 1, "msg": 1, "length": 1})
.sort(sort_by)
.skip((page_index - 1) * page_size)
.limit(page_size)))
result = await cursor.to_list(length=None)
@ -48,4 +52,4 @@ async def dirscan_data(request_data: dict, db=Depends(get_mongo_db), _: dict = D
except Exception as e:
logger.error(str(e))
# Handle exceptions as needed
return {"message": "error", "code": 500}
return {"message": "error", "code": 500}

253
api/export.py
View File

@ -124,7 +124,7 @@ async def export_data(request_data: dict, db=Depends(get_mongo_db), _: dict = De
"file_size": ""
})
if result.inserted_id:
background_tasks.add_task(export_data_from_mongodb, quantity, query, file_name, index, db)
background_tasks.add_task(export_data_from_mongodb, quantity, query, file_name, index)
return {"message": "Successfully added data export task", "code": 200}
else:
return {"message": "Failed to export data", "code": 500}
@ -154,133 +154,134 @@ async def fetch_data(db, collection, query, quantity, project_list):
return cursor
async def export_data_from_mongodb(quantity, query, file_name, index, db):
try:
cursor = await fetch_data(db, index, query, quantity, Project_List)
result = await cursor.to_list(length=None)
relative_path = f'file/{file_name}.xlsx'
file_path = os.path.join(os.getcwd(), relative_path)
if index == "asset":
http_columns = {
"timestamp": "时间",
"tlsdata": "TLS_Data",
"hashes": "Hash",
"cdnname": "Cdn_Name",
"port": "端口",
"url": "url",
"title": "标题",
"type": "类型",
"error": "错误",
"responsebody": "响应体",
"host": "IP",
"faviconmmh3": "图标Hash",
"faviconpath": "faviconpath",
"rawheaders": "响应头",
"jarm": "jarm",
"technologies": "technologies",
"statuscode": "响应码",
"contentlength": "contentlength",
"cdn": "cdn",
"webcheck": "webcheck",
"project": "项目",
"webfinger": "指纹",
"iconcontent": "图标",
"domain": "域名"
async def export_data_from_mongodb(quantity, query, file_name, index):
async for db in get_mongo_db():
try:
cursor = await fetch_data(db, index, query, quantity, Project_List)
result = await cursor.to_list(length=None)
relative_path = f'file/{file_name}.xlsx'
file_path = os.path.join(os.getcwd(), relative_path)
if index == "asset":
http_columns = {
"timestamp": "时间",
"tlsdata": "TLS_Data",
"hashes": "Hash",
"cdnname": "Cdn_Name",
"port": "端口",
"url": "url",
"title": "标题",
"type": "类型",
"error": "错误",
"responsebody": "响应体",
"host": "IP",
"faviconmmh3": "图标Hash",
"faviconpath": "faviconpath",
"rawheaders": "响应头",
"jarm": "jarm",
"technologies": "technologies",
"statuscode": "响应码",
"contentlength": "contentlength",
"cdn": "cdn",
"webcheck": "webcheck",
"project": "项目",
"webfinger": "指纹",
"iconcontent": "图标",
"domain": "域名"
}
other_columns = {
"timestamp": "时间",
"host": "域名",
"ip": "IP",
"port": "端口",
"protocol": "协议",
"tls": "TLS",
"transport": "transport",
"version": "版本",
"raw": "banner",
"project": "项目",
"type": "类型"
}
other_df = pd.DataFrame()
http_df = pd.DataFrame()
for doc in result:
if doc["type"] == "other":
other_df = pd.concat([other_df, pd.DataFrame([doc])], ignore_index=True)
else:
if doc['webfinger'] is not None:
webfinger = []
for webfinger_id in doc['webfinger']:
webfinger.append(APP[webfinger_id])
doc['webfinger'] = webfinger
http_df = pd.concat([http_df, pd.DataFrame([doc])], ignore_index=True)
try:
excel_writer = pd.ExcelWriter(file_path, engine='xlsxwriter')
http_df.rename(columns=http_columns, inplace=True)
http_df.to_excel(excel_writer, sheet_name='HTTP Data', index=False)
other_df.rename(columns=other_columns, inplace=True)
other_df.to_excel(excel_writer, sheet_name='Other Data', index=False)
excel_writer.close()
except IllegalCharacterError as e:
logger.error("导出内容有不可见字符，忽略此错误")
else:
columns = {}
if index == "subdomain":
columns = {'host': '域名', 'type': '解析类型', 'value': '解析值', 'ip': '解析IP', 'project': '项目',
'time': '时间'}
if index == "SubdoaminTakerResult":
columns = {
'input': '源域名', 'value': '解析值', 'cname': '接管类型', 'response': '响应体', 'project': '项目'
}
if index == "UrlScan":
columns = {
'input': '输入', 'source': '来源', 'outputtype': '输出类型', 'output': '输出',
'statuscode': 'statuscode', 'length': 'length', 'time': '时间', 'project': '项目'
}
if index == "crawler":
columns = {
'url': 'URL', 'method': 'Method', 'body': 'Body', 'project': '项目'
}
if index == "SensitiveResult":
columns = {
'url': 'URL', 'sid': '规则名称', 'match': '匹配内容', 'project': '项目', 'body': '响应体',
'color': '等级', 'time': '时间', 'md5': '响应体MD5'
}
if index == "DirScanResult":
columns = {
'url': 'URL', 'status': '响应码', 'msg': '跳转', 'project': '项目'
}
if index == "vulnerability":
columns = {
'url': 'URL', 'vulname': '漏洞', 'matched': '匹配', 'project': '项目', 'level': '危害等级',
'time': '时间', 'request': '请求', 'response': '响应'
}
if index == "PageMonitoring":
columns = {
'url': 'URL', 'content': '响应体', 'hash': '响应体Hash', 'diff': 'Diff',
'state': '状态', 'project': '项目', 'time': '时间'
}
try:
df = pd.DataFrame(result)
df.rename(columns=columns, inplace=True)
df.to_excel(file_path, index=False)
except IllegalCharacterError as e:
logger.error("导出内容有不可见字符，忽略此错误")
file_size = os.path.getsize(file_path) / (1024 * 1024) # kb
update_document = {
"$set": {
"state": 1,
"end_time": get_now_time(),
"file_size": str(round(file_size, 2))
}
}
other_columns = {
"timestamp": "时间",
"host": "域名",
"ip": "IP",
"port": "端口",
"protocol": "协议",
"tls": "TLS",
"transport": "transport",
"version": "版本",
"raw": "banner",
"project": "项目",
"type": "类型"
await db.export.update_one({"file_name": file_name}, update_document)
except Exception as e:
logger.error(str(e))
update_document = {
"$set": {
"state": 2,
}
}
other_df = pd.DataFrame()
http_df = pd.DataFrame()
for doc in result:
if doc["type"] == "other":
other_df = pd.concat([other_df, pd.DataFrame([doc])], ignore_index=True)
else:
if doc['webfinger'] is not None:
webfinger = []
for webfinger_id in doc['webfinger']:
webfinger.append(APP[webfinger_id])
doc['webfinger'] = webfinger
http_df = pd.concat([http_df, pd.DataFrame([doc])], ignore_index=True)
try:
excel_writer = pd.ExcelWriter(file_path, engine='xlsxwriter')
http_df.rename(columns=http_columns, inplace=True)
http_df.to_excel(excel_writer, sheet_name='HTTP Data', index=False)
other_df.rename(columns=other_columns, inplace=True)
other_df.to_excel(excel_writer, sheet_name='Other Data', index=False)
excel_writer.close()
except IllegalCharacterError as e:
logger.error("导出内容有不可见字符，忽略此错误")
else:
columns = {}
if index == "subdomain":
columns = {'host': '域名', 'type': '解析类型', 'value': '解析值', 'ip': '解析IP', 'project': '项目',
'time': '时间'}
if index == "SubdoaminTakerResult":
columns = {
'input': '源域名', 'value': '解析值', 'cname': '接管类型', 'response': '响应体', 'project': '项目'
}
if index == "UrlScan":
columns = {
'input': '输入', 'source': '来源', 'outputtype': '输出类型', 'output': '输出',
'statuscode': 'statuscode', 'length': 'length', 'time': '时间', 'project': '项目'
}
if index == "crawler":
columns = {
'url': 'URL', 'method': 'Method', 'body': 'Body', 'project': '项目'
}
if index == "SensitiveResult":
columns = {
'url': 'URL', 'sid': '规则名称', 'match': '匹配内容', 'project': '项目', 'body': '响应体',
'color': '等级', 'time': '时间', 'md5': '响应体MD5'
}
if index == "DirScanResult":
columns = {
'url': 'URL', 'status': '响应码', 'msg': '跳转', 'project': '项目'
}
if index == "vulnerability":
columns = {
'url': 'URL', 'vulname': '漏洞', 'matched': '匹配', 'project': '项目', 'level': '危害等级',
'time': '时间', 'request': '请求', 'response': '响应'
}
if index == "PageMonitoring":
columns = {
'url': 'URL', 'content': '响应体', 'hash': '响应体Hash', 'diff': 'Diff',
'state': '状态', 'project': '项目', 'time': '时间'
}
try:
df = pd.DataFrame(result)
df.rename(columns=columns, inplace=True)
df.to_excel(file_path, index=False)
except IllegalCharacterError as e:
logger.error("导出内容有不可见字符，忽略此错误")
file_size = os.path.getsize(file_path) / (1024 * 1024) # kb
update_document = {
"$set": {
"state": 1,
"end_time": get_now_time(),
"file_size": str(round(file_size, 2))
}
}
await db.export.update_one({"file_name": file_name}, update_document)
except Exception as e:
logger.error(str(e))
update_document = {
"$set": {
"state": 2,
}
}
await db.export.update_one({"file_name": file_name}, update_document)
await db.export.update_one({"file_name": file_name}, update_document)
@router.get("/export/record")

6
api/poc.py
View File

@ -11,7 +11,6 @@ from pymongo import ASCENDING, DESCENDING
from loguru import logger
from core.redis_handler import refresh_config
from core.util import *
from core.config import POC_LIST
router = APIRouter()
@ -124,7 +123,6 @@ async def update_poc_data(request_data: dict, db=Depends(get_mongo_db), _: dict
result = await db.PocList.update_one({"_id": ObjectId(poc_id)}, update_document)
# Check if the update was successful
if result:
POC_LIST[poc_id] = level
await refresh_config('all', 'poc')
return {"message": "Data updated successfully", "code": 200}
else:
@ -160,7 +158,6 @@ async def add_poc_data(request_data: dict, db=Depends(get_mongo_db), _: dict = D
# Check if the insertion was successful
if result.inserted_id:
POC_LIST[str(result.inserted_id)] = level
await refresh_config('all', 'poc')
return {"message": "Data added successfully", "code": 200}
else:
@ -186,9 +183,6 @@ async def delete_poc_rules(request_data: dict, db=Depends(get_mongo_db), _: dict
# Check if the deletion was successful
if result.deleted_count > 0:
for pid in poc_ids:
if pid in POC_LIST:
del POC_LIST[pid]
return {"code": 200, "message": "Poc deleted successfully"}
else:
return {"code": 404, "message": "Poc not found"}

21
api/project.py
View File

@ -68,7 +68,7 @@ async def get_projects_data(request_data: dict, db=Depends(get_mongo_db), _: dic
for result in results:
result["AssetCount"] = result.get("AssetCount", 0)
result_list[tag].append(result)
background_tasks.add_task(update_project_count, db=db, id=result["id"])
background_tasks.add_task(update_project_count, id=result["id"])
return {
"code": 200,
@ -79,15 +79,16 @@ async def get_projects_data(request_data: dict, db=Depends(get_mongo_db), _: dic
}
async def update_project_count(db, id):
query = {"project": {"$eq": id}}
total_count = await db['asset'].count_documents(query)
update_document = {
"$set": {
"AssetCount": total_count
async def update_project_count(id):
async for db in get_mongo_db():
query = {"project": {"$eq": id}}
total_count = await db['asset'].count_documents(query)
update_document = {
"$set": {
"AssetCount": total_count
}
}
}
await db.project.update_one({"_id": ObjectId(id)}, update_document)
await db.project.update_one({"_id": ObjectId(id)}, update_document)
@router.post("/project/content")
@ -190,7 +191,7 @@ async def delete_project_rules(request_data: dict, db=Depends(get_mongo_db), _:
pro_ids = request_data.get("ids", [])
delA = request_data.get("delA", False)
if delA:
background_tasks.add_task(delete_asset, pro_ids, db, True)
background_tasks.add_task(delete_asset, pro_ids, True)
obj_ids = [ObjectId(poc_id) for poc_id in pro_ids]
result = await db.project.delete_many({"_id": {"$in": obj_ids}})
await db.ProjectTargetData.delete_many({"id": {"$in": pro_ids}})

18
api/sensitive.py
View File

@ -16,7 +16,7 @@ from core.db import get_mongo_db
from core.redis_handler import refresh_config
from loguru import logger
from core.util import search_to_mongodb
from core.util import search_to_mongodb, get_search_query
router = APIRouter()
@ -223,23 +223,13 @@ async def get_sensitive_result_rules(request_data: dict, db=Depends(get_mongo_db
@router.post("/sensitive/result/data2")
async def get_sensitive_result_rules(request_data: dict, db=Depends(get_mongo_db), _: dict = Depends(verify_token)):
async def get_sensitive_result_data2(request_data: dict, db=Depends(get_mongo_db), _: dict = Depends(verify_token)):
try:
search_query = request_data.get("search", "")
page_index = request_data.get("pageIndex", 1)
page_size = request_data.get("pageSize", 10)
keyword = {
'url': 'url',
'sname': 'sid',
"body": "body",
"info": "match",
'project': 'project',
'md5': 'md5'
}
query = await search_to_mongodb(search_query, keyword)
if query == "" or query is None:
query = await get_search_query("sens", request_data)
if query == "":
return {"message": "Search condition parsing error", "code": 500}
query = query[0]
total_count = await db['SensitiveResult'].count_documents(query)
pipeline = [
{

35
api/task.py
View File

@ -181,7 +181,7 @@ async def delete_task(request_data: dict, db=Depends(get_mongo_db), _: dict = De
scheduler.remove_job(task_id)
await db.ScheduledTasks.delete_many({"id": {"$in": task_ids}})
if delA:
background_tasks.add_task(delete_asset, task_ids, db)
background_tasks.add_task(delete_asset, task_ids)
await redis_con.delete(*redis_key)
# Delete the SensitiveRule documents based on the provided IDs
result = await db.task.delete_many({"_id": {"$in": obj_ids}})
@ -558,19 +558,20 @@ async def scheduler_scan_task(id):
await create_scan_task(doc, task_id, targetList, redis)
async def delete_asset(task_ids, db, is_project = False):
key = ["asset", "subdomain", "SubdoaminTakerResult", "UrlScan", "crawler", "SensitiveResult", "DirScanResult", "vulnerability", "PageMonitoring"]
del_query = {"taskId": {"$in": task_ids}}
if is_project:
del_query = {
"$or": [
{"taskId": {"$in": task_ids}},
{"project": {"$in": task_ids}}
]
}
for k in key:
result = await db[k].delete_many(del_query)
if result.deleted_count > 0:
logger.info("Deleted {} {} documents".format(k, result.deleted_count))
else:
logger.info("Deleted {} None documents".format(k))
async def delete_asset(task_ids, is_project = False):
async for db in get_mongo_db():
key = ["asset", "subdomain", "SubdoaminTakerResult", "UrlScan", "crawler", "SensitiveResult", "DirScanResult", "vulnerability", "PageMonitoring"]
del_query = {"taskId": {"$in": task_ids}}
if is_project:
del_query = {
"$or": [
{"taskId": {"$in": task_ids}},
{"project": {"$in": task_ids}}
]
}
for k in key:
result = await db[k].delete_many(del_query)
if result.deleted_count > 0:
logger.info("Deleted {} {} documents".format(k, result.deleted_count))
else:
logger.info("Deleted {} None documents".format(k))

24
api/vulnerability.py
View File

@ -9,33 +9,20 @@ from fastapi import APIRouter, Depends
from motor.motor_asyncio import AsyncIOMotorCursor
from pymongo import DESCENDING
from api.users import verify_token
from core.config import POC_LIST
from core.db import get_mongo_db
from core.util import search_to_mongodb
from core.util import search_to_mongodb, get_search_query
from loguru import logger
router = APIRouter()
@router.post("/vul/data")
async def get_vul_data(request_data: dict, db=Depends(get_mongo_db), _: dict = Depends(verify_token)):
try:
search_query = request_data.get("search", "")
page_index = request_data.get("pageIndex", 1)
page_size = request_data.get("pageSize", 10)
# MongoDB collection for SensitiveRule
# Fuzzy search based on the name field
keyword = {
'url': 'url',
'vulname': 'vulname',
'project': 'project',
'matched': 'matched',
'request': 'request',
'response': 'response',
'level': 'level'
}
query = await search_to_mongodb(search_query, keyword)
if query == "" or query is None:
query = await get_search_query("vul", request_data)
if query == "":
return {"message": "Search condition parsing error", "code": 500}
query = query[0]
# Get the total count of documents matching the search criteria
total_count = await db.vulnerability.count_documents(query)
if total_count == 0:
@ -60,9 +47,8 @@ async def get_vul_data(request_data: dict, db=Depends(get_mongo_db), _: dict = D
"time": doc["time"],
"request": doc["request"],
"response": doc["response"],
"level": doc['level']
}
if doc["vulnid"] in POC_LIST:
data["level"] = POC_LIST[doc["vulnid"]]
response_data.append(data)
return {
"code": 200,

1
core/config.py
View File

@ -27,7 +27,6 @@ NODE_TIMEOUT = 50
TOTAL_LOGS = 1000
APP = {}
SensitiveRuleList = {}
POC_LIST = {}
Project_List = {}
def set_timezone(t):
global TIMEZONE

9
core/db.py
View File

@ -133,7 +133,6 @@ async def create_database():
{"id": "page_monitoring", "name": "Page Monitoring", 'hour': 24, 'type': 'Page Monitoring', 'state': True})
await get_fingerprint(client[DATABASE_NAME])
await get_sens_rule(client[DATABASE_NAME])
await get_pocList(client[DATABASE_NAME])
await get_project(client[DATABASE_NAME])
except Exception as e:
# 处理异常
@ -166,14 +165,6 @@ async def get_sens_rule(client):
}
async def get_pocList(client):
collection = client["PocList"]
cursor = collection.find({}, {"_id": 1, "level": 1})
async for document in cursor:
document['id'] = str(document['_id'])
POC_LIST[document['id']] = document['level']
async def get_project(client):
collection = client["project"]
cursor = collection.find({}, {"_id": 1, "name": 1})

49
core/util.py
View File

@ -321,6 +321,55 @@ async def search_to_mongodb(expression_raw, keyword):
logger.error(e)
return ""
async def get_search_query(name, request_data):
search_query = request_data.get("search", "")
search_key_v = {
'sens':{
'url': 'url',
'sname': 'sid',
"body": "body",
"info": "match",
'project': 'project',
'md5': 'md5'
},
'dir': {
'project': 'project',
'statuscode': 'status',
'url': 'url',
'redirect': 'msg',
'length': 'length'
},
'vul': {
'url': 'url',
'vulname': 'vulname',
'project': 'project',
'matched': 'matched',
'request': 'request',
'response': 'response',
'level': 'level'
}
}
keyword = search_key_v[name]
query = await search_to_mongodb(search_query, keyword)
if query == "" or query is None:
return ""
query = query[0]
filter_key = ['color', 'status', 'level']
filter = request_data.get("filter", {})
if filter:
query["$and"] = []
for f in filter:
if f in filter_key:
tmp_or = []
for v in filter[f]:
tmp_or.append({f: v})
if len(tmp_or) != 0:
query["$and"].append({"$or": tmp_or})
if "$and" in query:
if len(query["$and"]) == 0:
query.pop("$and")
return query
def get_root_domain(url):
# 如果URL不带协议，添加一个默认的http协议