add
This commit is contained in:
parent
8b0c77609f
commit
7f3a3c271f
|
@ -0,0 +1,196 @@
|
||||||
|
# -------------------------------------
|
||||||
|
# @file : export.py
|
||||||
|
# @author : Autumn
|
||||||
|
# @contact : rainy-autumn@outlook.com
|
||||||
|
# @time : 2024/6/16 16:11
|
||||||
|
# -------------------------------------------
|
||||||
|
import os
|
||||||
|
|
||||||
|
from bson import ObjectId
|
||||||
|
from fastapi import APIRouter, Depends, BackgroundTasks
|
||||||
|
from api.users import verify_token
|
||||||
|
from motor.motor_asyncio import AsyncIOMotorCursor
|
||||||
|
from core.db import get_mongo_db
|
||||||
|
import pandas as pd
|
||||||
|
from core.util import *
|
||||||
|
from pymongo import ASCENDING, DESCENDING
|
||||||
|
from loguru import logger
|
||||||
|
|
||||||
|
router = APIRouter()
|
||||||
|
|
||||||
|
keywords = {
|
||||||
|
"asset": {
|
||||||
|
'app': '',
|
||||||
|
'body': 'responsebody',
|
||||||
|
'header': 'rawheaders',
|
||||||
|
'project': 'project',
|
||||||
|
'title': 'title',
|
||||||
|
'statuscode': 'statuscode',
|
||||||
|
'icon': 'faviconmmh3',
|
||||||
|
'ip': ['host', 'ip'],
|
||||||
|
'domain': ['host', 'url', 'domain'],
|
||||||
|
'port': 'port',
|
||||||
|
'protocol': ['protocol', 'type'],
|
||||||
|
'banner': 'raw',
|
||||||
|
},
|
||||||
|
"subdomain": {
|
||||||
|
'domain': 'host',
|
||||||
|
'ip': 'ip',
|
||||||
|
'type': 'type',
|
||||||
|
'project': 'project',
|
||||||
|
'value': 'value'
|
||||||
|
},
|
||||||
|
"SubdoaminTakerResult": {
|
||||||
|
'domain': 'input',
|
||||||
|
'value': 'value',
|
||||||
|
'type': 'cname',
|
||||||
|
'response': 'response',
|
||||||
|
'project': 'project',
|
||||||
|
},
|
||||||
|
"UrlScan": {
|
||||||
|
'url': 'output',
|
||||||
|
'project': 'project',
|
||||||
|
'input': 'input',
|
||||||
|
'source': 'source',
|
||||||
|
"type": "outputtype"
|
||||||
|
},
|
||||||
|
"crawler": {
|
||||||
|
'url': 'url',
|
||||||
|
'method': 'method',
|
||||||
|
'body': 'body',
|
||||||
|
'project': 'project'
|
||||||
|
},
|
||||||
|
"SensitiveResult": {
|
||||||
|
'url': 'url',
|
||||||
|
'sname': 'sid',
|
||||||
|
"body": "body",
|
||||||
|
"info": "match",
|
||||||
|
'project': 'project',
|
||||||
|
'md5': 'md5'
|
||||||
|
},
|
||||||
|
"DirScanResult": {
|
||||||
|
'project': 'project',
|
||||||
|
'statuscode': 'status',
|
||||||
|
'url': 'url',
|
||||||
|
'redirect': 'msg'
|
||||||
|
},
|
||||||
|
"vulnerability": {
|
||||||
|
'url': 'url',
|
||||||
|
'vulname': 'vulname',
|
||||||
|
'project': 'project',
|
||||||
|
'matched': 'matched',
|
||||||
|
'request': 'request',
|
||||||
|
'response': 'response',
|
||||||
|
'level': 'level'
|
||||||
|
},
|
||||||
|
"PageMonitoring": {
|
||||||
|
'url': 'url',
|
||||||
|
'project': 'project',
|
||||||
|
'hash': 'hash',
|
||||||
|
'diff': 'diff',
|
||||||
|
'response': 'response'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/export")
|
||||||
|
async def export_data(request_data: dict, db=Depends(get_mongo_db), _: dict = Depends(verify_token), background_tasks: BackgroundTasks = BackgroundTasks()):
|
||||||
|
index = request_data.get("index", "")
|
||||||
|
quantity = int(request_data.get("quantity", 0))
|
||||||
|
export_type = request_data.get("type", "")
|
||||||
|
search_query = request_data.get("search", "")
|
||||||
|
if index == "" or quantity == 0 or export_type == "":
|
||||||
|
return {"code": 500, "message": f"get index, quantity, export_type null"}
|
||||||
|
query = {}
|
||||||
|
if export_type == "search":
|
||||||
|
query = await search_to_mongodb(search_query, keywords[index])
|
||||||
|
if query == "" or query is None:
|
||||||
|
return {"message": "Search condition parsing error", "code": 500}
|
||||||
|
query = query[0]
|
||||||
|
if index == "PageMonitoring":
|
||||||
|
query["diff"] = {"$ne": []}
|
||||||
|
file_name = generate_random_string(16)
|
||||||
|
result = await db.export.insert_one({
|
||||||
|
"file_name": file_name,
|
||||||
|
"create_time": get_now_time(),
|
||||||
|
"quantity": quantity,
|
||||||
|
"data_type": index,
|
||||||
|
"state": 0,
|
||||||
|
"end_time": ""
|
||||||
|
})
|
||||||
|
if result.inserted_id:
|
||||||
|
background_tasks.add_task(export_data_from_mongodb, quantity, query, file_name, index, db)
|
||||||
|
return {"message": "Successfully added data export task", "code": 200}
|
||||||
|
else:
|
||||||
|
return {"message": "Failed to export data", "code": 500}
|
||||||
|
|
||||||
|
|
||||||
|
async def fetch_data(db, collection, query, quantity, project_list):
|
||||||
|
# 构造替换字段值的pipeline
|
||||||
|
branches = []
|
||||||
|
for new_value, original_value in project_list.items():
|
||||||
|
branches.append({"case": {"$eq": ["$project", original_value]}, "then": new_value})
|
||||||
|
|
||||||
|
pipeline = [
|
||||||
|
{"$match": query},
|
||||||
|
{"$limit": quantity},
|
||||||
|
{"$addFields": {
|
||||||
|
"project": {
|
||||||
|
"$switch": {
|
||||||
|
"branches": branches,
|
||||||
|
"default": "$project"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}},
|
||||||
|
{"$project": {"_id": 0, "vulnid": 0}}
|
||||||
|
]
|
||||||
|
|
||||||
|
cursor = db[collection].aggregate(pipeline)
|
||||||
|
return cursor
|
||||||
|
|
||||||
|
async def export_data_from_mongodb(quantity, query, file_name, index, db):
|
||||||
|
cursor = await fetch_data(db, index, query, quantity, Project_List)
|
||||||
|
result = await cursor.to_list(length=None)
|
||||||
|
relative_path = f'file/{file_name}.xlsx'
|
||||||
|
file_path = os.path.join(os.getcwd(), relative_path)
|
||||||
|
if index == "asset":
|
||||||
|
http_df = None
|
||||||
|
other_df = None
|
||||||
|
http_columns = ['时间', 'TLS_Data', 'Hash', 'Cdn_Name', '端口', 'url', '标题', '类型', '错误', '响应体', 'IP', '图标Hash', 'faviconpath', '响应头', 'jarm', 'technologies', '响应码', 'contentlength', 'cdn', 'webcheck', '项目', '指纹', '域名']
|
||||||
|
other_columns = ['时间', '域名', 'IP', '端口', '协议', "TLS", '版本', 'banner', '项目', '类型']
|
||||||
|
for doc in result:
|
||||||
|
if doc["type"] == "other":
|
||||||
|
other_df = pd.DataFrame(doc, columns=other_columns)
|
||||||
|
else:
|
||||||
|
if len(doc['webfinger']) != 0:
|
||||||
|
webfinger = []
|
||||||
|
for webfinger_id in doc['webfinger']:
|
||||||
|
webfinger.append(APP[webfinger_id])
|
||||||
|
doc['webfinger'] = webfinger
|
||||||
|
http_df = pd.DataFrame(doc, columns=http_columns)
|
||||||
|
excel_writer = pd.ExcelWriter(file_path, engine='xlsxwriter')
|
||||||
|
http_df.to_excel(excel_writer, sheet_name='HTTP Data', index=False)
|
||||||
|
other_df.to_excel(excel_writer, sheet_name='Other Data', index=False)
|
||||||
|
excel_writer.save()
|
||||||
|
else:
|
||||||
|
columns = []
|
||||||
|
if index == "subdomain":
|
||||||
|
columns = ['域名', '解析类型', '解析值', '解析IP', '项目']
|
||||||
|
if index == "SubdoaminTakerResult":
|
||||||
|
columns = ['源域名', '解析值', '接管类型', '响应体', '项目']
|
||||||
|
if index == "UrlScan":
|
||||||
|
columns = ['输入', '来源', '输出', 'statuscode', 'length', '时间', '项目']
|
||||||
|
if index == "crawler":
|
||||||
|
columns = ['URL', 'Method', 'Body', '项目']
|
||||||
|
if index == "SensitiveResult":
|
||||||
|
columns = ['URL', '规则名称', '匹配内容', '项目', '响应体', '等级', '时间', '响应体MD5']
|
||||||
|
if index == "DirScanResult":
|
||||||
|
columns = ['URL', '响应码', '跳转', '项目']
|
||||||
|
if index == "vulnerability":
|
||||||
|
columns = ['URL', '漏洞', '匹配', '项目', '危害等级', '时间', '请求', '响应']
|
||||||
|
if index == "PageMonitoring":
|
||||||
|
columns = ['URL', '响应体', '响应体Hash', 'Diff', '状态', '项目', '时间']
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -140,12 +140,12 @@ async def add_project_rule(request_data: dict, db=Depends(get_mongo_db), _: dict
|
||||||
root_domains = []
|
root_domains = []
|
||||||
for t in target.split('\n'):
|
for t in target.split('\n'):
|
||||||
if t not in t_list:
|
if t not in t_list:
|
||||||
targetTmp = t.replace('http://', "").replace('https://', "").strip() + '\n'
|
targetTmp = t.replace('http://', "").replace('https://', "").strip()
|
||||||
if targetTmp != "":
|
if targetTmp != "":
|
||||||
root_domain = get_root_domain(targetTmp)
|
root_domain = get_root_domain(targetTmp)
|
||||||
if root_domain not in root_domains:
|
if root_domain not in root_domains:
|
||||||
root_domains.append(root_domain)
|
root_domains.append(root_domain)
|
||||||
tmsg += targetTmp
|
tmsg += targetTmp + '\n'
|
||||||
# Create a new SensitiveRule document
|
# Create a new SensitiveRule document
|
||||||
tmsg = tmsg.strip().strip("\n")
|
tmsg = tmsg.strip().strip("\n")
|
||||||
request_data["root_domains"] = root_domains
|
request_data["root_domains"] = root_domains
|
||||||
|
|
|
@ -0,0 +1,6 @@
|
||||||
|
# -------------------------------------
|
||||||
|
# @file : update.py
|
||||||
|
# @author : Autumn
|
||||||
|
# @contact : rainy-autumn@outlook.com
|
||||||
|
# @time : 2024/6/16 14:14
|
||||||
|
# -------------------------------------------
|
4
main.py
4
main.py
|
@ -54,7 +54,7 @@ async def http_exception_handler(request, exc):
|
||||||
os.chdir(os.path.dirname(os.path.abspath(__file__)))
|
os.chdir(os.path.dirname(os.path.abspath(__file__)))
|
||||||
|
|
||||||
from api import users, sensitive, dictionary, poc, configuration, fingerprint, node, project, task, asset_info, \
|
from api import users, sensitive, dictionary, poc, configuration, fingerprint, node, project, task, asset_info, \
|
||||||
page_monitoring, vulnerability, SubdoaminTaker, scheduled_tasks, notification, system
|
page_monitoring, vulnerability, SubdoaminTaker, scheduled_tasks, notification, system, export
|
||||||
|
|
||||||
app.include_router(users.router, prefix='/api')
|
app.include_router(users.router, prefix='/api')
|
||||||
app.include_router(sensitive.router, prefix='/api')
|
app.include_router(sensitive.router, prefix='/api')
|
||||||
|
@ -73,7 +73,7 @@ app.include_router(scheduled_tasks.router, prefix='/api')
|
||||||
app.include_router(dirscan.router, prefix='/api')
|
app.include_router(dirscan.router, prefix='/api')
|
||||||
app.include_router(notification.router, prefix='/api')
|
app.include_router(notification.router, prefix='/api')
|
||||||
app.include_router(system.router, prefix='/api')
|
app.include_router(system.router, prefix='/api')
|
||||||
|
app.include_router(export.router, prefix='/api')
|
||||||
app.mount("/assets", StaticFiles(directory="static/assets"), name="assets")
|
app.mount("/assets", StaticFiles(directory="static/assets"), name="assets")
|
||||||
|
|
||||||
@app.get("/logo.png", response_class=FileResponse)
|
@app.get("/logo.png", response_class=FileResponse)
|
||||||
|
|
BIN
requirements.txt
BIN
requirements.txt
Binary file not shown.
Loading…
Reference in New Issue