diff --git a/api/SubdoaminTaker.py b/api/SubdoaminTaker.py index 2ecb0a0..13ae60f 100644 --- a/api/SubdoaminTaker.py +++ b/api/SubdoaminTaker.py @@ -6,9 +6,7 @@ # ------------------------------------------- from fastapi import APIRouter, Depends from motor.motor_asyncio import AsyncIOMotorCursor -from pymongo import DESCENDING from api.users import verify_token -from core.config import POC_LIST from core.db import get_mongo_db from core.util import search_to_mongodb from loguru import logger diff --git a/api/configuration.py b/api/configuration.py index 801f39d..cb31936 100644 --- a/api/configuration.py +++ b/api/configuration.py @@ -177,14 +177,14 @@ async def do_asset_deduplication(): "filters": [], "groups": ["url", "status", "msg"] }, - "PageMonitoring": { - "filters": [], - "groups": ["url"] - }, - "SensitiveResult": { - "filters": [], - "groups": ["url"] - },############# + # "PageMonitoring": { + # "filters": [], + # "groups": ["url"] + # }, + # "SensitiveResult": { + # "filters": [], + # "groups": ["url"] + # }, "SubdoaminTakerResult": { "filters": [], "groups": ["input", "value"] @@ -196,7 +196,7 @@ async def do_asset_deduplication(): "asset": { "filters": [], "groups": [""] - },################ + }, "crawler": { "filters": [], "groups": ["url", "body"] @@ -205,14 +205,24 @@ async def do_asset_deduplication(): "filters": [], "groups": ["host", "type", "ip"] }, - "vulnerability": { - "filters": [], - "groups": ["url", "vulnid", "matched"] - } + # "vulnerability": { + # "filters": [], + # "groups": ["url", "vulnid", "matched"] + # } } for r in result: if result[r]: - await asset_data_dedup(db, r, ) + if r in f_g_k: + if r == "asset": + # http资产去重 + http_filter = [{"type": {"$ne": "other"}}] + http_group = ["url", "statuscode", "hashes.body_mmh3"] + await asset_data_dedup(db, r, http_filter, http_group) + other_filter = [{"type":"other"}] + other_group = ["host", "ip", "protocol"] + await asset_data_dedup(db, r, other_filter, other_group) + else: + await asset_data_dedup(db, r, f_g_k[r]['filters'], f_g_k[r]['groups']) async def asset_data_dedup(db, collection_name, filters, groups): diff --git a/api/dirscan.py b/api/dirscan.py index 3a43ab2..ab35254 100644 --- a/api/dirscan.py +++ b/api/dirscan.py @@ -10,31 +10,35 @@ from fastapi import APIRouter, Depends from motor.motor_asyncio import AsyncIOMotorCursor from api.users import verify_token from core.db import get_mongo_db -from core.util import search_to_mongodb +from core.util import search_to_mongodb, get_search_query from loguru import logger + router = APIRouter() @router.post("/dirscan/result/data") async def dirscan_data(request_data: dict, db=Depends(get_mongo_db), _: dict = Depends(verify_token)): try: - search_query = request_data.get("search", "") page_index = request_data.get("pageIndex", 1) page_size = request_data.get("pageSize", 10) - keyword = { - 'project': 'project', - 'statuscode': 'status', - 'url': 'url', - 'redirect': 'msg', - 'length': 'length' - } - query = await search_to_mongodb(search_query, keyword) - if query == "" or query is None: + query = await get_search_query("dir", request_data) + if query == "": return {"message": "Search condition parsing error", "code": 500} - query = query[0] total_count = await db['DirScanResult'].count_documents(query) - cursor: AsyncIOMotorCursor = ((db['DirScanResult'].find(query, {"_id": 0, "id": {"$toString": "$_id"}, "url": 1, "status": 1, "msg":1, "length": 1}) - .sort([('_id', -1)]) + sort = request_data.get("sort", {}) + sort_by = [('_id', -1)] + if sort != {}: + if 'length' in sort: + sort_value = sort['length'] + if sort_value is not None: + if sort_value == "ascending": + sort_value = 1 + else: + sort_value = -1 + sort_by = [('length', sort_value)] + cursor: AsyncIOMotorCursor = ((db['DirScanResult'].find(query, {"_id": 0, "id": {"$toString": "$_id"}, "url": 1, + "status": 1, "msg": 1, "length": 1}) + .sort(sort_by) .skip((page_index - 1) * page_size) .limit(page_size))) result = await cursor.to_list(length=None) @@ -48,4 +52,4 @@ async def dirscan_data(request_data: dict, db=Depends(get_mongo_db), _: dict = D except Exception as e: logger.error(str(e)) # Handle exceptions as needed - return {"message": "error", "code": 500} \ No newline at end of file + return {"message": "error", "code": 500} diff --git a/api/export.py b/api/export.py index e576610..d9b98d7 100644 --- a/api/export.py +++ b/api/export.py @@ -124,7 +124,7 @@ async def export_data(request_data: dict, db=Depends(get_mongo_db), _: dict = De "file_size": "" }) if result.inserted_id: - background_tasks.add_task(export_data_from_mongodb, quantity, query, file_name, index, db) + background_tasks.add_task(export_data_from_mongodb, quantity, query, file_name, index) return {"message": "Successfully added data export task", "code": 200} else: return {"message": "Failed to export data", "code": 500} @@ -154,133 +154,134 @@ async def fetch_data(db, collection, query, quantity, project_list): return cursor -async def export_data_from_mongodb(quantity, query, file_name, index, db): - try: - cursor = await fetch_data(db, index, query, quantity, Project_List) - result = await cursor.to_list(length=None) - relative_path = f'file/{file_name}.xlsx' - file_path = os.path.join(os.getcwd(), relative_path) - if index == "asset": - http_columns = { - "timestamp": "时间", - "tlsdata": "TLS_Data", - "hashes": "Hash", - "cdnname": "Cdn_Name", - "port": "端口", - "url": "url", - "title": "标题", - "type": "类型", - "error": "错误", - "responsebody": "响应体", - "host": "IP", - "faviconmmh3": "图标Hash", - "faviconpath": "faviconpath", - "rawheaders": "响应头", - "jarm": "jarm", - "technologies": "technologies", - "statuscode": "响应码", - "contentlength": "contentlength", - "cdn": "cdn", - "webcheck": "webcheck", - "project": "项目", - "webfinger": "指纹", - "iconcontent": "图标", - "domain": "域名" +async def export_data_from_mongodb(quantity, query, file_name, index): + async for db in get_mongo_db(): + try: + cursor = await fetch_data(db, index, query, quantity, Project_List) + result = await cursor.to_list(length=None) + relative_path = f'file/{file_name}.xlsx' + file_path = os.path.join(os.getcwd(), relative_path) + if index == "asset": + http_columns = { + "timestamp": "时间", + "tlsdata": "TLS_Data", + "hashes": "Hash", + "cdnname": "Cdn_Name", + "port": "端口", + "url": "url", + "title": "标题", + "type": "类型", + "error": "错误", + "responsebody": "响应体", + "host": "IP", + "faviconmmh3": "图标Hash", + "faviconpath": "faviconpath", + "rawheaders": "响应头", + "jarm": "jarm", + "technologies": "technologies", + "statuscode": "响应码", + "contentlength": "contentlength", + "cdn": "cdn", + "webcheck": "webcheck", + "project": "项目", + "webfinger": "指纹", + "iconcontent": "图标", + "domain": "域名" + } + other_columns = { + "timestamp": "时间", + "host": "域名", + "ip": "IP", + "port": "端口", + "protocol": "协议", + "tls": "TLS", + "transport": "transport", + "version": "版本", + "raw": "banner", + "project": "项目", + "type": "类型" + } + other_df = pd.DataFrame() + http_df = pd.DataFrame() + for doc in result: + if doc["type"] == "other": + other_df = pd.concat([other_df, pd.DataFrame([doc])], ignore_index=True) + else: + if doc['webfinger'] is not None: + webfinger = [] + for webfinger_id in doc['webfinger']: + webfinger.append(APP[webfinger_id]) + doc['webfinger'] = webfinger + http_df = pd.concat([http_df, pd.DataFrame([doc])], ignore_index=True) + try: + excel_writer = pd.ExcelWriter(file_path, engine='xlsxwriter') + http_df.rename(columns=http_columns, inplace=True) + http_df.to_excel(excel_writer, sheet_name='HTTP Data', index=False) + other_df.rename(columns=other_columns, inplace=True) + other_df.to_excel(excel_writer, sheet_name='Other Data', index=False) + excel_writer.close() + except IllegalCharacterError as e: + logger.error("导出内容有不可见字符，忽略此错误") + else: + columns = {} + if index == "subdomain": + columns = {'host': '域名', 'type': '解析类型', 'value': '解析值', 'ip': '解析IP', 'project': '项目', + 'time': '时间'} + if index == "SubdoaminTakerResult": + columns = { + 'input': '源域名', 'value': '解析值', 'cname': '接管类型', 'response': '响应体', 'project': '项目' + } + if index == "UrlScan": + columns = { + 'input': '输入', 'source': '来源', 'outputtype': '输出类型', 'output': '输出', + 'statuscode': 'statuscode', 'length': 'length', 'time': '时间', 'project': '项目' + } + if index == "crawler": + columns = { + 'url': 'URL', 'method': 'Method', 'body': 'Body', 'project': '项目' + } + if index == "SensitiveResult": + columns = { + 'url': 'URL', 'sid': '规则名称', 'match': '匹配内容', 'project': '项目', 'body': '响应体', + 'color': '等级', 'time': '时间', 'md5': '响应体MD5' + } + if index == "DirScanResult": + columns = { + 'url': 'URL', 'status': '响应码', 'msg': '跳转', 'project': '项目' + } + if index == "vulnerability": + columns = { + 'url': 'URL', 'vulname': '漏洞', 'matched': '匹配', 'project': '项目', 'level': '危害等级', + 'time': '时间', 'request': '请求', 'response': '响应' + } + if index == "PageMonitoring": + columns = { + 'url': 'URL', 'content': '响应体', 'hash': '响应体Hash', 'diff': 'Diff', + 'state': '状态', 'project': '项目', 'time': '时间' + } + try: + df = pd.DataFrame(result) + df.rename(columns=columns, inplace=True) + df.to_excel(file_path, index=False) + except IllegalCharacterError as e: + logger.error("导出内容有不可见字符，忽略此错误") + file_size = os.path.getsize(file_path) / (1024 * 1024) # kb + update_document = { + "$set": { + "state": 1, + "end_time": get_now_time(), + "file_size": str(round(file_size, 2)) + } } - other_columns = { - "timestamp": "时间", - "host": "域名", - "ip": "IP", - "port": "端口", - "protocol": "协议", - "tls": "TLS", - "transport": "transport", - "version": "版本", - "raw": "banner", - "project": "项目", - "type": "类型" + await db.export.update_one({"file_name": file_name}, update_document) + except Exception as e: + logger.error(str(e)) + update_document = { + "$set": { + "state": 2, + } } - other_df = pd.DataFrame() - http_df = pd.DataFrame() - for doc in result: - if doc["type"] == "other": - other_df = pd.concat([other_df, pd.DataFrame([doc])], ignore_index=True) - else: - if doc['webfinger'] is not None: - webfinger = [] - for webfinger_id in doc['webfinger']: - webfinger.append(APP[webfinger_id]) - doc['webfinger'] = webfinger - http_df = pd.concat([http_df, pd.DataFrame([doc])], ignore_index=True) - try: - excel_writer = pd.ExcelWriter(file_path, engine='xlsxwriter') - http_df.rename(columns=http_columns, inplace=True) - http_df.to_excel(excel_writer, sheet_name='HTTP Data', index=False) - other_df.rename(columns=other_columns, inplace=True) - other_df.to_excel(excel_writer, sheet_name='Other Data', index=False) - excel_writer.close() - except IllegalCharacterError as e: - logger.error("导出内容有不可见字符，忽略此错误") - else: - columns = {} - if index == "subdomain": - columns = {'host': '域名', 'type': '解析类型', 'value': '解析值', 'ip': '解析IP', 'project': '项目', - 'time': '时间'} - if index == "SubdoaminTakerResult": - columns = { - 'input': '源域名', 'value': '解析值', 'cname': '接管类型', 'response': '响应体', 'project': '项目' - } - if index == "UrlScan": - columns = { - 'input': '输入', 'source': '来源', 'outputtype': '输出类型', 'output': '输出', - 'statuscode': 'statuscode', 'length': 'length', 'time': '时间', 'project': '项目' - } - if index == "crawler": - columns = { - 'url': 'URL', 'method': 'Method', 'body': 'Body', 'project': '项目' - } - if index == "SensitiveResult": - columns = { - 'url': 'URL', 'sid': '规则名称', 'match': '匹配内容', 'project': '项目', 'body': '响应体', - 'color': '等级', 'time': '时间', 'md5': '响应体MD5' - } - if index == "DirScanResult": - columns = { - 'url': 'URL', 'status': '响应码', 'msg': '跳转', 'project': '项目' - } - if index == "vulnerability": - columns = { - 'url': 'URL', 'vulname': '漏洞', 'matched': '匹配', 'project': '项目', 'level': '危害等级', - 'time': '时间', 'request': '请求', 'response': '响应' - } - if index == "PageMonitoring": - columns = { - 'url': 'URL', 'content': '响应体', 'hash': '响应体Hash', 'diff': 'Diff', - 'state': '状态', 'project': '项目', 'time': '时间' - } - try: - df = pd.DataFrame(result) - df.rename(columns=columns, inplace=True) - df.to_excel(file_path, index=False) - except IllegalCharacterError as e: - logger.error("导出内容有不可见字符，忽略此错误") - file_size = os.path.getsize(file_path) / (1024 * 1024) # kb - update_document = { - "$set": { - "state": 1, - "end_time": get_now_time(), - "file_size": str(round(file_size, 2)) - } - } - await db.export.update_one({"file_name": file_name}, update_document) - except Exception as e: - logger.error(str(e)) - update_document = { - "$set": { - "state": 2, - } - } - await db.export.update_one({"file_name": file_name}, update_document) + await db.export.update_one({"file_name": file_name}, update_document) @router.get("/export/record") diff --git a/api/poc.py b/api/poc.py index 6f88a35..0c26e2c 100644 --- a/api/poc.py +++ b/api/poc.py @@ -11,7 +11,6 @@ from pymongo import ASCENDING, DESCENDING from loguru import logger from core.redis_handler import refresh_config from core.util import * -from core.config import POC_LIST router = APIRouter() @@ -124,7 +123,6 @@ async def update_poc_data(request_data: dict, db=Depends(get_mongo_db), _: dict result = await db.PocList.update_one({"_id": ObjectId(poc_id)}, update_document) # Check if the update was successful if result: - POC_LIST[poc_id] = level await refresh_config('all', 'poc') return {"message": "Data updated successfully", "code": 200} else: @@ -160,7 +158,6 @@ async def add_poc_data(request_data: dict, db=Depends(get_mongo_db), _: dict = D # Check if the insertion was successful if result.inserted_id: - POC_LIST[str(result.inserted_id)] = level await refresh_config('all', 'poc') return {"message": "Data added successfully", "code": 200} else: @@ -186,9 +183,6 @@ async def delete_poc_rules(request_data: dict, db=Depends(get_mongo_db), _: dict # Check if the deletion was successful if result.deleted_count > 0: - for pid in poc_ids: - if pid in POC_LIST: - del POC_LIST[pid] return {"code": 200, "message": "Poc deleted successfully"} else: return {"code": 404, "message": "Poc not found"} diff --git a/api/project.py b/api/project.py index cfd2c3e..bea6681 100644 --- a/api/project.py +++ b/api/project.py @@ -68,7 +68,7 @@ async def get_projects_data(request_data: dict, db=Depends(get_mongo_db), _: dic for result in results: result["AssetCount"] = result.get("AssetCount", 0) result_list[tag].append(result) - background_tasks.add_task(update_project_count, db=db, id=result["id"]) + background_tasks.add_task(update_project_count, id=result["id"]) return { "code": 200, @@ -79,15 +79,16 @@ async def get_projects_data(request_data: dict, db=Depends(get_mongo_db), _: dic } -async def update_project_count(db, id): - query = {"project": {"$eq": id}} - total_count = await db['asset'].count_documents(query) - update_document = { - "$set": { - "AssetCount": total_count +async def update_project_count(id): + async for db in get_mongo_db(): + query = {"project": {"$eq": id}} + total_count = await db['asset'].count_documents(query) + update_document = { + "$set": { + "AssetCount": total_count + } } - } - await db.project.update_one({"_id": ObjectId(id)}, update_document) + await db.project.update_one({"_id": ObjectId(id)}, update_document) @router.post("/project/content") @@ -190,7 +191,7 @@ async def delete_project_rules(request_data: dict, db=Depends(get_mongo_db), _: pro_ids = request_data.get("ids", []) delA = request_data.get("delA", False) if delA: - background_tasks.add_task(delete_asset, pro_ids, db, True) + background_tasks.add_task(delete_asset, pro_ids, True) obj_ids = [ObjectId(poc_id) for poc_id in pro_ids] result = await db.project.delete_many({"_id": {"$in": obj_ids}}) await db.ProjectTargetData.delete_many({"id": {"$in": pro_ids}}) diff --git a/api/sensitive.py b/api/sensitive.py index 2d98955..9e157bb 100644 --- a/api/sensitive.py +++ b/api/sensitive.py @@ -16,7 +16,7 @@ from core.db import get_mongo_db from core.redis_handler import refresh_config from loguru import logger -from core.util import search_to_mongodb +from core.util import search_to_mongodb, get_search_query router = APIRouter() @@ -223,23 +223,13 @@ async def get_sensitive_result_rules(request_data: dict, db=Depends(get_mongo_db @router.post("/sensitive/result/data2") -async def get_sensitive_result_rules(request_data: dict, db=Depends(get_mongo_db), _: dict = Depends(verify_token)): +async def get_sensitive_result_data2(request_data: dict, db=Depends(get_mongo_db), _: dict = Depends(verify_token)): try: - search_query = request_data.get("search", "") page_index = request_data.get("pageIndex", 1) page_size = request_data.get("pageSize", 10) - keyword = { - 'url': 'url', - 'sname': 'sid', - "body": "body", - "info": "match", - 'project': 'project', - 'md5': 'md5' - } - query = await search_to_mongodb(search_query, keyword) - if query == "" or query is None: + query = await get_search_query("sens", request_data) + if query == "": return {"message": "Search condition parsing error", "code": 500} - query = query[0] total_count = await db['SensitiveResult'].count_documents(query) pipeline = [ { diff --git a/api/task.py b/api/task.py index 42f6acc..6a6682f 100644 --- a/api/task.py +++ b/api/task.py @@ -181,7 +181,7 @@ async def delete_task(request_data: dict, db=Depends(get_mongo_db), _: dict = De scheduler.remove_job(task_id) await db.ScheduledTasks.delete_many({"id": {"$in": task_ids}}) if delA: - background_tasks.add_task(delete_asset, task_ids, db) + background_tasks.add_task(delete_asset, task_ids) await redis_con.delete(*redis_key) # Delete the SensitiveRule documents based on the provided IDs result = await db.task.delete_many({"_id": {"$in": obj_ids}}) @@ -558,19 +558,20 @@ async def scheduler_scan_task(id): await create_scan_task(doc, task_id, targetList, redis) -async def delete_asset(task_ids, db, is_project = False): - key = ["asset", "subdomain", "SubdoaminTakerResult", "UrlScan", "crawler", "SensitiveResult", "DirScanResult", "vulnerability", "PageMonitoring"] - del_query = {"taskId": {"$in": task_ids}} - if is_project: - del_query = { - "$or": [ - {"taskId": {"$in": task_ids}}, - {"project": {"$in": task_ids}} - ] - } - for k in key: - result = await db[k].delete_many(del_query) - if result.deleted_count > 0: - logger.info("Deleted {} {} documents".format(k, result.deleted_count)) - else: - logger.info("Deleted {} None documents".format(k)) \ No newline at end of file +async def delete_asset(task_ids, is_project = False): + async for db in get_mongo_db(): + key = ["asset", "subdomain", "SubdoaminTakerResult", "UrlScan", "crawler", "SensitiveResult", "DirScanResult", "vulnerability", "PageMonitoring"] + del_query = {"taskId": {"$in": task_ids}} + if is_project: + del_query = { + "$or": [ + {"taskId": {"$in": task_ids}}, + {"project": {"$in": task_ids}} + ] + } + for k in key: + result = await db[k].delete_many(del_query) + if result.deleted_count > 0: + logger.info("Deleted {} {} documents".format(k, result.deleted_count)) + else: + logger.info("Deleted {} None documents".format(k)) \ No newline at end of file diff --git a/api/vulnerability.py b/api/vulnerability.py index 8489052..f9243f2 100644 --- a/api/vulnerability.py +++ b/api/vulnerability.py @@ -9,33 +9,20 @@ from fastapi import APIRouter, Depends from motor.motor_asyncio import AsyncIOMotorCursor from pymongo import DESCENDING from api.users import verify_token -from core.config import POC_LIST from core.db import get_mongo_db -from core.util import search_to_mongodb +from core.util import search_to_mongodb, get_search_query from loguru import logger router = APIRouter() + @router.post("/vul/data") async def get_vul_data(request_data: dict, db=Depends(get_mongo_db), _: dict = Depends(verify_token)): try: - search_query = request_data.get("search", "") page_index = request_data.get("pageIndex", 1) page_size = request_data.get("pageSize", 10) - # MongoDB collection for SensitiveRule - # Fuzzy search based on the name field - keyword = { - 'url': 'url', - 'vulname': 'vulname', - 'project': 'project', - 'matched': 'matched', - 'request': 'request', - 'response': 'response', - 'level': 'level' - } - query = await search_to_mongodb(search_query, keyword) - if query == "" or query is None: + query = await get_search_query("vul", request_data) + if query == "": return {"message": "Search condition parsing error", "code": 500} - query = query[0] # Get the total count of documents matching the search criteria total_count = await db.vulnerability.count_documents(query) if total_count == 0: @@ -60,9 +47,8 @@ async def get_vul_data(request_data: dict, db=Depends(get_mongo_db), _: dict = D "time": doc["time"], "request": doc["request"], "response": doc["response"], + "level": doc['level'] } - if doc["vulnid"] in POC_LIST: - data["level"] = POC_LIST[doc["vulnid"]] response_data.append(data) return { "code": 200, diff --git a/core/config.py b/core/config.py index f0995a1..362bc00 100644 --- a/core/config.py +++ b/core/config.py @@ -27,7 +27,6 @@ NODE_TIMEOUT = 50 TOTAL_LOGS = 1000 APP = {} SensitiveRuleList = {} -POC_LIST = {} Project_List = {} def set_timezone(t): global TIMEZONE diff --git a/core/db.py b/core/db.py index af6c07e..5df1c44 100644 --- a/core/db.py +++ b/core/db.py @@ -133,7 +133,6 @@ async def create_database(): {"id": "page_monitoring", "name": "Page Monitoring", 'hour': 24, 'type': 'Page Monitoring', 'state': True}) await get_fingerprint(client[DATABASE_NAME]) await get_sens_rule(client[DATABASE_NAME]) - await get_pocList(client[DATABASE_NAME]) await get_project(client[DATABASE_NAME]) except Exception as e: # 处理异常 @@ -166,14 +165,6 @@ async def get_sens_rule(client): } -async def get_pocList(client): - collection = client["PocList"] - cursor = collection.find({}, {"_id": 1, "level": 1}) - async for document in cursor: - document['id'] = str(document['_id']) - POC_LIST[document['id']] = document['level'] - - async def get_project(client): collection = client["project"] cursor = collection.find({}, {"_id": 1, "name": 1}) diff --git a/core/util.py b/core/util.py index 0ccd0dc..ae5e8bd 100644 --- a/core/util.py +++ b/core/util.py @@ -321,6 +321,55 @@ async def search_to_mongodb(expression_raw, keyword): logger.error(e) return "" +async def get_search_query(name, request_data): + search_query = request_data.get("search", "") + search_key_v = { + 'sens':{ + 'url': 'url', + 'sname': 'sid', + "body": "body", + "info": "match", + 'project': 'project', + 'md5': 'md5' + }, + 'dir': { + 'project': 'project', + 'statuscode': 'status', + 'url': 'url', + 'redirect': 'msg', + 'length': 'length' + }, + 'vul': { + 'url': 'url', + 'vulname': 'vulname', + 'project': 'project', + 'matched': 'matched', + 'request': 'request', + 'response': 'response', + 'level': 'level' + } + } + keyword = search_key_v[name] + query = await search_to_mongodb(search_query, keyword) + if query == "" or query is None: + return "" + query = query[0] + filter_key = ['color', 'status', 'level'] + filter = request_data.get("filter", {}) + if filter: + query["$and"] = [] + for f in filter: + if f in filter_key: + tmp_or = [] + for v in filter[f]: + tmp_or.append({f: v}) + if len(tmp_or) != 0: + query["$and"].append({"$or": tmp_or}) + if "$and" in query: + if len(query["$and"]) == 0: + query.pop("$and") + return query + def get_root_domain(url): # 如果URL不带协议，添加一个默认的http协议