原创 Eastmount 2026-06-26 19:51 贵州
第3篇文章介绍智能体赋能网络威胁分析,希望您喜欢!
AUTHORIZATION_STRING="Bearer bce-v3/ALTAK-eGwh*****ec6"
import sys
import requests
from bs4 import BeautifulSoup
import csv
import re
import time
import json
import random
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.neighbors import NearestNeighbors
# ==========================================
# 百度千帆 API 参数:直接写在代码里
# ==========================================
API_URL = "https://qianfan.baidubce.com/v2/chat/completions"
AUTHORIZATION_STRING = "Bearer bce-v3/ALTAK-eGwh****ec6"
MAX_RETRIES = 5
BASE_DELAY = 1.0
HEADERS = {
"Content-Type": "application/json",
"Authorization": AUTHORIZATION_STRING
}
# System prompt
LOCAL_SYSTEM_PROMPT = """You are an intelligent assistant specializing in APT (Advanced Persistent Threat) threat intelligence analysis.
Based on the following threat intelligence sentence, identify and extract entities and relationships using only the specified types.
All outputs must be in English and follow the strict format below.
If entities and relationships are found, output in this format:
Entities:(name, entity type), (name, entity type)...
Relations:(entity1, relation, entity2), (entity1, relation, entity2)...
If only entities are found:
Entities:(name, entity type), (name, entity type)...
Relations:No relevant relations
If nothing found:
No relevant entities or relations identified
"""
class LocalLLMWithKNN:
def __init__(self, base_model_path, lora_model_path=None, jsonl_path=None, system_prompt=None, top_k=3):
"""
这里保留原来的类名和参数名。
但是 base_model_path 不再表示本地模型路径,而是百度在线模型名称。
lora_model_path 保留参数,但在线 API 不需要使用。
"""
self.top_k = top_k
self.system_prompt = LOCAL_SYSTEM_PROMPT
# 百度在线模型名
self.model_name = base_model_path
# 加载JSONL数据构建KNN
self.example_instructions = []
self.example_outputs = []
if jsonl_path:
self._load_jsonl_data(jsonl_path)
print(f"Loaded {len(self.example_instructions)} examples from {jsonl_path}")
# KNN相关初始化
if self.example_instructions:
self.vectorizer = TfidfVectorizer()
self.knn = self._build_knn_model()
print(f"KNN model built with {len(self.example_instructions)} examples")
else:
self.vectorizer = None
self.knn = None
print("No examples loaded, KNN will not be used")
print(f"Using Baidu online model: {self.model_name}")
if lora_model_path:
print("Notice: lora_model_path is ignored because Baidu API model is used online.")
def _load_jsonl_data(self, jsonl_path):
"""从JSONL文件加载示例数据:保持你的原逻辑"""
try:
with open(jsonl_path, 'r', encoding='utf-8') as f:
for line in f:
line = line.strip()
if not line:
continue
data = json.loads(line)
if 'instruction' in data and 'output' in data:
self.example_instructions.append(data['instruction'])
self.example_outputs.append(data['output'])
except Exception as e:
print(f"Error loading JSONL data from {jsonl_path}: {e}")
def _build_knn_model(self):
"""构建KNN模型:保持你的原逻辑"""
if not self.example_instructions:
return None
vectors = self.vectorizer.fit_transform(self.example_instructions)
n_neighbors = min(self.top_k + 1, len(self.example_instructions))
knn = NearestNeighbors(
n_neighbors=n_neighbors,
metric='cosine'
)
knn.fit(vectors)
return knn
def get_similar_examples(self, input_sentence):
"""获取相似的示例:保持你的原逻辑"""
if not self.knn or not self.vectorizer or not self.example_instructions:
return []
try:
input_vector = self.vectorizer.transform([input_sentence])
distances, indices = self.knn.kneighbors(input_vector)
similar_examples = []
for j in range(1, min(len(indices[0]), self.top_k + 1)):
idx = indices[0][j]
if idx < len(self.example_instructions):
similar_examples.append({
'instruction': self.example_instructions[idx],
'output': self.example_outputs[idx],
'distance': distances[0][j]
})
return similar_examples
except Exception as e:
print(f"Error getting similar examples: {e}")
return []
def call_baidu_api_with_retry(self, user_text):
"""
调用百度千帆 API,替代原来的本地 self.pipe 推理
"""
payload = {
"model": self.model_name,
"messages": [
{
"role": "system",
"content": self.system_prompt
},
{
"role": "user",
"content": user_text
}
],
"temperature": 0.0,
"top_p": 1.0
}
retries = 0
while retries < MAX_RETRIES:
try:
response = requests.post(
API_URL,
headers=HEADERS,
json=payload,
timeout=60
)
if response.status_code == 200:
return response.json()
elif response.status_code == 429:
wait_time = (2 ** retries) + random.random()
print(f" [!] 触发限速 429,等待 {wait_time:.2f} 秒后进行第 {retries + 1} 次重试...")
time.sleep(wait_time)
retries += 1
else:
print(f" [-] API 报错,状态码 {response.status_code}: {response.text}")
return None
except requests.exceptions.RequestException as e:
wait_time = (2 ** retries) + random.random()
print(f" [-] 网络异常: {e},等待 {wait_time:.2f} 秒后重试...")
time.sleep(wait_time)
retries += 1
print(" [-] 已达到最大重试次数,放弃该条数据。")
return None
def generate_response(self, input_sentence):
"""生成响应:保持你的执行逻辑,只把模型调用换成百度 API"""
# 获取相似示例
similar_examples = self.get_similar_examples(input_sentence)
# 构建prompt:保持你的原逻辑
if similar_examples:
examples_parts = []
for i, ex in enumerate(similar_examples, 1):
examples_parts.append(
f"Example {i}:\nInput: {ex['instruction']}\n {ex['output']}"
)
examples_text = "\n\n".join(examples_parts)
userText = "\nExamples:\n" + examples_text + "\n\nInput: " + input_sentence
else:
userText = "\n\nInput: " + input_sentence
try:
print("\n[Prompt Sent to Baidu API]:")
print(userText)
api_result = self.call_baidu_api_with_retry(userText)
generated_text = ""
if api_result and "choices" in api_result:
generated_text = api_result["choices"][0]["message"]["content"].strip()
else:
generated_text = "ERROR_OR_SKIPPED"
print(f"\n[Model Response]:")
print(generated_text)
time.sleep(BASE_DELAY)
return generated_text, similar_examples
except Exception as e:
print(f"Error generating response: {e}")
return f"Error: {e}", []
class MITREExtractor:
"""MITRE数据抽取器"""
def __init__(self, llm_model=None):
"""初始化MITRE抽取器"""
self.llm_model = llm_model
self.items = []
def _load_items(self, csv_file='groups.csv'):
"""从CSV文件加载项目"""
items = []
try:
with open(csv_file, 'r', encoding='utf-8-sig') as file:
reader = csv.reader(file)
next(reader)
for row in reader:
if len(row) >= 2:
name = row[0].strip()
url = row[1].strip()
if name and url:
items.append({
'name': name,
'url': url
})
print(f"Loaded {len(items)} items from {csv_file}")
self.items = items
return items
except Exception as e:
print(f"Error loading CSV file {csv_file}: {e}")
return []
def _save_result_to_jsonl(self, instruction, output, filename='extraction_results.jsonl'):
"""将抽取结果保存到JSONL文件"""
try:
data = {
"instruction": instruction,
"input": None,
"output": output
}
with open(filename, 'a', encoding='utf-8') as f:
f.write(json.dumps(data, ensure_ascii=False) + '\n')
print(f" [Saved to {filename}]")
except Exception as e:
print(f" [Error saving to JSONL: {e}]")
def _extract_item(self, item_name, url):
"""抽取单个项目的信息"""
if not self.llm_model:
print("Error: LLM model not set. Please set llm_model first.")
return
try:
response = requests.get(url, timeout=30)
soup = BeautifulSoup(response.text, 'html.parser')
table = soup.find(
'table',
class_='table techniques-used background table-bordered'
)
if not table:
print(f"No techniques table found for {item_name}")
return
rows = table.find_all('tr')
for row_idx, row in enumerate(rows):
td_elements = row.find_all('td')
if td_elements:
last_td_text = td_elements[-1].get_text(strip=True)
cleaned_text = re.sub(r'\[\d+\]', '', last_td_text).strip()
if not cleaned_text:
continue
print(f"\n--- Sentence {row_idx}/{len(rows) - 1} ---")
print(f"[Input Sentence]: {cleaned_text[:120]}{'...'iflen(cleaned_text) > 120else''}")
# 使用百度在线模型生成回答
llm_response, similar_examples = self.llm_model.generate_response(cleaned_text)
# 将结果保存到JSONL文件
if llm_response and not llm_response.startswith("Error:") and llm_response != "ERROR_OR_SKIPPED":
self._save_result_to_jsonl(cleaned_text, llm_response)
time.sleep(0.1)
except Exception as e:
print(f"Error extracting data from {item_name} ({url}): {e}")
def extract(self, csv_file='groups.csv'):
"""抽取数据"""
if not self.llm_model:
print("Error: LLM model not set. Please set llm_model first.")
return
# 从CSV文件加载项目
self._load_items(csv_file)
if not self.items:
print("Error: No items loaded from CSV file.")
return
# 抽取每个项目的信息
for index, item in enumerate(self.items, 1):
print(f"\n{'=' * 60}")
print(f"Item {index}/{len(self.items)}: {item['name']}")
self._extract_item(item['name'], item['url'])
if index < len(self.items):
time.sleep(0.5)
def run_extraction(base_model_path, lora_model_path=None, jsonl_path=None):
"""运行MITRE数据抽取流程"""
# 这里仍然保留原来的调用方式
# 但是 base_model_path 现在是百度在线模型名称
llm_model = LocalLLMWithKNN(
base_model_path=base_model_path,
lora_model_path=lora_model_path,
jsonl_path=jsonl_path,
top_k=3
)
# 创建MITRE抽取器,传入LLM模型
mitre_extractor = MITREExtractor(llm_model)
# 抽取数据
mitre_extractor.extract()
def main():
# 这里不再写本地模型路径,而是写百度平台模型名称
base_model_path = "百度配置的模型名称"
# 在线 API 不需要 LoRA 路径,保留变量只是为了不改变原执行逻辑
lora_model_path = None
jsonl_path = "APTtest-Final-440.jsonl"
print(f"Starting extraction")
print(f"Base model: {base_model_path}")
print(f"LoRA adapter: {lora_model_path if lora_model_path else'Not used'}")
print(f"KNN data: {jsonl_path if jsonl_path else'Not used'}")
run_extraction(base_model_path, lora_model_path, jsonl_path)
print("\nExtraction complete!")
if __name__ == "__main__":
main()
{"instruction": "APT32enumerated administrative users using the commandsnet localgroup administrators.", "input": null, "output": "Entities: (APT32, Attacker), (net, Tool)\nRelations: (APT32, Use, net)"}
{"instruction": "APT32has set up and operated websites to gather information and deliver malware.", "input": null, "output": "Entities: (APT32, Attacker), (information gathering, Technique), (malware delivery, Technique)\nRelations: (APT32, Use, information gathering), (APT32, Use, malware delivery)"}
{"instruction": "APT32has set up Dropbox, Amazon S3, and Google Drive to host malicious downloads.", "input": null, "output": "Entities: (APT32, Attacker), (Dropbox, Tool), (Amazon S3, Tool), (Google Drive, Tool)\nRelations: (APT32, Use, Dropbox), (APT32, Use, Amazon S3), (APT32, Use, Google Drive)"}
{"instruction": "APT32has used JavaScript that communicates over HTTP or HTTPS to attacker controlled domains to download additional frameworks. The group has also used downloaded encrypted payloads over HTTP.", "input": null, "output": "Entities: (APT32, Attacker), (JavaScript, Tool), (HTTP, Technique), (HTTPS, Technique)\nRelations: (APT32, Use, JavaScript), (APT32, Use, HTTP), (APT32, Use, HTTPS)"}
最终其将抽取的知识存储至本地Json文件,为后续知识图谱构建提供支撑。
▼点击下方,即可购书