← 返回列表

[智能体攻防实战] 三.基于精调大模型的网络威胁知识自动抽取与分析(CodeBuddy+千帆)

单站 娜璋AI安全之家 发布 Fri, 26 Jun 2026 19:51:00 +0800 入库 2026-11-18T12:26:05

原创 Eastmount 2026-06-26 19:51 贵州

第3篇文章介绍智能体赋能网络威胁分析,希望您喜欢!

AUTHORIZATION_STRING="Bearer bce-v3/ALTAK-eGwh*****ec6"
import sys

import requests

from bs4 import BeautifulSoup

import csv

import re

import time

import json

import random

from sklearn.feature_extraction.text import TfidfVectorizer

from sklearn.neighbors import NearestNeighbors

# ==========================================

# 百度千帆 API 参数:直接写在代码里

# ==========================================

API_URL = "https://qianfan.baidubce.com/v2/chat/completions"

AUTHORIZATION_STRING = "Bearer bce-v3/ALTAK-eGwh****ec6"

MAX_RETRIES = 5

BASE_DELAY = 1.0

HEADERS = {

"Content-Type": "application/json",

"Authorization": AUTHORIZATION_STRING

}

# System prompt

LOCAL_SYSTEM_PROMPT = """You are an intelligent assistant specializing in APT (Advanced Persistent Threat) threat intelligence analysis.

Based on the following threat intelligence sentence, identify and extract entities and relationships using only the specified types. 

All outputs must be in English and follow the strict format below.

If entities and relationships are found, output in this format:

Entities:(name, entity type), (name, entity type)...

Relations:(entity1, relation, entity2), (entity1, relation, entity2)...

If only entities are found:

Entities:(name, entity type), (name, entity type)...

Relations:No relevant relations

If nothing found:

No relevant entities or relations identified

"""

class LocalLLMWithKNN:

def __init__(self, base_model_path, lora_model_path=None, jsonl_path=None, system_prompt=None, top_k=3):

"""

这里保留原来的类名和参数名。

但是 base_model_path 不再表示本地模型路径,而是百度在线模型名称。

lora_model_path 保留参数,但在线 API 不需要使用。

"""

self.top_k = top_k

self.system_prompt = LOCAL_SYSTEM_PROMPT

# 百度在线模型名

self.model_name = base_model_path

# 加载JSONL数据构建KNN

self.example_instructions = []

self.example_outputs = []

if jsonl_path:

self._load_jsonl_data(jsonl_path)

print(f"Loaded {len(self.example_instructions)} examples from {jsonl_path}")

# KNN相关初始化

if self.example_instructions:

self.vectorizer = TfidfVectorizer()

self.knn = self._build_knn_model()

print(f"KNN model built with {len(self.example_instructions)} examples")

else:

self.vectorizer = None

self.knn = None

print("No examples loaded, KNN will not be used")

print(f"Using Baidu online model: {self.model_name}")

if lora_model_path:

print("Notice: lora_model_path is ignored because Baidu API model is used online.")

def _load_jsonl_data(self, jsonl_path):

"""从JSONL文件加载示例数据:保持你的原逻辑"""

try:

with open(jsonl_path, 'r', encoding='utf-8') as f:

for line in f:

line = line.strip()

if not line:

continue

data = json.loads(line)

if 'instruction' in data and 'output' in data:

self.example_instructions.append(data['instruction'])

self.example_outputs.append(data['output'])

except Exception as e:

print(f"Error loading JSONL data from {jsonl_path}: {e}")

def _build_knn_model(self):

"""构建KNN模型:保持你的原逻辑"""

if not self.example_instructions:

return None

vectors = self.vectorizer.fit_transform(self.example_instructions)

n_neighbors = min(self.top_k + 1, len(self.example_instructions))

knn = NearestNeighbors(

n_neighbors=n_neighbors,

metric='cosine'

)

knn.fit(vectors)

return knn

def get_similar_examples(self, input_sentence):

"""获取相似的示例:保持你的原逻辑"""

if not self.knn or not self.vectorizer or not self.example_instructions:

return []

try:

input_vector = self.vectorizer.transform([input_sentence])

distances, indices = self.knn.kneighbors(input_vector)

similar_examples = []

for j in range(1, min(len(indices[0]), self.top_k + 1)):

idx = indices[0][j]

if idx < len(self.example_instructions):

similar_examples.append({

'instruction': self.example_instructions[idx],

'output': self.example_outputs[idx],

'distance': distances[0][j]

})

return similar_examples

except Exception as e:

print(f"Error getting similar examples: {e}")

return []

def call_baidu_api_with_retry(self, user_text):

"""

调用百度千帆 API,替代原来的本地 self.pipe 推理

"""

payload = {

"model": self.model_name,

"messages": [

{

"role": "system",

"content": self.system_prompt

},

{

"role": "user",

"content": user_text

}

],

"temperature": 0.0,

"top_p": 1.0

}

retries = 0

while retries < MAX_RETRIES:

try:

response = requests.post(

API_URL,

headers=HEADERS,

json=payload,

timeout=60

)

if response.status_code == 200:

return response.json()

elif response.status_code == 429:

wait_time = (2 ** retries) + random.random()

print(f"    [!] 触发限速 429,等待 {wait_time:.2f} 秒后进行第 {retries + 1} 次重试...")

time.sleep(wait_time)

retries += 1

else:

print(f"    [-] API 报错,状态码 {response.status_code}: {response.text}")

return None

except requests.exceptions.RequestException as e:

wait_time = (2 ** retries) + random.random()

print(f"    [-] 网络异常: {e},等待 {wait_time:.2f} 秒后重试...")

time.sleep(wait_time)

retries += 1

print("    [-] 已达到最大重试次数,放弃该条数据。")

return None

def generate_response(self, input_sentence):

"""生成响应:保持你的执行逻辑,只把模型调用换成百度 API"""

# 获取相似示例

similar_examples = self.get_similar_examples(input_sentence)

# 构建prompt:保持你的原逻辑

if similar_examples:

examples_parts = []

for i, ex in enumerate(similar_examples, 1):

examples_parts.append(

f"Example {i}:\nInput: {ex['instruction']}\n {ex['output']}"

)

examples_text = "\n\n".join(examples_parts)

userText = "\nExamples:\n" + examples_text + "\n\nInput: " + input_sentence

else:

userText = "\n\nInput: " + input_sentence

try:

print("\n[Prompt Sent to Baidu API]:")

print(userText)

api_result = self.call_baidu_api_with_retry(userText)

generated_text = ""

if api_result and "choices" in api_result:

generated_text = api_result["choices"][0]["message"]["content"].strip()

else:

generated_text = "ERROR_OR_SKIPPED"

print(f"\n[Model Response]:")

print(generated_text)

time.sleep(BASE_DELAY)

return generated_text, similar_examples

except Exception as e:

print(f"Error generating response: {e}")

return f"Error: {e}", []

class MITREExtractor:

"""MITRE数据抽取器"""

def __init__(self, llm_model=None):

"""初始化MITRE抽取器"""

self.llm_model = llm_model

self.items = []

def _load_items(self, csv_file='groups.csv'):

"""从CSV文件加载项目"""

items = []

try:

with open(csv_file, 'r', encoding='utf-8-sig') as file:

reader = csv.reader(file)

next(reader)

for row in reader:

if len(row) >= 2:

name = row[0].strip()

url = row[1].strip()

if name and url:

items.append({

'name': name,

'url': url

})

print(f"Loaded {len(items)} items from {csv_file}")

self.items = items

return items

except Exception as e:

print(f"Error loading CSV file {csv_file}: {e}")

return []

def _save_result_to_jsonl(self, instruction, output, filename='extraction_results.jsonl'):

"""将抽取结果保存到JSONL文件"""

try:

data = {

"instruction": instruction,

"input": None,

"output": output

}

with open(filename, 'a', encoding='utf-8') as f:

f.write(json.dumps(data, ensure_ascii=False) + '\n')

print(f"  [Saved to {filename}]")

except Exception as e:

print(f"  [Error saving to JSONL: {e}]")

def _extract_item(self, item_name, url):

"""抽取单个项目的信息"""

if not self.llm_model:

print("Error: LLM model not set. Please set llm_model first.")

return

try:

response = requests.get(url, timeout=30)

soup = BeautifulSoup(response.text, 'html.parser')

table = soup.find(

'table',

class_='table techniques-used background table-bordered'

)

if not table:

print(f"No techniques table found for {item_name}")

return

rows = table.find_all('tr')

for row_idx, row in enumerate(rows):

td_elements = row.find_all('td')

if td_elements:

last_td_text = td_elements[-1].get_text(strip=True)

cleaned_text = re.sub(r'\[\d+\]', '', last_td_text).strip()

if not cleaned_text:

continue

print(f"\n--- Sentence {row_idx}/{len(rows) - 1} ---")

print(f"[Input Sentence]: {cleaned_text[:120]}{'...'iflen(cleaned_text) > 120else''}")

# 使用百度在线模型生成回答

llm_response, similar_examples = self.llm_model.generate_response(cleaned_text)

# 将结果保存到JSONL文件

if llm_response and not llm_response.startswith("Error:") and llm_response != "ERROR_OR_SKIPPED":

self._save_result_to_jsonl(cleaned_text, llm_response)

time.sleep(0.1)

except Exception as e:

print(f"Error extracting data from {item_name} ({url}): {e}")

def extract(self, csv_file='groups.csv'):

"""抽取数据"""

if not self.llm_model:

print("Error: LLM model not set. Please set llm_model first.")

return

# 从CSV文件加载项目

self._load_items(csv_file)

if not self.items:

print("Error: No items loaded from CSV file.")

return

# 抽取每个项目的信息

for index, item in enumerate(self.items, 1):

print(f"\n{'=' * 60}")

print(f"Item {index}/{len(self.items)}: {item['name']}")

self._extract_item(item['name'], item['url'])

if index < len(self.items):

time.sleep(0.5)

def run_extraction(base_model_path, lora_model_path=None, jsonl_path=None):

"""运行MITRE数据抽取流程"""

# 这里仍然保留原来的调用方式

# 但是 base_model_path 现在是百度在线模型名称

llm_model = LocalLLMWithKNN(

base_model_path=base_model_path,

lora_model_path=lora_model_path,

jsonl_path=jsonl_path,

top_k=3

)

# 创建MITRE抽取器,传入LLM模型

mitre_extractor = MITREExtractor(llm_model)

# 抽取数据

mitre_extractor.extract()

def main():

# 这里不再写本地模型路径,而是写百度平台模型名称

base_model_path = "百度配置的模型名称"

# 在线 API 不需要 LoRA 路径,保留变量只是为了不改变原执行逻辑

lora_model_path = None

jsonl_path = "APTtest-Final-440.jsonl"

print(f"Starting extraction")

print(f"Base model: {base_model_path}")

print(f"LoRA adapter: {lora_model_path if lora_model_path else'Not used'}")

print(f"KNN data: {jsonl_path if jsonl_path else'Not used'}")

run_extraction(base_model_path, lora_model_path, jsonl_path)

print("\nExtraction complete!")

if __name__ == "__main__":

main()

{"instruction": "APT32enumerated administrative users using the commandsnet localgroup administrators.", "input": null, "output": "Entities: (APT32, Attacker), (net, Tool)\nRelations: (APT32, Use, net)"}

{"instruction": "APT32has set up and operated websites to gather information and deliver malware.", "input": null, "output": "Entities: (APT32, Attacker), (information gathering, Technique), (malware delivery, Technique)\nRelations: (APT32, Use, information gathering), (APT32, Use, malware delivery)"}

{"instruction": "APT32has set up Dropbox, Amazon S3, and Google Drive to host malicious downloads.", "input": null, "output": "Entities: (APT32, Attacker), (Dropbox, Tool), (Amazon S3, Tool), (Google Drive, Tool)\nRelations: (APT32, Use, Dropbox), (APT32, Use, Amazon S3), (APT32, Use, Google Drive)"}

{"instruction": "APT32has used JavaScript that communicates over HTTP or HTTPS to attacker controlled domains to download additional frameworks. The group has also used downloaded encrypted payloads over HTTP.", "input": null, "output": "Entities: (APT32, Attacker), (JavaScript, Tool), (HTTP, Technique), (HTTPS, Technique)\nRelations: (APT32, Use, JavaScript), (APT32, Use, HTTP), (APT32, Use, HTTPS)"}

最终其将抽取的知识存储至本地Json文件,为后续知识图谱构建提供支撑。

▼点击下方,即可购书

原文链接:https://mp.weixin.qq.com/s?__biz=Mzg5MTM5ODU2Mg==&mid=2247502984&idx=1&sn=d3d7c0e5aec42ce8129a3e94af657291