backup materials and knowledge-base docs
This commit is contained in:
@@ -0,0 +1,81 @@
|
||||
# Citation Verification Scripts
|
||||
|
||||
## 状态说明
|
||||
|
||||
**这些脚本是参考实现,不是主要工作流的一部分。**
|
||||
|
||||
本目录中的Python脚本提供了基于API的引用验证实现,但**实际的引用验证工作流使用WebSearch和Google Scholar**,而不是这些脚本。
|
||||
|
||||
## 为什么保留这些脚本?
|
||||
|
||||
这些脚本作为**参考实现**保留,用于:
|
||||
|
||||
1. **理解验证逻辑** - 展示引用验证的完整逻辑和步骤
|
||||
2. **学习API使用** - 了解如何使用CrossRef、arXiv、Semantic Scholar等API
|
||||
3. **高级用例** - 对于需要批量验证或自动化的场景,可以参考这些实现
|
||||
|
||||
## 主要工作流
|
||||
|
||||
**实际的引用验证应该使用 `ml-paper-writing` skill 中的 Citation Workflow:**
|
||||
|
||||
1. 使用 WebSearch 查找论文
|
||||
2. 在 Google Scholar 上验证
|
||||
3. 从 Google Scholar 获取 BibTeX
|
||||
4. 验证声明(如需要)
|
||||
5. 添加到 bibliography
|
||||
|
||||
详见 `ml-paper-writing` skill 的 "Citation Workflow (Hallucination Prevention)" 部分。
|
||||
|
||||
## 脚本说明
|
||||
|
||||
### verify-citations.py
|
||||
|
||||
完整的引用验证脚本,包含:
|
||||
- 四层验证机制(格式、存在性、信息匹配、内容验证)
|
||||
- 多API支持(CrossRef、arXiv、Semantic Scholar)
|
||||
- 报告生成
|
||||
|
||||
**用途**: 参考实现,了解完整的验证逻辑
|
||||
|
||||
### api-clients.py
|
||||
|
||||
API客户端库,包含:
|
||||
- CrossRefClient - DOI验证
|
||||
- ArXivClient - arXiv论文验证
|
||||
- SemanticScholarClient - 通用学术搜索
|
||||
- CitationAPIManager - 统一API管理
|
||||
|
||||
**用途**: 参考实现,了解如何使用学术API
|
||||
|
||||
### format-checker.py
|
||||
|
||||
BibTeX和LaTeX格式检查工具,包含:
|
||||
- BibTeX格式验证
|
||||
- LaTeX引用检查
|
||||
- 格式错误报告
|
||||
|
||||
**用途**: 参考实现,了解格式检查逻辑
|
||||
|
||||
## 使用建议
|
||||
|
||||
**对于日常论文写作**:
|
||||
- ✅ 使用 `ml-paper-writing` skill 的 Citation Workflow
|
||||
- ✅ 使用 WebSearch 和 Google Scholar
|
||||
- ❌ 不要使用这些Python脚本
|
||||
|
||||
**对于批量验证或自动化**:
|
||||
- 可以参考这些脚本的实现
|
||||
- 根据需要修改和使用
|
||||
- 注意API速率限制
|
||||
|
||||
## 依赖安装
|
||||
|
||||
如果需要运行这些脚本(仅用于参考或高级用例):
|
||||
|
||||
```bash
|
||||
pip install bibtexparser requests semanticscholar arxiv
|
||||
```
|
||||
|
||||
## 更多信息
|
||||
|
||||
详见 `citation-verification` skill 的 SKILL.md 文件。
|
||||
@@ -0,0 +1,542 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
API Clients for Citation Verification
|
||||
|
||||
提供三个主要 API 客户端:
|
||||
1. CrossRefClient - DOI 验证
|
||||
2. ArXivClient - arXiv 论文验证
|
||||
3. SemanticScholarClient - 通用学术搜索
|
||||
|
||||
每个客户端都包含:
|
||||
- 错误处理
|
||||
- 重试机制
|
||||
- 速率限制
|
||||
- 结果标准化
|
||||
"""
|
||||
|
||||
import time
|
||||
import requests
|
||||
from typing import Dict, List, Optional
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
|
||||
class RateLimiter:
|
||||
"""速率限制器"""
|
||||
|
||||
def __init__(self, calls_per_minute: int):
|
||||
self.calls_per_minute = calls_per_minute
|
||||
self.last_call = 0
|
||||
self.min_interval = 60.0 / calls_per_minute
|
||||
|
||||
def wait_if_needed(self):
|
||||
"""如果需要,等待以满足速率限制"""
|
||||
elapsed = time.time() - self.last_call
|
||||
if elapsed < self.min_interval:
|
||||
time.sleep(self.min_interval - elapsed)
|
||||
self.last_call = time.time()
|
||||
|
||||
|
||||
class APIClient(ABC):
|
||||
"""API 客户端基类"""
|
||||
|
||||
def __init__(self, rate_limit: int = 20):
|
||||
"""
|
||||
Args:
|
||||
rate_limit: 每分钟最大请求数
|
||||
"""
|
||||
self.rate_limiter = RateLimiter(rate_limit)
|
||||
|
||||
@abstractmethod
|
||||
def search(self, **kwargs) -> Optional[Dict]:
|
||||
"""搜索论文"""
|
||||
pass
|
||||
|
||||
def _retry_request(self, func, max_retries: int = 3):
|
||||
"""带重试的请求"""
|
||||
for i in range(max_retries):
|
||||
try:
|
||||
self.rate_limiter.wait_if_needed()
|
||||
return func()
|
||||
except requests.exceptions.RequestException as e:
|
||||
if i == max_retries - 1:
|
||||
raise
|
||||
time.sleep(2 ** i) # 指数退避
|
||||
return None
|
||||
|
||||
|
||||
class CrossRefClient(APIClient):
|
||||
"""CrossRef API 客户端
|
||||
|
||||
用于通过 DOI 验证论文信息
|
||||
|
||||
API 文档: https://api.crossref.org/
|
||||
"""
|
||||
|
||||
def __init__(self, rate_limit: int = 50):
|
||||
"""
|
||||
Args:
|
||||
rate_limit: 每分钟最大请求数 (CrossRef 限制较宽松)
|
||||
"""
|
||||
super().__init__(rate_limit)
|
||||
self.base_url = "https://api.crossref.org"
|
||||
|
||||
def search_by_doi(self, doi: str) -> Optional[Dict]:
|
||||
"""通过 DOI 搜索论文
|
||||
|
||||
Args:
|
||||
doi: DOI 标识符 (如 10.1038/nature12345)
|
||||
|
||||
Returns:
|
||||
标准化的论文信息字典,如果未找到则返回 None
|
||||
"""
|
||||
def request():
|
||||
url = f"{self.base_url}/works/{doi}"
|
||||
response = requests.get(url, timeout=10)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
try:
|
||||
data = self._retry_request(request)
|
||||
if data and 'message' in data:
|
||||
return self._normalize_result(data['message'])
|
||||
return None
|
||||
except Exception as e:
|
||||
print(f"CrossRef API 错误: {e}")
|
||||
return None
|
||||
|
||||
def search(self, doi: str = None, **kwargs) -> Optional[Dict]:
|
||||
"""搜索论文 (统一接口)"""
|
||||
if doi:
|
||||
return self.search_by_doi(doi)
|
||||
return None
|
||||
|
||||
def _normalize_result(self, data: Dict) -> Dict:
|
||||
"""标准化 CrossRef 返回结果"""
|
||||
# 提取标题
|
||||
title = data.get('title', [''])[0] if 'title' in data else ''
|
||||
|
||||
# 提取作者
|
||||
authors = []
|
||||
if 'author' in data:
|
||||
for author in data['author']:
|
||||
given = author.get('given', '')
|
||||
family = author.get('family', '')
|
||||
if given and family:
|
||||
authors.append(f"{given} {family}")
|
||||
elif family:
|
||||
authors.append(family)
|
||||
|
||||
# 提取年份
|
||||
year = None
|
||||
if 'published' in data:
|
||||
date_parts = data['published'].get('date-parts', [[]])[0]
|
||||
if date_parts:
|
||||
year = date_parts[0]
|
||||
elif 'created' in data:
|
||||
date_parts = data['created'].get('date-parts', [[]])[0]
|
||||
if date_parts:
|
||||
year = date_parts[0]
|
||||
|
||||
# 提取期刊/会议名称
|
||||
venue = ''
|
||||
if 'container-title' in data:
|
||||
venue = data['container-title'][0] if data['container-title'] else ''
|
||||
|
||||
return {
|
||||
'title': title,
|
||||
'authors': authors,
|
||||
'year': year,
|
||||
'venue': venue,
|
||||
'doi': data.get('DOI', ''),
|
||||
'type': data.get('type', ''),
|
||||
'source': 'crossref'
|
||||
}
|
||||
|
||||
def get_bibtex(self, doi: str) -> Optional[str]:
|
||||
"""通过 DOI 获取 BibTeX
|
||||
|
||||
Args:
|
||||
doi: DOI 标识符
|
||||
|
||||
Returns:
|
||||
BibTeX 字符串,如果失败则返回 None
|
||||
"""
|
||||
def request():
|
||||
url = f"https://doi.org/{doi}"
|
||||
headers = {"Accept": "application/x-bibtex"}
|
||||
response = requests.get(url, headers=headers, timeout=10)
|
||||
response.raise_for_status()
|
||||
return response.text
|
||||
|
||||
try:
|
||||
return self._retry_request(request)
|
||||
except Exception as e:
|
||||
print(f"获取 BibTeX 失败: {e}")
|
||||
return None
|
||||
|
||||
|
||||
class ArXivClient(APIClient):
|
||||
"""arXiv API 客户端
|
||||
|
||||
用于验证 arXiv 预印本论文
|
||||
|
||||
API 文档: https://info.arxiv.org/help/api/
|
||||
"""
|
||||
|
||||
def __init__(self, rate_limit: int = 20):
|
||||
"""
|
||||
Args:
|
||||
rate_limit: 每分钟最大请求数
|
||||
"""
|
||||
super().__init__(rate_limit)
|
||||
try:
|
||||
import arxiv
|
||||
self.arxiv = arxiv
|
||||
except ImportError:
|
||||
raise ImportError("需要安装 arxiv 库: pip install arxiv")
|
||||
|
||||
def search_by_id(self, arxiv_id: str) -> Optional[Dict]:
|
||||
"""通过 arXiv ID 搜索论文
|
||||
|
||||
Args:
|
||||
arxiv_id: arXiv 标识符 (如 2301.12345 或 cs/0703001)
|
||||
|
||||
Returns:
|
||||
标准化的论文信息字典,如果未找到则返回 None
|
||||
"""
|
||||
def request():
|
||||
search = self.arxiv.Search(id_list=[arxiv_id])
|
||||
paper = next(search.results())
|
||||
return paper
|
||||
|
||||
try:
|
||||
self.rate_limiter.wait_if_needed()
|
||||
paper = request()
|
||||
return self._normalize_result(paper)
|
||||
except StopIteration:
|
||||
print(f"arXiv 论文未找到: {arxiv_id}")
|
||||
return None
|
||||
except Exception as e:
|
||||
print(f"arXiv API 错误: {e}")
|
||||
return None
|
||||
|
||||
def search_by_title(self, title: str, max_results: int = 5) -> Optional[Dict]:
|
||||
"""通过标题搜索论文
|
||||
|
||||
Args:
|
||||
title: 论文标题
|
||||
max_results: 最大返回结果数
|
||||
|
||||
Returns:
|
||||
标准化的论文信息字典(第一个结果),如果未找到则返回 None
|
||||
"""
|
||||
def request():
|
||||
search = self.arxiv.Search(
|
||||
query=f'ti:"{title}"',
|
||||
max_results=max_results,
|
||||
sort_by=self.arxiv.SortCriterion.Relevance
|
||||
)
|
||||
results = list(search.results())
|
||||
return results[0] if results else None
|
||||
|
||||
try:
|
||||
self.rate_limiter.wait_if_needed()
|
||||
paper = request()
|
||||
if paper:
|
||||
return self._normalize_result(paper)
|
||||
return None
|
||||
except Exception as e:
|
||||
print(f"arXiv API 错误: {e}")
|
||||
return None
|
||||
|
||||
def search(self, arxiv_id: str = None, title: str = None, **kwargs) -> Optional[Dict]:
|
||||
"""搜索论文 (统一接口)"""
|
||||
if arxiv_id:
|
||||
return self.search_by_id(arxiv_id)
|
||||
elif title:
|
||||
return self.search_by_title(title)
|
||||
return None
|
||||
|
||||
def _normalize_result(self, paper) -> Dict:
|
||||
"""标准化 arXiv 返回结果"""
|
||||
# 提取 arXiv ID
|
||||
arxiv_id = paper.entry_id.split('/')[-1]
|
||||
|
||||
return {
|
||||
'title': paper.title,
|
||||
'authors': [a.name for a in paper.authors],
|
||||
'year': paper.published.year,
|
||||
'venue': 'arXiv',
|
||||
'arxiv_id': arxiv_id,
|
||||
'doi': paper.doi if hasattr(paper, 'doi') else None,
|
||||
'abstract': paper.summary,
|
||||
'pdf_url': paper.pdf_url,
|
||||
'source': 'arxiv'
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def extract_arxiv_id(text: str) -> Optional[str]:
|
||||
"""从文本中提取 arXiv ID
|
||||
|
||||
Args:
|
||||
text: 包含 arXiv ID 的文本
|
||||
|
||||
Returns:
|
||||
arXiv ID,如果未找到则返回 None
|
||||
"""
|
||||
import re
|
||||
|
||||
# 匹配新格式: YYMM.NNNNN
|
||||
match = re.search(r'\d{4}\.\d{4,5}', text)
|
||||
if match:
|
||||
return match.group()
|
||||
|
||||
# 匹配旧格式: arch-ive/YYMMNNN
|
||||
match = re.search(r'[a-z-]+/\d{7}', text)
|
||||
if match:
|
||||
return match.group()
|
||||
|
||||
return None
|
||||
|
||||
|
||||
class SemanticScholarClient(APIClient):
|
||||
"""Semantic Scholar API 客户端
|
||||
|
||||
用于通用学术论文搜索和验证
|
||||
|
||||
API 文档: https://api.semanticscholar.org/api-docs/
|
||||
"""
|
||||
|
||||
def __init__(self, rate_limit: int = 20):
|
||||
"""
|
||||
Args:
|
||||
rate_limit: 每分钟最大请求数 (Semantic Scholar 限制: 100 requests/5min)
|
||||
"""
|
||||
super().__init__(rate_limit)
|
||||
try:
|
||||
from semanticscholar import SemanticScholar
|
||||
self.sch = SemanticScholar()
|
||||
except ImportError:
|
||||
raise ImportError("需要安装 semanticscholar 库: pip install semanticscholar")
|
||||
|
||||
def search_by_title(self, title: str, max_results: int = 5) -> Optional[Dict]:
|
||||
"""通过标题搜索论文
|
||||
|
||||
Args:
|
||||
title: 论文标题
|
||||
max_results: 最大返回结果数
|
||||
|
||||
Returns:
|
||||
标准化的论文信息字典(第一个结果),如果未找到则返回 None
|
||||
"""
|
||||
try:
|
||||
self.rate_limiter.wait_if_needed()
|
||||
results = self.sch.search_paper(title, limit=max_results)
|
||||
|
||||
if not results:
|
||||
return None
|
||||
|
||||
# 返回第一个结果
|
||||
paper = results[0]
|
||||
return self._normalize_result(paper)
|
||||
except Exception as e:
|
||||
print(f"Semantic Scholar API 错误: {e}")
|
||||
return None
|
||||
|
||||
def search_by_doi(self, doi: str) -> Optional[Dict]:
|
||||
"""通过 DOI 搜索论文
|
||||
|
||||
Args:
|
||||
doi: DOI 标识符
|
||||
|
||||
Returns:
|
||||
标准化的论文信息字典,如果未找到则返回 None
|
||||
"""
|
||||
try:
|
||||
self.rate_limiter.wait_if_needed()
|
||||
paper = self.sch.get_paper(f"DOI:{doi}")
|
||||
if paper:
|
||||
return self._normalize_result(paper)
|
||||
return None
|
||||
except Exception as e:
|
||||
print(f"Semantic Scholar API 错误: {e}")
|
||||
return None
|
||||
|
||||
def search(self, title: str = None, doi: str = None, **kwargs) -> Optional[Dict]:
|
||||
"""搜索论文 (统一接口)"""
|
||||
if doi:
|
||||
return self.search_by_doi(doi)
|
||||
elif title:
|
||||
return self.search_by_title(title)
|
||||
return None
|
||||
|
||||
def _normalize_result(self, paper) -> Dict:
|
||||
"""标准化 Semantic Scholar 返回结果"""
|
||||
# 提取作者
|
||||
authors = []
|
||||
if paper.authors:
|
||||
authors = [a.name for a in paper.authors]
|
||||
|
||||
# 提取外部 ID
|
||||
external_ids = paper.externalIds if hasattr(paper, 'externalIds') else {}
|
||||
doi = external_ids.get('DOI') if external_ids else None
|
||||
arxiv_id = external_ids.get('ArXiv') if external_ids else None
|
||||
|
||||
return {
|
||||
'title': paper.title,
|
||||
'authors': authors,
|
||||
'year': paper.year,
|
||||
'venue': paper.venue if hasattr(paper, 'venue') else '',
|
||||
'paperId': paper.paperId,
|
||||
'doi': doi,
|
||||
'arxiv_id': arxiv_id,
|
||||
'citationCount': paper.citationCount if hasattr(paper, 'citationCount') else 0,
|
||||
'abstract': paper.abstract if hasattr(paper, 'abstract') else '',
|
||||
'source': 'semantic_scholar'
|
||||
}
|
||||
|
||||
|
||||
class CitationAPIManager:
|
||||
"""统一的 API 管理器
|
||||
|
||||
协调三个 API 客户端,实现智能的 API 选择策略
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""初始化所有 API 客户端"""
|
||||
self.crossref = None
|
||||
self.arxiv = None
|
||||
self.semantic_scholar = None
|
||||
|
||||
# 尝试初始化各个客户端
|
||||
try:
|
||||
self.crossref = CrossRefClient()
|
||||
except Exception as e:
|
||||
print(f"警告: CrossRef 客户端初始化失败: {e}")
|
||||
|
||||
try:
|
||||
self.arxiv = ArXivClient()
|
||||
except Exception as e:
|
||||
print(f"警告: arXiv 客户端初始化失败: {e}")
|
||||
|
||||
try:
|
||||
self.semantic_scholar = SemanticScholarClient()
|
||||
except Exception as e:
|
||||
print(f"警告: Semantic Scholar 客户端初始化失败: {e}")
|
||||
|
||||
def verify_citation(self, citation_info: Dict) -> tuple[bool, Optional[str], Optional[Dict]]:
|
||||
"""验证引用
|
||||
|
||||
实现 API 选择策略:
|
||||
1. DOI 优先 → CrossRef
|
||||
2. arXiv ID → arXiv
|
||||
3. 标题搜索 → Semantic Scholar
|
||||
|
||||
Args:
|
||||
citation_info: 引用信息字典,可能包含 doi, arxiv_id, title, authors 等字段
|
||||
|
||||
Returns:
|
||||
(exists, api_source, api_data)
|
||||
- exists: 论文是否存在
|
||||
- api_source: 验证来源 ('crossref', 'arxiv', 'semantic_scholar')
|
||||
- api_data: API 返回的标准化数据
|
||||
"""
|
||||
# 策略 1: DOI 优先
|
||||
if 'doi' in citation_info and self.crossref:
|
||||
data = self.crossref.search_by_doi(citation_info['doi'])
|
||||
if data:
|
||||
return True, 'crossref', data
|
||||
|
||||
# 策略 2: arXiv ID
|
||||
arxiv_id = citation_info.get('arxiv_id')
|
||||
if not arxiv_id and 'note' in citation_info:
|
||||
# 尝试从 note 字段提取 arXiv ID
|
||||
arxiv_id = ArXivClient.extract_arxiv_id(citation_info['note'])
|
||||
|
||||
if arxiv_id and self.arxiv:
|
||||
data = self.arxiv.search_by_id(arxiv_id)
|
||||
if data:
|
||||
return True, 'arxiv', data
|
||||
|
||||
# 策略 3: 通用搜索 (Semantic Scholar)
|
||||
if 'title' in citation_info and self.semantic_scholar:
|
||||
data = self.semantic_scholar.search_by_title(citation_info['title'])
|
||||
if data:
|
||||
return True, 'semantic_scholar', data
|
||||
|
||||
return False, None, None
|
||||
|
||||
def get_bibtex(self, doi: str) -> Optional[str]:
|
||||
"""通过 DOI 获取 BibTeX
|
||||
|
||||
Args:
|
||||
doi: DOI 标识符
|
||||
|
||||
Returns:
|
||||
BibTeX 字符串,如果失败则返回 None
|
||||
"""
|
||||
if self.crossref:
|
||||
return self.crossref.get_bibtex(doi)
|
||||
return None
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 使用示例
|
||||
# ============================================================================
|
||||
|
||||
if __name__ == '__main__':
|
||||
# 示例 1: 使用 CrossRef 客户端
|
||||
print("示例 1: CrossRef 客户端")
|
||||
print("-" * 60)
|
||||
crossref = CrossRefClient()
|
||||
result = crossref.search_by_doi("10.48550/arXiv.1706.03762")
|
||||
if result:
|
||||
print(f"标题: {result['title']}")
|
||||
print(f"作者: {', '.join(result['authors'][:3])}")
|
||||
print(f"年份: {result['year']}")
|
||||
print()
|
||||
|
||||
# 示例 2: 使用 arXiv 客户端
|
||||
print("示例 2: arXiv 客户端")
|
||||
print("-" * 60)
|
||||
try:
|
||||
arxiv_client = ArXivClient()
|
||||
result = arxiv_client.search_by_id("1706.03762")
|
||||
if result:
|
||||
print(f"标题: {result['title']}")
|
||||
print(f"作者: {', '.join(result['authors'][:3])}")
|
||||
print(f"年份: {result['year']}")
|
||||
except ImportError as e:
|
||||
print(f"跳过: {e}")
|
||||
print()
|
||||
|
||||
# 示例 3: 使用 Semantic Scholar 客户端
|
||||
print("示例 3: Semantic Scholar 客户端")
|
||||
print("-" * 60)
|
||||
try:
|
||||
ss_client = SemanticScholarClient()
|
||||
result = ss_client.search_by_title("Attention is All You Need")
|
||||
if result:
|
||||
print(f"标题: {result['title']}")
|
||||
print(f"作者: {', '.join(result['authors'][:3])}")
|
||||
print(f"年份: {result['year']}")
|
||||
print(f"引用数: {result['citationCount']}")
|
||||
except ImportError as e:
|
||||
print(f"跳过: {e}")
|
||||
print()
|
||||
|
||||
# 示例 4: 使用统一管理器
|
||||
print("示例 4: 统一 API 管理器")
|
||||
print("-" * 60)
|
||||
manager = CitationAPIManager()
|
||||
citation_info = {
|
||||
'title': 'Attention is All You Need',
|
||||
'authors': ['Vaswani', 'Shazeer'],
|
||||
'year': '2017'
|
||||
}
|
||||
exists, source, data = manager.verify_citation(citation_info)
|
||||
if exists:
|
||||
print(f"验证成功!")
|
||||
print(f"来源: {source}")
|
||||
print(f"标题: {data['title']}")
|
||||
print(f"年份: {data['year']}")
|
||||
@@ -0,0 +1,605 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
BibTeX and LaTeX Format Checker
|
||||
|
||||
独立的格式检查工具,用于验证 BibTeX 和 LaTeX 引用格式。
|
||||
|
||||
功能:
|
||||
1. BibTeX 格式检查 - 验证条目结构、必填字段、字段格式
|
||||
2. LaTeX 引用检查 - 提取引用、检查一致性
|
||||
3. 快速格式验证 - 无需 API 调用的快速检查
|
||||
|
||||
使用方法:
|
||||
python format-checker.py references.bib
|
||||
python format-checker.py paper.tex --check-latex
|
||||
python format-checker.py references.bib --strict
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Tuple, Optional
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
|
||||
# 尝试导入 bibtexparser
|
||||
try:
|
||||
import bibtexparser
|
||||
from bibtexparser.bparser import BibTexParser
|
||||
BIBTEX_AVAILABLE = True
|
||||
except ImportError:
|
||||
print("警告: bibtexparser 未安装,BibTeX 解析功能受限")
|
||||
print("运行: pip install bibtexparser")
|
||||
BIBTEX_AVAILABLE = False
|
||||
|
||||
|
||||
class ErrorLevel(Enum):
|
||||
"""错误级别"""
|
||||
ERROR = "error" # 严重错误,必须修复
|
||||
WARNING = "warning" # 警告,建议修复
|
||||
INFO = "info" # 信息,可选修复
|
||||
|
||||
|
||||
@dataclass
|
||||
class FormatError:
|
||||
"""格式错误数据类"""
|
||||
level: ErrorLevel
|
||||
location: str # 文件位置 (如 "entry:smith2020" 或 "line:42")
|
||||
field: Optional[str] # 字段名 (如 "author", "year")
|
||||
message: str # 错误描述
|
||||
suggestion: Optional[str] = None # 修复建议
|
||||
|
||||
|
||||
def parse_arguments():
|
||||
"""解析命令行参数"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description='检查 BibTeX 和 LaTeX 引用格式',
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
示例:
|
||||
%(prog)s references.bib
|
||||
%(prog)s paper.tex --check-latex
|
||||
%(prog)s references.bib --strict --output report.txt
|
||||
%(prog)s references.bib --fix-common
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'input_file',
|
||||
type=str,
|
||||
help='BibTeX 文件(.bib)或 LaTeX 文件(.tex)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--check-latex',
|
||||
action='store_true',
|
||||
help='检查 LaTeX 引用(需要提供 .tex 文件)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--strict',
|
||||
action='store_true',
|
||||
help='严格模式 - 将警告视为错误'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--output',
|
||||
type=str,
|
||||
help='输出报告文件路径'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--fix-common',
|
||||
action='store_true',
|
||||
help='自动修复常见格式问题'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--verbose',
|
||||
action='store_true',
|
||||
help='显示详细信息'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--entry-type',
|
||||
type=str,
|
||||
help='只检查特定类型的条目(如 article, inproceedings)'
|
||||
)
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def load_bibtex_file(file_path: str) -> List[Dict]:
|
||||
"""加载 BibTeX 文件
|
||||
|
||||
Args:
|
||||
file_path: BibTeX 文件路径
|
||||
|
||||
Returns:
|
||||
BibTeX 条目列表
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: 文件不存在
|
||||
ValueError: 文件格式错误
|
||||
"""
|
||||
if not BIBTEX_AVAILABLE:
|
||||
raise ImportError("需要安装 bibtexparser: pip install bibtexparser")
|
||||
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
parser = BibTexParser(common_strings=True)
|
||||
bib_database = bibtexparser.load(f, parser)
|
||||
return bib_database.entries
|
||||
except FileNotFoundError:
|
||||
raise FileNotFoundError(f"文件不存在: {file_path}")
|
||||
except Exception as e:
|
||||
raise ValueError(f"无法解析 BibTeX 文件: {e}")
|
||||
|
||||
|
||||
def load_latex_file(file_path: str) -> str:
|
||||
"""加载 LaTeX 文件
|
||||
|
||||
Args:
|
||||
file_path: LaTeX 文件路径
|
||||
|
||||
Returns:
|
||||
文件内容
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: 文件不存在
|
||||
"""
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
return f.read()
|
||||
except FileNotFoundError:
|
||||
raise FileNotFoundError(f"文件不存在: {file_path}")
|
||||
except Exception as e:
|
||||
raise ValueError(f"无法读取 LaTeX 文件: {e}")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# BibTeX 格式检查函数
|
||||
# ============================================================================
|
||||
|
||||
def get_required_fields(entry_type: str) -> List[str]:
|
||||
"""获取 BibTeX 条目类型的必填字段
|
||||
|
||||
Args:
|
||||
entry_type: 条目类型 (如 'article', 'inproceedings')
|
||||
|
||||
Returns:
|
||||
必填字段列表
|
||||
"""
|
||||
required_fields = {
|
||||
'article': ['author', 'title', 'journal', 'year'],
|
||||
'inproceedings': ['author', 'title', 'booktitle', 'year'],
|
||||
'book': ['title', 'publisher', 'year'],
|
||||
'incollection': ['author', 'title', 'booktitle', 'publisher', 'year'],
|
||||
'inbook': ['author', 'title', 'chapter', 'publisher', 'year'],
|
||||
'proceedings': ['title', 'year'],
|
||||
'phdthesis': ['author', 'title', 'school', 'year'],
|
||||
'mastersthesis': ['author', 'title', 'school', 'year'],
|
||||
'techreport': ['author', 'title', 'institution', 'year'],
|
||||
'manual': ['title'],
|
||||
'misc': ['title'],
|
||||
'unpublished': ['author', 'title', 'note'],
|
||||
}
|
||||
return required_fields.get(entry_type.lower(), ['title'])
|
||||
|
||||
|
||||
def get_optional_fields(entry_type: str) -> List[str]:
|
||||
"""获取 BibTeX 条目类型的可选字段
|
||||
|
||||
Args:
|
||||
entry_type: 条目类型
|
||||
|
||||
Returns:
|
||||
可选字段列表
|
||||
"""
|
||||
optional_fields = {
|
||||
'article': ['volume', 'number', 'pages', 'month', 'doi', 'url'],
|
||||
'inproceedings': ['editor', 'volume', 'series', 'pages', 'address',
|
||||
'month', 'organization', 'publisher', 'doi', 'url'],
|
||||
'book': ['author', 'editor', 'volume', 'series', 'address',
|
||||
'edition', 'month', 'isbn', 'doi', 'url'],
|
||||
}
|
||||
return optional_fields.get(entry_type.lower(), [])
|
||||
|
||||
|
||||
def check_entry_structure(entry: Dict) -> List[FormatError]:
|
||||
"""检查 BibTeX 条目基本结构
|
||||
|
||||
Args:
|
||||
entry: BibTeX 条目字典
|
||||
|
||||
Returns:
|
||||
错误列表
|
||||
"""
|
||||
errors = []
|
||||
|
||||
# 检查条目类型
|
||||
if 'ENTRYTYPE' not in entry:
|
||||
errors.append(FormatError(
|
||||
level=ErrorLevel.ERROR,
|
||||
location=f"entry:{entry.get('ID', 'unknown')}",
|
||||
field='ENTRYTYPE',
|
||||
message="缺少条目类型",
|
||||
suggestion="添加条目类型,如 @article, @inproceedings"
|
||||
))
|
||||
return errors
|
||||
|
||||
# 检查 ID
|
||||
if 'ID' not in entry or not entry['ID'].strip():
|
||||
errors.append(FormatError(
|
||||
level=ErrorLevel.ERROR,
|
||||
location="entry:unknown",
|
||||
field='ID',
|
||||
message="缺少 citation key",
|
||||
suggestion="添加唯一的 citation key"
|
||||
))
|
||||
|
||||
# 检查必填字段
|
||||
entry_type = entry.get('ENTRYTYPE', '')
|
||||
required = get_required_fields(entry_type)
|
||||
for field in required:
|
||||
if field not in entry or not entry[field].strip():
|
||||
errors.append(FormatError(
|
||||
level=ErrorLevel.ERROR,
|
||||
location=f"entry:{entry.get('ID', 'unknown')}",
|
||||
field=field,
|
||||
message=f"缺少必填字段: {field}",
|
||||
suggestion=f"添加 {field} 字段"
|
||||
))
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def check_field_formats(entry: Dict) -> List[FormatError]:
|
||||
"""检查字段格式
|
||||
|
||||
Args:
|
||||
entry: BibTeX 条目字典
|
||||
|
||||
Returns:
|
||||
错误列表
|
||||
"""
|
||||
errors = []
|
||||
entry_id = entry.get('ID', 'unknown')
|
||||
|
||||
# 年份格式检查
|
||||
if 'year' in entry:
|
||||
year = entry['year'].strip()
|
||||
if not year.isdigit():
|
||||
errors.append(FormatError(
|
||||
level=ErrorLevel.ERROR,
|
||||
location=f"entry:{entry_id}",
|
||||
field='year',
|
||||
message=f"年份格式错误: {year} (应为4位数字)",
|
||||
suggestion="使用4位数字年份,如 2023"
|
||||
))
|
||||
elif len(year) != 4:
|
||||
errors.append(FormatError(
|
||||
level=ErrorLevel.ERROR,
|
||||
location=f"entry:{entry_id}",
|
||||
field='year',
|
||||
message=f"年份格式错误: {year} (应为4位数字)",
|
||||
suggestion="使用4位数字年份,如 2023"
|
||||
))
|
||||
else:
|
||||
year_int = int(year)
|
||||
if year_int < 1900 or year_int > 2030:
|
||||
errors.append(FormatError(
|
||||
level=ErrorLevel.WARNING,
|
||||
location=f"entry:{entry_id}",
|
||||
field='year',
|
||||
message=f"年份超出合理范围: {year}",
|
||||
suggestion="检查年份是否正确"
|
||||
))
|
||||
|
||||
# DOI 格式检查
|
||||
if 'doi' in entry:
|
||||
doi = entry['doi'].strip()
|
||||
if not doi.startswith('10.'):
|
||||
errors.append(FormatError(
|
||||
level=ErrorLevel.ERROR,
|
||||
location=f"entry:{entry_id}",
|
||||
field='doi',
|
||||
message=f"DOI 格式错误: {doi}",
|
||||
suggestion="DOI 应以 '10.' 开头,如 10.1038/nature12345"
|
||||
))
|
||||
# 检查是否包含 URL 前缀
|
||||
if 'doi.org' in doi or 'dx.doi.org' in doi:
|
||||
errors.append(FormatError(
|
||||
level=ErrorLevel.WARNING,
|
||||
location=f"entry:{entry_id}",
|
||||
field='doi',
|
||||
message=f"DOI 包含 URL 前缀: {doi}",
|
||||
suggestion="只保留 DOI 本身,移除 https://doi.org/ 前缀"
|
||||
))
|
||||
|
||||
# 作者名格式检查
|
||||
if 'author' in entry:
|
||||
author = entry['author'].strip()
|
||||
# 检查是否为空
|
||||
if not author:
|
||||
errors.append(FormatError(
|
||||
level=ErrorLevel.ERROR,
|
||||
location=f"entry:{entry_id}",
|
||||
field='author',
|
||||
message="作者字段为空",
|
||||
suggestion="添加作者信息"
|
||||
))
|
||||
# 检查格式一致性
|
||||
elif ' and ' in author:
|
||||
authors = author.split(' and ')
|
||||
formats = []
|
||||
for a in authors:
|
||||
if ',' in a:
|
||||
formats.append('last_first') # "Last, First"
|
||||
else:
|
||||
formats.append('first_last') # "First Last"
|
||||
|
||||
if len(set(formats)) > 1:
|
||||
errors.append(FormatError(
|
||||
level=ErrorLevel.WARNING,
|
||||
location=f"entry:{entry_id}",
|
||||
field='author',
|
||||
message="作者名格式不一致",
|
||||
suggestion="统一使用 'Last, First' 或 'First Last' 格式"
|
||||
))
|
||||
|
||||
# 页码格式检查
|
||||
if 'pages' in entry:
|
||||
pages = entry['pages'].strip()
|
||||
# 检查是否使用了正确的分隔符
|
||||
if '-' in pages and '--' not in pages:
|
||||
errors.append(FormatError(
|
||||
level=ErrorLevel.INFO,
|
||||
location=f"entry:{entry_id}",
|
||||
field='pages',
|
||||
message=f"页码使用单连字符: {pages}",
|
||||
suggestion="建议使用双连字符 '--',如 123--145"
|
||||
))
|
||||
|
||||
# URL 格式检查
|
||||
if 'url' in entry:
|
||||
url = entry['url'].strip()
|
||||
if not url.startswith(('http://', 'https://')):
|
||||
errors.append(FormatError(
|
||||
level=ErrorLevel.WARNING,
|
||||
location=f"entry:{entry_id}",
|
||||
field='url',
|
||||
message=f"URL 缺少协议前缀: {url}",
|
||||
suggestion="添加 http:// 或 https:// 前缀"
|
||||
))
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def check_consistency(entries: List[Dict]) -> List[FormatError]:
|
||||
"""检查条目间的一致性
|
||||
|
||||
Args:
|
||||
entries: BibTeX 条目列表
|
||||
|
||||
Returns:
|
||||
错误列表
|
||||
"""
|
||||
errors = []
|
||||
|
||||
# 检查重复的 citation key
|
||||
ids = [e.get('ID', '') for e in entries]
|
||||
duplicates = [id for id in ids if ids.count(id) > 1]
|
||||
if duplicates:
|
||||
for dup_id in set(duplicates):
|
||||
errors.append(FormatError(
|
||||
level=ErrorLevel.ERROR,
|
||||
location=f"entry:{dup_id}",
|
||||
field='ID',
|
||||
message=f"重复的 citation key: {dup_id}",
|
||||
suggestion="使用唯一的 citation key"
|
||||
))
|
||||
|
||||
# 检查作者名格式一致性
|
||||
author_formats = {}
|
||||
for entry in entries:
|
||||
if 'author' in entry and ' and ' in entry['author']:
|
||||
entry_id = entry.get('ID', 'unknown')
|
||||
authors = entry['author'].split(' and ')
|
||||
for author in authors:
|
||||
if ',' in author:
|
||||
author_formats[entry_id] = 'last_first'
|
||||
else:
|
||||
author_formats[entry_id] = 'first_last'
|
||||
break
|
||||
|
||||
if len(set(author_formats.values())) > 1:
|
||||
errors.append(FormatError(
|
||||
level=ErrorLevel.WARNING,
|
||||
location="global",
|
||||
field='author',
|
||||
message="不同条目使用了不同的作者名格式",
|
||||
suggestion="统一使用 'Last, First' 或 'First Last' 格式"
|
||||
))
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# LaTeX 引用检查函数
|
||||
# ============================================================================
|
||||
|
||||
def extract_latex_citations(tex_content: str) -> List[str]:
|
||||
"""从 LaTeX 文件中提取引用
|
||||
|
||||
Args:
|
||||
tex_content: LaTeX 文件内容
|
||||
|
||||
Returns:
|
||||
引用 key 列表
|
||||
"""
|
||||
# 匹配 \cite{...} 命令
|
||||
cite_pattern = r'\\cite(?:\[[^\]]*\])?(?:\[[^\]]*\])?\{([^}]+)\}'
|
||||
citations = re.findall(cite_pattern, tex_content)
|
||||
|
||||
# 展开多个引用
|
||||
all_keys = []
|
||||
for cite in citations:
|
||||
keys = [k.strip() for k in cite.split(',')]
|
||||
all_keys.extend(keys)
|
||||
|
||||
return list(set(all_keys)) # 去重
|
||||
|
||||
|
||||
def check_latex_consistency(tex_keys: List[str], bib_keys: List[str]) -> List[FormatError]:
|
||||
"""检查 LaTeX 引用与 BibTeX 的一致性
|
||||
|
||||
Args:
|
||||
tex_keys: LaTeX 中的引用 key 列表
|
||||
bib_keys: BibTeX 中的 key 列表
|
||||
|
||||
Returns:
|
||||
错误列表
|
||||
"""
|
||||
errors = []
|
||||
|
||||
tex_set = set(tex_keys)
|
||||
bib_set = set(bib_keys)
|
||||
|
||||
# 未定义的引用
|
||||
undefined = tex_set - bib_set
|
||||
if undefined:
|
||||
for key in sorted(undefined):
|
||||
errors.append(FormatError(
|
||||
level=ErrorLevel.ERROR,
|
||||
location=f"latex:cite",
|
||||
field=key,
|
||||
message=f"未定义的引用: {key}",
|
||||
suggestion=f"在 BibTeX 文件中添加 {key} 条目"
|
||||
))
|
||||
|
||||
# 未使用的引用
|
||||
unused = bib_set - tex_set
|
||||
if unused:
|
||||
for key in sorted(unused):
|
||||
errors.append(FormatError(
|
||||
level=ErrorLevel.WARNING,
|
||||
location=f"bibtex:entry",
|
||||
field=key,
|
||||
message=f"未使用的引用: {key}",
|
||||
suggestion=f"在 LaTeX 文件中引用 {key} 或从 BibTeX 中删除"
|
||||
))
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 报告生成函数
|
||||
# ============================================================================
|
||||
|
||||
def print_errors(errors: List[FormatError], verbose: bool = False):
|
||||
"""打印错误列表
|
||||
|
||||
Args:
|
||||
errors: 错误列表
|
||||
verbose: 是否显示详细信息
|
||||
"""
|
||||
if not errors:
|
||||
print("✅ 未发现格式错误")
|
||||
return
|
||||
|
||||
# 按级别分组
|
||||
errors_by_level = {
|
||||
ErrorLevel.ERROR: [],
|
||||
ErrorLevel.WARNING: [],
|
||||
ErrorLevel.INFO: []
|
||||
}
|
||||
|
||||
for error in errors:
|
||||
errors_by_level[error.level].append(error)
|
||||
|
||||
# 打印统计
|
||||
print("\n" + "="*60)
|
||||
print("格式检查结果")
|
||||
print("="*60)
|
||||
print(f"❌ 错误: {len(errors_by_level[ErrorLevel.ERROR])}")
|
||||
print(f"⚠️ 警告: {len(errors_by_level[ErrorLevel.WARNING])}")
|
||||
print(f"ℹ️ 信息: {len(errors_by_level[ErrorLevel.INFO])}")
|
||||
print("="*60)
|
||||
|
||||
# 打印详细错误
|
||||
for level in [ErrorLevel.ERROR, ErrorLevel.WARNING, ErrorLevel.INFO]:
|
||||
level_errors = errors_by_level[level]
|
||||
if not level_errors:
|
||||
continue
|
||||
|
||||
level_symbol = {
|
||||
ErrorLevel.ERROR: "❌",
|
||||
ErrorLevel.WARNING: "⚠️",
|
||||
ErrorLevel.INFO: "ℹ️"
|
||||
}[level]
|
||||
|
||||
print(f"\n{level_symbol} {level.value.upper()} ({len(level_errors)}):\n")
|
||||
|
||||
for error in level_errors:
|
||||
print(f" [{error.location}]", end="")
|
||||
if error.field:
|
||||
print(f" {error.field}:", end="")
|
||||
print(f" {error.message}")
|
||||
|
||||
if verbose and error.suggestion:
|
||||
print(f" 💡 建议: {error.suggestion}")
|
||||
print()
|
||||
|
||||
|
||||
def generate_report(errors: List[FormatError], output_file: str):
|
||||
"""生成文本格式的检查报告
|
||||
|
||||
Args:
|
||||
errors: 错误列表
|
||||
output_file: 输出文件路径
|
||||
"""
|
||||
# 按级别分组
|
||||
errors_by_level = {
|
||||
ErrorLevel.ERROR: [],
|
||||
ErrorLevel.WARNING: [],
|
||||
ErrorLevel.INFO: []
|
||||
}
|
||||
|
||||
for error in errors:
|
||||
errors_by_level[error.level].append(error)
|
||||
|
||||
with open(output_file, 'w', encoding='utf-8') as f:
|
||||
f.write("# BibTeX/LaTeX 格式检查报告\n\n")
|
||||
|
||||
# 总体统计
|
||||
f.write("## 总体统计\n\n")
|
||||
f.write(f"- **错误**: {len(errors_by_level[ErrorLevel.ERROR])}\n")
|
||||
f.write(f"- **警告**: {len(errors_by_level[ErrorLevel.WARNING])}\n")
|
||||
f.write(f"- **信息**: {len(errors_by_level[ErrorLevel.INFO])}\n\n")
|
||||
|
||||
# 详细错误
|
||||
for level in [ErrorLevel.ERROR, ErrorLevel.WARNING, ErrorLevel.INFO]:
|
||||
level_errors = errors_by_level[level]
|
||||
if not level_errors:
|
||||
continue
|
||||
|
||||
level_name = {
|
||||
ErrorLevel.ERROR: "错误",
|
||||
ErrorLevel.WARNING: "警告",
|
||||
ErrorLevel.INFO: "信息"
|
||||
}[level]
|
||||
|
||||
f.write(f"## {level_name} ({len(level_errors)})\n\n")
|
||||
|
||||
for error in level_errors:
|
||||
f.write(f"### [{error.location}]")
|
||||
if error.field:
|
||||
f.write(f" {error.field}")
|
||||
f.write("\n\n")
|
||||
f.write(f"**问题**: {error.message}\n\n")
|
||||
if error.suggestion:
|
||||
f.write(f"**建议**: {error.suggestion}\n\n")
|
||||
|
||||
print(f"\n报告已保存到: {output_file}")
|
||||
@@ -0,0 +1,678 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Citation Verification Script
|
||||
|
||||
四层验证机制:
|
||||
1. Format Validation - BibTeX 格式检查
|
||||
2. Existence Verification - API 验证论文存在性
|
||||
3. Information Matching - 信息匹配(标题、作者、年份)
|
||||
4. Content Validation - 综合评分和判定
|
||||
|
||||
使用方法:
|
||||
python verify-citations.py references.bib
|
||||
python verify-citations.py paper.tex --check-latex
|
||||
python verify-citations.py references.bib --verbose --output report.md
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from dataclasses import dataclass, asdict
|
||||
import re
|
||||
from difflib import SequenceMatcher
|
||||
|
||||
# 尝试导入 bibtexparser
|
||||
try:
|
||||
import bibtexparser
|
||||
from bibtexparser.bparser import BibTexParser
|
||||
except ImportError:
|
||||
print("错误: 需要安装 bibtexparser")
|
||||
print("运行: pip install bibtexparser")
|
||||
sys.exit(1)
|
||||
|
||||
# 尝试导入 API 客户端库
|
||||
try:
|
||||
from semanticscholar import SemanticScholar
|
||||
except ImportError:
|
||||
print("警告: semanticscholar 未安装,将跳过 Semantic Scholar 验证")
|
||||
print("运行: pip install semanticscholar")
|
||||
|
||||
try:
|
||||
import arxiv
|
||||
except ImportError:
|
||||
print("警告: arxiv 未安装,将跳过 arXiv 验证")
|
||||
print("运行: pip install arxiv")
|
||||
|
||||
try:
|
||||
import requests
|
||||
except ImportError:
|
||||
print("错误: 需要安装 requests")
|
||||
print("运行: pip install requests")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
@dataclass
|
||||
class VerificationResult:
|
||||
"""验证结果数据类"""
|
||||
citation_key: str
|
||||
status: str # verified, partial_match, low_match, failed, not_found
|
||||
confidence: str # high_confidence, medium_confidence, low_confidence, no_confidence
|
||||
match_score: float
|
||||
format_errors: List[str]
|
||||
api_source: Optional[str] # crossref, arxiv, semantic_scholar
|
||||
message: str
|
||||
|
||||
|
||||
def parse_arguments():
|
||||
"""解析命令行参数"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description='验证 BibTeX 引用的准确性和完整性',
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
示例:
|
||||
%(prog)s references.bib
|
||||
%(prog)s paper.tex --check-latex
|
||||
%(prog)s references.bib --verbose --output report.md
|
||||
%(prog)s references.bib --api-only
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'input_file',
|
||||
type=str,
|
||||
help='BibTeX 文件(.bib)或 LaTeX 文件(.tex)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--check-latex',
|
||||
action='store_true',
|
||||
help='检查 LaTeX 引用一致性(需要提供 .tex 文件)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--verbose',
|
||||
action='store_true',
|
||||
help='显示详细验证信息'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--output',
|
||||
type=str,
|
||||
help='输出报告文件路径(Markdown 格式)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--api-only',
|
||||
action='store_true',
|
||||
help='仅进行 API 验证,跳过格式检查'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--format-only',
|
||||
action='store_true',
|
||||
help='仅进行格式检查,跳过 API 验证'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--threshold',
|
||||
type=float,
|
||||
default=0.85,
|
||||
help='匹配阈值(0.0-1.0),默认 0.85'
|
||||
)
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def load_bibtex(file_path: str) -> List[Dict]:
|
||||
"""加载 BibTeX 文件"""
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
parser = BibTexParser(common_strings=True)
|
||||
bib_database = bibtexparser.load(f, parser)
|
||||
return bib_database.entries
|
||||
except FileNotFoundError:
|
||||
print(f"错误: 文件不存在: {file_path}")
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(f"错误: 无法解析 BibTeX 文件: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def extract_latex_citations(tex_file: str) -> List[str]:
|
||||
"""从 LaTeX 文件中提取引用"""
|
||||
try:
|
||||
with open(tex_file, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
|
||||
# 匹配 \cite{...} 命令
|
||||
cite_pattern = r'\\cite(?:\[[^\]]*\])?\{([^}]+)\}'
|
||||
citations = re.findall(cite_pattern, content)
|
||||
|
||||
# 展开多个引用
|
||||
all_keys = []
|
||||
for cite in citations:
|
||||
keys = [k.strip() for k in cite.split(',')]
|
||||
all_keys.extend(keys)
|
||||
|
||||
return list(set(all_keys)) # 去重
|
||||
except FileNotFoundError:
|
||||
print(f"错误: 文件不存在: {tex_file}")
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(f"错误: 无法解析 LaTeX 文件: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Layer 1: Format Validation (格式验证)
|
||||
# ============================================================================
|
||||
|
||||
def get_required_fields(entry_type: str) -> List[str]:
|
||||
"""获取 BibTeX 条目类型的必填字段"""
|
||||
required_fields = {
|
||||
'article': ['author', 'title', 'journal', 'year'],
|
||||
'inproceedings': ['author', 'title', 'booktitle', 'year'],
|
||||
'book': ['title', 'publisher', 'year'],
|
||||
'misc': ['title'],
|
||||
'phdthesis': ['author', 'title', 'school', 'year'],
|
||||
'mastersthesis': ['author', 'title', 'school', 'year'],
|
||||
'techreport': ['author', 'title', 'institution', 'year'],
|
||||
}
|
||||
return required_fields.get(entry_type.lower(), ['title'])
|
||||
|
||||
|
||||
def check_bibtex_format(entry: Dict) -> List[str]:
|
||||
"""检查 BibTeX 条目格式
|
||||
|
||||
Returns:
|
||||
错误列表
|
||||
"""
|
||||
errors = []
|
||||
|
||||
# 检查条目类型
|
||||
if 'ENTRYTYPE' not in entry:
|
||||
errors.append("缺少条目类型")
|
||||
return errors
|
||||
|
||||
# 检查 ID
|
||||
if 'ID' not in entry:
|
||||
errors.append("缺少 citation key")
|
||||
|
||||
# 检查必填字段
|
||||
entry_type = entry.get('ENTRYTYPE', '')
|
||||
required = get_required_fields(entry_type)
|
||||
for field in required:
|
||||
if field not in entry or not entry[field].strip():
|
||||
errors.append(f"缺少必填字段: {field}")
|
||||
|
||||
# 年份格式检查
|
||||
if 'year' in entry:
|
||||
year = entry['year'].strip()
|
||||
if not year.isdigit() or len(year) != 4:
|
||||
errors.append(f"年份格式错误: {year}")
|
||||
else:
|
||||
year_int = int(year)
|
||||
if year_int < 1900 or year_int > 2030:
|
||||
errors.append(f"年份超出合理范围: {year}")
|
||||
|
||||
# DOI 格式检查
|
||||
if 'doi' in entry:
|
||||
doi = entry['doi'].strip()
|
||||
if not doi.startswith('10.'):
|
||||
errors.append(f"DOI 格式错误: {doi}")
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def check_citation_consistency(tex_keys: List[str], bib_keys: List[str]) -> Dict:
|
||||
"""检查 LaTeX 引用与 BibTeX 的一致性
|
||||
|
||||
Returns:
|
||||
{'undefined': [...], 'unused': [...]}
|
||||
"""
|
||||
tex_set = set(tex_keys)
|
||||
bib_set = set(bib_keys)
|
||||
|
||||
return {
|
||||
'undefined': list(tex_set - bib_set),
|
||||
'unused': list(bib_set - tex_set)
|
||||
}
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Layer 2: Existence Verification (存在性验证)
|
||||
# ============================================================================
|
||||
|
||||
def verify_with_crossref(doi: str) -> Optional[Dict]:
|
||||
"""通过 CrossRef API 验证 DOI"""
|
||||
try:
|
||||
url = f"https://api.crossref.org/works/{doi}"
|
||||
response = requests.get(url, timeout=10)
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
return data.get('message')
|
||||
return None
|
||||
except Exception as e:
|
||||
print(f"CrossRef API 错误: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def verify_with_arxiv(arxiv_id: str) -> Optional[Dict]:
|
||||
"""通过 arXiv API 验证"""
|
||||
try:
|
||||
search = arxiv.Search(id_list=[arxiv_id])
|
||||
paper = next(search.results())
|
||||
return {
|
||||
'title': paper.title,
|
||||
'authors': [a.name for a in paper.authors],
|
||||
'year': paper.published.year,
|
||||
'arxiv_id': arxiv_id
|
||||
}
|
||||
except Exception as e:
|
||||
print(f"arXiv API 错误: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def verify_with_semantic_scholar(title: str, authors: Optional[List[str]] = None) -> Optional[Dict]:
|
||||
"""通过 Semantic Scholar API 验证"""
|
||||
try:
|
||||
sch = SemanticScholar()
|
||||
results = sch.search_paper(title, limit=5)
|
||||
|
||||
if not results:
|
||||
return None
|
||||
|
||||
# 返回第一个结果
|
||||
paper = results[0]
|
||||
return {
|
||||
'title': paper.title,
|
||||
'authors': [a.name for a in paper.authors] if paper.authors else [],
|
||||
'year': paper.year,
|
||||
'paperId': paper.paperId
|
||||
}
|
||||
except Exception as e:
|
||||
print(f"Semantic Scholar API 错误: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def verify_existence(entry: Dict) -> Tuple[bool, Optional[str], Optional[Dict]]:
|
||||
"""验证论文存在性
|
||||
|
||||
Returns:
|
||||
(exists, api_source, api_data)
|
||||
"""
|
||||
# 策略 1: DOI 优先
|
||||
if 'doi' in entry:
|
||||
data = verify_with_crossref(entry['doi'])
|
||||
if data:
|
||||
return True, 'crossref', data
|
||||
|
||||
# 策略 2: arXiv ID
|
||||
if 'eprint' in entry or 'arxiv' in entry.get('note', '').lower():
|
||||
arxiv_id = entry.get('eprint', '')
|
||||
if not arxiv_id:
|
||||
# 尝试从 note 中提取
|
||||
match = re.search(r'arXiv:(\d{4}\.\d{4,5})', entry.get('note', ''))
|
||||
if match:
|
||||
arxiv_id = match.group(1)
|
||||
|
||||
if arxiv_id:
|
||||
data = verify_with_arxiv(arxiv_id)
|
||||
if data:
|
||||
return True, 'arxiv', data
|
||||
|
||||
# 策略 3: 通用搜索
|
||||
if 'title' in entry:
|
||||
authors = entry.get('author', '').split(' and ') if 'author' in entry else None
|
||||
data = verify_with_semantic_scholar(entry['title'], authors)
|
||||
if data:
|
||||
return True, 'semantic_scholar', data
|
||||
|
||||
return False, None, None
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Layer 3 & 4: Information Matching & Content Validation (信息匹配和内容验证)
|
||||
# ============================================================================
|
||||
|
||||
def normalize_text(text: str) -> str:
|
||||
"""标准化文本用于匹配"""
|
||||
text = text.lower()
|
||||
text = re.sub(r'[^\w\s]', '', text)
|
||||
return ' '.join(text.split())
|
||||
|
||||
|
||||
def match_title(title1: str, title2: str, threshold: float = 0.85) -> Dict:
|
||||
"""标题匹配"""
|
||||
t1 = normalize_text(title1)
|
||||
t2 = normalize_text(title2)
|
||||
|
||||
ratio = SequenceMatcher(None, t1, t2).ratio()
|
||||
|
||||
return {
|
||||
'match': ratio >= threshold,
|
||||
'similarity': ratio
|
||||
}
|
||||
|
||||
|
||||
def normalize_author_name(name: str) -> str:
|
||||
"""标准化作者名"""
|
||||
parts = name.replace(',', '').split()
|
||||
return ' '.join(sorted(parts)).lower()
|
||||
|
||||
|
||||
def match_authors(authors1: List[str], authors2: List[str], threshold: float = 0.7) -> Dict:
|
||||
"""作者匹配"""
|
||||
names1 = [normalize_author_name(a) for a in authors1]
|
||||
names2 = [normalize_author_name(a) for a in authors2]
|
||||
|
||||
set1 = set(names1)
|
||||
set2 = set(names2)
|
||||
intersection = len(set1 & set2)
|
||||
union = len(set1 | set2)
|
||||
|
||||
if union == 0:
|
||||
return {'match': False, 'similarity': 0}
|
||||
|
||||
ratio = intersection / union
|
||||
|
||||
return {
|
||||
'match': ratio >= threshold,
|
||||
'similarity': ratio
|
||||
}
|
||||
|
||||
|
||||
def match_year(year1: str, year2: int, tolerance: int = 1) -> Dict:
|
||||
"""年份匹配"""
|
||||
try:
|
||||
y1 = int(year1)
|
||||
y2 = int(year2)
|
||||
diff = abs(y1 - y2)
|
||||
return {
|
||||
'match': diff <= tolerance,
|
||||
'difference': diff
|
||||
}
|
||||
except (ValueError, TypeError):
|
||||
return {'match': False, 'difference': None}
|
||||
|
||||
|
||||
def calculate_match_score(entry: Dict, api_data: Dict, threshold: float) -> float:
|
||||
"""计算综合匹配分数"""
|
||||
scores = {}
|
||||
weights = {
|
||||
'title': 0.4,
|
||||
'authors': 0.3,
|
||||
'year': 0.2,
|
||||
'venue': 0.1
|
||||
}
|
||||
|
||||
# 标题匹配
|
||||
if 'title' in entry and 'title' in api_data:
|
||||
result = match_title(entry['title'], api_data['title'], threshold)
|
||||
scores['title'] = result['similarity']
|
||||
|
||||
# 作者匹配
|
||||
if 'author' in entry and 'authors' in api_data:
|
||||
entry_authors = entry['author'].split(' and ')
|
||||
result = match_authors(entry_authors, api_data['authors'])
|
||||
scores['authors'] = result['similarity']
|
||||
|
||||
# 年份匹配
|
||||
if 'year' in entry and 'year' in api_data:
|
||||
result = match_year(entry['year'], api_data['year'])
|
||||
scores['year'] = 1.0 if result['match'] else 0.0
|
||||
|
||||
# 计算加权总分
|
||||
total_score = 0
|
||||
total_weight = 0
|
||||
for key, weight in weights.items():
|
||||
if key in scores:
|
||||
total_score += scores[key] * weight
|
||||
total_weight += weight
|
||||
|
||||
if total_weight == 0:
|
||||
return 0
|
||||
|
||||
return total_score / total_weight
|
||||
|
||||
|
||||
def judge_verification_result(match_score: float) -> Dict:
|
||||
"""判定验证结果"""
|
||||
if match_score >= 0.9:
|
||||
return {
|
||||
'status': 'verified',
|
||||
'level': 'high_confidence',
|
||||
'message': '✅ 验证通过 - 信息完全匹配'
|
||||
}
|
||||
elif match_score >= 0.7:
|
||||
return {
|
||||
'status': 'partial_match',
|
||||
'level': 'medium_confidence',
|
||||
'message': '⚠️ 部分匹配 - 信息有轻微差异,建议人工确认'
|
||||
}
|
||||
elif match_score >= 0.5:
|
||||
return {
|
||||
'status': 'low_match',
|
||||
'level': 'low_confidence',
|
||||
'message': '❌ 匹配度低 - 信息差异较大,需要人工验证'
|
||||
}
|
||||
else:
|
||||
return {
|
||||
'status': 'failed',
|
||||
'level': 'no_confidence',
|
||||
'message': '❌ 验证失败 - 信息严重不匹配或论文不存在'
|
||||
}
|
||||
|
||||
|
||||
def verify_citation(entry: Dict, args) -> VerificationResult:
|
||||
"""完整的引用验证流程"""
|
||||
citation_key = entry.get('ID', 'unknown')
|
||||
|
||||
# Layer 1: 格式验证
|
||||
format_errors = []
|
||||
if not args.api_only:
|
||||
format_errors = check_bibtex_format(entry)
|
||||
|
||||
# Layer 2: 存在性验证
|
||||
if args.format_only:
|
||||
return VerificationResult(
|
||||
citation_key=citation_key,
|
||||
status='format_checked',
|
||||
confidence='n/a',
|
||||
match_score=0.0,
|
||||
format_errors=format_errors,
|
||||
api_source=None,
|
||||
message='仅格式检查'
|
||||
)
|
||||
|
||||
exists, api_source, api_data = verify_existence(entry)
|
||||
|
||||
if not exists:
|
||||
return VerificationResult(
|
||||
citation_key=citation_key,
|
||||
status='not_found',
|
||||
confidence='no_confidence',
|
||||
match_score=0.0,
|
||||
format_errors=format_errors,
|
||||
api_source=None,
|
||||
message='❌ 论文不存在 - 无法通过任何 API 验证'
|
||||
)
|
||||
|
||||
# Layer 3 & 4: 信息匹配和内容验证
|
||||
match_score = calculate_match_score(entry, api_data, args.threshold)
|
||||
judgment = judge_verification_result(match_score)
|
||||
|
||||
return VerificationResult(
|
||||
citation_key=citation_key,
|
||||
status=judgment['status'],
|
||||
confidence=judgment['level'],
|
||||
match_score=match_score,
|
||||
format_errors=format_errors,
|
||||
api_source=api_source,
|
||||
message=judgment['message']
|
||||
)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Report Generation (报告生成)
|
||||
# ============================================================================
|
||||
|
||||
def print_summary(results: List[VerificationResult], verbose: bool = False):
|
||||
"""打印验证摘要"""
|
||||
total = len(results)
|
||||
verified = sum(1 for r in results if r.status == 'verified')
|
||||
partial = sum(1 for r in results if r.status == 'partial_match')
|
||||
low = sum(1 for r in results if r.status == 'low_match')
|
||||
failed = sum(1 for r in results if r.status in ['failed', 'not_found'])
|
||||
|
||||
print("\n" + "="*60)
|
||||
print("验证摘要")
|
||||
print("="*60)
|
||||
print(f"总引用数: {total}")
|
||||
print(f"✅ 验证通过: {verified} ({verified/total*100:.1f}%)")
|
||||
print(f"⚠️ 部分匹配: {partial} ({partial/total*100:.1f}%)")
|
||||
print(f"❌ 匹配度低: {low} ({low/total*100:.1f}%)")
|
||||
print(f"❌ 验证失败: {failed} ({failed/total*100:.1f}%)")
|
||||
print("="*60)
|
||||
|
||||
if verbose:
|
||||
print("\n详细结果:\n")
|
||||
for result in results:
|
||||
print(f"[{result.citation_key}]")
|
||||
print(f" 状态: {result.message}")
|
||||
print(f" 匹配分数: {result.match_score:.2f}")
|
||||
if result.api_source:
|
||||
print(f" 验证源: {result.api_source}")
|
||||
if result.format_errors:
|
||||
print(f" 格式错误: {', '.join(result.format_errors)}")
|
||||
print()
|
||||
|
||||
|
||||
def generate_markdown_report(results: List[VerificationResult], output_file: str):
|
||||
"""生成 Markdown 格式的验证报告"""
|
||||
total = len(results)
|
||||
verified = sum(1 for r in results if r.status == 'verified')
|
||||
partial = sum(1 for r in results if r.status == 'partial_match')
|
||||
low = sum(1 for r in results if r.status == 'low_match')
|
||||
failed = sum(1 for r in results if r.status in ['failed', 'not_found'])
|
||||
|
||||
with open(output_file, 'w', encoding='utf-8') as f:
|
||||
f.write("# Citation Verification Report\n\n")
|
||||
|
||||
# 总体统计
|
||||
f.write("## 总体统计\n\n")
|
||||
f.write(f"- **总引用数**: {total}\n")
|
||||
f.write(f"- **✅ 验证通过**: {verified} ({verified/total*100:.1f}%)\n")
|
||||
f.write(f"- **⚠️ 部分匹配**: {partial} ({partial/total*100:.1f}%)\n")
|
||||
f.write(f"- **❌ 匹配度低**: {low} ({low/total*100:.1f}%)\n")
|
||||
f.write(f"- **❌ 验证失败**: {failed} ({failed/total*100:.1f}%)\n\n")
|
||||
|
||||
# 详细结果
|
||||
f.write("## 详细结果\n\n")
|
||||
|
||||
# 按状态分组
|
||||
for status, emoji, title in [
|
||||
('verified', '✅', '验证通过'),
|
||||
('partial_match', '⚠️', '部分匹配'),
|
||||
('low_match', '❌', '匹配度低'),
|
||||
('failed', '❌', '验证失败'),
|
||||
('not_found', '❌', '论文不存在')
|
||||
]:
|
||||
status_results = [r for r in results if r.status == status]
|
||||
if status_results:
|
||||
f.write(f"### {emoji} {title} ({len(status_results)})\n\n")
|
||||
for result in status_results:
|
||||
f.write(f"#### `{result.citation_key}`\n\n")
|
||||
f.write(f"- **状态**: {result.message}\n")
|
||||
f.write(f"- **匹配分数**: {result.match_score:.2f}\n")
|
||||
f.write(f"- **置信度**: {result.confidence}\n")
|
||||
if result.api_source:
|
||||
f.write(f"- **验证源**: {result.api_source}\n")
|
||||
if result.format_errors:
|
||||
f.write(f"- **格式错误**:\n")
|
||||
for error in result.format_errors:
|
||||
f.write(f" - {error}\n")
|
||||
f.write("\n")
|
||||
|
||||
# 建议操作
|
||||
f.write("## 建议操作\n\n")
|
||||
if failed > 0:
|
||||
f.write("### 需要修正的引用\n\n")
|
||||
failed_results = [r for r in results if r.status in ['failed', 'not_found']]
|
||||
for result in failed_results:
|
||||
f.write(f"- `{result.citation_key}`: {result.message}\n")
|
||||
f.write("\n")
|
||||
|
||||
if partial > 0 or low > 0:
|
||||
f.write("### 需要人工确认的引用\n\n")
|
||||
check_results = [r for r in results if r.status in ['partial_match', 'low_match']]
|
||||
for result in check_results:
|
||||
f.write(f"- `{result.citation_key}`: {result.message}\n")
|
||||
f.write("\n")
|
||||
|
||||
print(f"\n报告已保存到: {output_file}")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Main Function (主函数)
|
||||
# ============================================================================
|
||||
|
||||
def main():
|
||||
"""主函数"""
|
||||
args = parse_arguments()
|
||||
|
||||
# 加载 BibTeX 文件
|
||||
print(f"正在加载 BibTeX 文件: {args.input_file}")
|
||||
entries = load_bibtex(args.input_file)
|
||||
print(f"找到 {len(entries)} 个引用条目")
|
||||
|
||||
# LaTeX 一致性检查
|
||||
if args.check_latex:
|
||||
tex_file = args.input_file.replace('.bib', '.tex')
|
||||
if Path(tex_file).exists():
|
||||
print(f"\n正在检查 LaTeX 引用一致性: {tex_file}")
|
||||
tex_keys = extract_latex_citations(tex_file)
|
||||
bib_keys = [e['ID'] for e in entries]
|
||||
consistency = check_citation_consistency(tex_keys, bib_keys)
|
||||
|
||||
if consistency['undefined']:
|
||||
print(f"⚠️ 未定义的引用 ({len(consistency['undefined'])}): {', '.join(consistency['undefined'])}")
|
||||
if consistency['unused']:
|
||||
print(f"⚠️ 未使用的引用 ({len(consistency['unused'])}): {', '.join(consistency['unused'])}")
|
||||
if not consistency['undefined'] and not consistency['unused']:
|
||||
print("✅ LaTeX 引用与 BibTeX 完全一致")
|
||||
|
||||
# 验证每个引用
|
||||
print("\n开始验证引用...")
|
||||
results = []
|
||||
for i, entry in enumerate(entries, 1):
|
||||
citation_key = entry.get('ID', 'unknown')
|
||||
print(f"[{i}/{len(entries)}] 验证 {citation_key}...", end=' ')
|
||||
|
||||
result = verify_citation(entry, args)
|
||||
results.append(result)
|
||||
|
||||
# 简短状态输出
|
||||
if result.status == 'verified':
|
||||
print("✅")
|
||||
elif result.status == 'partial_match':
|
||||
print("⚠️")
|
||||
else:
|
||||
print("❌")
|
||||
|
||||
# 打印摘要
|
||||
print_summary(results, args.verbose)
|
||||
|
||||
# 生成报告
|
||||
if args.output:
|
||||
generate_markdown_report(results, args.output)
|
||||
|
||||
# 返回退出码
|
||||
failed_count = sum(1 for r in results if r.status in ['failed', 'not_found'])
|
||||
return 0 if failed_count == 0 else 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user