100 lines
3.1 KiB
Python
Executable File
100 lines
3.1 KiB
Python
Executable File
from langchain.docstore.document import Document
|
||
import re
|
||
|
||
|
||
def under_non_alpha_ratio(text: str, threshold: float = 0.5):
|
||
"""Checks if the proportion of non-alpha characters in the text snippet exceeds a given
|
||
threshold. This helps prevent text like "-----------BREAK---------" from being tagged
|
||
as a title or narrative text. The ratio does not count spaces.
|
||
|
||
Parameters
|
||
----------
|
||
text
|
||
The input string to test
|
||
threshold
|
||
If the proportion of non-alpha characters exceeds this threshold, the function
|
||
returns False
|
||
"""
|
||
if len(text) == 0:
|
||
return False
|
||
|
||
alpha_count = len([char for char in text if char.strip() and char.isalpha()])
|
||
total_count = len([char for char in text if char.strip()])
|
||
try:
|
||
ratio = alpha_count / total_count
|
||
return ratio < threshold
|
||
except:
|
||
return False
|
||
|
||
|
||
def is_possible_title(
|
||
text: str,
|
||
title_max_word_length: int = 20,
|
||
non_alpha_threshold: float = 0.5,
|
||
) -> bool:
|
||
"""Checks to see if the text passes all of the checks for a valid title.
|
||
|
||
Parameters
|
||
----------
|
||
text
|
||
The input text to check
|
||
title_max_word_length
|
||
The maximum number of words a title can contain
|
||
non_alpha_threshold
|
||
The minimum number of alpha characters the text needs to be considered a title
|
||
"""
|
||
|
||
# 文本长度为0的话，肯定不是title
|
||
if len(text) == 0:
|
||
print("Not a title. Text is empty.")
|
||
return False
|
||
|
||
# 文本中有标点符号，就不是title
|
||
ENDS_IN_PUNCT_PATTERN = r"[^\w\s]\Z"
|
||
ENDS_IN_PUNCT_RE = re.compile(ENDS_IN_PUNCT_PATTERN)
|
||
if ENDS_IN_PUNCT_RE.search(text) is not None:
|
||
return False
|
||
|
||
# 文本长度不能超过设定值，默认20
|
||
# NOTE(robinson) - splitting on spaces here instead of word tokenizing because it
|
||
# is less expensive and actual tokenization doesn't add much value for the length check
|
||
if len(text) > title_max_word_length:
|
||
return False
|
||
|
||
# 文本中数字的占比不能太高，否则不是title
|
||
if under_non_alpha_ratio(text, threshold=non_alpha_threshold):
|
||
return False
|
||
|
||
# NOTE(robinson) - Prevent flagging salutations like "To My Dearest Friends," as titles
|
||
if text.endswith((",", ".", "，", "。")):
|
||
return False
|
||
|
||
if text.isnumeric():
|
||
print(f"Not a title. Text is all numeric:\n\n{text}") # type: ignore
|
||
return False
|
||
|
||
# 开头的字符内应该有数字，默认5个字符内
|
||
if len(text) < 5:
|
||
text_5 = text
|
||
else:
|
||
text_5 = text[:5]
|
||
alpha_in_text_5 = sum(list(map(lambda x: x.isnumeric(), list(text_5))))
|
||
if not alpha_in_text_5:
|
||
return False
|
||
|
||
return True
|
||
|
||
|
||
def zh_title_enhance(docs: Document) -> Document:
|
||
title = None
|
||
if len(docs) > 0:
|
||
for doc in docs:
|
||
if is_possible_title(doc.page_content):
|
||
doc.metadata['category'] = 'cn_Title'
|
||
title = doc.page_content
|
||
elif title:
|
||
doc.page_content = f"下文与({title})有关。{doc.page_content}"
|
||
return docs
|
||
else:
|
||
print("文件不存在")
|