48 lines
1.1 KiB
Python
48 lines
1.1 KiB
Python
import math
|
|
import re
|
|
from collections import Counter
|
|
|
|
import numpy as np
|
|
|
|
|
|
def shannon_entropy(s: str) -> float:
|
|
if not s:
|
|
return 0.0
|
|
counts = Counter(s)
|
|
length = len(s)
|
|
return -sum((c / length) * math.log2(c / length) for c in counts.values())
|
|
|
|
|
|
def special_char_ratio(s: str) -> float:
|
|
if not s:
|
|
return 0.0
|
|
specials = sum(1 for ch in s if not ch.isalnum() and not ch.isspace())
|
|
return specials / len(s)
|
|
|
|
|
|
def digit_ratio(s: str) -> float:
|
|
if not s:
|
|
return 0.0
|
|
return sum(ch.isdigit() for ch in s) / len(s)
|
|
|
|
|
|
def keyword_hit_count(s: str, keywords) -> int:
|
|
low = s.lower()
|
|
return sum(1 for kw in keywords if kw in low)
|
|
|
|
|
|
def handcrafted_feature_matrix(texts, keywords):
|
|
rows = []
|
|
for t in texts:
|
|
rows.append([
|
|
len(t),
|
|
shannon_entropy(t),
|
|
special_char_ratio(t),
|
|
digit_ratio(t),
|
|
keyword_hit_count(t, keywords),
|
|
])
|
|
return np.array(rows, dtype=float)
|
|
|
|
|
|
FEATURE_NAMES = ["length", "entropy", "special_char_ratio", "digit_ratio", "keyword_hits"]
|