P3-6.2 当特征本身还不够时，还可以加入什么中间表示¶

# 这个例子在原始日志和最终特征之间加入中间表示，用来追踪计算依据。
import csv
from collections import defaultdict
from pathlib import Path

data_path = Path("docs/assets/part-03/chapter-06/p3_6_2_segment_slopes.csv")
token_settings = {
    "sensitive": {"strong_threshold": 0.80, "weak_threshold": 0.20},
    "conservative": {"strong_threshold": 0.90, "weak_threshold": 0.30},
}


def slope_to_token(slope: float, strong_threshold: float, weak_threshold: float) -> str:
    if slope >= strong_threshold:
        return "UP2"
    if slope >= weak_threshold:
        return "UP1"
    if slope <= -strong_threshold:
        return "DOWN2"
    if slope <= -weak_threshold:
        return "DOWN1"
    return "FLAT"


rows = list(csv.DictReader(data_path.open(encoding="utf-8")))
for row in rows:
    row["segment_order"] = int(row["segment_order"])
    row["slope"] = float(row["slope"])

events = defaultdict(list)
for row in rows:
    events[row["event_id"]].append(row)

reports = {}
for setting_name, thresholds in token_settings.items():
    event_reports = []
    token_counts = defaultdict(int)
    for event_id, event_rows in sorted(events.items()):
        ordered_rows = sorted(event_rows, key=lambda row: row["segment_order"])
        slopes = [row["slope"] for row in ordered_rows]
        tokens = [
            slope_to_token(
                slope,
                thresholds["strong_threshold"],
                thresholds["weak_threshold"],
            )
            for slope in slopes
        ]
        for token in tokens:
            token_counts[token] += 1
        event_reports.append(
            {
                "event_id": event_id,
                "slopes": slopes,
                "tokens": tokens,
            }
        )
    reports[setting_name] = {
        "thresholds": thresholds,
        "events": event_reports,
        "token_counts": dict(sorted(token_counts.items())),
    }

sensitive_tokens = {
    report["event_id"]: report["tokens"]
    for report in reports["sensitive"]["events"]
}
changed_events = []
for report in reports["conservative"]["events"]:
    event_id = report["event_id"]
    if report["tokens"] != sensitive_tokens[event_id]:
        changed_events.append(event_id)

print("1) input rows:", len(rows))
print("2) event count:", len(events))
for setting_name, report in reports.items():
    print(f"[{setting_name}] thresholds =", report["thresholds"])
    print("token_counts =", report["token_counts"])
    for event in report["events"][:3]:
        print(event)
    print()
print("3) events changed when thresholds become conservative:", changed_events)

1) input rows: 40
2) event count: 8
[sensitive] thresholds = {'strong_threshold': 0.8, 'weak_threshold': 0.2}
token_counts = {'DOWN1': 9, 'DOWN2': 3, 'FLAT': 15, 'UP1': 9, 'UP2': 4}
{'event_id': 'A', 'slopes': [0.92, 0.31, 0.05, -0.42, -1.0], 'tokens': ['UP2', 'UP1', 'FLAT', 'DOWN1', 'DOWN2']}
{'event_id': 'B', 'slopes': [0.62, 0.24, 0.01, -0.22, -0.74], 'tokens': ['UP1', 'UP1', 'FLAT', 'DOWN1', 'DOWN1']}
{'event_id': 'C', 'slopes': [0.18, 0.12, 0.04, -0.1, -0.18], 'tokens': ['FLAT', 'FLAT', 'FLAT', 'FLAT', 'FLAT']}

[conservative] thresholds = {'strong_threshold': 0.9, 'weak_threshold': 0.3}
token_counts = {'DOWN1': 7, 'DOWN2': 1, 'FLAT': 23, 'UP1': 7, 'UP2': 2}
{'event_id': 'A', 'slopes': [0.92, 0.31, 0.05, -0.42, -1.0], 'tokens': ['UP2', 'UP1', 'FLAT', 'DOWN1', 'DOWN2']}
{'event_id': 'B', 'slopes': [0.62, 0.24, 0.01, -0.22, -0.74], 'tokens': ['UP1', 'FLAT', 'FLAT', 'FLAT', 'DOWN1']}
{'event_id': 'C', 'slopes': [0.18, 0.12, 0.04, -0.1, -0.18], 'tokens': ['FLAT', 'FLAT', 'FLAT', 'FLAT', 'FLAT']}

3) events changed when thresholds become conservative: ['B', 'D', 'E', 'F', 'H']

import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity

patterns = pd.DataFrame(
    [
        {"event_id": "A", "overall_mean": 2.5, "token_sequence": "UP2 UP1 FLAT DOWN1 DOWN2"},
        {"event_id": "B", "overall_mean": 2.5, "token_sequence": "FLAT FLAT FLAT FLAT FLAT"},
        {"event_id": "C", "overall_mean": 2.4, "token_sequence": "UP1 UP1 FLAT DOWN1 DOWN1"},
        {"event_id": "D", "overall_mean": 2.8, "token_sequence": "DOWN2 DOWN1 FLAT UP1 UP2"},
    ]
)

query_mean = 2.5
query_text = "UP2 UP1 FLAT DOWN1 DOWN2"

patterns["mean_distance"] = (patterns["overall_mean"] - query_mean).abs()
mean_rank = patterns.sort_values(["mean_distance", "event_id"])[
    ["event_id", "overall_mean", "mean_distance"]
]

vectorizer = TfidfVectorizer(ngram_range=(1, 2))
matrix = vectorizer.fit_transform(patterns["token_sequence"])
query_vector = vectorizer.transform([query_text])
patterns["token_similarity"] = cosine_similarity(query_vector, matrix)[0]
token_rank = patterns.sort_values(["token_similarity", "event_id"], ascending=[False, True])[
    ["event_id", "token_sequence", "token_similarity"]
]

print("rank by numeric mean")
print(mean_rank.to_string(index=False))
print()
print("rank by token sequence")
print(token_rank.to_string(index=False))

rank by numeric mean
event_id  overall_mean  mean_distance
       A           2.5            0.0
       B           2.5            0.0
       C           2.4            0.1
       D           2.8            0.3

rank by token sequence
event_id           token_sequence  token_similarity
       A UP2 UP1 FLAT DOWN1 DOWN2          1.000000
       C UP1 UP1 FLAT DOWN1 DOWN1          0.511833
       D DOWN2 DOWN1 FLAT UP1 UP2          0.392319
       B FLAT FLAT FLAT FLAT FLAT          0.120765