Этап: SQLite и запуск пайплайна

tests/test_quality.py

@@ -0,0 +1,80 @@
+import codecs
+import csv
+import json
+
+import pytest
+
+from src.extract import FIELDS, read_records
+from src.quality import build_report, save_outputs
+from src.transform import clean_orders
+
+
+def read_csv(path):
+    assert path.read_bytes().startswith(codecs.BOM_UTF8)
+    with path.open(encoding="utf-8-sig", newline="") as source:
+        return list(csv.reader(source))
+
+
+def test_export_and_replace_snapshot(tmp_path):
+    path = tmp_path / "raw.csv"
+    header = ("category", "order_id", "quantity", "order_date", "unit_price")
+    bad = ['дом, "сад"\nспорт', "2", "", "?", "0"]
+    good = [" КНИГИ ", "7", "3", "29.02.2024", "0,07"]
+    extra = good + ["лишнее"]
+    rows = [good, bad, bad, extra, [], ["", "4", "1", "2024-01-01", "10"]]
+    with path.open("w", encoding="utf-8", newline="") as output:
+        writer = csv.writer(output)
+        writer.writerow(header)
+        writer.writerows(rows)
+    records = list(read_records(path))
+    accepted, rejected = clean_orders(records)
+    directory = tmp_path / "outputs"
+    save_outputs(accepted[::-1], rejected[::-1], len(records), directory)
+    clean = directory / "clean_orders.csv"
+    rejects = directory / "rejected_rows.csv"
+    report_file = directory / "quality_report.json"
+    clean_header = ["order_id", "order_date", "category", "quantity", "unit_price_kopecks", "revenue_kopecks"]
+    reject_header = ["record_number", "source_header", "source_values", "reasons"]
+    assert read_csv(clean) == [
+        clean_header, ["7", "2024-02-29", "Книги", "3", "7", "21"],
+        ["4", "2024-01-01", "Не указана", "1", "1000", "1000"],
+    ]
+    columns, *exported = read_csv(rejects)
+    assert columns == reject_header
+    assert [int(row[0]) for row in exported] == [2, 3, 4, 5]
+    for row, rejection in zip(exported, rejected, strict=True):
+        assert json.loads(row[1]) == list(header)
+        assert json.loads(row[2]) == rows[int(row[0]) - 1]
+        assert json.loads(row[3]) == rejection.reasons
+
+    report = json.loads(report_file.read_text(encoding="utf-8"))
+    assert (report["source_count"], report["accepted_count"], report["rejected_count"]) == (6, 2, 4)
+    assert report["accepted_share"] == 2 / 6
+    assert report["final_reason_counts"] == {
+        "order_date: ожидается YYYY-MM-DD или DD.MM.YYYY": 1, "Точный дубль": 1,
+        "Число полей: ожидалось 5, получено 6": 1, "Пустая запись CSV": 1,
+    }
+    assert report["problem_counts"] == report["final_reason_counts"] | {
+        "quantity: пропущено обязательное значение": 1, "unit_price: цена должна быть больше нуля": 1,
+    }
+    before = {file: file.read_bytes() for file in directory.iterdir()}
+    save_outputs(accepted, rejected, len(records), directory)
+    assert before == {file: file.read_bytes() for file in directory.iterdir()}
+
+    save_outputs([], [], 0, directory)
+    assert read_csv(clean) == [clean_header]
+    assert read_csv(rejects) == [reject_header]
+    report = json.loads(report_file.read_text(encoding="utf-8"))
+    assert report["accepted_share"] is None
+    assert report["source_count"] == report["accepted_count"] == report["rejected_count"] == 0
+    assert report["final_reason_counts"] == report["problem_counts"] == {}
+
+
+def test_invalid_accounting(tmp_path):
+    source = tmp_path / "raw.csv"
+    source.write_text(",".join(FIELDS) + "\n1,2024-01-01,x,1,1\n", encoding="utf-8")
+    accepted, rejected = clean_orders(read_records(source))
+    with pytest.raises(ValueError, match="сумме"):
+        build_report(accepted, rejected, 2)
+    with pytest.raises(ValueError, match="без повторов"):
+        build_report(accepted * 2, rejected, 2)