Этап: SQLite и запуск пайплайна

tests/test_extract_transform.py

@@ -1,4 +1,5 @@
 import csv
+from pathlib import Path
 
 import pytest
 
@@ -116,3 +117,62 @@ def test_missing_or_damaged_file(tmp_path):
     path.write_text(",".join(FIELDS) + '\n1,2024-02-29,"незакрытое поле', encoding="utf-8")
     with pytest.raises(CSVFileError, match="ошибка структуры CSV"):
         clean_orders(read_records(path))
+
+
+def test_invalid_utf8(tmp_path):
+    path = tmp_path / "orders.csv"
+    path.write_bytes(",".join(FIELDS).encode() + b"\n1,2024-01-01,\xff,1,1\n")
+    with pytest.raises(CSVFileError, match="UTF-8") as error:
+        list(read_records(path))
+    assert str(path) in str(error.value)
+
+
+def test_access_denied(tmp_path, monkeypatch):
+    path = tmp_path / "private.csv"
+
+    def deny_access(self, *args, **kwargs):
+        raise PermissionError("access denied")
+
+    monkeypatch.setattr(Path, "open", deny_access)
+    with pytest.raises(CSVFileError, match="ошибка чтения") as error:
+        list(read_records(path))
+    assert str(path) in str(error.value)
+    assert isinstance(error.value.__cause__, PermissionError)
+
+
+def test_logical_record_numbers_across_multiline_field(tmp_path):
+    path = tmp_path / "orders.csv"
+    path.write_bytes((",".join(FIELDS) + '\r\n1,2024-02-29,"дом,\r\nсад",1,10\r\n'
+                      '\r\n2,2024-03-01,книги,2,3\r\n').encode())
+    records = list(read_records(path))
+    assert [r.record_number for r in records] == [1, 2, 3]
+    assert records[0].values[2] == "дом,\r\nсад"
+    assert records[1].structure_error == "Пустая запись CSV"
+    assert records[2].values[0] == "2"
+
+
+def test_order_and_reasons_across_processing_stages(tmp_path):
+    conflict = order(order_id="9")
+    bad = order(quantity="-1", unit_price="0.001")
+    accepted, rejected = process(
+        tmp_path, conflict, order(order_id="8"), bad, conflict,
+        conflict | {"order_date": "29.02.2024"}, order(), bad,
+        conflict | {"quantity": "2"}, order(order_id="08"),
+    )
+    assert [(o.record_number, o.order_id) for o in accepted] == [(2, 8), (6, 1)]
+    assert [(r.source.record_number, r.reasons) for r in rejected] == [
+        (1, ["Конфликт order_id"]),
+        (3, ["quantity: ожидается положительное целое число", "unit_price: больше двух дробных знаков"]),
+        (4, ["Точный дубль"]),
+        (5, ["Конфликт order_id"]),
+        (7, ["Точный дубль"]),
+        (8, ["Конфликт order_id"]),
+        (9, ["Нормализованный дубль"]),
+    ]
+
+
+def test_large_price_preserves_last_kopeck(tmp_path):
+    accepted, rejected = process(tmp_path, order(unit_price="90071992547409,93", quantity="3"))
+    assert not rejected
+    assert accepted[0].unit_price_kopecks == 9_007_199_254_740_993
+    assert accepted[0].revenue_kopecks == 27_021_597_764_222_979

tests/test_generate_data.py

@@ -1,4 +1,6 @@
 import csv
+from collections import defaultdict
+from datetime import date, datetime
 
 from src.generate_data import generate_data
 
@@ -18,3 +20,36 @@ def test_repeated_generation_is_byte_identical(tmp_path):
     first_run = path.read_bytes()
     generate_data(path)
     assert path.read_bytes() == first_run
+
+
+def test_planned_examples_are_present_in_generated_csv(tmp_path):
+    path = tmp_path / "orders.csv"
+    generate_data(path)
+    with path.open(encoding="utf-8", newline="") as source:
+        rows = list(csv.DictReader(source))
+
+    expected_values = {
+        "order_id": {""},
+        "order_date": {"", "2024-02-30", "03/15/2024"},
+        "category": {"", "  тОВАРЫ   для  ДОМА  "},
+        "quantity": {"", "1.5", "0", "-2", "три"},
+        "unit_price": {"", "0", "-12.50", "бесплатно", "12.340", "12,50"},
+    }
+    for field, expected in expected_values.items():
+        assert expected <= {row[field] for row in rows}, field
+
+    by_id = defaultdict(list)
+    for row in rows:
+        by_id[row["order_id"]].append(row)
+    first, repeat = by_id["1"]
+    assert first == repeat
+
+    iso, local = sorted(by_id["2"], key=lambda row: "." in row["order_date"])
+    assert date.fromisoformat(iso["order_date"]) == datetime.strptime(
+        local["order_date"], "%d.%m.%Y"
+    ).date()
+    assert iso == local | {"order_date": iso["order_date"]}
+
+    first, conflict = by_id["3"]
+    assert int(first["quantity"]) != int(conflict["quantity"])
+    assert first == conflict | {"quantity": first["quantity"]}

tests/test_pipeline.py

@@ -0,0 +1,130 @@
+import csv
+import json
+import os
+from pathlib import Path
+import shutil
+import sqlite3
+import subprocess
+import sys
+from contextlib import closing
+
+import pytest
+
+from src.extract import FIELDS
+
+ROOT = Path(__file__).resolve().parents[1]
+ROWS = [
+    (8, "2024-01-01", "Книги", 2, "10,25"),
+    (3, "01.01.2024", " книги ", 1, "0.50"),
+    (5, "2024-01-01", "Дом", 3, "1"),
+    (2, "2024-01-02", "Книги", 1, "5"),
+    (1, "2024-01-02", "Дом", 0, "2"),
+]
+
+
+def write_input(project, rows):
+    with (project / "data/raw_orders.csv").open("w", encoding="utf-8", newline="") as out:
+        writer = csv.writer(out)
+        writer.writerow(FIELDS)
+        writer.writerows(rows)
+
+
+@pytest.fixture
+def project(tmp_path):
+    folder = tmp_path / "project"
+    (folder / "data").mkdir(parents=True)
+    shutil.copy(ROOT / "main.py", folder)
+    shutil.copytree(ROOT / "sql", folder / "sql")
+    shutil.copytree(ROOT / "src", folder / "src", ignore=shutil.ignore_patterns("__pycache__"))
+    write_input(folder, ROWS)
+    return folder
+
+
+def run(project, *, success=True):
+    result = subprocess.run(
+        [sys.executable, str(project / "main.py")], cwd=project.parent,
+        env={**os.environ, "PYTHONIOENCODING": "utf-8"}, capture_output=True, encoding="utf-8",
+    )
+    if success:
+        assert result.returncode == 0, result.stderr
+    else:
+        assert result.returncode != 0
+        assert not result.stdout
+    return result
+
+
+def snapshot(project):
+    with closing(sqlite3.connect(project / "outputs/orders.db")) as db:
+        orders = db.execute("SELECT * FROM orders ORDER BY order_id").fetchall()
+        daily = db.execute("SELECT * FROM daily_category_revenue ORDER BY order_date, category").fetchall()
+    return orders, daily
+
+
+def test_full_run_and_repeat(project):
+    run(project)
+    orders, daily = snapshot(project)
+    assert len(orders) == 4
+    assert daily == [
+        ("2024-01-01", "Дом", 1, 300), ("2024-01-01", "Книги", 2, 2100),
+        ("2024-01-02", "Книги", 1, 500),
+    ]
+    for day, category, count, revenue in daily:
+        amounts = [row[5] for row in orders if (row[1], row[2]) == (day, category)]
+        assert (len(amounts), sum(amounts)) == (count, revenue)
+
+    outputs = project / "outputs"
+    with (outputs / "clean_orders.csv").open(encoding="utf-8-sig", newline="") as source:
+        clean = list(csv.DictReader(source))
+    assert sorted((int(r["order_id"]), r["order_date"], r["category"], int(r["quantity"]),
+                   int(r["unit_price_kopecks"]), int(r["revenue_kopecks"])) for r in clean) == orders
+    with (outputs / "rejected_rows.csv").open(encoding="utf-8-sig", newline="") as source:
+        assert [r["record_number"] for r in csv.DictReader(source)] == ["5"]
+    report = json.loads((outputs / "quality_report.json").read_text(encoding="utf-8"))
+    assert (report["source_count"], report["accepted_count"], report["rejected_count"]) == (5, 4, 1)
+    run(project)
+    assert snapshot(project) == (orders, daily)
+
+
+def test_aggregate_overflow_rolls_back_both_tables(project):
+    run(project)
+    before = snapshot(project)
+    write_input(project, [
+        (99, "2025-01-01", "Дом", 1, "92233720368547758.07"),
+        (100, "2025-01-01", "Дом", 1, "0.01"),
+    ])
+    assert "integer overflow" in run(project, success=False).stderr
+    assert snapshot(project) == before
+    report = json.loads((project / "outputs/quality_report.json").read_text(encoding="utf-8"))
+    assert report["source_count"] == len(ROWS)
+
+
+def test_damaged_input_preserves_database(project):
+    run(project)
+    before = snapshot(project)
+    with (project / "data/raw_orders.csv").open("a", encoding="utf-8") as out:
+        out.write('123,2024-01-20,"незакрытое поле')
+    assert "ошибка структуры CSV" in run(project, success=False).stderr
+    assert snapshot(project) == before
+
+
+def test_no_accepted_orders_clears_both_tables(project):
+    run(project)
+    write_input(project, [ROWS[-1]])
+    run(project)
+    assert snapshot(project) == ([], [])
+    report = json.loads((project / "outputs/quality_report.json").read_text(encoding="utf-8"))
+    assert report["rejected_count"] == 1
+
+
+def test_export_error_after_commit(project):
+    run(project)
+    write_input(project, [ROWS[2]])
+    blocked = project / "outputs/clean_orders.csv"
+    blocked.unlink()
+    blocked.mkdir()
+    result = run(project, success=False)
+    assert "База уже обновлена" in result.stderr
+    assert "clean_orders.csv" in result.stderr
+    assert snapshot(project) == (
+        [(5, "2024-01-01", "Дом", 3, 100, 300)], [("2024-01-01", "Дом", 1, 300)]
+    )

tests/test_quality.py

@@ -0,0 +1,80 @@
+import codecs
+import csv
+import json
+
+import pytest
+
+from src.extract import FIELDS, read_records
+from src.quality import build_report, save_outputs
+from src.transform import clean_orders
+
+
+def read_csv(path):
+    assert path.read_bytes().startswith(codecs.BOM_UTF8)
+    with path.open(encoding="utf-8-sig", newline="") as source:
+        return list(csv.reader(source))
+
+
+def test_export_and_replace_snapshot(tmp_path):
+    path = tmp_path / "raw.csv"
+    header = ("category", "order_id", "quantity", "order_date", "unit_price")
+    bad = ['дом, "сад"\nспорт', "2", "", "?", "0"]
+    good = [" КНИГИ ", "7", "3", "29.02.2024", "0,07"]
+    extra = good + ["лишнее"]
+    rows = [good, bad, bad, extra, [], ["", "4", "1", "2024-01-01", "10"]]
+    with path.open("w", encoding="utf-8", newline="") as output:
+        writer = csv.writer(output)
+        writer.writerow(header)
+        writer.writerows(rows)
+    records = list(read_records(path))
+    accepted, rejected = clean_orders(records)
+    directory = tmp_path / "outputs"
+    save_outputs(accepted[::-1], rejected[::-1], len(records), directory)
+    clean = directory / "clean_orders.csv"
+    rejects = directory / "rejected_rows.csv"
+    report_file = directory / "quality_report.json"
+    clean_header = ["order_id", "order_date", "category", "quantity", "unit_price_kopecks", "revenue_kopecks"]
+    reject_header = ["record_number", "source_header", "source_values", "reasons"]
+    assert read_csv(clean) == [
+        clean_header, ["7", "2024-02-29", "Книги", "3", "7", "21"],
+        ["4", "2024-01-01", "Не указана", "1", "1000", "1000"],
+    ]
+    columns, *exported = read_csv(rejects)
+    assert columns == reject_header
+    assert [int(row[0]) for row in exported] == [2, 3, 4, 5]
+    for row, rejection in zip(exported, rejected, strict=True):
+        assert json.loads(row[1]) == list(header)
+        assert json.loads(row[2]) == rows[int(row[0]) - 1]
+        assert json.loads(row[3]) == rejection.reasons
+
+    report = json.loads(report_file.read_text(encoding="utf-8"))
+    assert (report["source_count"], report["accepted_count"], report["rejected_count"]) == (6, 2, 4)
+    assert report["accepted_share"] == 2 / 6
+    assert report["final_reason_counts"] == {
+        "order_date: ожидается YYYY-MM-DD или DD.MM.YYYY": 1, "Точный дубль": 1,
+        "Число полей: ожидалось 5, получено 6": 1, "Пустая запись CSV": 1,
+    }
+    assert report["problem_counts"] == report["final_reason_counts"] | {
+        "quantity: пропущено обязательное значение": 1, "unit_price: цена должна быть больше нуля": 1,
+    }
+    before = {file: file.read_bytes() for file in directory.iterdir()}
+    save_outputs(accepted, rejected, len(records), directory)
+    assert before == {file: file.read_bytes() for file in directory.iterdir()}
+
+    save_outputs([], [], 0, directory)
+    assert read_csv(clean) == [clean_header]
+    assert read_csv(rejects) == [reject_header]
+    report = json.loads(report_file.read_text(encoding="utf-8"))
+    assert report["accepted_share"] is None
+    assert report["source_count"] == report["accepted_count"] == report["rejected_count"] == 0
+    assert report["final_reason_counts"] == report["problem_counts"] == {}
+
+
+def test_invalid_accounting(tmp_path):
+    source = tmp_path / "raw.csv"
+    source.write_text(",".join(FIELDS) + "\n1,2024-01-01,x,1,1\n", encoding="utf-8")
+    accepted, rejected = clean_orders(read_records(source))
+    with pytest.raises(ValueError, match="сумме"):
+        build_report(accepted, rejected, 2)
+    with pytest.raises(ValueError, match="без повторов"):
+        build_report(accepted * 2, rejected, 2)