576a14c82f
Adds ivona (invoice-checker, timesheet-checker) and timesheet (check-my-timesheet, invoice-prep) as new plugins, plus invoice-prep-test — a one-off experiment that requests the Tempo API token via plugin userConfig at install time instead of a .env file, to check whether that value actually reaches the skill's script. Also adds the root LICENSE (MIT) referenced from the README.
1243 lines
48 KiB
Python
1243 lines
48 KiB
Python
#!/usr/bin/env python3
|
||
"""Fetch purchase invoices from KSeF v2 API, check contractors and MF white list."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import base64
|
||
import json
|
||
import os
|
||
import re
|
||
import sys
|
||
import time
|
||
import urllib.error
|
||
import urllib.request
|
||
import xml.etree.ElementTree as ET
|
||
from datetime import date, datetime, timezone
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
|
||
# ── Dependency check ─────────────────────────────────────────────────────────
|
||
|
||
def _require_cryptography() -> None:
|
||
try:
|
||
import cryptography # noqa: F401
|
||
except ImportError:
|
||
print(
|
||
"Error: the 'cryptography' package is required.\n"
|
||
"Install it with: pip3 install cryptography",
|
||
file=sys.stderr,
|
||
)
|
||
sys.exit(1)
|
||
|
||
|
||
# ── .env loading ──────────────────────────────────────────────────────────────
|
||
|
||
def load_dotenv(*paths: Path) -> Path | None:
|
||
for path in paths:
|
||
if not path.exists():
|
||
continue
|
||
for raw in path.read_text(encoding="utf-8").splitlines():
|
||
line = raw.strip()
|
||
if not line or line.startswith("#") or "=" not in line:
|
||
continue
|
||
key, _, value = line.partition("=")
|
||
key = key.strip()
|
||
value = value.strip().strip('"').strip("'")
|
||
if key and key not in os.environ:
|
||
os.environ[key] = value
|
||
return path
|
||
return None
|
||
|
||
|
||
# ── Utilities ─────────────────────────────────────────────────────────────────
|
||
|
||
def clean_text(value: Any) -> str:
|
||
return re.sub(r"\s+", " ", str(value or "")).strip()
|
||
|
||
|
||
def normalize_key(value: Any) -> str:
|
||
return re.sub(r"[^a-z0-9]+", "_", clean_text(value).lower()).strip("_")
|
||
|
||
|
||
def normalize_iban(raw: str) -> str:
|
||
s = re.sub(r"\s+", "", raw.upper())
|
||
if re.fullmatch(r"\d{26}", s):
|
||
s = "PL" + s
|
||
return s
|
||
|
||
|
||
def iso_to_ms(value: Any) -> int:
|
||
"""Convert ISO timestamp string or epoch-ms integer to epoch milliseconds."""
|
||
if isinstance(value, (int, float)):
|
||
return int(value)
|
||
try:
|
||
dt = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
||
return int(dt.timestamp() * 1000)
|
||
except Exception:
|
||
return int(time.time() * 1000)
|
||
|
||
|
||
# ── Period parsing ────────────────────────────────────────────────────────────
|
||
|
||
def parse_period(period_str: str) -> tuple[date, date]:
|
||
s = period_str.strip()
|
||
if ":" in s:
|
||
parts = s.split(":", 1)
|
||
return date.fromisoformat(parts[0].strip()), date.fromisoformat(parts[1].strip())
|
||
raise ValueError(f"Invalid period '{period_str}'. Use YYYY-MM-DD:YYYY-MM-DD.")
|
||
|
||
|
||
# ── KSeF base URLs ────────────────────────────────────────────────────────────
|
||
|
||
KSEF_URLS = {
|
||
"prod": "https://api.ksef.mf.gov.pl/api/v2",
|
||
"test": "https://api.ksef-test.mf.gov.pl/api/v2",
|
||
}
|
||
|
||
|
||
# ── HTTP helpers ──────────────────────────────────────────────────────────────
|
||
|
||
def _http(
|
||
url: str,
|
||
method: str = "GET",
|
||
body: Any = None,
|
||
headers: dict | None = None,
|
||
accept: str = "application/json",
|
||
raw_body: bytes | None = None,
|
||
) -> tuple[int, bytes, dict]:
|
||
"""Return (status_code, body_bytes, response_headers)."""
|
||
data: bytes | None = None
|
||
h = {"Accept": accept}
|
||
if raw_body is not None:
|
||
data = raw_body
|
||
h["Content-Type"] = "application/octet-stream"
|
||
elif body is not None:
|
||
data = json.dumps(body).encode()
|
||
h["Content-Type"] = "application/json"
|
||
if headers:
|
||
h.update(headers)
|
||
req = urllib.request.Request(url, data=data, headers=h, method=method)
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||
return resp.status, resp.read(), dict(resp.headers)
|
||
except urllib.error.HTTPError as exc:
|
||
body_bytes = exc.read()
|
||
raise RuntimeError(
|
||
f"HTTP {exc.code} {method} {url}: "
|
||
f"{body_bytes.decode('utf-8', errors='replace')[:400]}"
|
||
) from exc
|
||
|
||
|
||
def _json(url: str, method: str = "GET", body: Any = None,
|
||
headers: dict | None = None) -> Any:
|
||
_, raw, _ = _http(url, method=method, body=body, headers=headers)
|
||
return json.loads(raw)
|
||
|
||
|
||
# ── KSeF v2 authentication ────────────────────────────────────────────────────
|
||
|
||
def get_public_key_cert(base_url: str) -> str:
|
||
"""
|
||
GET /security/public-key-certificates
|
||
Returns PEM string for the KsefTokenEncryption certificate.
|
||
"""
|
||
data = _json(f"{base_url}/security/public-key-certificates")
|
||
certs: list[dict] = []
|
||
if isinstance(data, list):
|
||
certs = data
|
||
elif isinstance(data, dict):
|
||
certs = data.get("data", data.get("certificates", [data]))
|
||
|
||
def has_usage(c: dict, needle: str) -> bool:
|
||
u = c.get("usage", [])
|
||
if isinstance(u, list):
|
||
return any(needle.lower() in str(x).lower() for x in u)
|
||
return needle.lower() in str(u).lower()
|
||
|
||
cert = (
|
||
next((c for c in certs if has_usage(c, "token")), None)
|
||
or next((c for c in certs if has_usage(c, "auth")), None)
|
||
or (certs[0] if certs else None)
|
||
)
|
||
if not cert:
|
||
raise RuntimeError(f"No public key certificates returned. Response: {data}")
|
||
|
||
b64 = cert.get("certificate") or cert.get("publicKeyCertificate") or cert.get("value", "")
|
||
if not b64:
|
||
raise RuntimeError(f"Certificate payload missing: {cert}")
|
||
|
||
if "BEGIN CERTIFICATE" in b64:
|
||
return b64
|
||
wrapped = "\n".join(b64[i:i+64] for i in range(0, len(b64), 64))
|
||
return f"-----BEGIN CERTIFICATE-----\n{wrapped}\n-----END CERTIFICATE-----"
|
||
|
||
|
||
def get_challenge(base_url: str) -> dict:
|
||
"""
|
||
POST /auth/challenge (no request body)
|
||
Returns {challenge: str, timestamp: str|int}
|
||
"""
|
||
return _json(f"{base_url}/auth/challenge", method="POST")
|
||
|
||
|
||
def encrypt_token_rsa_oaep(token: str, timestamp_ms: int, cert_pem: str) -> str:
|
||
"""
|
||
RSA-OAEP-SHA256 encrypt f'{token}|{timestamp_ms}' using the KSeF public key
|
||
extracted from the X.509 certificate. Returns base64-encoded ciphertext.
|
||
"""
|
||
from cryptography.hazmat.primitives import hashes, serialization
|
||
from cryptography.hazmat.primitives.asymmetric import padding as asym_padding
|
||
from cryptography.x509 import load_pem_x509_certificate
|
||
|
||
plaintext = f"{token}|{timestamp_ms}".encode("utf-8")
|
||
cert = load_pem_x509_certificate(cert_pem.encode())
|
||
public_key = cert.public_key()
|
||
ciphertext = public_key.encrypt(
|
||
plaintext,
|
||
asym_padding.OAEP(
|
||
mgf=asym_padding.MGF1(algorithm=hashes.SHA256()),
|
||
algorithm=hashes.SHA256(),
|
||
label=None,
|
||
),
|
||
)
|
||
return base64.b64encode(ciphertext).decode()
|
||
|
||
|
||
def init_auth_ksef_token(
|
||
nip: str, challenge: str, encrypted_token: str, base_url: str
|
||
) -> dict:
|
||
"""
|
||
POST /auth/ksef-token
|
||
Body: {challenge, contextIdentifier: {type: 'nip', value: nip}, encryptedToken}
|
||
Returns {referenceNumber, authenticationToken: {token}}
|
||
"""
|
||
return _json(
|
||
f"{base_url}/auth/ksef-token",
|
||
method="POST",
|
||
body={
|
||
"challenge": challenge,
|
||
"contextIdentifier": {"type": "nip", "value": nip},
|
||
"encryptedToken": encrypted_token,
|
||
},
|
||
)
|
||
|
||
|
||
def check_auth_status(reference_number: str, auth_token: str, base_url: str) -> dict:
|
||
"""
|
||
GET /auth/{referenceNumber}
|
||
Returns {status: {code: int, description: str}, ...}
|
||
"""
|
||
return _json(
|
||
f"{base_url}/auth/{reference_number}",
|
||
headers={"Authorization": f"Bearer {auth_token}"},
|
||
)
|
||
|
||
|
||
def redeem_access_token(auth_token: str, base_url: str) -> str:
|
||
"""
|
||
POST /auth/token/redeem
|
||
Returns the access token string from {accessToken: {token}}.
|
||
"""
|
||
data = _json(
|
||
f"{base_url}/auth/token/redeem",
|
||
method="POST",
|
||
headers={"Authorization": f"Bearer {auth_token}"},
|
||
)
|
||
token = data.get("accessToken", {}).get("token", "")
|
||
if not token:
|
||
raise RuntimeError(f"No accessToken in redeem response: {data}")
|
||
return token
|
||
|
||
|
||
def authenticate(nip: str, ksef_token: str, base_url: str) -> str:
|
||
"""
|
||
Full KSeF v2 auth flow. Returns an access token ready for API calls.
|
||
|
||
Flow:
|
||
1. GET /security/public-key-certificates
|
||
2. POST /auth/challenge
|
||
3. RSA-OAEP-SHA256 encrypt '{ksef_token}|{timestamp_ms}'
|
||
4. POST /auth/ksef-token → {referenceNumber, authenticationToken}
|
||
5. Poll GET /auth/{referenceNumber} until status.code == 200
|
||
6. POST /auth/token/redeem → accessToken
|
||
"""
|
||
print("Fetching KSeF public key certificate...", flush=True)
|
||
cert_pem = get_public_key_cert(base_url)
|
||
|
||
print("Requesting auth challenge...", flush=True)
|
||
challenge_resp = get_challenge(base_url)
|
||
challenge = challenge_resp.get("challenge", "")
|
||
# Prefer the pre-computed integer field; fall back to ISO string conversion.
|
||
# KSeF verifies the exact ms value used in the encrypted plaintext, so
|
||
# converting the ISO string can introduce a sub-ms discrepancy that causes 450.
|
||
timestamp_ms = iso_to_ms(
|
||
challenge_resp.get("timestampMs") or challenge_resp.get("timestamp", int(time.time() * 1000))
|
||
)
|
||
if not challenge:
|
||
raise RuntimeError(f"No challenge in response: {challenge_resp}")
|
||
|
||
print("Encrypting KSeF token (RSA-OAEP-SHA256)...", flush=True)
|
||
encrypted_token = encrypt_token_rsa_oaep(ksef_token, timestamp_ms, cert_pem)
|
||
|
||
print("Initiating auth session...", flush=True)
|
||
init_resp = init_auth_ksef_token(nip, challenge, encrypted_token, base_url)
|
||
reference_number = init_resp.get("referenceNumber", "")
|
||
auth_token = init_resp.get("authenticationToken", {}).get("token", "")
|
||
if not reference_number or not auth_token:
|
||
raise RuntimeError(f"Unexpected init response: {init_resp}")
|
||
print(f" Reference: {reference_number}", flush=True)
|
||
|
||
# Poll until auth is ready (max ~30 s)
|
||
for attempt in range(10):
|
||
time.sleep(3)
|
||
print(f" Checking auth status (attempt {attempt + 1})...", flush=True)
|
||
status_resp = check_auth_status(reference_number, auth_token, base_url)
|
||
code = status_resp.get("status", {}).get("code", 0)
|
||
if code == 200:
|
||
print(" Auth ready.", flush=True)
|
||
break
|
||
desc = status_resp.get("status", {}).get("description", "")
|
||
print(f" Status {code}: {desc}", flush=True)
|
||
else:
|
||
raise RuntimeError("KSeF auth did not complete within the timeout.")
|
||
|
||
print("Redeeming access token...", flush=True)
|
||
access_token = redeem_access_token(auth_token, base_url)
|
||
print("Access token obtained.", flush=True)
|
||
return access_token
|
||
|
||
|
||
# ── Invoice fetching ──────────────────────────────────────────────────────────
|
||
|
||
def fetch_invoice_metadata_page(
|
||
access_token: str,
|
||
subject_type: str,
|
||
date_from: str,
|
||
date_to: str,
|
||
base_url: str,
|
||
page_size: int = 100,
|
||
page_offset: int = 0,
|
||
) -> dict:
|
||
"""
|
||
POST /invoices/query/metadata
|
||
Body: {subjectType, dateRange: {dateType, from, to}}
|
||
Returns the raw response dict.
|
||
"""
|
||
return _json(
|
||
f"{base_url}/invoices/query/metadata"
|
||
f"?pageSize={page_size}&pageOffset={page_offset}",
|
||
method="POST",
|
||
body={
|
||
"subjectType": subject_type,
|
||
"dateRange": {
|
||
"dateType": "invoicing",
|
||
"from": date_from,
|
||
"to": date_to,
|
||
},
|
||
},
|
||
headers={
|
||
"Authorization": f"Bearer {access_token}",
|
||
},
|
||
)
|
||
|
||
|
||
def fetch_all_invoice_metadata(
|
||
access_token: str,
|
||
subject_type: str,
|
||
start: date,
|
||
end: date,
|
||
base_url: str,
|
||
page_size: int = 100,
|
||
) -> list[dict]:
|
||
"""Paginated fetch of all invoice metadata for the period."""
|
||
date_from = f"{start}T00:00:00.000Z"
|
||
date_to = f"{end}T23:59:59.999Z"
|
||
all_invoices: list[dict] = []
|
||
offset = 0
|
||
|
||
while True:
|
||
resp = fetch_invoice_metadata_page(
|
||
access_token, subject_type, date_from, date_to,
|
||
base_url, page_size=page_size, page_offset=offset,
|
||
)
|
||
batch = resp.get("invoices", [])
|
||
all_invoices.extend(batch)
|
||
total = resp.get("totalCount", resp.get("count", len(batch)))
|
||
print(f" Fetched {len(all_invoices)} / {total} invoice references", flush=True)
|
||
offset += page_size
|
||
if offset >= total or not batch:
|
||
break
|
||
|
||
return all_invoices
|
||
|
||
|
||
def fetch_invoice_xml(ksef_ref: str, access_token: str, base_url: str) -> bytes:
|
||
"""GET /invoices/ksef/{ref} — returns raw XML bytes."""
|
||
_, raw, _ = _http(
|
||
f"{base_url}/invoices/ksef/{ksef_ref}",
|
||
headers={"Authorization": f"Bearer {access_token}"},
|
||
accept="application/xml",
|
||
)
|
||
return raw
|
||
|
||
|
||
# ── Invoice XML parsing ───────────────────────────────────────────────────────
|
||
|
||
def parse_invoice(raw: bytes, meta: dict) -> dict[str, Any]:
|
||
"""
|
||
Parse FA(2)/FA(3) invoice XML. Auto-detects the XML namespace so it works
|
||
regardless of schema version. Combines KSeF metadata with XML fields.
|
||
Saves raw XML to the debug dir (invoices-output/xml/) for inspection.
|
||
"""
|
||
ksef_ref = meta.get("ksefReferenceNumber") or meta.get("ksefNumber", "")
|
||
result: dict[str, Any] = {
|
||
"ksef_reference": ksef_ref,
|
||
"acquisition_timestamp": meta.get("acquisitionTimestamp", ""),
|
||
"invoice_number": None,
|
||
"issue_date": None,
|
||
"sale_date": None,
|
||
"due_date": None,
|
||
"seller_nip": None,
|
||
"seller_name": None,
|
||
"buyer_nip": None,
|
||
"buyer_name": None,
|
||
"gross_amount": None,
|
||
"net_amount": None,
|
||
"vat_amount": None,
|
||
"currency": "PLN",
|
||
"bank_account": None,
|
||
"invoice_type": "VAT",
|
||
}
|
||
|
||
xml_bytes = raw
|
||
if raw[:2] == b"\x1f\x8b":
|
||
import gzip
|
||
xml_bytes = gzip.decompress(raw)
|
||
|
||
# Save raw XML for debugging
|
||
try:
|
||
xml_dir = Path("invoices-output/xml")
|
||
xml_dir.mkdir(parents=True, exist_ok=True)
|
||
safe_ref = re.sub(r"[^A-Za-z0-9_-]", "_", ksef_ref)
|
||
(xml_dir / f"{safe_ref}.xml").write_bytes(xml_bytes)
|
||
except Exception:
|
||
pass
|
||
|
||
try:
|
||
root = ET.fromstring(xml_bytes)
|
||
except ET.ParseError as exc:
|
||
print(f" Warning: XML parse error for {ksef_ref}: {exc}", file=sys.stderr)
|
||
return result
|
||
|
||
# Auto-detect namespace from the root tag, e.g. {http://...}Faktura
|
||
ns_match = re.match(r"\{([^}]+)\}", root.tag)
|
||
nsp = f"{{{ns_match.group(1)}}}" if ns_match else ""
|
||
if nsp:
|
||
print(f" XML namespace: {ns_match.group(1)}", flush=True)
|
||
|
||
def find(path: str) -> str | None:
|
||
el = root.find(path)
|
||
return el.text.strip() if el is not None and el.text else None
|
||
|
||
def num(v: str | None) -> float | None:
|
||
if v is None:
|
||
return None
|
||
try:
|
||
return float(v.replace(",", "."))
|
||
except ValueError:
|
||
return None
|
||
|
||
# Seller (Podmiot1)
|
||
result["seller_nip"] = find(f".//{nsp}Podmiot1/{nsp}DaneIdentyfikacyjne/{nsp}NIP")
|
||
result["seller_name"] = (
|
||
find(f".//{nsp}Podmiot1/{nsp}DaneIdentyfikacyjne/{nsp}NazwaPodmiotu")
|
||
or find(f".//{nsp}Podmiot1/{nsp}DaneIdentyfikacyjne/{nsp}Nazwa")
|
||
)
|
||
|
||
# Buyer (Podmiot2)
|
||
result["buyer_nip"] = find(f".//{nsp}Podmiot2/{nsp}DaneIdentyfikacyjne/{nsp}NIP")
|
||
result["buyer_name"] = (
|
||
find(f".//{nsp}Podmiot2/{nsp}DaneIdentyfikacyjne/{nsp}NazwaPodmiotu")
|
||
or find(f".//{nsp}Podmiot2/{nsp}DaneIdentyfikacyjne/{nsp}Nazwa")
|
||
)
|
||
|
||
# Invoice header — FA(3) stores RodzajFaktury inside Fa, FA(2) in Naglowek
|
||
result["invoice_number"] = (
|
||
find(f".//{nsp}Fa/{nsp}P_2")
|
||
or find(f".//{nsp}Fa/{nsp}P_2A")
|
||
or find(f".//{nsp}Fa/{nsp}P_2B")
|
||
)
|
||
result["issue_date"] = find(f".//{nsp}Fa/{nsp}P_1")
|
||
result["sale_date"] = find(f".//{nsp}Fa/{nsp}P_6")
|
||
result["currency"] = find(f".//{nsp}Fa/{nsp}KodWaluty") or "PLN"
|
||
result["invoice_type"] = (
|
||
find(f".//{nsp}Fa/{nsp}RodzajFaktury")
|
||
or find(f".//{nsp}Naglowek/{nsp}RodzajFaktury")
|
||
or "VAT"
|
||
)
|
||
|
||
# Amounts
|
||
result["gross_amount"] = num(find(f".//{nsp}Fa/{nsp}P_15"))
|
||
for field, tags in [
|
||
("net_amount", ["P_13_1", "P_13_2", "P_13_3", "P_13_7"]),
|
||
("vat_amount", ["P_14_1", "P_14_2", "P_14_3", "P_14_7"]),
|
||
]:
|
||
for tag in tags:
|
||
v = num(find(f".//{nsp}Fa/{nsp}{tag}"))
|
||
if v is not None:
|
||
result[field] = v
|
||
break
|
||
|
||
# Due date
|
||
result["due_date"] = (
|
||
find(f".//{nsp}Fa/{nsp}Platnosc/{nsp}TerminPlatnosci/{nsp}Termin")
|
||
or find(f".//{nsp}Fa/{nsp}TerminPlatnosci/{nsp}Termin")
|
||
or find(f".//{nsp}Fa/{nsp}TerminPlatnosci")
|
||
)
|
||
|
||
# Bank account
|
||
raw_iban = find(f".//{nsp}Platnosc/{nsp}RachunekBankowy/{nsp}NrRB") \
|
||
or find(f".//{nsp}Fa/{nsp}Platnosc/{nsp}RachunekBankowy/{nsp}NrRB")
|
||
if raw_iban:
|
||
result["bank_account"] = normalize_iban(raw_iban)
|
||
|
||
# Line items — FA(3): Fa/Wiersze/FaWiersz, FA(2): Fa/FaWiersz
|
||
positions = []
|
||
fa_el = root.find(f".//{nsp}Fa")
|
||
if fa_el is not None:
|
||
wiersze_el = fa_el.find(f"{nsp}Wiersze")
|
||
item_parent = wiersze_el if wiersze_el is not None else fa_el
|
||
for w in item_parent.findall(f"{nsp}FaWiersz"):
|
||
def wfind(tag: str, _w: ET.Element = w) -> str | None:
|
||
child = _w.find(f"{nsp}{tag}")
|
||
return child.text.strip() if child is not None and child.text else None
|
||
positions.append({
|
||
"line_number": int(wfind("NrWierszaFa") or "0"),
|
||
"description": wfind("P_7"),
|
||
"unit": wfind("P_8A"),
|
||
"quantity": num(wfind("P_8B")),
|
||
"unit_price": num(wfind("P_9A")),
|
||
"net_value": num(wfind("P_11")),
|
||
"vat_rate": wfind("P_12"),
|
||
})
|
||
result["positions"] = positions
|
||
|
||
return result
|
||
|
||
|
||
# ── Contractors config ────────────────────────────────────────────────────────
|
||
|
||
def is_table_row(line: str) -> bool:
|
||
s = line.strip()
|
||
return s.startswith("|") and s.endswith("|")
|
||
|
||
|
||
def split_table_row(line: str) -> list[str]:
|
||
return [c.strip() for c in line.strip().lstrip("|").rstrip("|").split("|")]
|
||
|
||
|
||
def is_separator_row(cells: list[str]) -> bool:
|
||
return all(re.fullmatch(r":?-{3,}:?", c.strip()) for c in cells if c.strip())
|
||
|
||
|
||
def load_contractors(path: Path) -> list[dict[str, Any]]:
|
||
lines = path.read_text(encoding="utf-8").splitlines()
|
||
headers: list[str] = []
|
||
rows: list[dict[str, str]] = []
|
||
in_table = False
|
||
for line in lines:
|
||
if not is_table_row(line):
|
||
in_table = False
|
||
continue
|
||
cells = split_table_row(line)
|
||
if not headers:
|
||
if is_separator_row(cells):
|
||
continue
|
||
headers = [normalize_key(c) for c in cells]
|
||
in_table = True
|
||
continue
|
||
if is_separator_row(cells):
|
||
continue
|
||
if not in_table:
|
||
headers = []
|
||
continue
|
||
if len(cells) != len(headers):
|
||
continue
|
||
rows.append(dict(zip(headers, cells)))
|
||
|
||
def parse_limit(raw: str) -> float | None:
|
||
s = clean_text(raw).replace(",", ".").replace(" ", "").replace(" ", "")
|
||
try:
|
||
return float(s) if s else None
|
||
except ValueError:
|
||
return None
|
||
|
||
contractors = []
|
||
for row in rows:
|
||
nip = re.sub(r"\D", "", clean_text(row.get("nip", "")))
|
||
name = clean_text(row.get("name", ""))
|
||
if not nip or not name:
|
||
continue
|
||
raw_accounts = clean_text(row.get("bank_accounts", ""))
|
||
accounts = [
|
||
normalize_iban(a)
|
||
for a in re.split(r"[;,]", raw_accounts)
|
||
if a.strip()
|
||
]
|
||
raw_jira = clean_text(row.get("check_with_jira", "")).lower()
|
||
contractors.append({
|
||
"name": name,
|
||
"nip": nip,
|
||
"bank_accounts": accounts,
|
||
"max_invoice_net_value": parse_limit(row.get("max_invoice_net_value", "")),
|
||
"max_position_net_rate": parse_limit(row.get("max_position_net_rate", "")),
|
||
"max_total_quantity": parse_limit(row.get("max_total_quantity", "")),
|
||
"email": clean_text(row.get("email", "")) or None,
|
||
"check_with_jira": raw_jira in ("yes", "true", "1", "tak"),
|
||
"notes": clean_text(row.get("notes", "")),
|
||
})
|
||
return contractors
|
||
|
||
|
||
# ── Cache ─────────────────────────────────────────────────────────────────────
|
||
|
||
def load_cache(path: Path) -> dict[str, dict]:
|
||
"""Return {ksef_reference: {first_seen, whitelist_request_id, whitelist_checked_at}}."""
|
||
if not path.exists():
|
||
return {}
|
||
try:
|
||
raw = json.loads(path.read_text(encoding="utf-8"))
|
||
# Backward compat: old format was a plain list of reference strings.
|
||
if isinstance(raw, list):
|
||
return {ref: {} for ref in raw}
|
||
return raw if isinstance(raw, dict) else {}
|
||
except Exception:
|
||
return {}
|
||
|
||
|
||
def save_cache(path: Path, cache: dict[str, dict]) -> None:
|
||
path.write_text(json.dumps(cache, indent=2, ensure_ascii=False), encoding="utf-8")
|
||
|
||
|
||
# ── White list (Biała lista) ──────────────────────────────────────────────────
|
||
|
||
WL_BASE = "https://wl-api.mf.gov.pl/api"
|
||
|
||
|
||
def check_whitelist(nip: str, bank_account: str, check_date: str) -> dict[str, Any]:
|
||
# White list API expects the 26-digit national number, not the full IBAN with country code.
|
||
account_clean = re.sub(r"\s+", "", bank_account)
|
||
if re.match(r"^[A-Z]{2}\d", account_clean):
|
||
account_clean = account_clean[2:]
|
||
url = f"{WL_BASE}/check/nip/{nip}/bank-account/{account_clean}?date={check_date}"
|
||
req = urllib.request.Request(url, headers={"Accept": "application/json"})
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||
data = json.loads(resp.read())
|
||
result = data.get("result", {})
|
||
assigned_raw = result.get("accountAssigned", "")
|
||
assigned = str(assigned_raw).upper() in ("TAK", "TRUE", "YES", "1")
|
||
return {
|
||
"status": "ok" if assigned else "failed",
|
||
"account_assigned": assigned,
|
||
"request_id": result.get("requestId"),
|
||
"checked_at": result.get("requestDateTime"),
|
||
"error": None,
|
||
}
|
||
except urllib.error.HTTPError as exc:
|
||
body = exc.read().decode("utf-8", errors="replace")
|
||
return {
|
||
"status": "error", "account_assigned": None,
|
||
"request_id": None, "checked_at": None,
|
||
"error": f"HTTP {exc.code}: {body[:200]}",
|
||
}
|
||
except Exception as exc:
|
||
return {
|
||
"status": "error", "account_assigned": None,
|
||
"request_id": None, "checked_at": None,
|
||
"error": str(exc),
|
||
}
|
||
|
||
|
||
# ── Jira compliance check ────────────────────────────────────────────────────
|
||
|
||
_PROJECT_CODE_RE = re.compile(r"\b([A-Z][A-Z0-9]{1,9})-\d+\b")
|
||
|
||
|
||
def extract_project_codes(text: str) -> list[str]:
|
||
"""Return unique Jira project codes found in text (e.g. 'IAA' from 'IAA-1 work')."""
|
||
return list(dict.fromkeys(m.group(1) for m in _PROJECT_CODE_RE.finditer(text or "")))
|
||
|
||
|
||
def resolve_jira_account_by_email(
|
||
email: str,
|
||
jira_base_url: str,
|
||
jira_auth_email: str,
|
||
jira_token: str,
|
||
) -> str | None:
|
||
"""Return Jira accountId for the given email address, or None if not found."""
|
||
credentials = base64.b64encode(f"{jira_auth_email}:{jira_token}".encode()).decode()
|
||
headers = {"Authorization": f"Basic {credentials}", "Accept": "application/json"}
|
||
url = f"{jira_base_url.rstrip('/')}/rest/api/3/user/search?query={email}&maxResults=5"
|
||
req = urllib.request.Request(url, headers=headers)
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||
users = json.loads(resp.read())
|
||
match = next(
|
||
(u for u in users if u.get("emailAddress", "").lower() == email.lower()),
|
||
None,
|
||
)
|
||
return match["accountId"] if match else None
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def _resolve_issue_keys_from_worklogs(
|
||
worklogs: list[dict],
|
||
jira_auth_email: str,
|
||
jira_token: str,
|
||
) -> dict[int, str]:
|
||
"""
|
||
Resolve Jira issue IDs → keys by calling each issue's self URL directly.
|
||
Uses the self URL already present in each Tempo worklog (avoids JQL search).
|
||
"""
|
||
creds = base64.b64encode(f"{jira_auth_email}:{jira_token}".encode()).decode()
|
||
headers = {"Authorization": f"Basic {creds}", "Accept": "application/json"}
|
||
result: dict[int, str] = {}
|
||
for wl in worklogs:
|
||
issue = wl.get("issue", {})
|
||
issue_id = issue.get("id")
|
||
if not issue_id or int(issue_id) in result:
|
||
continue
|
||
self_url = issue.get("self", "")
|
||
if not self_url:
|
||
continue
|
||
url = re.sub(r"/rest/api/\d+/", "/rest/api/3/", self_url) + "?fields=key"
|
||
req = urllib.request.Request(url, headers=headers)
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||
data = json.loads(resp.read())
|
||
result[int(issue_id)] = data.get("key", "")
|
||
except Exception as exc:
|
||
print(f"Warning: could not resolve issue {issue_id}: {exc}", file=sys.stderr)
|
||
return result
|
||
|
||
|
||
def fetch_tempo_hours_by_project(
|
||
account_id: str,
|
||
year_month: str,
|
||
tempo_token: str,
|
||
tempo_base_url: str = "https://api.tempo.io/4",
|
||
jira_base_url: str = "",
|
||
jira_auth_email: str = "",
|
||
jira_token: str = "",
|
||
) -> dict[str, float]:
|
||
"""
|
||
Return {project_code: total_hours} for the given Jira account and calendar month.
|
||
year_month format: 'YYYY-MM'.
|
||
Resolves issue keys from Jira when jira_* credentials are provided.
|
||
"""
|
||
import calendar as _cal
|
||
year, month = int(year_month[:4]), int(year_month[5:7])
|
||
last_day = _cal.monthrange(year, month)[1]
|
||
start = f"{year_month}-01"
|
||
end = f"{year_month}-{last_day:02d}"
|
||
|
||
raw_worklogs: list[dict] = []
|
||
headers = {"Authorization": f"Bearer {tempo_token}", "Accept": "application/json"}
|
||
next_url: str | None = (
|
||
f"{tempo_base_url}/worklogs/user/{account_id}"
|
||
f"?from={start}&to={end}&limit=1000"
|
||
)
|
||
while next_url:
|
||
req = urllib.request.Request(next_url, headers=headers)
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||
data = json.loads(resp.read())
|
||
except urllib.error.HTTPError as exc:
|
||
raise RuntimeError(
|
||
f"Tempo API {exc.code}: {exc.read().decode('utf-8', errors='replace')[:200]}"
|
||
) from exc
|
||
raw_worklogs.extend(data.get("results", []))
|
||
next_url = data.get("metadata", {}).get("next")
|
||
|
||
# Resolve issue IDs → keys via self URLs (Tempo v4 omits key; JQL search returns 410)
|
||
issue_id_map: dict[int, str] = {}
|
||
if jira_auth_email and jira_token:
|
||
issue_id_map = _resolve_issue_keys_from_worklogs(raw_worklogs, jira_auth_email, jira_token)
|
||
|
||
totals: dict[str, float] = {}
|
||
for wl in raw_worklogs:
|
||
issue_id = int(wl.get("issue", {}).get("id", 0))
|
||
issue_key = issue_id_map.get(issue_id, "")
|
||
codes = extract_project_codes(issue_key)
|
||
hours = wl.get("timeSpentSeconds", 0) / 3600
|
||
for code in codes:
|
||
totals[code] = totals.get(code, 0.0) + hours
|
||
return totals
|
||
|
||
|
||
def check_jira_compliance(
|
||
inv: dict,
|
||
contractor: dict,
|
||
jira_base_url: str,
|
||
jira_auth_email: str,
|
||
jira_token: str,
|
||
tempo_token: str,
|
||
tempo_base_url: str = "https://api.tempo.io/4",
|
||
) -> dict:
|
||
"""
|
||
For a contractor with check_with_jira=True, verify that position quantities
|
||
on the invoice match hours logged in Tempo for each Jira project code,
|
||
for the calendar month of the invoice sale date.
|
||
|
||
Returns a dict of compliance fields to merge into the invoice record.
|
||
"""
|
||
base: dict[str, Any] = {
|
||
"jira_checked": False,
|
||
"jira_compliant": None,
|
||
"jira_account_id": None,
|
||
"jira_violations": [],
|
||
"jira_no_project_positions": [],
|
||
"jira_error": None,
|
||
}
|
||
|
||
email = contractor.get("email")
|
||
if not email:
|
||
base["jira_error"] = "No email on contractor — cannot resolve Jira account"
|
||
return base
|
||
|
||
account_id = resolve_jira_account_by_email(
|
||
email, jira_base_url, jira_auth_email, jira_token
|
||
)
|
||
if not account_id:
|
||
base["jira_error"] = f"Jira account not found for {email}"
|
||
return base
|
||
base["jira_account_id"] = account_id
|
||
|
||
sale_date = inv.get("sale_date") or inv.get("issue_date") or ""
|
||
if len(sale_date) < 7:
|
||
base["jira_error"] = "Invoice has no sale date for month matching"
|
||
return base
|
||
year_month = sale_date[:7]
|
||
|
||
positions = inv.get("positions", [])
|
||
if not positions:
|
||
base["jira_error"] = "Invoice has no line items"
|
||
return base
|
||
|
||
base["jira_checked"] = True
|
||
|
||
# Classify positions: group qty by project code, flag those with no code
|
||
project_qty: dict[str, float] = {}
|
||
no_project: list[dict] = []
|
||
for pos in positions:
|
||
codes = extract_project_codes(pos.get("description") or "")
|
||
if not codes:
|
||
no_project.append({
|
||
"line_number": pos.get("line_number"),
|
||
"description": pos.get("description") or "",
|
||
})
|
||
else:
|
||
qty = pos.get("quantity") or 0.0
|
||
for code in codes:
|
||
project_qty[code] = project_qty.get(code, 0.0) + qty
|
||
|
||
base["jira_no_project_positions"] = no_project
|
||
|
||
# Fetch Tempo hours for contractor + month
|
||
try:
|
||
tempo_hours = fetch_tempo_hours_by_project(
|
||
account_id, year_month, tempo_token, tempo_base_url,
|
||
jira_base_url=jira_base_url,
|
||
jira_auth_email=jira_auth_email,
|
||
jira_token=jira_token,
|
||
)
|
||
except RuntimeError as exc:
|
||
base["jira_error"] = str(exc)
|
||
return base
|
||
|
||
# Compare quantities vs logged hours
|
||
violations = []
|
||
for project, inv_qty in project_qty.items():
|
||
jira_h = round(tempo_hours.get(project, 0.0), 4)
|
||
if round(inv_qty, 4) != jira_h:
|
||
violations.append({
|
||
"project": project,
|
||
"invoice_qty": round(inv_qty, 4),
|
||
"jira_hours": jira_h,
|
||
"month": year_month,
|
||
})
|
||
|
||
base["jira_violations"] = violations
|
||
base["jira_compliant"] = not violations and not no_project
|
||
return base
|
||
|
||
|
||
# ── Duplicate detection ───────────────────────────────────────────────────────
|
||
|
||
def detect_month_duplicates(invoices: dict[str, dict]) -> None:
|
||
"""
|
||
Flag invoices that are the 2nd+ from the same seller in the same calendar month.
|
||
If the gross amount also matches an earlier invoice in the group, flag as potential duplicate.
|
||
Operates in-place on the invoices dict (keyed by ksef_reference).
|
||
"""
|
||
from collections import defaultdict
|
||
|
||
# Reset flags on every run so they stay accurate as new invoices are added
|
||
for inv in invoices.values():
|
||
inv["second_in_month"] = False
|
||
inv["potential_duplicate"] = False
|
||
|
||
groups: dict[tuple, list[dict]] = defaultdict(list)
|
||
for inv in invoices.values():
|
||
nip = inv.get("seller_nip") or ""
|
||
date_str = inv.get("issue_date") or ""
|
||
if nip and len(date_str) >= 7:
|
||
groups[(nip, date_str[:7])].append(inv) # "YYYY-MM"
|
||
|
||
for group in groups.values():
|
||
if len(group) <= 1:
|
||
continue
|
||
group.sort(key=lambda i: (i.get("issue_date") or "", i.get("ksef_reference") or ""))
|
||
for idx, inv in enumerate(group):
|
||
if idx == 0:
|
||
continue
|
||
inv["second_in_month"] = True
|
||
current_gross = inv.get("gross_amount")
|
||
if current_gross is not None:
|
||
for earlier in group[:idx]:
|
||
if earlier.get("gross_amount") == current_gross:
|
||
inv["potential_duplicate"] = True
|
||
break
|
||
|
||
|
||
# ── Main ──────────────────────────────────────────────────────────────────────
|
||
|
||
def run(args: argparse.Namespace) -> int:
|
||
_require_cryptography()
|
||
|
||
skill_dir = Path(__file__).parent.parent
|
||
loaded = load_dotenv(Path.cwd() / ".env", skill_dir / ".env")
|
||
if loaded:
|
||
print(f"Loaded credentials from {loaded}")
|
||
|
||
nip = re.sub(r"\D", "", (args.nip or os.environ.get("KSEF_NIP", "")).strip())
|
||
token = (args.token or os.environ.get("KSEF_TOKEN", "")).strip()
|
||
env = (args.ksef_env or os.environ.get("KSEF_ENV", "prod")).strip().lower()
|
||
|
||
missing = [(n, v) for n, v in [("KSEF_NIP", nip), ("KSEF_TOKEN", token)] if not v]
|
||
if missing:
|
||
print(
|
||
f"Error: missing credentials: {', '.join(n for n, _ in missing)}\n"
|
||
"Set them in your .env file.",
|
||
file=sys.stderr,
|
||
)
|
||
return 1
|
||
if len(nip) != 10:
|
||
print(f"Error: KSEF_NIP must be 10 digits (got '{nip}').", file=sys.stderr)
|
||
return 1
|
||
if env not in KSEF_URLS:
|
||
print(f"Error: KSEF_ENV must be 'test' or 'prod' (got '{env}').", file=sys.stderr)
|
||
return 1
|
||
|
||
base_url = KSEF_URLS[env]
|
||
print(f"KSeF environment: {env} ({base_url})")
|
||
|
||
start, end = parse_period(args.period)
|
||
print(f"Period: {start} to {end}")
|
||
|
||
# Contractors
|
||
contractors_path: Path | None = None
|
||
if args.contractors:
|
||
contractors_path = Path(args.contractors)
|
||
else:
|
||
for candidate in (
|
||
Path.cwd() / "contractors.md",
|
||
skill_dir / "references" / "contractors.md",
|
||
):
|
||
if candidate.exists():
|
||
contractors_path = candidate
|
||
break
|
||
if contractors_path is None:
|
||
print("Warning: no contractors.md found — all contractors will show as unknown.",
|
||
file=sys.stderr)
|
||
contractors: list[dict] = []
|
||
else:
|
||
contractors = load_contractors(contractors_path)
|
||
print(f"Contractors loaded: {len(contractors)} (from {contractors_path})")
|
||
contractors_by_nip = {c["nip"]: c for c in contractors}
|
||
|
||
# Cache
|
||
output_dir = Path(args.output_dir)
|
||
output_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
cache_path = output_dir / "cache" / "invoices_cache.json"
|
||
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
||
cache: dict[str, dict] = load_cache(cache_path)
|
||
print(f"Invoice cache: {len(cache)} previously seen references")
|
||
|
||
# ── Auth ─────────────────────────────────────────────────────────────────
|
||
try:
|
||
access_token = authenticate(nip, token, base_url)
|
||
except Exception as exc:
|
||
print(f"Error: KSeF authentication failed: {exc}", file=sys.stderr)
|
||
return 1
|
||
|
||
# ── Fetch metadata ────────────────────────────────────────────────────────
|
||
print(f"\nFetching invoice metadata for {start} – {end}...")
|
||
try:
|
||
all_meta = fetch_all_invoice_metadata(
|
||
access_token, "subject2", start, end, base_url
|
||
)
|
||
except Exception as exc:
|
||
print(f"Error fetching invoice metadata: {exc}", file=sys.stderr)
|
||
return 1
|
||
|
||
print(f"Total invoices in KSeF for period: {len(all_meta)}")
|
||
|
||
# Identify new invoices
|
||
all_refs = {
|
||
(m.get("ksefReferenceNumber") or m.get("ksefNumber", "")): m
|
||
for m in all_meta
|
||
if (m.get("ksefReferenceNumber") or m.get("ksefNumber"))
|
||
}
|
||
new_refs = {ref: meta for ref, meta in all_refs.items() if ref not in cache}
|
||
known_count = len(all_refs) - len(new_refs)
|
||
print(f"New: {len(new_refs)} | Already seen: {known_count}")
|
||
|
||
# ── Process each invoice ──────────────────────────────────────────────────
|
||
invoice_records: list[dict] = []
|
||
|
||
for ref, meta in all_refs.items():
|
||
is_new = ref in new_refs
|
||
|
||
if not is_new:
|
||
invoice_records.append({"ksef_reference": ref, "is_new": False})
|
||
continue
|
||
|
||
print(f"\n Processing {ref} (new)...", flush=True)
|
||
|
||
# Download and parse XML
|
||
try:
|
||
raw_xml = fetch_invoice_xml(ref, access_token, base_url)
|
||
inv = parse_invoice(raw_xml, meta)
|
||
except Exception as exc:
|
||
print(f" Warning: could not fetch/parse invoice: {exc}", file=sys.stderr)
|
||
inv = {
|
||
"ksef_reference": ref,
|
||
"acquisition_timestamp": meta.get("acquisitionTimestamp", ""),
|
||
}
|
||
|
||
inv["is_new"] = True
|
||
|
||
# Contractor check
|
||
seller_nip = re.sub(r"\D", "", inv.get("seller_nip") or "")
|
||
contractor = contractors_by_nip.get(seller_nip)
|
||
inv["contractor_known"] = contractor is not None
|
||
|
||
inv_account = inv.get("bank_account")
|
||
if contractor and inv_account and contractor["bank_accounts"]:
|
||
inv["account_in_contractors"] = inv_account in contractor["bank_accounts"]
|
||
else:
|
||
inv["account_in_contractors"] = None
|
||
|
||
# White list check
|
||
check_date = (inv.get("issue_date") or str(start))[:10]
|
||
if inv_account and seller_nip:
|
||
print(f" White list: NIP {seller_nip}, account {inv_account}...", flush=True)
|
||
wl = check_whitelist(seller_nip, inv_account, check_date)
|
||
inv["whitelist_status"] = wl["status"]
|
||
inv["whitelist_account_assigned"] = wl["account_assigned"]
|
||
inv["whitelist_request_id"] = wl["request_id"]
|
||
inv["whitelist_checked_at"] = wl["checked_at"]
|
||
inv["whitelist_error"] = wl["error"]
|
||
label = {"ok": "✓ OK", "failed": "✗ FAILED", "error": "? error"}.get(
|
||
wl["status"], wl["status"]
|
||
)
|
||
print(f" White list: {label}")
|
||
elif not inv_account:
|
||
inv["whitelist_status"] = "no_account"
|
||
inv["whitelist_account_assigned"] = None
|
||
inv["whitelist_request_id"] = None
|
||
inv["whitelist_checked_at"] = None
|
||
inv["whitelist_error"] = None
|
||
print(" White list: — (no bank account on invoice)")
|
||
else:
|
||
inv["whitelist_status"] = "no_nip"
|
||
inv["whitelist_account_assigned"] = None
|
||
inv["whitelist_request_id"] = None
|
||
inv["whitelist_checked_at"] = None
|
||
inv["whitelist_error"] = "seller NIP missing from invoice"
|
||
|
||
# Rule violations (only checked for known contractors with limits set)
|
||
violations: list[dict] = []
|
||
if contractor:
|
||
max_inv = contractor.get("max_invoice_net_value")
|
||
if max_inv is not None:
|
||
net = inv.get("net_amount")
|
||
if net is not None and net > max_inv:
|
||
violations.append({
|
||
"rule": "max_invoice_net_value",
|
||
"limit": max_inv,
|
||
"actual": round(net, 2),
|
||
})
|
||
max_pos = contractor.get("max_position_net_rate")
|
||
if max_pos is not None:
|
||
for pos in inv.get("positions", []):
|
||
rate = pos.get("unit_price")
|
||
if rate is not None and rate > max_pos:
|
||
violations.append({
|
||
"rule": "max_position_net_rate",
|
||
"limit": max_pos,
|
||
"actual": round(rate, 2),
|
||
"position": pos.get("line_number"),
|
||
"description": pos.get("description") or "",
|
||
})
|
||
max_qty = contractor.get("max_total_quantity")
|
||
if max_qty is not None:
|
||
total_qty = sum(pos.get("quantity") or 0 for pos in inv.get("positions", []))
|
||
if total_qty > max_qty:
|
||
violations.append({
|
||
"rule": "max_total_quantity",
|
||
"limit": max_qty,
|
||
"actual": round(total_qty, 4),
|
||
})
|
||
inv["rule_violations"] = violations
|
||
if violations:
|
||
print(f" ⚠ Rule violations: {len(violations)}", flush=True)
|
||
|
||
# Jira compliance check
|
||
if contractor and contractor.get("check_with_jira"):
|
||
jira_base = os.environ.get("JIRA_BASE_URL", "")
|
||
jira_auth_email = os.environ.get("JIRA_EMAIL", "")
|
||
jira_tok = os.environ.get("JIRA_API_TOKEN", "")
|
||
tempo_tok = os.environ.get("TEMPO_API_TOKEN", "")
|
||
if all([jira_base, jira_auth_email, jira_tok, tempo_tok]):
|
||
print(" Checking Jira compliance...", flush=True)
|
||
jira_result = check_jira_compliance(
|
||
inv, contractor, jira_base, jira_auth_email, jira_tok, tempo_tok
|
||
)
|
||
inv.update(jira_result)
|
||
if jira_result.get("jira_error"):
|
||
print(f" Jira: ? error — {jira_result['jira_error']}", flush=True)
|
||
elif jira_result.get("jira_compliant"):
|
||
print(" Jira: ✓ compliant", flush=True)
|
||
else:
|
||
n = len(jira_result.get("jira_violations", [])) + len(jira_result.get("jira_no_project_positions", []))
|
||
print(f" Jira: ✗ non-compliant ({n} issue(s))", flush=True)
|
||
else:
|
||
inv.update({
|
||
"jira_checked": False, "jira_compliant": None,
|
||
"jira_account_id": None, "jira_violations": [],
|
||
"jira_no_project_positions": [],
|
||
"jira_error": "Missing JIRA_BASE_URL / JIRA_EMAIL / JIRA_API_TOKEN / TEMPO_API_TOKEN",
|
||
})
|
||
|
||
invoice_records.append(inv)
|
||
|
||
# Update cache — add new refs with whitelist audit trail
|
||
now_iso = datetime.now(timezone.utc).isoformat()
|
||
for inv in invoice_records:
|
||
ref = inv.get("ksef_reference", "")
|
||
if not ref:
|
||
continue
|
||
if ref not in cache:
|
||
cache[ref] = {"first_seen": now_iso}
|
||
if inv.get("is_new") and inv.get("whitelist_request_id"):
|
||
cache[ref]["whitelist_request_id"] = inv["whitelist_request_id"]
|
||
cache[ref]["whitelist_checked_at"] = inv.get("whitelist_checked_at")
|
||
save_cache(cache_path, cache)
|
||
|
||
# Update cumulative all-invoices store
|
||
all_inv_path = output_dir / "invoices_all.json"
|
||
try:
|
||
existing = json.loads(all_inv_path.read_text(encoding="utf-8")) if all_inv_path.exists() else {}
|
||
except Exception:
|
||
existing = {}
|
||
stored: dict[str, dict] = existing.get("invoices", {})
|
||
for inv in invoice_records:
|
||
ref = inv.get("ksef_reference", "")
|
||
if inv.get("is_new") and ref:
|
||
stored[ref] = inv
|
||
# Detect duplicates across ALL known invoices, then propagate flags to current run
|
||
detect_month_duplicates(stored)
|
||
for inv in invoice_records:
|
||
ref = inv.get("ksef_reference", "")
|
||
if ref in stored:
|
||
inv["second_in_month"] = stored[ref].get("second_in_month", False)
|
||
inv["potential_duplicate"] = stored[ref].get("potential_duplicate", False)
|
||
else:
|
||
inv.setdefault("second_in_month", False)
|
||
inv.setdefault("potential_duplicate", False)
|
||
|
||
all_inv_out = {
|
||
"updated_at": datetime.now(timezone.utc).isoformat(),
|
||
"total": len(stored),
|
||
"invoices": stored,
|
||
}
|
||
all_inv_path.write_text(json.dumps(all_inv_out, indent=2, ensure_ascii=False), encoding="utf-8")
|
||
print(f"Updated {all_inv_path} ({len(stored)} total invoices)")
|
||
|
||
# Summary
|
||
new_invoices = [r for r in invoice_records if r.get("is_new")]
|
||
summary = {
|
||
"total_invoices": len(invoice_records),
|
||
"new_invoices": len(new_invoices),
|
||
"known_contractors": sum(1 for r in new_invoices if r.get("contractor_known")),
|
||
"unknown_contractors": sum(1 for r in new_invoices if not r.get("contractor_known")),
|
||
"whitelist_ok": sum(1 for r in new_invoices if r.get("whitelist_status") == "ok"),
|
||
"whitelist_failed": sum(1 for r in new_invoices if r.get("whitelist_status") == "failed"),
|
||
"whitelist_error": sum(1 for r in new_invoices if r.get("whitelist_status") == "error"),
|
||
"account_mismatch": sum(1 for r in new_invoices if r.get("account_in_contractors") is False),
|
||
"rule_violations": sum(1 for r in new_invoices if r.get("rule_violations")),
|
||
"second_invoices": sum(1 for r in new_invoices if r.get("second_in_month")),
|
||
"potential_duplicates": sum(1 for r in new_invoices if r.get("potential_duplicate")),
|
||
"jira_non_compliant": sum(1 for r in new_invoices if r.get("jira_checked") and not r.get("jira_compliant")),
|
||
}
|
||
|
||
output = {
|
||
"period": {"start": str(start), "end": str(end)},
|
||
"ksef_env": env,
|
||
"summary": summary,
|
||
"invoices": invoice_records,
|
||
"known_invoices_count": known_count,
|
||
}
|
||
|
||
out_path = output_dir / "invoices_data.json"
|
||
out_path.write_text(json.dumps(output, indent=2, ensure_ascii=False), encoding="utf-8")
|
||
print(f"\nWrote {out_path}")
|
||
|
||
print(f"\nSummary: {summary['new_invoices']} new invoice(s)")
|
||
if summary["whitelist_failed"]:
|
||
print(f" ✗ WHITE LIST FAILURES: {summary['whitelist_failed']} — review before payment!")
|
||
if summary["unknown_contractors"]:
|
||
print(f" ✦ Unknown contractors: {summary['unknown_contractors']}")
|
||
if summary["account_mismatch"]:
|
||
print(f" ⚠ Account mismatch: {summary['account_mismatch']}")
|
||
if summary["rule_violations"]:
|
||
print(f" ⚠ Rule violations: {summary['rule_violations']}")
|
||
if summary["jira_non_compliant"]:
|
||
print(f" ✗ Jira non-compliant: {summary['jira_non_compliant']}")
|
||
|
||
return 0
|
||
|
||
|
||
def build_parser() -> argparse.ArgumentParser:
|
||
parser = argparse.ArgumentParser(
|
||
description="Fetch KSeF v2 invoices and run contractor/white-list checks."
|
||
)
|
||
parser.add_argument("--period", required=True, help="YYYY-MM-DD:YYYY-MM-DD")
|
||
parser.add_argument("--contractors", help="Path to contractors.md.")
|
||
parser.add_argument("--output-dir", required=True)
|
||
parser.add_argument("--nip", help="Company NIP. Defaults to KSEF_NIP env var.")
|
||
parser.add_argument("--token", help="KSeF token. Defaults to KSEF_TOKEN env var.")
|
||
parser.add_argument("--ksef-env", choices=["test", "prod"],
|
||
help="Defaults to KSEF_ENV env var or 'prod'.")
|
||
return parser
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(run(build_parser().parse_args()))
|