c9336941ac
Add four skills for Tempo/KSeF timesheet and invoice workflows: - check-my-timesheet: show/log the current user's Tempo time entries - timesheet-checker: audit timesheet completion across all reporters - invoice-checker: pull KSeF invoices, check contractors + MF white list - invoice-prep: summarize Tempo hours per Jira project for invoicing Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1243 lines
48 KiB
Python
1243 lines
48 KiB
Python
#!/usr/bin/env python3
|
||
"""Fetch purchase invoices from KSeF v2 API, check contractors and MF white list."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import base64
|
||
import json
|
||
import os
|
||
import re
|
||
import sys
|
||
import time
|
||
import urllib.error
|
||
import urllib.request
|
||
import xml.etree.ElementTree as ET
|
||
from datetime import date, datetime, timezone
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
|
||
# ── Dependency check ─────────────────────────────────────────────────────────
|
||
|
||
def _require_cryptography() -> None:
|
||
try:
|
||
import cryptography # noqa: F401
|
||
except ImportError:
|
||
print(
|
||
"Error: the 'cryptography' package is required.\n"
|
||
"Install it with: pip3 install cryptography",
|
||
file=sys.stderr,
|
||
)
|
||
sys.exit(1)
|
||
|
||
|
||
# ── .env loading ──────────────────────────────────────────────────────────────
|
||
|
||
def load_dotenv(*paths: Path) -> Path | None:
|
||
for path in paths:
|
||
if not path.exists():
|
||
continue
|
||
for raw in path.read_text(encoding="utf-8").splitlines():
|
||
line = raw.strip()
|
||
if not line or line.startswith("#") or "=" not in line:
|
||
continue
|
||
key, _, value = line.partition("=")
|
||
key = key.strip()
|
||
value = value.strip().strip('"').strip("'")
|
||
if key and key not in os.environ:
|
||
os.environ[key] = value
|
||
return path
|
||
return None
|
||
|
||
|
||
# ── Utilities ─────────────────────────────────────────────────────────────────
|
||
|
||
def clean_text(value: Any) -> str:
|
||
return re.sub(r"\s+", " ", str(value or "")).strip()
|
||
|
||
|
||
def normalize_key(value: Any) -> str:
|
||
return re.sub(r"[^a-z0-9]+", "_", clean_text(value).lower()).strip("_")
|
||
|
||
|
||
def normalize_iban(raw: str) -> str:
|
||
s = re.sub(r"\s+", "", raw.upper())
|
||
if re.fullmatch(r"\d{26}", s):
|
||
s = "PL" + s
|
||
return s
|
||
|
||
|
||
def iso_to_ms(value: Any) -> int:
|
||
"""Convert ISO timestamp string or epoch-ms integer to epoch milliseconds."""
|
||
if isinstance(value, (int, float)):
|
||
return int(value)
|
||
try:
|
||
dt = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
||
return int(dt.timestamp() * 1000)
|
||
except Exception:
|
||
return int(time.time() * 1000)
|
||
|
||
|
||
# ── Period parsing ────────────────────────────────────────────────────────────
|
||
|
||
def parse_period(period_str: str) -> tuple[date, date]:
|
||
s = period_str.strip()
|
||
if ":" in s:
|
||
parts = s.split(":", 1)
|
||
return date.fromisoformat(parts[0].strip()), date.fromisoformat(parts[1].strip())
|
||
raise ValueError(f"Invalid period '{period_str}'. Use YYYY-MM-DD:YYYY-MM-DD.")
|
||
|
||
|
||
# ── KSeF base URLs ────────────────────────────────────────────────────────────
|
||
|
||
KSEF_URLS = {
|
||
"prod": "https://api.ksef.mf.gov.pl/api/v2",
|
||
"test": "https://api.ksef-test.mf.gov.pl/api/v2",
|
||
}
|
||
|
||
|
||
# ── HTTP helpers ──────────────────────────────────────────────────────────────
|
||
|
||
def _http(
|
||
url: str,
|
||
method: str = "GET",
|
||
body: Any = None,
|
||
headers: dict | None = None,
|
||
accept: str = "application/json",
|
||
raw_body: bytes | None = None,
|
||
) -> tuple[int, bytes, dict]:
|
||
"""Return (status_code, body_bytes, response_headers)."""
|
||
data: bytes | None = None
|
||
h = {"Accept": accept}
|
||
if raw_body is not None:
|
||
data = raw_body
|
||
h["Content-Type"] = "application/octet-stream"
|
||
elif body is not None:
|
||
data = json.dumps(body).encode()
|
||
h["Content-Type"] = "application/json"
|
||
if headers:
|
||
h.update(headers)
|
||
req = urllib.request.Request(url, data=data, headers=h, method=method)
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||
return resp.status, resp.read(), dict(resp.headers)
|
||
except urllib.error.HTTPError as exc:
|
||
body_bytes = exc.read()
|
||
raise RuntimeError(
|
||
f"HTTP {exc.code} {method} {url}: "
|
||
f"{body_bytes.decode('utf-8', errors='replace')[:400]}"
|
||
) from exc
|
||
|
||
|
||
def _json(url: str, method: str = "GET", body: Any = None,
|
||
headers: dict | None = None) -> Any:
|
||
_, raw, _ = _http(url, method=method, body=body, headers=headers)
|
||
return json.loads(raw)
|
||
|
||
|
||
# ── KSeF v2 authentication ────────────────────────────────────────────────────
|
||
|
||
def get_public_key_cert(base_url: str) -> str:
|
||
"""
|
||
GET /security/public-key-certificates
|
||
Returns PEM string for the KsefTokenEncryption certificate.
|
||
"""
|
||
data = _json(f"{base_url}/security/public-key-certificates")
|
||
certs: list[dict] = []
|
||
if isinstance(data, list):
|
||
certs = data
|
||
elif isinstance(data, dict):
|
||
certs = data.get("data", data.get("certificates", [data]))
|
||
|
||
def has_usage(c: dict, needle: str) -> bool:
|
||
u = c.get("usage", [])
|
||
if isinstance(u, list):
|
||
return any(needle.lower() in str(x).lower() for x in u)
|
||
return needle.lower() in str(u).lower()
|
||
|
||
cert = (
|
||
next((c for c in certs if has_usage(c, "token")), None)
|
||
or next((c for c in certs if has_usage(c, "auth")), None)
|
||
or (certs[0] if certs else None)
|
||
)
|
||
if not cert:
|
||
raise RuntimeError(f"No public key certificates returned. Response: {data}")
|
||
|
||
b64 = cert.get("certificate") or cert.get("publicKeyCertificate") or cert.get("value", "")
|
||
if not b64:
|
||
raise RuntimeError(f"Certificate payload missing: {cert}")
|
||
|
||
if "BEGIN CERTIFICATE" in b64:
|
||
return b64
|
||
wrapped = "\n".join(b64[i:i+64] for i in range(0, len(b64), 64))
|
||
return f"-----BEGIN CERTIFICATE-----\n{wrapped}\n-----END CERTIFICATE-----"
|
||
|
||
|
||
def get_challenge(base_url: str) -> dict:
|
||
"""
|
||
POST /auth/challenge (no request body)
|
||
Returns {challenge: str, timestamp: str|int}
|
||
"""
|
||
return _json(f"{base_url}/auth/challenge", method="POST")
|
||
|
||
|
||
def encrypt_token_rsa_oaep(token: str, timestamp_ms: int, cert_pem: str) -> str:
|
||
"""
|
||
RSA-OAEP-SHA256 encrypt f'{token}|{timestamp_ms}' using the KSeF public key
|
||
extracted from the X.509 certificate. Returns base64-encoded ciphertext.
|
||
"""
|
||
from cryptography.hazmat.primitives import hashes, serialization
|
||
from cryptography.hazmat.primitives.asymmetric import padding as asym_padding
|
||
from cryptography.x509 import load_pem_x509_certificate
|
||
|
||
plaintext = f"{token}|{timestamp_ms}".encode("utf-8")
|
||
cert = load_pem_x509_certificate(cert_pem.encode())
|
||
public_key = cert.public_key()
|
||
ciphertext = public_key.encrypt(
|
||
plaintext,
|
||
asym_padding.OAEP(
|
||
mgf=asym_padding.MGF1(algorithm=hashes.SHA256()),
|
||
algorithm=hashes.SHA256(),
|
||
label=None,
|
||
),
|
||
)
|
||
return base64.b64encode(ciphertext).decode()
|
||
|
||
|
||
def init_auth_ksef_token(
|
||
nip: str, challenge: str, encrypted_token: str, base_url: str
|
||
) -> dict:
|
||
"""
|
||
POST /auth/ksef-token
|
||
Body: {challenge, contextIdentifier: {type: 'nip', value: nip}, encryptedToken}
|
||
Returns {referenceNumber, authenticationToken: {token}}
|
||
"""
|
||
return _json(
|
||
f"{base_url}/auth/ksef-token",
|
||
method="POST",
|
||
body={
|
||
"challenge": challenge,
|
||
"contextIdentifier": {"type": "nip", "value": nip},
|
||
"encryptedToken": encrypted_token,
|
||
},
|
||
)
|
||
|
||
|
||
def check_auth_status(reference_number: str, auth_token: str, base_url: str) -> dict:
|
||
"""
|
||
GET /auth/{referenceNumber}
|
||
Returns {status: {code: int, description: str}, ...}
|
||
"""
|
||
return _json(
|
||
f"{base_url}/auth/{reference_number}",
|
||
headers={"Authorization": f"Bearer {auth_token}"},
|
||
)
|
||
|
||
|
||
def redeem_access_token(auth_token: str, base_url: str) -> str:
|
||
"""
|
||
POST /auth/token/redeem
|
||
Returns the access token string from {accessToken: {token}}.
|
||
"""
|
||
data = _json(
|
||
f"{base_url}/auth/token/redeem",
|
||
method="POST",
|
||
headers={"Authorization": f"Bearer {auth_token}"},
|
||
)
|
||
token = data.get("accessToken", {}).get("token", "")
|
||
if not token:
|
||
raise RuntimeError(f"No accessToken in redeem response: {data}")
|
||
return token
|
||
|
||
|
||
def authenticate(nip: str, ksef_token: str, base_url: str) -> str:
|
||
"""
|
||
Full KSeF v2 auth flow. Returns an access token ready for API calls.
|
||
|
||
Flow:
|
||
1. GET /security/public-key-certificates
|
||
2. POST /auth/challenge
|
||
3. RSA-OAEP-SHA256 encrypt '{ksef_token}|{timestamp_ms}'
|
||
4. POST /auth/ksef-token → {referenceNumber, authenticationToken}
|
||
5. Poll GET /auth/{referenceNumber} until status.code == 200
|
||
6. POST /auth/token/redeem → accessToken
|
||
"""
|
||
print("Fetching KSeF public key certificate...", flush=True)
|
||
cert_pem = get_public_key_cert(base_url)
|
||
|
||
print("Requesting auth challenge...", flush=True)
|
||
challenge_resp = get_challenge(base_url)
|
||
challenge = challenge_resp.get("challenge", "")
|
||
# Prefer the pre-computed integer field; fall back to ISO string conversion.
|
||
# KSeF verifies the exact ms value used in the encrypted plaintext, so
|
||
# converting the ISO string can introduce a sub-ms discrepancy that causes 450.
|
||
timestamp_ms = iso_to_ms(
|
||
challenge_resp.get("timestampMs") or challenge_resp.get("timestamp", int(time.time() * 1000))
|
||
)
|
||
if not challenge:
|
||
raise RuntimeError(f"No challenge in response: {challenge_resp}")
|
||
|
||
print("Encrypting KSeF token (RSA-OAEP-SHA256)...", flush=True)
|
||
encrypted_token = encrypt_token_rsa_oaep(ksef_token, timestamp_ms, cert_pem)
|
||
|
||
print("Initiating auth session...", flush=True)
|
||
init_resp = init_auth_ksef_token(nip, challenge, encrypted_token, base_url)
|
||
reference_number = init_resp.get("referenceNumber", "")
|
||
auth_token = init_resp.get("authenticationToken", {}).get("token", "")
|
||
if not reference_number or not auth_token:
|
||
raise RuntimeError(f"Unexpected init response: {init_resp}")
|
||
print(f" Reference: {reference_number}", flush=True)
|
||
|
||
# Poll until auth is ready (max ~30 s)
|
||
for attempt in range(10):
|
||
time.sleep(3)
|
||
print(f" Checking auth status (attempt {attempt + 1})...", flush=True)
|
||
status_resp = check_auth_status(reference_number, auth_token, base_url)
|
||
code = status_resp.get("status", {}).get("code", 0)
|
||
if code == 200:
|
||
print(" Auth ready.", flush=True)
|
||
break
|
||
desc = status_resp.get("status", {}).get("description", "")
|
||
print(f" Status {code}: {desc}", flush=True)
|
||
else:
|
||
raise RuntimeError("KSeF auth did not complete within the timeout.")
|
||
|
||
print("Redeeming access token...", flush=True)
|
||
access_token = redeem_access_token(auth_token, base_url)
|
||
print("Access token obtained.", flush=True)
|
||
return access_token
|
||
|
||
|
||
# ── Invoice fetching ──────────────────────────────────────────────────────────
|
||
|
||
def fetch_invoice_metadata_page(
|
||
access_token: str,
|
||
subject_type: str,
|
||
date_from: str,
|
||
date_to: str,
|
||
base_url: str,
|
||
page_size: int = 100,
|
||
page_offset: int = 0,
|
||
) -> dict:
|
||
"""
|
||
POST /invoices/query/metadata
|
||
Body: {subjectType, dateRange: {dateType, from, to}}
|
||
Returns the raw response dict.
|
||
"""
|
||
return _json(
|
||
f"{base_url}/invoices/query/metadata"
|
||
f"?pageSize={page_size}&pageOffset={page_offset}",
|
||
method="POST",
|
||
body={
|
||
"subjectType": subject_type,
|
||
"dateRange": {
|
||
"dateType": "invoicing",
|
||
"from": date_from,
|
||
"to": date_to,
|
||
},
|
||
},
|
||
headers={
|
||
"Authorization": f"Bearer {access_token}",
|
||
},
|
||
)
|
||
|
||
|
||
def fetch_all_invoice_metadata(
|
||
access_token: str,
|
||
subject_type: str,
|
||
start: date,
|
||
end: date,
|
||
base_url: str,
|
||
page_size: int = 100,
|
||
) -> list[dict]:
|
||
"""Paginated fetch of all invoice metadata for the period."""
|
||
date_from = f"{start}T00:00:00.000Z"
|
||
date_to = f"{end}T23:59:59.999Z"
|
||
all_invoices: list[dict] = []
|
||
offset = 0
|
||
|
||
while True:
|
||
resp = fetch_invoice_metadata_page(
|
||
access_token, subject_type, date_from, date_to,
|
||
base_url, page_size=page_size, page_offset=offset,
|
||
)
|
||
batch = resp.get("invoices", [])
|
||
all_invoices.extend(batch)
|
||
total = resp.get("totalCount", resp.get("count", len(batch)))
|
||
print(f" Fetched {len(all_invoices)} / {total} invoice references", flush=True)
|
||
offset += page_size
|
||
if offset >= total or not batch:
|
||
break
|
||
|
||
return all_invoices
|
||
|
||
|
||
def fetch_invoice_xml(ksef_ref: str, access_token: str, base_url: str) -> bytes:
|
||
"""GET /invoices/ksef/{ref} — returns raw XML bytes."""
|
||
_, raw, _ = _http(
|
||
f"{base_url}/invoices/ksef/{ksef_ref}",
|
||
headers={"Authorization": f"Bearer {access_token}"},
|
||
accept="application/xml",
|
||
)
|
||
return raw
|
||
|
||
|
||
# ── Invoice XML parsing ───────────────────────────────────────────────────────
|
||
|
||
def parse_invoice(raw: bytes, meta: dict) -> dict[str, Any]:
|
||
"""
|
||
Parse FA(2)/FA(3) invoice XML. Auto-detects the XML namespace so it works
|
||
regardless of schema version. Combines KSeF metadata with XML fields.
|
||
Saves raw XML to the debug dir (invoices-output/xml/) for inspection.
|
||
"""
|
||
ksef_ref = meta.get("ksefReferenceNumber") or meta.get("ksefNumber", "")
|
||
result: dict[str, Any] = {
|
||
"ksef_reference": ksef_ref,
|
||
"acquisition_timestamp": meta.get("acquisitionTimestamp", ""),
|
||
"invoice_number": None,
|
||
"issue_date": None,
|
||
"sale_date": None,
|
||
"due_date": None,
|
||
"seller_nip": None,
|
||
"seller_name": None,
|
||
"buyer_nip": None,
|
||
"buyer_name": None,
|
||
"gross_amount": None,
|
||
"net_amount": None,
|
||
"vat_amount": None,
|
||
"currency": "PLN",
|
||
"bank_account": None,
|
||
"invoice_type": "VAT",
|
||
}
|
||
|
||
xml_bytes = raw
|
||
if raw[:2] == b"\x1f\x8b":
|
||
import gzip
|
||
xml_bytes = gzip.decompress(raw)
|
||
|
||
# Save raw XML for debugging
|
||
try:
|
||
xml_dir = Path("invoices-output/xml")
|
||
xml_dir.mkdir(parents=True, exist_ok=True)
|
||
safe_ref = re.sub(r"[^A-Za-z0-9_-]", "_", ksef_ref)
|
||
(xml_dir / f"{safe_ref}.xml").write_bytes(xml_bytes)
|
||
except Exception:
|
||
pass
|
||
|
||
try:
|
||
root = ET.fromstring(xml_bytes)
|
||
except ET.ParseError as exc:
|
||
print(f" Warning: XML parse error for {ksef_ref}: {exc}", file=sys.stderr)
|
||
return result
|
||
|
||
# Auto-detect namespace from the root tag, e.g. {http://...}Faktura
|
||
ns_match = re.match(r"\{([^}]+)\}", root.tag)
|
||
nsp = f"{{{ns_match.group(1)}}}" if ns_match else ""
|
||
if nsp:
|
||
print(f" XML namespace: {ns_match.group(1)}", flush=True)
|
||
|
||
def find(path: str) -> str | None:
|
||
el = root.find(path)
|
||
return el.text.strip() if el is not None and el.text else None
|
||
|
||
def num(v: str | None) -> float | None:
|
||
if v is None:
|
||
return None
|
||
try:
|
||
return float(v.replace(",", "."))
|
||
except ValueError:
|
||
return None
|
||
|
||
# Seller (Podmiot1)
|
||
result["seller_nip"] = find(f".//{nsp}Podmiot1/{nsp}DaneIdentyfikacyjne/{nsp}NIP")
|
||
result["seller_name"] = (
|
||
find(f".//{nsp}Podmiot1/{nsp}DaneIdentyfikacyjne/{nsp}NazwaPodmiotu")
|
||
or find(f".//{nsp}Podmiot1/{nsp}DaneIdentyfikacyjne/{nsp}Nazwa")
|
||
)
|
||
|
||
# Buyer (Podmiot2)
|
||
result["buyer_nip"] = find(f".//{nsp}Podmiot2/{nsp}DaneIdentyfikacyjne/{nsp}NIP")
|
||
result["buyer_name"] = (
|
||
find(f".//{nsp}Podmiot2/{nsp}DaneIdentyfikacyjne/{nsp}NazwaPodmiotu")
|
||
or find(f".//{nsp}Podmiot2/{nsp}DaneIdentyfikacyjne/{nsp}Nazwa")
|
||
)
|
||
|
||
# Invoice header — FA(3) stores RodzajFaktury inside Fa, FA(2) in Naglowek
|
||
result["invoice_number"] = (
|
||
find(f".//{nsp}Fa/{nsp}P_2")
|
||
or find(f".//{nsp}Fa/{nsp}P_2A")
|
||
or find(f".//{nsp}Fa/{nsp}P_2B")
|
||
)
|
||
result["issue_date"] = find(f".//{nsp}Fa/{nsp}P_1")
|
||
result["sale_date"] = find(f".//{nsp}Fa/{nsp}P_6")
|
||
result["currency"] = find(f".//{nsp}Fa/{nsp}KodWaluty") or "PLN"
|
||
result["invoice_type"] = (
|
||
find(f".//{nsp}Fa/{nsp}RodzajFaktury")
|
||
or find(f".//{nsp}Naglowek/{nsp}RodzajFaktury")
|
||
or "VAT"
|
||
)
|
||
|
||
# Amounts
|
||
result["gross_amount"] = num(find(f".//{nsp}Fa/{nsp}P_15"))
|
||
for field, tags in [
|
||
("net_amount", ["P_13_1", "P_13_2", "P_13_3", "P_13_7"]),
|
||
("vat_amount", ["P_14_1", "P_14_2", "P_14_3", "P_14_7"]),
|
||
]:
|
||
for tag in tags:
|
||
v = num(find(f".//{nsp}Fa/{nsp}{tag}"))
|
||
if v is not None:
|
||
result[field] = v
|
||
break
|
||
|
||
# Due date
|
||
result["due_date"] = (
|
||
find(f".//{nsp}Fa/{nsp}Platnosc/{nsp}TerminPlatnosci/{nsp}Termin")
|
||
or find(f".//{nsp}Fa/{nsp}TerminPlatnosci/{nsp}Termin")
|
||
or find(f".//{nsp}Fa/{nsp}TerminPlatnosci")
|
||
)
|
||
|
||
# Bank account
|
||
raw_iban = find(f".//{nsp}Platnosc/{nsp}RachunekBankowy/{nsp}NrRB") \
|
||
or find(f".//{nsp}Fa/{nsp}Platnosc/{nsp}RachunekBankowy/{nsp}NrRB")
|
||
if raw_iban:
|
||
result["bank_account"] = normalize_iban(raw_iban)
|
||
|
||
# Line items — FA(3): Fa/Wiersze/FaWiersz, FA(2): Fa/FaWiersz
|
||
positions = []
|
||
fa_el = root.find(f".//{nsp}Fa")
|
||
if fa_el is not None:
|
||
wiersze_el = fa_el.find(f"{nsp}Wiersze")
|
||
item_parent = wiersze_el if wiersze_el is not None else fa_el
|
||
for w in item_parent.findall(f"{nsp}FaWiersz"):
|
||
def wfind(tag: str, _w: ET.Element = w) -> str | None:
|
||
child = _w.find(f"{nsp}{tag}")
|
||
return child.text.strip() if child is not None and child.text else None
|
||
positions.append({
|
||
"line_number": int(wfind("NrWierszaFa") or "0"),
|
||
"description": wfind("P_7"),
|
||
"unit": wfind("P_8A"),
|
||
"quantity": num(wfind("P_8B")),
|
||
"unit_price": num(wfind("P_9A")),
|
||
"net_value": num(wfind("P_11")),
|
||
"vat_rate": wfind("P_12"),
|
||
})
|
||
result["positions"] = positions
|
||
|
||
return result
|
||
|
||
|
||
# ── Contractors config ────────────────────────────────────────────────────────
|
||
|
||
def is_table_row(line: str) -> bool:
|
||
s = line.strip()
|
||
return s.startswith("|") and s.endswith("|")
|
||
|
||
|
||
def split_table_row(line: str) -> list[str]:
|
||
return [c.strip() for c in line.strip().lstrip("|").rstrip("|").split("|")]
|
||
|
||
|
||
def is_separator_row(cells: list[str]) -> bool:
|
||
return all(re.fullmatch(r":?-{3,}:?", c.strip()) for c in cells if c.strip())
|
||
|
||
|
||
def load_contractors(path: Path) -> list[dict[str, Any]]:
|
||
lines = path.read_text(encoding="utf-8").splitlines()
|
||
headers: list[str] = []
|
||
rows: list[dict[str, str]] = []
|
||
in_table = False
|
||
for line in lines:
|
||
if not is_table_row(line):
|
||
in_table = False
|
||
continue
|
||
cells = split_table_row(line)
|
||
if not headers:
|
||
if is_separator_row(cells):
|
||
continue
|
||
headers = [normalize_key(c) for c in cells]
|
||
in_table = True
|
||
continue
|
||
if is_separator_row(cells):
|
||
continue
|
||
if not in_table:
|
||
headers = []
|
||
continue
|
||
if len(cells) != len(headers):
|
||
continue
|
||
rows.append(dict(zip(headers, cells)))
|
||
|
||
def parse_limit(raw: str) -> float | None:
|
||
s = clean_text(raw).replace(",", ".").replace(" ", "").replace(" ", "")
|
||
try:
|
||
return float(s) if s else None
|
||
except ValueError:
|
||
return None
|
||
|
||
contractors = []
|
||
for row in rows:
|
||
nip = re.sub(r"\D", "", clean_text(row.get("nip", "")))
|
||
name = clean_text(row.get("name", ""))
|
||
if not nip or not name:
|
||
continue
|
||
raw_accounts = clean_text(row.get("bank_accounts", ""))
|
||
accounts = [
|
||
normalize_iban(a)
|
||
for a in re.split(r"[;,]", raw_accounts)
|
||
if a.strip()
|
||
]
|
||
raw_jira = clean_text(row.get("check_with_jira", "")).lower()
|
||
contractors.append({
|
||
"name": name,
|
||
"nip": nip,
|
||
"bank_accounts": accounts,
|
||
"max_invoice_net_value": parse_limit(row.get("max_invoice_net_value", "")),
|
||
"max_position_net_rate": parse_limit(row.get("max_position_net_rate", "")),
|
||
"max_total_quantity": parse_limit(row.get("max_total_quantity", "")),
|
||
"email": clean_text(row.get("email", "")) or None,
|
||
"check_with_jira": raw_jira in ("yes", "true", "1", "tak"),
|
||
"notes": clean_text(row.get("notes", "")),
|
||
})
|
||
return contractors
|
||
|
||
|
||
# ── Cache ─────────────────────────────────────────────────────────────────────
|
||
|
||
def load_cache(path: Path) -> dict[str, dict]:
|
||
"""Return {ksef_reference: {first_seen, whitelist_request_id, whitelist_checked_at}}."""
|
||
if not path.exists():
|
||
return {}
|
||
try:
|
||
raw = json.loads(path.read_text(encoding="utf-8"))
|
||
# Backward compat: old format was a plain list of reference strings.
|
||
if isinstance(raw, list):
|
||
return {ref: {} for ref in raw}
|
||
return raw if isinstance(raw, dict) else {}
|
||
except Exception:
|
||
return {}
|
||
|
||
|
||
def save_cache(path: Path, cache: dict[str, dict]) -> None:
|
||
path.write_text(json.dumps(cache, indent=2, ensure_ascii=False), encoding="utf-8")
|
||
|
||
|
||
# ── White list (Biała lista) ──────────────────────────────────────────────────
|
||
|
||
WL_BASE = "https://wl-api.mf.gov.pl/api"
|
||
|
||
|
||
def check_whitelist(nip: str, bank_account: str, check_date: str) -> dict[str, Any]:
|
||
# White list API expects the 26-digit national number, not the full IBAN with country code.
|
||
account_clean = re.sub(r"\s+", "", bank_account)
|
||
if re.match(r"^[A-Z]{2}\d", account_clean):
|
||
account_clean = account_clean[2:]
|
||
url = f"{WL_BASE}/check/nip/{nip}/bank-account/{account_clean}?date={check_date}"
|
||
req = urllib.request.Request(url, headers={"Accept": "application/json"})
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||
data = json.loads(resp.read())
|
||
result = data.get("result", {})
|
||
assigned_raw = result.get("accountAssigned", "")
|
||
assigned = str(assigned_raw).upper() in ("TAK", "TRUE", "YES", "1")
|
||
return {
|
||
"status": "ok" if assigned else "failed",
|
||
"account_assigned": assigned,
|
||
"request_id": result.get("requestId"),
|
||
"checked_at": result.get("requestDateTime"),
|
||
"error": None,
|
||
}
|
||
except urllib.error.HTTPError as exc:
|
||
body = exc.read().decode("utf-8", errors="replace")
|
||
return {
|
||
"status": "error", "account_assigned": None,
|
||
"request_id": None, "checked_at": None,
|
||
"error": f"HTTP {exc.code}: {body[:200]}",
|
||
}
|
||
except Exception as exc:
|
||
return {
|
||
"status": "error", "account_assigned": None,
|
||
"request_id": None, "checked_at": None,
|
||
"error": str(exc),
|
||
}
|
||
|
||
|
||
# ── Jira compliance check ────────────────────────────────────────────────────
|
||
|
||
_PROJECT_CODE_RE = re.compile(r"\b([A-Z][A-Z0-9]{1,9})-\d+\b")
|
||
|
||
|
||
def extract_project_codes(text: str) -> list[str]:
|
||
"""Return unique Jira project codes found in text (e.g. 'IAA' from 'IAA-1 work')."""
|
||
return list(dict.fromkeys(m.group(1) for m in _PROJECT_CODE_RE.finditer(text or "")))
|
||
|
||
|
||
def resolve_jira_account_by_email(
|
||
email: str,
|
||
jira_base_url: str,
|
||
jira_auth_email: str,
|
||
jira_token: str,
|
||
) -> str | None:
|
||
"""Return Jira accountId for the given email address, or None if not found."""
|
||
credentials = base64.b64encode(f"{jira_auth_email}:{jira_token}".encode()).decode()
|
||
headers = {"Authorization": f"Basic {credentials}", "Accept": "application/json"}
|
||
url = f"{jira_base_url.rstrip('/')}/rest/api/3/user/search?query={email}&maxResults=5"
|
||
req = urllib.request.Request(url, headers=headers)
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||
users = json.loads(resp.read())
|
||
match = next(
|
||
(u for u in users if u.get("emailAddress", "").lower() == email.lower()),
|
||
None,
|
||
)
|
||
return match["accountId"] if match else None
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def _resolve_issue_keys_from_worklogs(
|
||
worklogs: list[dict],
|
||
jira_auth_email: str,
|
||
jira_token: str,
|
||
) -> dict[int, str]:
|
||
"""
|
||
Resolve Jira issue IDs → keys by calling each issue's self URL directly.
|
||
Uses the self URL already present in each Tempo worklog (avoids JQL search).
|
||
"""
|
||
creds = base64.b64encode(f"{jira_auth_email}:{jira_token}".encode()).decode()
|
||
headers = {"Authorization": f"Basic {creds}", "Accept": "application/json"}
|
||
result: dict[int, str] = {}
|
||
for wl in worklogs:
|
||
issue = wl.get("issue", {})
|
||
issue_id = issue.get("id")
|
||
if not issue_id or int(issue_id) in result:
|
||
continue
|
||
self_url = issue.get("self", "")
|
||
if not self_url:
|
||
continue
|
||
url = re.sub(r"/rest/api/\d+/", "/rest/api/3/", self_url) + "?fields=key"
|
||
req = urllib.request.Request(url, headers=headers)
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||
data = json.loads(resp.read())
|
||
result[int(issue_id)] = data.get("key", "")
|
||
except Exception as exc:
|
||
print(f"Warning: could not resolve issue {issue_id}: {exc}", file=sys.stderr)
|
||
return result
|
||
|
||
|
||
def fetch_tempo_hours_by_project(
|
||
account_id: str,
|
||
year_month: str,
|
||
tempo_token: str,
|
||
tempo_base_url: str = "https://api.tempo.io/4",
|
||
jira_base_url: str = "",
|
||
jira_auth_email: str = "",
|
||
jira_token: str = "",
|
||
) -> dict[str, float]:
|
||
"""
|
||
Return {project_code: total_hours} for the given Jira account and calendar month.
|
||
year_month format: 'YYYY-MM'.
|
||
Resolves issue keys from Jira when jira_* credentials are provided.
|
||
"""
|
||
import calendar as _cal
|
||
year, month = int(year_month[:4]), int(year_month[5:7])
|
||
last_day = _cal.monthrange(year, month)[1]
|
||
start = f"{year_month}-01"
|
||
end = f"{year_month}-{last_day:02d}"
|
||
|
||
raw_worklogs: list[dict] = []
|
||
headers = {"Authorization": f"Bearer {tempo_token}", "Accept": "application/json"}
|
||
next_url: str | None = (
|
||
f"{tempo_base_url}/worklogs/user/{account_id}"
|
||
f"?from={start}&to={end}&limit=1000"
|
||
)
|
||
while next_url:
|
||
req = urllib.request.Request(next_url, headers=headers)
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||
data = json.loads(resp.read())
|
||
except urllib.error.HTTPError as exc:
|
||
raise RuntimeError(
|
||
f"Tempo API {exc.code}: {exc.read().decode('utf-8', errors='replace')[:200]}"
|
||
) from exc
|
||
raw_worklogs.extend(data.get("results", []))
|
||
next_url = data.get("metadata", {}).get("next")
|
||
|
||
# Resolve issue IDs → keys via self URLs (Tempo v4 omits key; JQL search returns 410)
|
||
issue_id_map: dict[int, str] = {}
|
||
if jira_auth_email and jira_token:
|
||
issue_id_map = _resolve_issue_keys_from_worklogs(raw_worklogs, jira_auth_email, jira_token)
|
||
|
||
totals: dict[str, float] = {}
|
||
for wl in raw_worklogs:
|
||
issue_id = int(wl.get("issue", {}).get("id", 0))
|
||
issue_key = issue_id_map.get(issue_id, "")
|
||
codes = extract_project_codes(issue_key)
|
||
hours = wl.get("timeSpentSeconds", 0) / 3600
|
||
for code in codes:
|
||
totals[code] = totals.get(code, 0.0) + hours
|
||
return totals
|
||
|
||
|
||
def check_jira_compliance(
|
||
inv: dict,
|
||
contractor: dict,
|
||
jira_base_url: str,
|
||
jira_auth_email: str,
|
||
jira_token: str,
|
||
tempo_token: str,
|
||
tempo_base_url: str = "https://api.tempo.io/4",
|
||
) -> dict:
|
||
"""
|
||
For a contractor with check_with_jira=True, verify that position quantities
|
||
on the invoice match hours logged in Tempo for each Jira project code,
|
||
for the calendar month of the invoice sale date.
|
||
|
||
Returns a dict of compliance fields to merge into the invoice record.
|
||
"""
|
||
base: dict[str, Any] = {
|
||
"jira_checked": False,
|
||
"jira_compliant": None,
|
||
"jira_account_id": None,
|
||
"jira_violations": [],
|
||
"jira_no_project_positions": [],
|
||
"jira_error": None,
|
||
}
|
||
|
||
email = contractor.get("email")
|
||
if not email:
|
||
base["jira_error"] = "No email on contractor — cannot resolve Jira account"
|
||
return base
|
||
|
||
account_id = resolve_jira_account_by_email(
|
||
email, jira_base_url, jira_auth_email, jira_token
|
||
)
|
||
if not account_id:
|
||
base["jira_error"] = f"Jira account not found for {email}"
|
||
return base
|
||
base["jira_account_id"] = account_id
|
||
|
||
sale_date = inv.get("sale_date") or inv.get("issue_date") or ""
|
||
if len(sale_date) < 7:
|
||
base["jira_error"] = "Invoice has no sale date for month matching"
|
||
return base
|
||
year_month = sale_date[:7]
|
||
|
||
positions = inv.get("positions", [])
|
||
if not positions:
|
||
base["jira_error"] = "Invoice has no line items"
|
||
return base
|
||
|
||
base["jira_checked"] = True
|
||
|
||
# Classify positions: group qty by project code, flag those with no code
|
||
project_qty: dict[str, float] = {}
|
||
no_project: list[dict] = []
|
||
for pos in positions:
|
||
codes = extract_project_codes(pos.get("description") or "")
|
||
if not codes:
|
||
no_project.append({
|
||
"line_number": pos.get("line_number"),
|
||
"description": pos.get("description") or "",
|
||
})
|
||
else:
|
||
qty = pos.get("quantity") or 0.0
|
||
for code in codes:
|
||
project_qty[code] = project_qty.get(code, 0.0) + qty
|
||
|
||
base["jira_no_project_positions"] = no_project
|
||
|
||
# Fetch Tempo hours for contractor + month
|
||
try:
|
||
tempo_hours = fetch_tempo_hours_by_project(
|
||
account_id, year_month, tempo_token, tempo_base_url,
|
||
jira_base_url=jira_base_url,
|
||
jira_auth_email=jira_auth_email,
|
||
jira_token=jira_token,
|
||
)
|
||
except RuntimeError as exc:
|
||
base["jira_error"] = str(exc)
|
||
return base
|
||
|
||
# Compare quantities vs logged hours
|
||
violations = []
|
||
for project, inv_qty in project_qty.items():
|
||
jira_h = round(tempo_hours.get(project, 0.0), 4)
|
||
if round(inv_qty, 4) != jira_h:
|
||
violations.append({
|
||
"project": project,
|
||
"invoice_qty": round(inv_qty, 4),
|
||
"jira_hours": jira_h,
|
||
"month": year_month,
|
||
})
|
||
|
||
base["jira_violations"] = violations
|
||
base["jira_compliant"] = not violations and not no_project
|
||
return base
|
||
|
||
|
||
# ── Duplicate detection ───────────────────────────────────────────────────────
|
||
|
||
def detect_month_duplicates(invoices: dict[str, dict]) -> None:
|
||
"""
|
||
Flag invoices that are the 2nd+ from the same seller in the same calendar month.
|
||
If the gross amount also matches an earlier invoice in the group, flag as potential duplicate.
|
||
Operates in-place on the invoices dict (keyed by ksef_reference).
|
||
"""
|
||
from collections import defaultdict
|
||
|
||
# Reset flags on every run so they stay accurate as new invoices are added
|
||
for inv in invoices.values():
|
||
inv["second_in_month"] = False
|
||
inv["potential_duplicate"] = False
|
||
|
||
groups: dict[tuple, list[dict]] = defaultdict(list)
|
||
for inv in invoices.values():
|
||
nip = inv.get("seller_nip") or ""
|
||
date_str = inv.get("issue_date") or ""
|
||
if nip and len(date_str) >= 7:
|
||
groups[(nip, date_str[:7])].append(inv) # "YYYY-MM"
|
||
|
||
for group in groups.values():
|
||
if len(group) <= 1:
|
||
continue
|
||
group.sort(key=lambda i: (i.get("issue_date") or "", i.get("ksef_reference") or ""))
|
||
for idx, inv in enumerate(group):
|
||
if idx == 0:
|
||
continue
|
||
inv["second_in_month"] = True
|
||
current_gross = inv.get("gross_amount")
|
||
if current_gross is not None:
|
||
for earlier in group[:idx]:
|
||
if earlier.get("gross_amount") == current_gross:
|
||
inv["potential_duplicate"] = True
|
||
break
|
||
|
||
|
||
# ── Main ──────────────────────────────────────────────────────────────────────
|
||
|
||
def run(args: argparse.Namespace) -> int:
|
||
_require_cryptography()
|
||
|
||
skill_dir = Path(__file__).parent.parent
|
||
loaded = load_dotenv(Path.cwd() / ".env", skill_dir / ".env")
|
||
if loaded:
|
||
print(f"Loaded credentials from {loaded}")
|
||
|
||
nip = re.sub(r"\D", "", (args.nip or os.environ.get("KSEF_NIP", "")).strip())
|
||
token = (args.token or os.environ.get("KSEF_TOKEN", "")).strip()
|
||
env = (args.ksef_env or os.environ.get("KSEF_ENV", "prod")).strip().lower()
|
||
|
||
missing = [(n, v) for n, v in [("KSEF_NIP", nip), ("KSEF_TOKEN", token)] if not v]
|
||
if missing:
|
||
print(
|
||
f"Error: missing credentials: {', '.join(n for n, _ in missing)}\n"
|
||
"Set them in your .env file.",
|
||
file=sys.stderr,
|
||
)
|
||
return 1
|
||
if len(nip) != 10:
|
||
print(f"Error: KSEF_NIP must be 10 digits (got '{nip}').", file=sys.stderr)
|
||
return 1
|
||
if env not in KSEF_URLS:
|
||
print(f"Error: KSEF_ENV must be 'test' or 'prod' (got '{env}').", file=sys.stderr)
|
||
return 1
|
||
|
||
base_url = KSEF_URLS[env]
|
||
print(f"KSeF environment: {env} ({base_url})")
|
||
|
||
start, end = parse_period(args.period)
|
||
print(f"Period: {start} to {end}")
|
||
|
||
# Contractors
|
||
contractors_path: Path | None = None
|
||
if args.contractors:
|
||
contractors_path = Path(args.contractors)
|
||
else:
|
||
for candidate in (
|
||
Path.cwd() / "contractors.md",
|
||
skill_dir / "references" / "contractors.md",
|
||
):
|
||
if candidate.exists():
|
||
contractors_path = candidate
|
||
break
|
||
if contractors_path is None:
|
||
print("Warning: no contractors.md found — all contractors will show as unknown.",
|
||
file=sys.stderr)
|
||
contractors: list[dict] = []
|
||
else:
|
||
contractors = load_contractors(contractors_path)
|
||
print(f"Contractors loaded: {len(contractors)} (from {contractors_path})")
|
||
contractors_by_nip = {c["nip"]: c for c in contractors}
|
||
|
||
# Cache
|
||
output_dir = Path(args.output_dir)
|
||
output_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
cache_path = output_dir / "cache" / "invoices_cache.json"
|
||
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
||
cache: dict[str, dict] = load_cache(cache_path)
|
||
print(f"Invoice cache: {len(cache)} previously seen references")
|
||
|
||
# ── Auth ─────────────────────────────────────────────────────────────────
|
||
try:
|
||
access_token = authenticate(nip, token, base_url)
|
||
except Exception as exc:
|
||
print(f"Error: KSeF authentication failed: {exc}", file=sys.stderr)
|
||
return 1
|
||
|
||
# ── Fetch metadata ────────────────────────────────────────────────────────
|
||
print(f"\nFetching invoice metadata for {start} – {end}...")
|
||
try:
|
||
all_meta = fetch_all_invoice_metadata(
|
||
access_token, "subject2", start, end, base_url
|
||
)
|
||
except Exception as exc:
|
||
print(f"Error fetching invoice metadata: {exc}", file=sys.stderr)
|
||
return 1
|
||
|
||
print(f"Total invoices in KSeF for period: {len(all_meta)}")
|
||
|
||
# Identify new invoices
|
||
all_refs = {
|
||
(m.get("ksefReferenceNumber") or m.get("ksefNumber", "")): m
|
||
for m in all_meta
|
||
if (m.get("ksefReferenceNumber") or m.get("ksefNumber"))
|
||
}
|
||
new_refs = {ref: meta for ref, meta in all_refs.items() if ref not in cache}
|
||
known_count = len(all_refs) - len(new_refs)
|
||
print(f"New: {len(new_refs)} | Already seen: {known_count}")
|
||
|
||
# ── Process each invoice ──────────────────────────────────────────────────
|
||
invoice_records: list[dict] = []
|
||
|
||
for ref, meta in all_refs.items():
|
||
is_new = ref in new_refs
|
||
|
||
if not is_new:
|
||
invoice_records.append({"ksef_reference": ref, "is_new": False})
|
||
continue
|
||
|
||
print(f"\n Processing {ref} (new)...", flush=True)
|
||
|
||
# Download and parse XML
|
||
try:
|
||
raw_xml = fetch_invoice_xml(ref, access_token, base_url)
|
||
inv = parse_invoice(raw_xml, meta)
|
||
except Exception as exc:
|
||
print(f" Warning: could not fetch/parse invoice: {exc}", file=sys.stderr)
|
||
inv = {
|
||
"ksef_reference": ref,
|
||
"acquisition_timestamp": meta.get("acquisitionTimestamp", ""),
|
||
}
|
||
|
||
inv["is_new"] = True
|
||
|
||
# Contractor check
|
||
seller_nip = re.sub(r"\D", "", inv.get("seller_nip") or "")
|
||
contractor = contractors_by_nip.get(seller_nip)
|
||
inv["contractor_known"] = contractor is not None
|
||
|
||
inv_account = inv.get("bank_account")
|
||
if contractor and inv_account and contractor["bank_accounts"]:
|
||
inv["account_in_contractors"] = inv_account in contractor["bank_accounts"]
|
||
else:
|
||
inv["account_in_contractors"] = None
|
||
|
||
# White list check
|
||
check_date = (inv.get("issue_date") or str(start))[:10]
|
||
if inv_account and seller_nip:
|
||
print(f" White list: NIP {seller_nip}, account {inv_account}...", flush=True)
|
||
wl = check_whitelist(seller_nip, inv_account, check_date)
|
||
inv["whitelist_status"] = wl["status"]
|
||
inv["whitelist_account_assigned"] = wl["account_assigned"]
|
||
inv["whitelist_request_id"] = wl["request_id"]
|
||
inv["whitelist_checked_at"] = wl["checked_at"]
|
||
inv["whitelist_error"] = wl["error"]
|
||
label = {"ok": "✓ OK", "failed": "✗ FAILED", "error": "? error"}.get(
|
||
wl["status"], wl["status"]
|
||
)
|
||
print(f" White list: {label}")
|
||
elif not inv_account:
|
||
inv["whitelist_status"] = "no_account"
|
||
inv["whitelist_account_assigned"] = None
|
||
inv["whitelist_request_id"] = None
|
||
inv["whitelist_checked_at"] = None
|
||
inv["whitelist_error"] = None
|
||
print(" White list: — (no bank account on invoice)")
|
||
else:
|
||
inv["whitelist_status"] = "no_nip"
|
||
inv["whitelist_account_assigned"] = None
|
||
inv["whitelist_request_id"] = None
|
||
inv["whitelist_checked_at"] = None
|
||
inv["whitelist_error"] = "seller NIP missing from invoice"
|
||
|
||
# Rule violations (only checked for known contractors with limits set)
|
||
violations: list[dict] = []
|
||
if contractor:
|
||
max_inv = contractor.get("max_invoice_net_value")
|
||
if max_inv is not None:
|
||
net = inv.get("net_amount")
|
||
if net is not None and net > max_inv:
|
||
violations.append({
|
||
"rule": "max_invoice_net_value",
|
||
"limit": max_inv,
|
||
"actual": round(net, 2),
|
||
})
|
||
max_pos = contractor.get("max_position_net_rate")
|
||
if max_pos is not None:
|
||
for pos in inv.get("positions", []):
|
||
rate = pos.get("unit_price")
|
||
if rate is not None and rate > max_pos:
|
||
violations.append({
|
||
"rule": "max_position_net_rate",
|
||
"limit": max_pos,
|
||
"actual": round(rate, 2),
|
||
"position": pos.get("line_number"),
|
||
"description": pos.get("description") or "",
|
||
})
|
||
max_qty = contractor.get("max_total_quantity")
|
||
if max_qty is not None:
|
||
total_qty = sum(pos.get("quantity") or 0 for pos in inv.get("positions", []))
|
||
if total_qty > max_qty:
|
||
violations.append({
|
||
"rule": "max_total_quantity",
|
||
"limit": max_qty,
|
||
"actual": round(total_qty, 4),
|
||
})
|
||
inv["rule_violations"] = violations
|
||
if violations:
|
||
print(f" ⚠ Rule violations: {len(violations)}", flush=True)
|
||
|
||
# Jira compliance check
|
||
if contractor and contractor.get("check_with_jira"):
|
||
jira_base = os.environ.get("JIRA_BASE_URL", "")
|
||
jira_auth_email = os.environ.get("JIRA_EMAIL", "")
|
||
jira_tok = os.environ.get("JIRA_API_TOKEN", "")
|
||
tempo_tok = os.environ.get("TEMPO_API_TOKEN", "")
|
||
if all([jira_base, jira_auth_email, jira_tok, tempo_tok]):
|
||
print(" Checking Jira compliance...", flush=True)
|
||
jira_result = check_jira_compliance(
|
||
inv, contractor, jira_base, jira_auth_email, jira_tok, tempo_tok
|
||
)
|
||
inv.update(jira_result)
|
||
if jira_result.get("jira_error"):
|
||
print(f" Jira: ? error — {jira_result['jira_error']}", flush=True)
|
||
elif jira_result.get("jira_compliant"):
|
||
print(" Jira: ✓ compliant", flush=True)
|
||
else:
|
||
n = len(jira_result.get("jira_violations", [])) + len(jira_result.get("jira_no_project_positions", []))
|
||
print(f" Jira: ✗ non-compliant ({n} issue(s))", flush=True)
|
||
else:
|
||
inv.update({
|
||
"jira_checked": False, "jira_compliant": None,
|
||
"jira_account_id": None, "jira_violations": [],
|
||
"jira_no_project_positions": [],
|
||
"jira_error": "Missing JIRA_BASE_URL / JIRA_EMAIL / JIRA_API_TOKEN / TEMPO_API_TOKEN",
|
||
})
|
||
|
||
invoice_records.append(inv)
|
||
|
||
# Update cache — add new refs with whitelist audit trail
|
||
now_iso = datetime.now(timezone.utc).isoformat()
|
||
for inv in invoice_records:
|
||
ref = inv.get("ksef_reference", "")
|
||
if not ref:
|
||
continue
|
||
if ref not in cache:
|
||
cache[ref] = {"first_seen": now_iso}
|
||
if inv.get("is_new") and inv.get("whitelist_request_id"):
|
||
cache[ref]["whitelist_request_id"] = inv["whitelist_request_id"]
|
||
cache[ref]["whitelist_checked_at"] = inv.get("whitelist_checked_at")
|
||
save_cache(cache_path, cache)
|
||
|
||
# Update cumulative all-invoices store
|
||
all_inv_path = output_dir / "invoices_all.json"
|
||
try:
|
||
existing = json.loads(all_inv_path.read_text(encoding="utf-8")) if all_inv_path.exists() else {}
|
||
except Exception:
|
||
existing = {}
|
||
stored: dict[str, dict] = existing.get("invoices", {})
|
||
for inv in invoice_records:
|
||
ref = inv.get("ksef_reference", "")
|
||
if inv.get("is_new") and ref:
|
||
stored[ref] = inv
|
||
# Detect duplicates across ALL known invoices, then propagate flags to current run
|
||
detect_month_duplicates(stored)
|
||
for inv in invoice_records:
|
||
ref = inv.get("ksef_reference", "")
|
||
if ref in stored:
|
||
inv["second_in_month"] = stored[ref].get("second_in_month", False)
|
||
inv["potential_duplicate"] = stored[ref].get("potential_duplicate", False)
|
||
else:
|
||
inv.setdefault("second_in_month", False)
|
||
inv.setdefault("potential_duplicate", False)
|
||
|
||
all_inv_out = {
|
||
"updated_at": datetime.now(timezone.utc).isoformat(),
|
||
"total": len(stored),
|
||
"invoices": stored,
|
||
}
|
||
all_inv_path.write_text(json.dumps(all_inv_out, indent=2, ensure_ascii=False), encoding="utf-8")
|
||
print(f"Updated {all_inv_path} ({len(stored)} total invoices)")
|
||
|
||
# Summary
|
||
new_invoices = [r for r in invoice_records if r.get("is_new")]
|
||
summary = {
|
||
"total_invoices": len(invoice_records),
|
||
"new_invoices": len(new_invoices),
|
||
"known_contractors": sum(1 for r in new_invoices if r.get("contractor_known")),
|
||
"unknown_contractors": sum(1 for r in new_invoices if not r.get("contractor_known")),
|
||
"whitelist_ok": sum(1 for r in new_invoices if r.get("whitelist_status") == "ok"),
|
||
"whitelist_failed": sum(1 for r in new_invoices if r.get("whitelist_status") == "failed"),
|
||
"whitelist_error": sum(1 for r in new_invoices if r.get("whitelist_status") == "error"),
|
||
"account_mismatch": sum(1 for r in new_invoices if r.get("account_in_contractors") is False),
|
||
"rule_violations": sum(1 for r in new_invoices if r.get("rule_violations")),
|
||
"second_invoices": sum(1 for r in new_invoices if r.get("second_in_month")),
|
||
"potential_duplicates": sum(1 for r in new_invoices if r.get("potential_duplicate")),
|
||
"jira_non_compliant": sum(1 for r in new_invoices if r.get("jira_checked") and not r.get("jira_compliant")),
|
||
}
|
||
|
||
output = {
|
||
"period": {"start": str(start), "end": str(end)},
|
||
"ksef_env": env,
|
||
"summary": summary,
|
||
"invoices": invoice_records,
|
||
"known_invoices_count": known_count,
|
||
}
|
||
|
||
out_path = output_dir / "invoices_data.json"
|
||
out_path.write_text(json.dumps(output, indent=2, ensure_ascii=False), encoding="utf-8")
|
||
print(f"\nWrote {out_path}")
|
||
|
||
print(f"\nSummary: {summary['new_invoices']} new invoice(s)")
|
||
if summary["whitelist_failed"]:
|
||
print(f" ✗ WHITE LIST FAILURES: {summary['whitelist_failed']} — review before payment!")
|
||
if summary["unknown_contractors"]:
|
||
print(f" ✦ Unknown contractors: {summary['unknown_contractors']}")
|
||
if summary["account_mismatch"]:
|
||
print(f" ⚠ Account mismatch: {summary['account_mismatch']}")
|
||
if summary["rule_violations"]:
|
||
print(f" ⚠ Rule violations: {summary['rule_violations']}")
|
||
if summary["jira_non_compliant"]:
|
||
print(f" ✗ Jira non-compliant: {summary['jira_non_compliant']}")
|
||
|
||
return 0
|
||
|
||
|
||
def build_parser() -> argparse.ArgumentParser:
|
||
parser = argparse.ArgumentParser(
|
||
description="Fetch KSeF v2 invoices and run contractor/white-list checks."
|
||
)
|
||
parser.add_argument("--period", required=True, help="YYYY-MM-DD:YYYY-MM-DD")
|
||
parser.add_argument("--contractors", help="Path to contractors.md.")
|
||
parser.add_argument("--output-dir", required=True)
|
||
parser.add_argument("--nip", help="Company NIP. Defaults to KSEF_NIP env var.")
|
||
parser.add_argument("--token", help="KSeF token. Defaults to KSEF_TOKEN env var.")
|
||
parser.add_argument("--ksef-env", choices=["test", "prod"],
|
||
help="Defaults to KSEF_ENV env var or 'prod'.")
|
||
return parser
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(run(build_parser().parse_args()))
|