Read VAT and copy counts from court fee statements
3a8d893feat(probate)merged 1d ago4 review threads
Illustrative: based on a real change, edited for clarity and privacy.
src/app/services/fee_statement_parser.py+31 −12
| @@ | |||
| 10 | 10 | ||
| 11 | 11 | _REFERENCE_RE = re.compile(r"(\d{6}\.\d{3})") | |
| 12 | added | # Far above any real fee or copy count. | |
| 2 line 13, review thread 2 | added | _MAX_DIGITS = 12 | |
| 12 | 14 | ||
| 13 | 15 | # HMCTS month abbreviations, so parsing does not depend on locale. | |
| 14 | 16 | # September is SEPT, not the three-letter SEP. | |
| @@ def _parse_line(el: ET.Element) -> dict[str, Any]: | |||
| 95 | 97 | memo_name = _text(el, "MemoName") | |
| 96 | 98 | fee_type, fee_label = _classify_fee(memo_name) | |
| 97 | 99 | ||
| 98 | removed | line: dict[str, Any] = { | |
| 100 | added | return { | |
| 99 | 101 | "date": _parse_date(_text(el, "Date")), | |
| 100 | 102 | "amount": _format_money(_text(el, "Amount")), | |
| 103 | added | "vat": _format_money(_text(el, "VatAmount")), | |
| 101 | 104 | "reference": _extract_reference(description), | |
| 102 | 105 | "fee_type": fee_type, | |
| 1 line 106, review thread 1 | added | "fee_label": fee_label, | |
| 107 | added | "copies": ( | |
| 108 | added | _parse_quantity(_text(el, "QuantityInvoiced")) | |
| 109 | added | if fee_type == "copies" | |
| 110 | added | else None | |
| 111 | added | ), | |
| 103 | 112 | } | |
| 104 | removed | if fee_type is None: | |
| 105 | removed | line["fee_label"] = fee_label | |
| 106 | removed | return line | |
| 107 | 113 | ||
| 108 | 114 | ||
| 109 | 115 | def _extract_reference(description: str | None) -> str | None: | |
| @@ def _extract_reference(description: str | None) -> str | None: | |||
| 121 | 127 | def _classify_fee(memo_name: str | None) -> tuple[str | None, str | None]: | |
| 122 | 128 | if memo_name is None: | |
| 123 | 129 | return None, None | |
| 124 | removed | if ( | |
| 125 | removed | "issue probate" in memo_name | |
| 126 | removed | or "CC-Application for a grant" in memo_name | |
| 127 | removed | ): | |
| 130 | added | lowered = memo_name.lower() | |
| 131 | added | if "issue probate" in lowered or "cc-application for a grant" in lowered: | |
| 128 | 132 | return "court_fee", None | |
| 129 | removed | if "copies probate" in memo_name: | |
| 133 | added | if "copies probate" in lowered: | |
| 130 | 134 | return "copies", None | |
| 131 | 135 | return None, memo_name | |
| 132 | 136 | ||
| @@ def _date_from_parts(day_s: str, month_s: str, year_s: str) -> str | None: | |||
| 163 | 167 | return parsed.date().isoformat() | |
| 164 | 168 | ||
| 165 | 169 | ||
| 170 | added | def _parse_quantity(value: str | None) -> int | None: | |
| 171 | added | quantity = _parse_number(value) | |
| 3 line 172, review thread 3 | added | if quantity is None or quantity != quantity.to_integral_value(): | |
| 173 | added | return None | |
| 174 | added | return int(quantity) | |
| 175 | added | ||
| 176 | added | ||
| 166 | 177 | def _format_money(value: str | None) -> str | None: | |
| 178 | added | amount = _parse_number(value) | |
| 179 | added | return f"{amount:.2f}" if amount is not None else None | |
| 180 | added | ||
| 181 | added | ||
| 182 | added | def _parse_number(value: str | None) -> Decimal | None: | |
| 183 | added | """Reject huge exponents like 1e999999999 before they become huge.""" | |
| 167 | 184 | if value is None: | |
| 168 | 185 | return None | |
| 169 | removed | cleaned = value.replace(",", "").strip() | |
| 170 | 186 | try: | |
| 171 | removed | return f"{Decimal(cleaned):.2f}" | |
| 172 | removed | except (InvalidOperation, ValueError): | |
| 187 | added | number = Decimal(value.replace(",", "").strip()) | |
| 188 | added | except InvalidOperation: | |
| 189 | added | return None | |
| 4 line 190, review thread 4 | added | if not number.is_finite() or number.adjusted() > _MAX_DIGITS: | |
| 173 | 191 | return None | |
| 192 | added | return number | |
probate team
This saves us retyping copy counts from every statement. Could the fee label show on the case as well, so we can tell an application fee from extra copies at a glance?
author
The label comes through in this change. Showing it on the case is a separate change, so this one stays small. It’s next on the list.
reviewer
Is 12 digits enough headroom? The largest fee I can think of is nowhere near, but a test either side of the boundary would make the limit obvious to the next person.
author
Added tests at 12 and 13 digits. The comment on the constant says why it’s 12, too.
reviewer
What happens if a statement says
4.00copies? Looks like that comes back asNone, which reads as “no count” rather than four.author
The statements we’ve seen never do this, so I’m happy to assume whole numbers.
reviewer
Fair enough if that’s all we’ve seen. Worth a test pinning the behaviour either way, so it’s a decision rather than an accident.
author
Changed our mind on this one: whole decimals like
4.00now parse, and fractional ones returnNone, with tests for both.
AI reviewP1
Huge counts stall the worker
Decimalhappily accepts a compact value like1e999999999. It passes the checks, thenint(quantity)tries to build an enormous integer. Anyone who can upload a statement can tie up the extraction worker. Bound the size before converting.author
Good catch. Money and quantities now share
_parse_number, which rejects anything with more than 12 digits before the decimal point._format_moneyhad the same problem before this change (formatting1e999999999builds a string of about 1 GB), so that’s covered too, with tests.