diff --git a/src/FetchRequest/FetchRequestCreate.tsx b/src/FetchRequest/FetchRequestCreate.tsx new file mode 100644 index 0000000..23c682f --- /dev/null +++ b/src/FetchRequest/FetchRequestCreate.tsx @@ -0,0 +1,194 @@ +import React, { useEffect, useMemo, useState } from "react"; +import { + Box, Container, Typography, Paper, Button, Alert, + CircularProgress, +} from "@mui/material"; +import { useNavigate } from "react-router-dom"; +import { useAppContext, useResource, ListCellRenderer, FormFieldRenderer, getApi, applyDisplayFormat } from "../../react-openapi"; +import type { FieldConfig } from "../../react-openapi"; + +const CREATE_FIELDS = ["account", "format", "start_date", "end_date", "source"]; + +function FetchRequestList() { + const navigate = useNavigate(); + const { list, resource } = useResource("fetch-requests"); + const { resources: allResources } = useAppContext(); + const [rows, setRows] = useState(null); + + const columns = useMemo(() => { + if (!resource) return []; + return resource.listColumns + .map((n) => resource.fields.find((f) => f.name === n)) + .filter(Boolean) as FieldConfig[]; + }, [resource]); + + useEffect(() => { + if (!resource) return; + list({ limit: 20 }).then((res) => setRows(res.items ?? [])); + }, [resource?.name]); + + if (!rows) { + return ( + + + + ); + } + + if (rows.length === 0) { + return No fetch requests found.; + } + + return ( + + {rows.map((row, i) => { + const displayFormat = resource?.displayFormat ?? ""; + return ( + navigate(`/fetch-requests/${row.id}`)} + > + + {columns.map((col) => ( + + + {col.label} + + r.name === col.fk!.resource)?.displayFormat ?? displayFormat) + : displayFormat} + /> + + ))} + + + ); + })} + + ); +} + +export default function FetchRequestCreate() { + const { resources: allResources } = useAppContext(); + const resource = useMemo(() => allResources.find((r) => r.name === "fetch-requests"), [allResources]); + const { create } = useResource("fetch-requests"); + const [formData, setFormData] = useState>({}); + const [fkOptions, setFkOptions] = useState>({}); + const [loading, setLoading] = useState(false); + const [result, setResult] = useState<{ severity: "success" | "error"; message: string } | null>(null); + + useEffect(() => { + if (!resource) return; + resource.relationships.forEach((rel) => { + const targetRes = allResources.find((r) => r.name === rel.config.resource); + if (!targetRes) return; + (async () => { + try { + const api = getApi(); + const params: Record = {}; + if (targetRes.pagination) params.limit = 0; + const res = await api.get(targetRes.path, { params }); + const items = targetRes.pagination + ? (res.data.items ?? []) + : (Array.isArray(res.data) ? res.data : []); + const opts = items.map((item: any) => ({ + value: item[targetRes.primaryKey], + label: applyDisplayFormat(item, targetRes.displayFormat), + })); + setFkOptions((prev) => ({ ...prev, [rel.fieldName]: opts })); + } catch (e) { + console.warn(`Failed to load FK options for ${rel.fieldName}:`, e); + } + })(); + }); + }, [resource]); + + const formFields = useMemo(() => { + if (!resource) return []; + return resource.orderedFields.filter((f) => CREATE_FIELDS.includes(f.name)); + }, [resource]); + + const handleChange = (fieldName: string, value: any) => { + setFormData((prev) => ({ ...prev, [fieldName]: value })); + setResult(null); + }; + + const handleSubmit = async () => { + setLoading(true); + setResult(null); + try { + const created = await create(formData); + const display = applyDisplayFormat(created, resource!.displayFormat); + setResult({ severity: "success", message: `Created: ${display}` }); + setFormData({}); + } catch (e: any) { + const detail = e?.response?.data?.detail; + const msg = Array.isArray(detail) ? detail.map((d: any) => d.msg).join("; ") : (detail ?? e?.message ?? "Unknown error"); + setResult({ severity: "error", message: `Failed: ${msg}` }); + } finally { + setLoading(false); + } + }; + + if (!resource) { + return ( + + The fetch-requests resource was not found in this spec. + + ); + } + + return ( + + + Fetch Requests + + + + + + New Fetch Request + + + + + {formFields.map((field) => ( + handleChange(field.name, val)} + fkOptions={fkOptions[field.name]} + /> + ))} + + + + + + + + {result && ( + setResult(null)}> + {result.message} + + )} + + + + Recent Fetch Requests + + + + ); +} diff --git a/src/FetchRequest/FetchRequestDetail.tsx b/src/FetchRequest/FetchRequestDetail.tsx new file mode 100644 index 0000000..c3d7015 --- /dev/null +++ b/src/FetchRequest/FetchRequestDetail.tsx @@ -0,0 +1,338 @@ +import React, { useMemo, useState, useEffect, useRef } from "react"; +import { useParams, useNavigate } from "react-router-dom"; +import { + Box, + Container, + Paper, + Typography, + Button, + Chip, + CircularProgress, + Alert, + IconButton, + Snackbar, +} from "@mui/material"; +import ArrowBackIcon from "@mui/icons-material/ArrowBack"; +import ReplayIcon from "@mui/icons-material/Replay"; +import CheckCircleIcon from "@mui/icons-material/CheckCircle"; +import ErrorIcon from "@mui/icons-material/Error"; +import WarningAmberIcon from "@mui/icons-material/WarningAmber"; +import PlayArrowIcon from "@mui/icons-material/PlayArrow"; +import RemoveCircleOutlineIcon from "@mui/icons-material/RemoveCircleOutline"; +import FiberManualRecordIcon from "@mui/icons-material/FiberManualRecord"; +import { useResource, useItemSse, DetailFieldRenderer, applyDisplayFormat } from "../../react-openapi"; +import { useQuery, useMutation } from "@tanstack/react-query"; +import { RETRY_MAX, formatApiError } from "../features/fetch-requests"; +import type { FetchRequestStatus, SSEEvent, ProgressMessage } from "../features/fetch-requests"; +import { PipelineStepper } from "./components/PipelineStepper"; +import { AmbiguityResolver } from "./components/AmbiguityResolver"; + +const statusColors: Record = { + pending: "default", + processing: "info", + paused: "warning", + raw_expenses_done: "primary", + enriched_done: "warning", + completed: "success", + failed: "error", +}; + +const statusIcons: Record = { + pending: , + processing: , + paused: , + raw_expenses_done: , + enriched_done: , + completed: , + failed: , +}; + +function formatProgressMessage(msg: ProgressMessage): string { + if (msg.lines !== undefined) return `${msg.lines} lines`; + if (msg.blocks !== undefined) return `${msg.blocks} blocks`; + if (msg.count !== undefined && msg.unit) return `${msg.count} ${msg.unit}`; + if (msg.count !== undefined) return `${msg.count} items`; + if (msg.raw_ocr_line) return `"${msg.raw_ocr_line.slice(0, 60)}${msg.raw_ocr_line.length > 60 ? "…" : ""}"`; + if (msg.error) return msg.error.slice(0, 80); + return ""; +} + +function sseIcon(status: SSEEvent["status"]) { + switch (status) { + case "started": return ; + case "completed": return ; + case "failed": return ; + case "skipped": return ; + case "paused": return ; + case "progress": return ; + } +} + +export default function FetchRequestDetail() { + const { id } = useParams<{ id: string }>(); + const navigate = useNavigate(); + const { get, update, resource } = useResource("fetch-requests"); + const [stepStats, setStepStats] = useState>({}); + const [liveParsedCount, setLiveParsedCount] = useState(undefined); + const [failNotif, setFailNotif] = useState(null); + const feedRef = useRef(null); + + const { data: fetchRequest, isLoading, error: fetchError, refetch: refetchRequest } = useQuery({ + queryKey: ["fetch-requests", "detail", id], + queryFn: () => get(id!), + enabled: !!id, + }); + + const updateMutation = useMutation({ + mutationFn: ({ id: rid, data }: { id: string; data: any }) => update(rid, data), + }); + + const sseUrl = id ? `/fetch-requests/${id}/events` : null; + const { connected: sseConnected, events: sseEvents } = useItemSse(sseUrl, { + onEvent: (parsed: SSEEvent) => { + if (parsed.status === "progress" && parsed.message.count !== undefined) { + if (parsed.step === "txn_dicts") setLiveParsedCount(parsed.message.count); + if (parsed.step === "enrich") setStepStats((prev) => ({ ...prev, enrich_count: parsed.message.count! })); + if (parsed.step === "save_expenses") setStepStats((prev) => ({ ...prev, save_count: parsed.message.count! })); + } + + if (parsed.status === "completed" && parsed.message.count !== undefined) { + const stats: Record = {}; + if (parsed.step === "raw_lines" && parsed.message.lines !== undefined) stats.raw_lines = parsed.message.lines; + if (parsed.step === "txn_blocks" && parsed.message.blocks !== undefined) stats.txn_blocks = parsed.message.blocks; + if (parsed.step === "txn_dicts") stats.txn_dicts = parsed.message.count; + if (parsed.step === "enrich") stats.enrich_count = parsed.message.count; + if (parsed.step === "save_expenses") stats.save_count = parsed.message.count; + if (Object.keys(stats).length) { + setStepStats((prev) => ({ ...prev, ...stats })); + } + } + + if (parsed.status === "paused") { + refetchRequest(); + } + if (parsed.status === "failed") { + setFailNotif(parsed.message.error || "Fetch request failed"); + refetchRequest(); + } + if (parsed.status === "completed" || parsed.step === "resume_extract") { + refetchRequest(); + } + }, + }); + + useEffect(() => { + if (feedRef.current) { + feedRef.current.scrollTop = feedRef.current.scrollHeight; + } + }, [sseEvents]); + + const displayEvents = useMemo(() => { + const progressSteps = new Set(["txn_dicts", "enrich", "save_expenses"]); + const lastProgressIdx: Record = {}; + for (let i = sseEvents.length - 1; i >= 0; i--) { + const e = sseEvents[i]; + if (progressSteps.has(e.step) && e.status === "progress" && lastProgressIdx[e.step] === undefined) { + lastProgressIdx[e.step] = i; + } + } + + const terminalStatuses = new Set(["completed", "skipped", "paused", "failed"]); + return sseEvents.filter((e, i) => { + if (progressSteps.has(e.step) && e.status === "progress") return i === lastProgressIdx[e.step]; + if (e.status === "started") { + return !sseEvents.slice(i + 1).some( + (later) => later.step === e.step && terminalStatuses.has(later.status), + ); + } + return true; + }); + }, [sseEvents]); + + const handleRetry = async () => { + if (!id) return; + try { + await updateMutation.mutateAsync({ id, data: { status: "pending" } }); + refetchRequest(); + } catch (err: any) { + setFailNotif(formatApiError(err)); + } + }; + + if (isLoading) { + return ( + + + + ); + } + + if (fetchError || !fetchRequest) { + return ( + + + Failed to load fetch request + + ); + } + + const req = fetchRequest as any; + const retryCount = req.retry_count ?? 0; + const isRetryExhausted = retryCount >= RETRY_MAX; + const status = req.status as FetchRequestStatus; + + const detailFields = (resource?.orderedFields ?? []).filter( + (f) => f.name !== "source", + ); + + return ( + + + + + + + {req.account_name} + + + + + {detailFields.map((field) => ( + + ))} + + + + + + Retries: {retryCount}/{RETRY_MAX} + + + {status === "failed" && !isRetryExhausted && ( + + )} + + + + {status === "failed" && req.error_message && ( + + {req.error_message} + + )} + {isRetryExhausted && status === "failed" && ( + + Max retries reached — no further retry attempts will be made. + + )} + + + + + + + Progress Events + + + + {sseConnected ? "Connected" : "Disconnected"} + + + + {displayEvents.length === 0 ? ( + + Waiting for events... + + ) : ( + displayEvents.map((evt, i) => ( + + {sseIcon(evt.status)} + + + {evt.step.replace(/_/g, " ")} + + {evt.message && formatProgressMessage(evt.message) && ( + + {formatProgressMessage(evt.message)} + + )} + + + )) + )} + + + + + + setFailNotif(null)} + anchorOrigin={{ vertical: "bottom", horizontal: "center" }} + > + setFailNotif(null)} sx={{ borderRadius: 2 }}> + {failNotif} + + + + ); +} diff --git a/src/FetchRequest/components/AmbiguityResolver.tsx b/src/FetchRequest/components/AmbiguityResolver.tsx new file mode 100644 index 0000000..c224d40 --- /dev/null +++ b/src/FetchRequest/components/AmbiguityResolver.tsx @@ -0,0 +1,121 @@ +import React from "react"; +import { + Box, + Paper, + Typography, + Button, + Alert, +} from "@mui/material"; +import CheckCircleIcon from "@mui/icons-material/CheckCircle"; +import { useFetchRequestAmbiguities, useResolveAmbiguity } from "../../features/fetch-requests"; + +interface AmbiguityResolverProps { + fetchRequestId: string; +} + +export function AmbiguityResolver({ fetchRequestId }: AmbiguityResolverProps) { + const { data: ambiguities, refetch } = useFetchRequestAmbiguities(fetchRequestId); + const resolveMutation = useResolveAmbiguity(); + + const handleResolve = async (ambiguity: any, candidate: { amount: number; balance: number }) => { + await resolveMutation.mutateAsync({ + ambiguityId: ambiguity.id, + payload: { chosen: { amount: candidate.amount, balance: candidate.balance } }, + }); + refetch(); + }; + + if (!ambiguities || ambiguities.length === 0) return null; + + const allResolved = ambiguities.every((a: any) => a.status === "resolved"); + + return ( + + + Ambiguity Resolution + + + {allResolved ? ( + + All ambiguities resolved — pipeline will resume on next poll cycle + + ) : ( + + Pipeline paused — resolve ambiguities to continue + + )} + + + {ambiguities.map((ambiguity: any) => { + const isResolved = ambiguity.status === "resolved"; + return ( + + + {ambiguity.line} + + + + + OCR Amount + + ₹{ambiguity.ocr_amount} + + + + OCR Balance + + ₹{ambiguity.ocr_balance} + + + + Previous Balance + ₹{ambiguity.prev_balance} + + + + {isResolved ? ( + }> + Resolved: ₹{ambiguity.chosen?.amount} / ₹{ambiguity.chosen?.balance} + + ) : ( + + {ambiguity.candidates.map((candidate: any, ci: number) => { + const isCredit = candidate.amount > 0; + const isDebit = candidate.amount < 0; + const cColor = isCredit ? "success.main" : isDebit ? "error.main" : undefined; + return ( + + ); + })} + + )} + + ); + })} + + + ); +} diff --git a/src/FetchRequest/components/PipelineStepper.tsx b/src/FetchRequest/components/PipelineStepper.tsx new file mode 100644 index 0000000..fce4931 --- /dev/null +++ b/src/FetchRequest/components/PipelineStepper.tsx @@ -0,0 +1,209 @@ +import React, { useMemo } from "react"; +import { + Box, + Paper, + Typography, + Stepper, + Step, + StepLabel, + LinearProgress, +} from "@mui/material"; +import CheckCircleIcon from "@mui/icons-material/CheckCircle"; +import ErrorIcon from "@mui/icons-material/Error"; +import WarningAmberIcon from "@mui/icons-material/WarningAmber"; +import CircularProgress from "@mui/material/CircularProgress"; +import type { FetchRequestStatus, SSEEvent, ProgressMessage } from "../../features/fetch-requests"; + +const STEP_LABELS = ["Extract", "Raw Expense", "Enrich", "Save"]; + +function computeProgressPercent( + status: FetchRequestStatus, + liveCount: number, + seenSteps: Set, + stepStats: Record, + txnBlockCount: number, + txnDictCount: number, +): number { + if (status === "pending") return 0; + if (status === "completed") return 100; + + let pct = 0; + + if (seenSteps.has("raw_lines") || seenSteps.has("txn_blocks")) pct += 10; + + if (txnBlockCount > 0) { + const current = Math.max(liveCount, stepStats.txn_dicts ?? 0); + pct += Math.min(1, current / txnBlockCount) * 20; + } + + if (txnDictCount > 0) { + pct += Math.min(1, (stepStats.enrich_count ?? 0) / txnDictCount) * 50; + pct += Math.min(1, (stepStats.save_count ?? 0) / txnDictCount) * 20; + } + + return Math.round(Math.min(100, pct)); +} + +function computeActiveStep(status: FetchRequestStatus, seenSteps: Set): number { + if (status === "completed") return STEP_LABELS.length; + + if (seenSteps.has("save_expenses/completed") || seenSteps.has("complete/completed")) return STEP_LABELS.length; + if (seenSteps.has("save_expenses") || seenSteps.has("complete")) return 3; + + if (seenSteps.has("enrich/completed")) return 3; + if (seenSteps.has("enrich")) return 2; + + if (seenSteps.has("txn_dicts/completed") || status === "raw_expenses_done") return 2; + if (seenSteps.has("txn_dicts")) return 1; + + if (seenSteps.has("txn_blocks/completed")) return 1; + if (seenSteps.has("raw_lines") || seenSteps.has("txn_blocks")) return 0; + + if (status === "processing" || status === "paused") return 0; + + return -1; +} + +function computeStepMessages( + fetchRequest: any, + stepStats: Record, + liveParsedCount: number, + txnBlockCount: number, +): Record { + const msgs: Record = {}; + const source = fetchRequest?.source; + + const rawLineCount = stepStats.raw_lines ?? (source?.raw_lines?.length ?? 0); + if (rawLineCount) msgs[0] = `${rawLineCount}`; + + const sourceDictCount = source?.txn_dict_count ?? source?.txn_dicts_count ?? 0; + const dictLive = liveParsedCount ?? stepStats.txn_dicts ?? 0; + const dictCurrent = Math.max(dictLive, sourceDictCount); + if (dictCurrent && txnBlockCount) msgs[1] = `${dictCurrent}/${txnBlockCount}`; + else if (dictCurrent) msgs[1] = `${dictCurrent}`; + + const txnDictDenom = stepStats.txn_dicts ?? sourceDictCount; + if (stepStats.enrich_count && txnDictDenom) msgs[2] = `${stepStats.enrich_count}/${txnDictDenom}`; + else if (stepStats.enrich_count) msgs[2] = `${stepStats.enrich_count}`; + + if (stepStats.save_count && txnDictDenom) msgs[3] = `${stepStats.save_count}/${txnDictDenom}`; + else if (stepStats.save_count) msgs[3] = `${stepStats.save_count}`; + + return msgs; +} + +interface PipelineStepperProps { + fetchRequest: any; + sseEvents: SSEEvent[]; + stepStats: Record; + liveParsedCount: number; +} + +export function PipelineStepper({ fetchRequest, sseEvents, stepStats, liveParsedCount }: PipelineStepperProps) { + const status = (fetchRequest?.status ?? "pending") as FetchRequestStatus; + + const seenSteps = useMemo(() => { + const steps = new Set(); + for (const evt of sseEvents) { + steps.add(evt.step); + if (evt.status === "completed") steps.add(`${evt.step}/completed`); + if (evt.status === "failed") steps.add(`${evt.step}/failed`); + if (evt.status === "started") steps.add(`${evt.step}/started`); + if (evt.status === "progress") steps.add(`${evt.step}/progress`); + } + return steps; + }, [sseEvents]); + + const txnBlockCount = useMemo(() => { + const blocks = fetchRequest?.source?.txn_blocks; + if (!blocks) return 0; + return Object.values(blocks).reduce( + (sum: number, list: any) => sum + (Array.isArray(list) ? list.length : 0), + 0, + ); + }, [fetchRequest]); + + const txnDictCount = useMemo(() => { + const source = fetchRequest?.source; + if (stepStats.txn_dicts && stepStats.txn_dicts > 0) return stepStats.txn_dicts; + return source?.txn_dict_count ?? source?.txn_dicts_count ?? 0; + }, [fetchRequest, stepStats]); + + const displayParsedCount = useMemo(() => { + if (liveParsedCount && liveParsedCount > 0) return liveParsedCount; + const source = fetchRequest?.source; + const persistedCount = source?.txn_dict_count ?? source?.txn_dicts_count ?? 0; + if (persistedCount > 0) return persistedCount; + const dicts = source?.txn_dicts; + if (Array.isArray(dicts) && dicts.length > 0) return dicts.length; + return 0; + }, [liveParsedCount, fetchRequest]); + + const activeStep = computeActiveStep(status, seenSteps); + const progressPercent = computeProgressPercent(status, displayParsedCount, seenSteps, stepStats, txnBlockCount, txnDictCount); + const stepMessages = computeStepMessages(fetchRequest, stepStats, liveParsedCount, txnBlockCount); + + return ( + + + Pipeline Progress + + + {STEP_LABELS.map((label, index) => { + const isCompleted = index < activeStep; + const isActive = index === activeStep; + const isPaused = status === "paused" && isActive; + const isFailed = status === "failed" && isActive; + + let icon: React.ReactNode; + if (isCompleted) { + icon = ; + } else if (isFailed) { + icon = ; + } else if (isPaused) { + icon = ; + } else if (isActive) { + icon = ; + } else { + icon = {index + 1}; + } + + return ( + + {icon}} + > + {label} + {stepMessages[index] && ( + + {stepMessages[index]} + + )} + + + ); + })} + + + + + Overall Progress + {["processing", "paused"].includes(status) && displayParsedCount > 0 && ( + + Validated: {displayParsedCount} transactions + + )} + + + + {progressPercent}% + + + + ); +} diff --git a/src/main.jsx b/src/main.jsx index bb1a06d..ab91bf1 100644 --- a/src/main.jsx +++ b/src/main.jsx @@ -12,8 +12,8 @@ import { Toolbar } from "@mui/material"; import Home from './Home'; -import FetchRequests from './FetchRequests'; -import FetchRequestDetail from './FetchRequestDetail'; +import FetchRequests from './FetchRequest/FetchRequestCreate'; +import FetchRequestDetail from './FetchRequest/FetchRequestDetail'; import { RequireAuth } from './RequireAuth'; import { AppProvider, Admin } from '../react-openapi'; import { Buffer } from 'buffer';