new FetchRequest
This commit is contained in:
194
src/FetchRequest/FetchRequestCreate.tsx
Normal file
194
src/FetchRequest/FetchRequestCreate.tsx
Normal file
@@ -0,0 +1,194 @@
|
|||||||
|
import React, { useEffect, useMemo, useState } from "react";
|
||||||
|
import {
|
||||||
|
Box, Container, Typography, Paper, Button, Alert,
|
||||||
|
CircularProgress,
|
||||||
|
} from "@mui/material";
|
||||||
|
import { useNavigate } from "react-router-dom";
|
||||||
|
import { useAppContext, useResource, ListCellRenderer, FormFieldRenderer, getApi, applyDisplayFormat } from "../../react-openapi";
|
||||||
|
import type { FieldConfig } from "../../react-openapi";
|
||||||
|
|
||||||
|
const CREATE_FIELDS = ["account", "format", "start_date", "end_date", "source"];
|
||||||
|
|
||||||
|
function FetchRequestList() {
|
||||||
|
const navigate = useNavigate();
|
||||||
|
const { list, resource } = useResource("fetch-requests");
|
||||||
|
const { resources: allResources } = useAppContext();
|
||||||
|
const [rows, setRows] = useState<any[] | null>(null);
|
||||||
|
|
||||||
|
const columns = useMemo(() => {
|
||||||
|
if (!resource) return [];
|
||||||
|
return resource.listColumns
|
||||||
|
.map((n) => resource.fields.find((f) => f.name === n))
|
||||||
|
.filter(Boolean) as FieldConfig[];
|
||||||
|
}, [resource]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!resource) return;
|
||||||
|
list({ limit: 20 }).then((res) => setRows(res.items ?? []));
|
||||||
|
}, [resource?.name]);
|
||||||
|
|
||||||
|
if (!rows) {
|
||||||
|
return (
|
||||||
|
<Box sx={{ display: "flex", justifyContent: "center", py: 4 }}>
|
||||||
|
<CircularProgress size={24} />
|
||||||
|
</Box>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (rows.length === 0) {
|
||||||
|
return <Typography variant="body2" color="text.secondary">No fetch requests found.</Typography>;
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Box sx={{ display: "flex", flexDirection: "column", gap: 1 }}>
|
||||||
|
{rows.map((row, i) => {
|
||||||
|
const displayFormat = resource?.displayFormat ?? "";
|
||||||
|
return (
|
||||||
|
<Paper
|
||||||
|
key={row.id ?? i}
|
||||||
|
variant="outlined"
|
||||||
|
sx={{ p: 2, cursor: "pointer", "&:hover": { borderColor: "primary.main" } }}
|
||||||
|
onClick={() => navigate(`/fetch-requests/${row.id}`)}
|
||||||
|
>
|
||||||
|
<Box sx={{ display: "flex", gap: 2, flexWrap: "wrap", alignItems: "center" }}>
|
||||||
|
{columns.map((col) => (
|
||||||
|
<Box key={col.name} sx={{ minWidth: 120 }}>
|
||||||
|
<Typography variant="caption" color="text.secondary" sx={{ display: "block" }}>
|
||||||
|
{col.label}
|
||||||
|
</Typography>
|
||||||
|
<ListCellRenderer
|
||||||
|
field={col}
|
||||||
|
value={row[col.name]}
|
||||||
|
displayFormat={col.fk
|
||||||
|
? (allResources.find((r) => r.name === col.fk!.resource)?.displayFormat ?? displayFormat)
|
||||||
|
: displayFormat}
|
||||||
|
/>
|
||||||
|
</Box>
|
||||||
|
))}
|
||||||
|
</Box>
|
||||||
|
</Paper>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</Box>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export default function FetchRequestCreate() {
|
||||||
|
const { resources: allResources } = useAppContext();
|
||||||
|
const resource = useMemo(() => allResources.find((r) => r.name === "fetch-requests"), [allResources]);
|
||||||
|
const { create } = useResource("fetch-requests");
|
||||||
|
const [formData, setFormData] = useState<Record<string, any>>({});
|
||||||
|
const [fkOptions, setFkOptions] = useState<Record<string, { value: any; label: string }[]>>({});
|
||||||
|
const [loading, setLoading] = useState(false);
|
||||||
|
const [result, setResult] = useState<{ severity: "success" | "error"; message: string } | null>(null);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!resource) return;
|
||||||
|
resource.relationships.forEach((rel) => {
|
||||||
|
const targetRes = allResources.find((r) => r.name === rel.config.resource);
|
||||||
|
if (!targetRes) return;
|
||||||
|
(async () => {
|
||||||
|
try {
|
||||||
|
const api = getApi();
|
||||||
|
const params: Record<string, any> = {};
|
||||||
|
if (targetRes.pagination) params.limit = 0;
|
||||||
|
const res = await api.get(targetRes.path, { params });
|
||||||
|
const items = targetRes.pagination
|
||||||
|
? (res.data.items ?? [])
|
||||||
|
: (Array.isArray(res.data) ? res.data : []);
|
||||||
|
const opts = items.map((item: any) => ({
|
||||||
|
value: item[targetRes.primaryKey],
|
||||||
|
label: applyDisplayFormat(item, targetRes.displayFormat),
|
||||||
|
}));
|
||||||
|
setFkOptions((prev) => ({ ...prev, [rel.fieldName]: opts }));
|
||||||
|
} catch (e) {
|
||||||
|
console.warn(`Failed to load FK options for ${rel.fieldName}:`, e);
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
});
|
||||||
|
}, [resource]);
|
||||||
|
|
||||||
|
const formFields = useMemo(() => {
|
||||||
|
if (!resource) return [];
|
||||||
|
return resource.orderedFields.filter((f) => CREATE_FIELDS.includes(f.name));
|
||||||
|
}, [resource]);
|
||||||
|
|
||||||
|
const handleChange = (fieldName: string, value: any) => {
|
||||||
|
setFormData((prev) => ({ ...prev, [fieldName]: value }));
|
||||||
|
setResult(null);
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleSubmit = async () => {
|
||||||
|
setLoading(true);
|
||||||
|
setResult(null);
|
||||||
|
try {
|
||||||
|
const created = await create(formData);
|
||||||
|
const display = applyDisplayFormat(created, resource!.displayFormat);
|
||||||
|
setResult({ severity: "success", message: `Created: ${display}` });
|
||||||
|
setFormData({});
|
||||||
|
} catch (e: any) {
|
||||||
|
const detail = e?.response?.data?.detail;
|
||||||
|
const msg = Array.isArray(detail) ? detail.map((d: any) => d.msg).join("; ") : (detail ?? e?.message ?? "Unknown error");
|
||||||
|
setResult({ severity: "error", message: `Failed: ${msg}` });
|
||||||
|
} finally {
|
||||||
|
setLoading(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if (!resource) {
|
||||||
|
return (
|
||||||
|
<Alert severity="info">
|
||||||
|
The <strong>fetch-requests</strong> resource was not found in this spec.
|
||||||
|
</Alert>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Container maxWidth="lg" sx={{ py: 4 }}>
|
||||||
|
<Typography variant="h5" fontWeight={800} gutterBottom>
|
||||||
|
Fetch Requests
|
||||||
|
</Typography>
|
||||||
|
|
||||||
|
<Paper variant="outlined" sx={{ p: 3, mb: 4, borderRadius: 2, position: "relative", overflow: "hidden", "&::before": { content: '""', position: "absolute", left: 0, top: 0, bottom: 0, width: 4, bgcolor: "primary.main" } }}>
|
||||||
|
<Box sx={{ ml: 0.5, mb: 2.5 }}>
|
||||||
|
<Typography variant="subtitle1" fontWeight={700}>
|
||||||
|
New Fetch Request
|
||||||
|
</Typography>
|
||||||
|
</Box>
|
||||||
|
|
||||||
|
<Box sx={{ display: "flex", flexDirection: "column", gap: 2, maxWidth: 480 }}>
|
||||||
|
{formFields.map((field) => (
|
||||||
|
<FormFieldRenderer
|
||||||
|
key={field.name}
|
||||||
|
field={field}
|
||||||
|
value={formData[field.name] ?? ""}
|
||||||
|
onChange={(val) => handleChange(field.name, val)}
|
||||||
|
fkOptions={fkOptions[field.name]}
|
||||||
|
/>
|
||||||
|
))}
|
||||||
|
</Box>
|
||||||
|
|
||||||
|
<Box sx={{ display: "flex", justifyContent: "flex-end", mt: 2, gap: 1 }}>
|
||||||
|
<Button variant="outlined" onClick={() => { setFormData({}); setResult(null); }}>
|
||||||
|
Reset
|
||||||
|
</Button>
|
||||||
|
<Button variant="contained" onClick={handleSubmit} disabled={loading}>
|
||||||
|
{loading ? <CircularProgress size={20} sx={{ mr: 0.5 }} /> : null}
|
||||||
|
Create Fetch Request
|
||||||
|
</Button>
|
||||||
|
</Box>
|
||||||
|
|
||||||
|
{result && (
|
||||||
|
<Alert severity={result.severity} sx={{ mt: 1 }} onClose={() => setResult(null)}>
|
||||||
|
{result.message}
|
||||||
|
</Alert>
|
||||||
|
)}
|
||||||
|
</Paper>
|
||||||
|
|
||||||
|
<Typography variant="h6" fontWeight={700} sx={{ mb: 2 }}>
|
||||||
|
Recent Fetch Requests
|
||||||
|
</Typography>
|
||||||
|
<FetchRequestList />
|
||||||
|
</Container>
|
||||||
|
);
|
||||||
|
}
|
||||||
338
src/FetchRequest/FetchRequestDetail.tsx
Normal file
338
src/FetchRequest/FetchRequestDetail.tsx
Normal file
@@ -0,0 +1,338 @@
|
|||||||
|
import React, { useMemo, useState, useEffect, useRef } from "react";
|
||||||
|
import { useParams, useNavigate } from "react-router-dom";
|
||||||
|
import {
|
||||||
|
Box,
|
||||||
|
Container,
|
||||||
|
Paper,
|
||||||
|
Typography,
|
||||||
|
Button,
|
||||||
|
Chip,
|
||||||
|
CircularProgress,
|
||||||
|
Alert,
|
||||||
|
IconButton,
|
||||||
|
Snackbar,
|
||||||
|
} from "@mui/material";
|
||||||
|
import ArrowBackIcon from "@mui/icons-material/ArrowBack";
|
||||||
|
import ReplayIcon from "@mui/icons-material/Replay";
|
||||||
|
import CheckCircleIcon from "@mui/icons-material/CheckCircle";
|
||||||
|
import ErrorIcon from "@mui/icons-material/Error";
|
||||||
|
import WarningAmberIcon from "@mui/icons-material/WarningAmber";
|
||||||
|
import PlayArrowIcon from "@mui/icons-material/PlayArrow";
|
||||||
|
import RemoveCircleOutlineIcon from "@mui/icons-material/RemoveCircleOutline";
|
||||||
|
import FiberManualRecordIcon from "@mui/icons-material/FiberManualRecord";
|
||||||
|
import { useResource, useItemSse, DetailFieldRenderer, applyDisplayFormat } from "../../react-openapi";
|
||||||
|
import { useQuery, useMutation } from "@tanstack/react-query";
|
||||||
|
import { RETRY_MAX, formatApiError } from "../features/fetch-requests";
|
||||||
|
import type { FetchRequestStatus, SSEEvent, ProgressMessage } from "../features/fetch-requests";
|
||||||
|
import { PipelineStepper } from "./components/PipelineStepper";
|
||||||
|
import { AmbiguityResolver } from "./components/AmbiguityResolver";
|
||||||
|
|
||||||
|
const statusColors: Record<FetchRequestStatus, "default" | "primary" | "warning" | "info" | "success" | "error"> = {
|
||||||
|
pending: "default",
|
||||||
|
processing: "info",
|
||||||
|
paused: "warning",
|
||||||
|
raw_expenses_done: "primary",
|
||||||
|
enriched_done: "warning",
|
||||||
|
completed: "success",
|
||||||
|
failed: "error",
|
||||||
|
};
|
||||||
|
|
||||||
|
const statusIcons: Record<FetchRequestStatus, React.ReactNode> = {
|
||||||
|
pending: <PlayArrowIcon sx={{ fontSize: 16 }} />,
|
||||||
|
processing: <CircularProgress size={14} />,
|
||||||
|
paused: <WarningAmberIcon sx={{ fontSize: 16 }} />,
|
||||||
|
raw_expenses_done: <CheckCircleIcon sx={{ fontSize: 16 }} />,
|
||||||
|
enriched_done: <CheckCircleIcon sx={{ fontSize: 16 }} />,
|
||||||
|
completed: <CheckCircleIcon sx={{ fontSize: 16 }} />,
|
||||||
|
failed: <ErrorIcon sx={{ fontSize: 16 }} />,
|
||||||
|
};
|
||||||
|
|
||||||
|
function formatProgressMessage(msg: ProgressMessage): string {
|
||||||
|
if (msg.lines !== undefined) return `${msg.lines} lines`;
|
||||||
|
if (msg.blocks !== undefined) return `${msg.blocks} blocks`;
|
||||||
|
if (msg.count !== undefined && msg.unit) return `${msg.count} ${msg.unit}`;
|
||||||
|
if (msg.count !== undefined) return `${msg.count} items`;
|
||||||
|
if (msg.raw_ocr_line) return `"${msg.raw_ocr_line.slice(0, 60)}${msg.raw_ocr_line.length > 60 ? "…" : ""}"`;
|
||||||
|
if (msg.error) return msg.error.slice(0, 80);
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
function sseIcon(status: SSEEvent["status"]) {
|
||||||
|
switch (status) {
|
||||||
|
case "started": return <CircularProgress size={14} />;
|
||||||
|
case "completed": return <CheckCircleIcon sx={{ fontSize: 16, color: "success.main" }} />;
|
||||||
|
case "failed": return <ErrorIcon sx={{ fontSize: 16, color: "error.main" }} />;
|
||||||
|
case "skipped": return <RemoveCircleOutlineIcon sx={{ fontSize: 16, color: "text.disabled" }} />;
|
||||||
|
case "paused": return <WarningAmberIcon sx={{ fontSize: 16, color: "warning.main" }} />;
|
||||||
|
case "progress": return <FiberManualRecordIcon sx={{ fontSize: 14, color: "info.main" }} />;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export default function FetchRequestDetail() {
|
||||||
|
const { id } = useParams<{ id: string }>();
|
||||||
|
const navigate = useNavigate();
|
||||||
|
const { get, update, resource } = useResource("fetch-requests");
|
||||||
|
const [stepStats, setStepStats] = useState<Record<string, number>>({});
|
||||||
|
const [liveParsedCount, setLiveParsedCount] = useState<number | undefined>(undefined);
|
||||||
|
const [failNotif, setFailNotif] = useState<string | null>(null);
|
||||||
|
const feedRef = useRef<HTMLDivElement>(null);
|
||||||
|
|
||||||
|
const { data: fetchRequest, isLoading, error: fetchError, refetch: refetchRequest } = useQuery({
|
||||||
|
queryKey: ["fetch-requests", "detail", id],
|
||||||
|
queryFn: () => get(id!),
|
||||||
|
enabled: !!id,
|
||||||
|
});
|
||||||
|
|
||||||
|
const updateMutation = useMutation({
|
||||||
|
mutationFn: ({ id: rid, data }: { id: string; data: any }) => update(rid, data),
|
||||||
|
});
|
||||||
|
|
||||||
|
const sseUrl = id ? `/fetch-requests/${id}/events` : null;
|
||||||
|
const { connected: sseConnected, events: sseEvents } = useItemSse(sseUrl, {
|
||||||
|
onEvent: (parsed: SSEEvent) => {
|
||||||
|
if (parsed.status === "progress" && parsed.message.count !== undefined) {
|
||||||
|
if (parsed.step === "txn_dicts") setLiveParsedCount(parsed.message.count);
|
||||||
|
if (parsed.step === "enrich") setStepStats((prev) => ({ ...prev, enrich_count: parsed.message.count! }));
|
||||||
|
if (parsed.step === "save_expenses") setStepStats((prev) => ({ ...prev, save_count: parsed.message.count! }));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (parsed.status === "completed" && parsed.message.count !== undefined) {
|
||||||
|
const stats: Record<string, number> = {};
|
||||||
|
if (parsed.step === "raw_lines" && parsed.message.lines !== undefined) stats.raw_lines = parsed.message.lines;
|
||||||
|
if (parsed.step === "txn_blocks" && parsed.message.blocks !== undefined) stats.txn_blocks = parsed.message.blocks;
|
||||||
|
if (parsed.step === "txn_dicts") stats.txn_dicts = parsed.message.count;
|
||||||
|
if (parsed.step === "enrich") stats.enrich_count = parsed.message.count;
|
||||||
|
if (parsed.step === "save_expenses") stats.save_count = parsed.message.count;
|
||||||
|
if (Object.keys(stats).length) {
|
||||||
|
setStepStats((prev) => ({ ...prev, ...stats }));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (parsed.status === "paused") {
|
||||||
|
refetchRequest();
|
||||||
|
}
|
||||||
|
if (parsed.status === "failed") {
|
||||||
|
setFailNotif(parsed.message.error || "Fetch request failed");
|
||||||
|
refetchRequest();
|
||||||
|
}
|
||||||
|
if (parsed.status === "completed" || parsed.step === "resume_extract") {
|
||||||
|
refetchRequest();
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (feedRef.current) {
|
||||||
|
feedRef.current.scrollTop = feedRef.current.scrollHeight;
|
||||||
|
}
|
||||||
|
}, [sseEvents]);
|
||||||
|
|
||||||
|
const displayEvents = useMemo(() => {
|
||||||
|
const progressSteps = new Set(["txn_dicts", "enrich", "save_expenses"]);
|
||||||
|
const lastProgressIdx: Record<string, number> = {};
|
||||||
|
for (let i = sseEvents.length - 1; i >= 0; i--) {
|
||||||
|
const e = sseEvents[i];
|
||||||
|
if (progressSteps.has(e.step) && e.status === "progress" && lastProgressIdx[e.step] === undefined) {
|
||||||
|
lastProgressIdx[e.step] = i;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const terminalStatuses = new Set(["completed", "skipped", "paused", "failed"]);
|
||||||
|
return sseEvents.filter((e, i) => {
|
||||||
|
if (progressSteps.has(e.step) && e.status === "progress") return i === lastProgressIdx[e.step];
|
||||||
|
if (e.status === "started") {
|
||||||
|
return !sseEvents.slice(i + 1).some(
|
||||||
|
(later) => later.step === e.step && terminalStatuses.has(later.status),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
}, [sseEvents]);
|
||||||
|
|
||||||
|
const handleRetry = async () => {
|
||||||
|
if (!id) return;
|
||||||
|
try {
|
||||||
|
await updateMutation.mutateAsync({ id, data: { status: "pending" } });
|
||||||
|
refetchRequest();
|
||||||
|
} catch (err: any) {
|
||||||
|
setFailNotif(formatApiError(err));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if (isLoading) {
|
||||||
|
return (
|
||||||
|
<Box sx={{ display: "flex", justifyContent: "center", p: 8 }}>
|
||||||
|
<CircularProgress />
|
||||||
|
</Box>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (fetchError || !fetchRequest) {
|
||||||
|
return (
|
||||||
|
<Container sx={{ mt: 4 }}>
|
||||||
|
<Button startIcon={<ArrowBackIcon />} onClick={() => navigate("/fetch-requests")} sx={{ mb: 2 }}>
|
||||||
|
Back
|
||||||
|
</Button>
|
||||||
|
<Alert severity="error">Failed to load fetch request</Alert>
|
||||||
|
</Container>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const req = fetchRequest as any;
|
||||||
|
const retryCount = req.retry_count ?? 0;
|
||||||
|
const isRetryExhausted = retryCount >= RETRY_MAX;
|
||||||
|
const status = req.status as FetchRequestStatus;
|
||||||
|
|
||||||
|
const detailFields = (resource?.orderedFields ?? []).filter(
|
||||||
|
(f) => f.name !== "source",
|
||||||
|
);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Container sx={{ mt: 4, mb: 4 }}>
|
||||||
|
<Button startIcon={<ArrowBackIcon />} onClick={() => navigate("/fetch-requests")} sx={{ mb: 2 }}>
|
||||||
|
Back to Fetch Requests
|
||||||
|
</Button>
|
||||||
|
|
||||||
|
<Paper sx={{ p: 3, borderRadius: 4, mb: 3 }} variant="outlined">
|
||||||
|
<Box sx={{ display: "flex", alignItems: "center", gap: 2, mb: 2, flexWrap: "wrap" }}>
|
||||||
|
<Chip
|
||||||
|
icon={statusIcons[status] as any}
|
||||||
|
label={status.replace(/_/g, " ")}
|
||||||
|
color={statusColors[status]}
|
||||||
|
/>
|
||||||
|
<Typography variant="h6" fontWeight={600}>{req.account_name}</Typography>
|
||||||
|
<Chip
|
||||||
|
label={"path" in (req.source ?? {}) ? "File" : "Email"}
|
||||||
|
size="small"
|
||||||
|
variant="outlined"
|
||||||
|
color={"path" in (req.source ?? {}) ? "primary" : "secondary"}
|
||||||
|
/>
|
||||||
|
</Box>
|
||||||
|
|
||||||
|
<Box sx={{ display: "flex", gap: 4, flexWrap: "wrap", mb: 2 }}>
|
||||||
|
{detailFields.map((field) => (
|
||||||
|
<DetailFieldRenderer
|
||||||
|
key={field.name}
|
||||||
|
field={field}
|
||||||
|
value={req[field.name]}
|
||||||
|
displayFormat={resource?.displayFormat}
|
||||||
|
/>
|
||||||
|
))}
|
||||||
|
</Box>
|
||||||
|
|
||||||
|
<Box sx={{ display: "flex", alignItems: "center", gap: 2 }}>
|
||||||
|
<Box sx={{ flex: 1, maxWidth: 300 }}>
|
||||||
|
<Typography variant="caption" color="text.secondary">
|
||||||
|
Retries: {retryCount}/{RETRY_MAX}
|
||||||
|
</Typography>
|
||||||
|
</Box>
|
||||||
|
{status === "failed" && !isRetryExhausted && (
|
||||||
|
<Button
|
||||||
|
variant="outlined"
|
||||||
|
size="small"
|
||||||
|
startIcon={<ReplayIcon />}
|
||||||
|
onClick={handleRetry}
|
||||||
|
disabled={updateMutation.isPending}
|
||||||
|
>
|
||||||
|
Retry
|
||||||
|
</Button>
|
||||||
|
)}
|
||||||
|
</Box>
|
||||||
|
</Paper>
|
||||||
|
|
||||||
|
{status === "failed" && req.error_message && (
|
||||||
|
<Alert severity="error" sx={{ mb: 3, borderRadius: 2 }}>
|
||||||
|
{req.error_message}
|
||||||
|
</Alert>
|
||||||
|
)}
|
||||||
|
{isRetryExhausted && status === "failed" && (
|
||||||
|
<Alert severity="info" sx={{ mb: 3, borderRadius: 2 }}>
|
||||||
|
Max retries reached — no further retry attempts will be made.
|
||||||
|
</Alert>
|
||||||
|
)}
|
||||||
|
|
||||||
|
<PipelineStepper
|
||||||
|
fetchRequest={req}
|
||||||
|
sseEvents={sseEvents}
|
||||||
|
stepStats={stepStats}
|
||||||
|
liveParsedCount={liveParsedCount ?? 0}
|
||||||
|
/>
|
||||||
|
|
||||||
|
<Paper sx={{ borderRadius: 4, mb: 3 }} variant="outlined">
|
||||||
|
<Box sx={{ display: "flex", alignItems: "center", gap: 1, p: 2, pb: 0 }}>
|
||||||
|
<Typography variant="subtitle1" fontWeight={600} sx={{ flex: 1 }}>
|
||||||
|
Progress Events
|
||||||
|
</Typography>
|
||||||
|
<Box
|
||||||
|
sx={{
|
||||||
|
width: 10,
|
||||||
|
height: 10,
|
||||||
|
borderRadius: "50%",
|
||||||
|
bgcolor: sseConnected ? "success.main" : "error.main",
|
||||||
|
flexShrink: 0,
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
<Typography variant="caption" color="text.secondary">
|
||||||
|
{sseConnected ? "Connected" : "Disconnected"}
|
||||||
|
</Typography>
|
||||||
|
</Box>
|
||||||
|
<Box
|
||||||
|
ref={feedRef}
|
||||||
|
sx={{
|
||||||
|
maxHeight: 300,
|
||||||
|
overflowY: "auto",
|
||||||
|
p: 2,
|
||||||
|
display: "flex",
|
||||||
|
flexDirection: "column",
|
||||||
|
gap: 1,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{displayEvents.length === 0 ? (
|
||||||
|
<Typography variant="body2" color="text.disabled" sx={{ textAlign: "center", py: 2 }}>
|
||||||
|
Waiting for events...
|
||||||
|
</Typography>
|
||||||
|
) : (
|
||||||
|
displayEvents.map((evt, i) => (
|
||||||
|
<Box
|
||||||
|
key={i}
|
||||||
|
sx={{
|
||||||
|
display: "flex",
|
||||||
|
alignItems: "center",
|
||||||
|
gap: 1.5,
|
||||||
|
p: 1,
|
||||||
|
borderRadius: 2,
|
||||||
|
bgcolor: "action.hover",
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{sseIcon(evt.status)}
|
||||||
|
<Box sx={{ flex: 1 }}>
|
||||||
|
<Typography variant="body2" fontWeight={600}>
|
||||||
|
{evt.step.replace(/_/g, " ")}
|
||||||
|
</Typography>
|
||||||
|
{evt.message && formatProgressMessage(evt.message) && (
|
||||||
|
<Typography variant="caption" color="text.secondary">
|
||||||
|
{formatProgressMessage(evt.message)}
|
||||||
|
</Typography>
|
||||||
|
)}
|
||||||
|
</Box>
|
||||||
|
</Box>
|
||||||
|
))
|
||||||
|
)}
|
||||||
|
</Box>
|
||||||
|
</Paper>
|
||||||
|
|
||||||
|
<AmbiguityResolver fetchRequestId={id!} />
|
||||||
|
|
||||||
|
<Snackbar
|
||||||
|
open={!!failNotif}
|
||||||
|
autoHideDuration={6000}
|
||||||
|
onClose={() => setFailNotif(null)}
|
||||||
|
anchorOrigin={{ vertical: "bottom", horizontal: "center" }}
|
||||||
|
>
|
||||||
|
<Alert severity="error" onClose={() => setFailNotif(null)} sx={{ borderRadius: 2 }}>
|
||||||
|
{failNotif}
|
||||||
|
</Alert>
|
||||||
|
</Snackbar>
|
||||||
|
</Container>
|
||||||
|
);
|
||||||
|
}
|
||||||
121
src/FetchRequest/components/AmbiguityResolver.tsx
Normal file
121
src/FetchRequest/components/AmbiguityResolver.tsx
Normal file
@@ -0,0 +1,121 @@
|
|||||||
|
import React from "react";
|
||||||
|
import {
|
||||||
|
Box,
|
||||||
|
Paper,
|
||||||
|
Typography,
|
||||||
|
Button,
|
||||||
|
Alert,
|
||||||
|
} from "@mui/material";
|
||||||
|
import CheckCircleIcon from "@mui/icons-material/CheckCircle";
|
||||||
|
import { useFetchRequestAmbiguities, useResolveAmbiguity } from "../../features/fetch-requests";
|
||||||
|
|
||||||
|
interface AmbiguityResolverProps {
|
||||||
|
fetchRequestId: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function AmbiguityResolver({ fetchRequestId }: AmbiguityResolverProps) {
|
||||||
|
const { data: ambiguities, refetch } = useFetchRequestAmbiguities(fetchRequestId);
|
||||||
|
const resolveMutation = useResolveAmbiguity();
|
||||||
|
|
||||||
|
const handleResolve = async (ambiguity: any, candidate: { amount: number; balance: number }) => {
|
||||||
|
await resolveMutation.mutateAsync({
|
||||||
|
ambiguityId: ambiguity.id,
|
||||||
|
payload: { chosen: { amount: candidate.amount, balance: candidate.balance } },
|
||||||
|
});
|
||||||
|
refetch();
|
||||||
|
};
|
||||||
|
|
||||||
|
if (!ambiguities || ambiguities.length === 0) return null;
|
||||||
|
|
||||||
|
const allResolved = ambiguities.every((a: any) => a.status === "resolved");
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Paper sx={{ p: 3, borderRadius: 4, mb: 3 }} variant="outlined">
|
||||||
|
<Typography variant="subtitle1" fontWeight={600} gutterBottom>
|
||||||
|
Ambiguity Resolution
|
||||||
|
</Typography>
|
||||||
|
|
||||||
|
{allResolved ? (
|
||||||
|
<Alert severity="success" sx={{ mb: 2, borderRadius: 2 }}>
|
||||||
|
All ambiguities resolved — pipeline will resume on next poll cycle
|
||||||
|
</Alert>
|
||||||
|
) : (
|
||||||
|
<Alert severity="warning" sx={{ mb: 2, borderRadius: 2 }}>
|
||||||
|
Pipeline paused — resolve ambiguities to continue
|
||||||
|
</Alert>
|
||||||
|
)}
|
||||||
|
|
||||||
|
<Box sx={{ display: "flex", flexDirection: "column", gap: 2 }}>
|
||||||
|
{ambiguities.map((ambiguity: any) => {
|
||||||
|
const isResolved = ambiguity.status === "resolved";
|
||||||
|
return (
|
||||||
|
<Paper
|
||||||
|
key={ambiguity.id}
|
||||||
|
sx={{
|
||||||
|
p: 2,
|
||||||
|
borderRadius: 3,
|
||||||
|
border: 1,
|
||||||
|
borderColor: isResolved ? "success.main" : "divider",
|
||||||
|
opacity: isResolved ? 0.8 : 1,
|
||||||
|
}}
|
||||||
|
variant="outlined"
|
||||||
|
>
|
||||||
|
<Box sx={{ fontFamily: "monospace", fontSize: "0.85rem", mb: 1.5, p: 1, bgcolor: "grey.900", borderRadius: 1, color: "grey.100" }}>
|
||||||
|
{ambiguity.line}
|
||||||
|
</Box>
|
||||||
|
|
||||||
|
<Box sx={{ display: "flex", gap: 3, mb: 1.5, flexWrap: "wrap" }}>
|
||||||
|
<Box>
|
||||||
|
<Typography variant="caption" color="text.secondary">OCR Amount</Typography>
|
||||||
|
<Typography variant="body2" sx={{ textDecoration: "line-through", color: "text.secondary" }}>
|
||||||
|
₹{ambiguity.ocr_amount}
|
||||||
|
</Typography>
|
||||||
|
</Box>
|
||||||
|
<Box>
|
||||||
|
<Typography variant="caption" color="text.secondary">OCR Balance</Typography>
|
||||||
|
<Typography variant="body2" sx={{ textDecoration: "line-through", color: "text.secondary" }}>
|
||||||
|
₹{ambiguity.ocr_balance}
|
||||||
|
</Typography>
|
||||||
|
</Box>
|
||||||
|
<Box>
|
||||||
|
<Typography variant="caption" color="text.secondary">Previous Balance</Typography>
|
||||||
|
<Typography variant="body2">₹{ambiguity.prev_balance}</Typography>
|
||||||
|
</Box>
|
||||||
|
</Box>
|
||||||
|
|
||||||
|
{isResolved ? (
|
||||||
|
<Alert severity="success" sx={{ py: 0.5, borderRadius: 2 }} icon={<CheckCircleIcon />}>
|
||||||
|
Resolved: ₹{ambiguity.chosen?.amount} / ₹{ambiguity.chosen?.balance}
|
||||||
|
</Alert>
|
||||||
|
) : (
|
||||||
|
<Box sx={{ display: "flex", gap: 1, flexWrap: "wrap" }}>
|
||||||
|
{ambiguity.candidates.map((candidate: any, ci: number) => {
|
||||||
|
const isCredit = candidate.amount > 0;
|
||||||
|
const isDebit = candidate.amount < 0;
|
||||||
|
const cColor = isCredit ? "success.main" : isDebit ? "error.main" : undefined;
|
||||||
|
return (
|
||||||
|
<Button
|
||||||
|
key={ci}
|
||||||
|
variant="outlined"
|
||||||
|
size="small"
|
||||||
|
onClick={() => handleResolve(ambiguity, candidate)}
|
||||||
|
disabled={resolveMutation.isPending}
|
||||||
|
sx={{
|
||||||
|
borderColor: cColor,
|
||||||
|
color: cColor,
|
||||||
|
"&:hover": cColor ? { borderColor: cColor } : undefined,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
₹{candidate.amount} / ₹{candidate.balance}
|
||||||
|
</Button>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</Box>
|
||||||
|
)}
|
||||||
|
</Paper>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</Box>
|
||||||
|
</Paper>
|
||||||
|
);
|
||||||
|
}
|
||||||
209
src/FetchRequest/components/PipelineStepper.tsx
Normal file
209
src/FetchRequest/components/PipelineStepper.tsx
Normal file
@@ -0,0 +1,209 @@
|
|||||||
|
import React, { useMemo } from "react";
|
||||||
|
import {
|
||||||
|
Box,
|
||||||
|
Paper,
|
||||||
|
Typography,
|
||||||
|
Stepper,
|
||||||
|
Step,
|
||||||
|
StepLabel,
|
||||||
|
LinearProgress,
|
||||||
|
} from "@mui/material";
|
||||||
|
import CheckCircleIcon from "@mui/icons-material/CheckCircle";
|
||||||
|
import ErrorIcon from "@mui/icons-material/Error";
|
||||||
|
import WarningAmberIcon from "@mui/icons-material/WarningAmber";
|
||||||
|
import CircularProgress from "@mui/material/CircularProgress";
|
||||||
|
import type { FetchRequestStatus, SSEEvent, ProgressMessage } from "../../features/fetch-requests";
|
||||||
|
|
||||||
|
const STEP_LABELS = ["Extract", "Raw Expense", "Enrich", "Save"];
|
||||||
|
|
||||||
|
function computeProgressPercent(
|
||||||
|
status: FetchRequestStatus,
|
||||||
|
liveCount: number,
|
||||||
|
seenSteps: Set<string>,
|
||||||
|
stepStats: Record<string, number>,
|
||||||
|
txnBlockCount: number,
|
||||||
|
txnDictCount: number,
|
||||||
|
): number {
|
||||||
|
if (status === "pending") return 0;
|
||||||
|
if (status === "completed") return 100;
|
||||||
|
|
||||||
|
let pct = 0;
|
||||||
|
|
||||||
|
if (seenSteps.has("raw_lines") || seenSteps.has("txn_blocks")) pct += 10;
|
||||||
|
|
||||||
|
if (txnBlockCount > 0) {
|
||||||
|
const current = Math.max(liveCount, stepStats.txn_dicts ?? 0);
|
||||||
|
pct += Math.min(1, current / txnBlockCount) * 20;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (txnDictCount > 0) {
|
||||||
|
pct += Math.min(1, (stepStats.enrich_count ?? 0) / txnDictCount) * 50;
|
||||||
|
pct += Math.min(1, (stepStats.save_count ?? 0) / txnDictCount) * 20;
|
||||||
|
}
|
||||||
|
|
||||||
|
return Math.round(Math.min(100, pct));
|
||||||
|
}
|
||||||
|
|
||||||
|
function computeActiveStep(status: FetchRequestStatus, seenSteps: Set<string>): number {
|
||||||
|
if (status === "completed") return STEP_LABELS.length;
|
||||||
|
|
||||||
|
if (seenSteps.has("save_expenses/completed") || seenSteps.has("complete/completed")) return STEP_LABELS.length;
|
||||||
|
if (seenSteps.has("save_expenses") || seenSteps.has("complete")) return 3;
|
||||||
|
|
||||||
|
if (seenSteps.has("enrich/completed")) return 3;
|
||||||
|
if (seenSteps.has("enrich")) return 2;
|
||||||
|
|
||||||
|
if (seenSteps.has("txn_dicts/completed") || status === "raw_expenses_done") return 2;
|
||||||
|
if (seenSteps.has("txn_dicts")) return 1;
|
||||||
|
|
||||||
|
if (seenSteps.has("txn_blocks/completed")) return 1;
|
||||||
|
if (seenSteps.has("raw_lines") || seenSteps.has("txn_blocks")) return 0;
|
||||||
|
|
||||||
|
if (status === "processing" || status === "paused") return 0;
|
||||||
|
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
function computeStepMessages(
|
||||||
|
fetchRequest: any,
|
||||||
|
stepStats: Record<string, number>,
|
||||||
|
liveParsedCount: number,
|
||||||
|
txnBlockCount: number,
|
||||||
|
): Record<number, string> {
|
||||||
|
const msgs: Record<number, string> = {};
|
||||||
|
const source = fetchRequest?.source;
|
||||||
|
|
||||||
|
const rawLineCount = stepStats.raw_lines ?? (source?.raw_lines?.length ?? 0);
|
||||||
|
if (rawLineCount) msgs[0] = `${rawLineCount}`;
|
||||||
|
|
||||||
|
const sourceDictCount = source?.txn_dict_count ?? source?.txn_dicts_count ?? 0;
|
||||||
|
const dictLive = liveParsedCount ?? stepStats.txn_dicts ?? 0;
|
||||||
|
const dictCurrent = Math.max(dictLive, sourceDictCount);
|
||||||
|
if (dictCurrent && txnBlockCount) msgs[1] = `${dictCurrent}/${txnBlockCount}`;
|
||||||
|
else if (dictCurrent) msgs[1] = `${dictCurrent}`;
|
||||||
|
|
||||||
|
const txnDictDenom = stepStats.txn_dicts ?? sourceDictCount;
|
||||||
|
if (stepStats.enrich_count && txnDictDenom) msgs[2] = `${stepStats.enrich_count}/${txnDictDenom}`;
|
||||||
|
else if (stepStats.enrich_count) msgs[2] = `${stepStats.enrich_count}`;
|
||||||
|
|
||||||
|
if (stepStats.save_count && txnDictDenom) msgs[3] = `${stepStats.save_count}/${txnDictDenom}`;
|
||||||
|
else if (stepStats.save_count) msgs[3] = `${stepStats.save_count}`;
|
||||||
|
|
||||||
|
return msgs;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface PipelineStepperProps {
|
||||||
|
fetchRequest: any;
|
||||||
|
sseEvents: SSEEvent[];
|
||||||
|
stepStats: Record<string, number>;
|
||||||
|
liveParsedCount: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function PipelineStepper({ fetchRequest, sseEvents, stepStats, liveParsedCount }: PipelineStepperProps) {
|
||||||
|
const status = (fetchRequest?.status ?? "pending") as FetchRequestStatus;
|
||||||
|
|
||||||
|
const seenSteps = useMemo(() => {
|
||||||
|
const steps = new Set<string>();
|
||||||
|
for (const evt of sseEvents) {
|
||||||
|
steps.add(evt.step);
|
||||||
|
if (evt.status === "completed") steps.add(`${evt.step}/completed`);
|
||||||
|
if (evt.status === "failed") steps.add(`${evt.step}/failed`);
|
||||||
|
if (evt.status === "started") steps.add(`${evt.step}/started`);
|
||||||
|
if (evt.status === "progress") steps.add(`${evt.step}/progress`);
|
||||||
|
}
|
||||||
|
return steps;
|
||||||
|
}, [sseEvents]);
|
||||||
|
|
||||||
|
const txnBlockCount = useMemo(() => {
|
||||||
|
const blocks = fetchRequest?.source?.txn_blocks;
|
||||||
|
if (!blocks) return 0;
|
||||||
|
return Object.values(blocks).reduce(
|
||||||
|
(sum: number, list: any) => sum + (Array.isArray(list) ? list.length : 0),
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
}, [fetchRequest]);
|
||||||
|
|
||||||
|
const txnDictCount = useMemo(() => {
|
||||||
|
const source = fetchRequest?.source;
|
||||||
|
if (stepStats.txn_dicts && stepStats.txn_dicts > 0) return stepStats.txn_dicts;
|
||||||
|
return source?.txn_dict_count ?? source?.txn_dicts_count ?? 0;
|
||||||
|
}, [fetchRequest, stepStats]);
|
||||||
|
|
||||||
|
const displayParsedCount = useMemo(() => {
|
||||||
|
if (liveParsedCount && liveParsedCount > 0) return liveParsedCount;
|
||||||
|
const source = fetchRequest?.source;
|
||||||
|
const persistedCount = source?.txn_dict_count ?? source?.txn_dicts_count ?? 0;
|
||||||
|
if (persistedCount > 0) return persistedCount;
|
||||||
|
const dicts = source?.txn_dicts;
|
||||||
|
if (Array.isArray(dicts) && dicts.length > 0) return dicts.length;
|
||||||
|
return 0;
|
||||||
|
}, [liveParsedCount, fetchRequest]);
|
||||||
|
|
||||||
|
const activeStep = computeActiveStep(status, seenSteps);
|
||||||
|
const progressPercent = computeProgressPercent(status, displayParsedCount, seenSteps, stepStats, txnBlockCount, txnDictCount);
|
||||||
|
const stepMessages = computeStepMessages(fetchRequest, stepStats, liveParsedCount, txnBlockCount);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Paper sx={{ p: 3, borderRadius: 4, mb: 3 }} variant="outlined">
|
||||||
|
<Typography variant="subtitle1" fontWeight={600} gutterBottom>
|
||||||
|
Pipeline Progress
|
||||||
|
</Typography>
|
||||||
|
<Stepper activeStep={activeStep} alternativeLabel>
|
||||||
|
{STEP_LABELS.map((label, index) => {
|
||||||
|
const isCompleted = index < activeStep;
|
||||||
|
const isActive = index === activeStep;
|
||||||
|
const isPaused = status === "paused" && isActive;
|
||||||
|
const isFailed = status === "failed" && isActive;
|
||||||
|
|
||||||
|
let icon: React.ReactNode;
|
||||||
|
if (isCompleted) {
|
||||||
|
icon = <CheckCircleIcon sx={{ color: "success.main" }} />;
|
||||||
|
} else if (isFailed) {
|
||||||
|
icon = <ErrorIcon sx={{ color: "error.main" }} />;
|
||||||
|
} else if (isPaused) {
|
||||||
|
icon = <WarningAmberIcon sx={{ color: "warning.main" }} />;
|
||||||
|
} else if (isActive) {
|
||||||
|
icon = <CircularProgress size={20} />;
|
||||||
|
} else {
|
||||||
|
icon = <Typography variant="caption" color="text.disabled">{index + 1}</Typography>;
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Step key={label}>
|
||||||
|
<StepLabel
|
||||||
|
StepIconComponent={() => <Box sx={{ display: "flex", alignItems: "center" }}>{icon}</Box>}
|
||||||
|
>
|
||||||
|
<Typography variant="body2" fontWeight={600}>{label}</Typography>
|
||||||
|
{stepMessages[index] && (
|
||||||
|
<Typography variant="caption" color="text.secondary" sx={{ display: "block", lineHeight: 1.2 }}>
|
||||||
|
{stepMessages[index]}
|
||||||
|
</Typography>
|
||||||
|
)}
|
||||||
|
</StepLabel>
|
||||||
|
</Step>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</Stepper>
|
||||||
|
|
||||||
|
<Box sx={{ mt: 3, mb: 1 }}>
|
||||||
|
<Box sx={{ display: "flex", alignItems: "center", gap: 2, mb: 0.5 }}>
|
||||||
|
<Typography variant="caption" color="text.secondary">Overall Progress</Typography>
|
||||||
|
{["processing", "paused"].includes(status) && displayParsedCount > 0 && (
|
||||||
|
<Typography variant="caption" fontWeight={600} color="info.main">
|
||||||
|
Validated: {displayParsedCount} transactions
|
||||||
|
</Typography>
|
||||||
|
)}
|
||||||
|
</Box>
|
||||||
|
<LinearProgress
|
||||||
|
variant="determinate"
|
||||||
|
value={progressPercent}
|
||||||
|
color={status === "failed" ? "error" : status === "completed" ? "success" : "primary"}
|
||||||
|
sx={{ borderRadius: 1, height: 8, transition: "width 0.3s ease" }}
|
||||||
|
/>
|
||||||
|
<Typography variant="caption" color="text.secondary" sx={{ mt: 0.25, display: "block" }}>
|
||||||
|
{progressPercent}%
|
||||||
|
</Typography>
|
||||||
|
</Box>
|
||||||
|
</Paper>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -12,8 +12,8 @@ import {
|
|||||||
Toolbar
|
Toolbar
|
||||||
} from "@mui/material";
|
} from "@mui/material";
|
||||||
import Home from './Home';
|
import Home from './Home';
|
||||||
import FetchRequests from './FetchRequests';
|
import FetchRequests from './FetchRequest/FetchRequestCreate';
|
||||||
import FetchRequestDetail from './FetchRequestDetail';
|
import FetchRequestDetail from './FetchRequest/FetchRequestDetail';
|
||||||
import { RequireAuth } from './RequireAuth';
|
import { RequireAuth } from './RequireAuth';
|
||||||
import { AppProvider, Admin } from '../react-openapi';
|
import { AppProvider, Admin } from '../react-openapi';
|
||||||
import { Buffer } from 'buffer';
|
import { Buffer } from 'buffer';
|
||||||
|
|||||||
Reference in New Issue
Block a user