5 Commits

13 changed files with 139 additions and 474 deletions

View File

@@ -31,7 +31,6 @@ import VisibilityIcon from '@mui/icons-material/Visibility';
import MoreVertIcon from '@mui/icons-material/MoreVert'; import MoreVertIcon from '@mui/icons-material/MoreVert';
import { useNavigate } from 'react-router-dom'; import { useNavigate } from 'react-router-dom';
import { ResourceConfig } from '../types/config'; import { ResourceConfig } from '../types/config';
import { getFieldOptions, toGridValueOptions, resolveTemplate } from '../utils/options';
interface EnhancedTableProps { interface EnhancedTableProps {
config: ResourceConfig; config: ResourceConfig;
@@ -96,8 +95,9 @@ export default function EnhancedTable({
}; };
} }
if (muiType === 'singleSelect') { if (muiType === 'singleSelect' && field.options) {
col.valueOptions = toGridValueOptions(getFieldOptions(field)); // @ts-ignore
col.valueOptions = field.options;
} }
return col; return col;
@@ -274,9 +274,8 @@ function MobileCardRow({ row, config, onDelete, onNavigate, navigate }: any) {
); );
} }
function getFormattedDisplayValue(item: any, displayField?: string | string[], enumValue?: string) { function getFormattedDisplayValue(item: any, displayField?: string | string[]) {
if (!item) return ""; if (!item) return "";
if (enumValue) return resolveTemplate(enumValue, item);
if (!displayField) return item.name || item.title || item.label || item.id || JSON.stringify(item); if (!displayField) return item.name || item.title || item.label || item.id || JSON.stringify(item);
if (Array.isArray(displayField)) { if (Array.isArray(displayField)) {
@@ -298,7 +297,7 @@ function FieldRenderer({ params, field, fieldKey, config, onNavigate, navigate,
// 1. Single Relation // 1. Single Relation
if (field.relation && value && !Array.isArray(value)) { if (field.relation && value && !Array.isArray(value)) {
const relationId = typeof value === 'object' ? (value.id || value._id || value.pk) : value; const relationId = typeof value === 'object' ? (value.id || value._id || value.pk) : value;
const displayValue = getFormattedDisplayValue(value, field.displayField, field.enumOption?.value); const displayValue = getFormattedDisplayValue(value, field.displayField);
return ( return (
<Chip <Chip
@@ -317,8 +316,7 @@ function FieldRenderer({ params, field, fieldKey, config, onNavigate, navigate,
// 2. Multi-Select (Array of relations or simple strings) // 2. Multi-Select (Array of relations or simple strings)
if (field.type === 'array' && Array.isArray(value)) { if (field.type === 'array' && Array.isArray(value)) {
const enumValue = field.enumOption?.value; const tooltipTitle = value.map((item) => getFormattedDisplayValue(item, field.displayField)).join(', ');
const tooltipTitle = value.map((item) => getFormattedDisplayValue(item, field.displayField, enumValue)).join(', ');
return ( return (
<Tooltip title={tooltipTitle} arrow placement="top"> <Tooltip title={tooltipTitle} arrow placement="top">
@@ -326,7 +324,7 @@ function FieldRenderer({ params, field, fieldKey, config, onNavigate, navigate,
{value.map((item, idx) => ( {value.map((item, idx) => (
<Chip <Chip
key={idx} key={idx}
label={getFormattedDisplayValue(item, field.displayField, enumValue)} label={getFormattedDisplayValue(item, field.displayField)}
size="small" size="small"
variant="filled" variant="filled"
sx={{ maxWidth: 120 }} sx={{ maxWidth: 120 }}
@@ -346,7 +344,7 @@ function FieldRenderer({ params, field, fieldKey, config, onNavigate, navigate,
// 3. Simple Objects // 3. Simple Objects
if (field.type === 'object' && value) { if (field.type === 'object' && value) {
return getFormattedDisplayValue(value, field.displayField, field.enumOption?.value) || (isMobile ? 'Object' : JSON.stringify(value)); return getFormattedDisplayValue(value, field.displayField) || (isMobile ? 'Object' : JSON.stringify(value));
} }
if (field.type === 'number' && typeof value === 'number') { if (field.type === 'number' && typeof value === 'number') {
@@ -381,11 +379,6 @@ function FieldRenderer({ params, field, fieldKey, config, onNavigate, navigate,
if (field.type === 'datetime' || field.type === 'date') return value ? new Date(value).toLocaleString() : ''; if (field.type === 'datetime' || field.type === 'date') return value ? new Date(value).toLocaleString() : '';
if (field.type === 'enum') {
const opt = getFieldOptions(field).find(o => o.key === value);
return opt?.value ?? value;
}
if (isPk && !isMobile) { if (isPk && !isMobile) {
return ( return (
<Chip <Chip

View File

@@ -11,7 +11,6 @@ import {
import DoneIcon from "@mui/icons-material/Done"; import DoneIcon from "@mui/icons-material/Done";
import FilterListIcon from "@mui/icons-material/FilterList"; import FilterListIcon from "@mui/icons-material/FilterList";
import { ResourceField, ResourceMode } from "../types/config"; import { ResourceField, ResourceMode } from "../types/config";
import { getFieldOptions, resolveTemplate } from "../utils/options";
function FilterAutocomplete({ function FilterAutocomplete({
options, options,
@@ -111,9 +110,7 @@ function extractOptions(
): string[] { ): string[] {
const values = new Set<string>(); const values = new Set<string>();
if (field.type === 'enum') { if (field.options) return field.options;
return getFieldOptions(field).map(o => o.value);
}
if (!data) return []; if (!data) return [];
const pull = (item: any): string | null => { const pull = (item: any): string | null => {
@@ -121,18 +118,18 @@ function extractOptions(
if (typeof item === "string") return item; if (typeof item === "string") return item;
if (typeof item !== "object") return String(item); if (typeof item !== "object") return String(item);
if (field.enumOption?.value) return resolveTemplate(field.enumOption.value, item);
const df = field.displayField; const df = field.displayField;
if (!df) return null; if (!df) { debugger; return null; }
if (Array.isArray(df)) { if (Array.isArray(df)) {
const parts = df.map((k) => item[k]).filter((v) => v != null); const parts = df.map((k) => item[k]).filter((v) => v != null);
if (parts.length > 0) return parts.join(" "); if (parts.length > 0) return parts.join(" ");
} } else {
const v = item[df]; const v = item[df];
if (v != null) return String(v); if (v != null) return String(v);
}
debugger;
return null; return null;
}; };

View File

@@ -3,7 +3,6 @@ import { Box, Paper, CircularProgress } from '@mui/material';
import { ResourceConfig } from '../types/config'; import { ResourceConfig } from '../types/config';
import type { ResourceField } from '../types/config'; import type { ResourceField } from '../types/config';
import { useResource } from '../hooks/useResource'; import { useResource } from '../hooks/useResource';
import { resolveTemplate } from '../utils/options';
import GenericForm from './GenericForm'; import GenericForm from './GenericForm';
import EnhancedTable from './EnhancedTable'; import EnhancedTable from './EnhancedTable';
import FilterBar from './FilterBar'; import FilterBar from './FilterBar';
@@ -16,16 +15,11 @@ interface ResourceViewProps {
import { GridPaginationModel } from '@mui/x-data-grid'; import { GridPaginationModel } from '@mui/x-data-grid';
function getDisplayString(item: any, field: ResourceField): string { function getFilterDisplayFields(field: ResourceField): string[] {
if (item == null || typeof item !== 'object') return String(item ?? ''); if (!field.displayField) return [];
if (field.enumOption?.value) return resolveTemplate(field.enumOption.value, item); return (Array.isArray(field.displayField) ? field.displayField : [field.displayField]).filter(
const df = field.displayField; (df): df is string => !!df
if (!df) return item.name ?? item.title ?? item.label ?? item.id ?? JSON.stringify(item); );
if (Array.isArray(df)) {
const parts = df.map((k: string) => item[k]).filter((v: any) => v != null);
return parts.length > 0 ? parts.join(' ') : '';
}
return String(item[df] ?? '');
} }
function applyClientFilters( function applyClientFilters(
@@ -65,12 +59,18 @@ function applyClientFilters(
if (Array.isArray(filterValue)) { if (Array.isArray(filterValue)) {
if (field.type === "array" && Array.isArray(itemValue)) { if (field.type === "array" && Array.isArray(itemValue)) {
return itemValue.some((el: any) => return itemValue.some((el: any) => {
filterValue.includes(getDisplayString(el, field)) if (el != null && typeof el === "object") {
); const dispFields = getFilterDisplayFields(field);
return dispFields.some((df) => filterValue.includes(String(el[df])));
}
return filterValue.includes(String(el));
});
} }
if (itemValue && typeof itemValue === "object") { if (itemValue && typeof itemValue === "object") {
return filterValue.includes(getDisplayString(itemValue, field)); const dispFields = getFilterDisplayFields(field);
const itemDisplay = dispFields.map((df) => itemValue[df]).filter((v) => v != null).join(" ");
return filterValue.includes(itemDisplay);
} }
return filterValue.includes(String(itemValue)); return filterValue.includes(String(itemValue));
} }
@@ -82,13 +82,18 @@ function applyClientFilters(
} }
if (field.type === "array" && Array.isArray(itemValue)) { if (field.type === "array" && Array.isArray(itemValue)) {
return itemValue.some((el: any) => return itemValue.some((el: any) => {
getDisplayString(el, field) === String(filterValue) if (el != null && typeof el === "object") {
); const dispFields = getFilterDisplayFields(field);
return dispFields.some((df) => String(el[df]) === String(filterValue));
}
return String(el) === String(filterValue);
});
} }
if (itemValue && typeof itemValue === "object") { if (itemValue && typeof itemValue === "object") {
return getDisplayString(itemValue, field) === String(filterValue); const dispFields = getFilterDisplayFields(field);
return dispFields.some((df) => String(itemValue[df]) === String(filterValue));
} }
return String(itemValue) === String(filterValue); return String(itemValue) === String(filterValue);

View File

@@ -12,7 +12,6 @@ import {
Divider, Divider,
} from '@mui/material'; } from '@mui/material';
import { ResourceField } from '../../types/config'; import { ResourceField } from '../../types/config';
import { getFieldOptions } from '../../utils/options';
import ImageUploadField from './ImageUploadField'; import ImageUploadField from './ImageUploadField';
interface FormFieldProps { interface FormFieldProps {
@@ -74,40 +73,40 @@ export default function FormField({
if (field.relation && relationDataMap[field.relation]) { if (field.relation && relationDataMap[field.relation]) {
const relationData = relationDataMap[field.relation].data; const relationData = relationDataMap[field.relation].data;
const isArrayRelation = field.type === 'array'; const isArrayRelation = field.type === 'array';
const options = getFieldOptions(field, relationData);
const keyField = field.enumOption?.key ?? 'id';
// Normalize value: API returns whole objects on GET, but form uses key strings // Determine how to display the related item
const normalizedValue = (() => { const getOptionLabel = (option: any) => {
if (isArrayRelation && Array.isArray(value)) { if (!option) return "";
return value.map((v: any) => (v != null && typeof v === 'object' ? String(v[keyField] ?? '') : String(v))); if (field.displayField && option[field.displayField]) return option[field.displayField];
} // Standard naming fields
if (value != null && typeof value === 'object') { return option.name || option.title || option.label || option.id || JSON.stringify(option);
return String(value[keyField] ?? ''); };
}
return value ?? (isArrayRelation ? [] : ""); const getOptionValue = (option: any) => {
})(); // Return the whole object to maintain identity
return option;
};
return ( return (
<FormControl fullWidth> <FormControl fullWidth>
<InputLabel shrink>{label}</InputLabel> <InputLabel shrink>{label}</InputLabel>
<Select <Select
multiple={isArrayRelation} multiple={isArrayRelation}
value={normalizedValue} value={value || (isArrayRelation ? [] : "")}
label={label} label={label}
displayEmpty displayEmpty
onChange={(e) => onChange(e.target.value)} onChange={(e) => onChange(e.target.value)}
disabled={disabled} disabled={disabled}
renderValue={(selected: any) => { renderValue={(selected: any) => {
if (isArrayRelation) { if (isArrayRelation) {
return (selected as string[]).map(k => options.find(o => o.key === k)?.value ?? k).join(', '); return (selected as any[]).map(getOptionLabel).join(', ');
} }
return options.find(o => o.key === selected)?.value ?? selected; return getOptionLabel(selected);
}} }}
> >
{options.map((opt) => ( {relationData.map((option) => (
<MenuItem key={opt.key} value={opt.key}> <MenuItem key={option.id || JSON.stringify(option)} value={getOptionValue(option)}>
{opt.value} {getOptionLabel(option)}
</MenuItem> </MenuItem>
))} ))}
</Select> </Select>
@@ -149,8 +148,7 @@ export default function FormField({
} }
// 5. Enum Handling // 5. Enum Handling
if (field.type === 'enum') { if (field.type === 'enum' && field.options) {
const options = getFieldOptions(field);
return ( return (
<FormControl fullWidth> <FormControl fullWidth>
<InputLabel>{label}</InputLabel> <InputLabel>{label}</InputLabel>
@@ -160,9 +158,9 @@ export default function FormField({
onChange={(e) => onChange(e.target.value)} onChange={(e) => onChange(e.target.value)}
disabled={disabled} disabled={disabled}
> >
{options.map((opt) => ( {field.options.map((opt: string) => (
<MenuItem key={opt.key} value={opt.key}> <MenuItem key={opt} value={opt}>
{opt.value} {opt}
</MenuItem> </MenuItem>
))} ))}
</Select> </Select>

View File

@@ -10,16 +10,6 @@ export type FieldType =
| 'object' | 'object'
| 'array'; | 'array';
export interface SelectOption {
key: string;
value: string;
}
export interface EnumOption {
key: string;
value: string;
}
export interface ResourceField { export interface ResourceField {
type: FieldType; type: FieldType;
label: string; label: string;
@@ -29,10 +19,8 @@ export interface ResourceField {
schema?: Record<string, ResourceField>; schema?: Record<string, ResourceField>;
displayField?: string | string[]; displayField?: string | string[];
formatter?: (value: any) => string; formatter?: (value: any) => string;
relation?: string; relation?: string; // Name of the target resource
filterType?: "autocomplete" | "multiselect" | "number-range" | "date-range"; filterType?: "autocomplete" | "multiselect" | "number-range" | "date-range";
enumOption?: EnumOption;
enumLabels?: Record<string, string>;
} }
export type ResourceMode = "server" | "client"; export type ResourceMode = "server" | "client";
@@ -50,14 +38,12 @@ export interface ResourceConfig {
mode?: ResourceMode; mode?: ResourceMode;
fields?: string[]; fields?: string[];
}; };
enumOption?: EnumOption;
} }
export interface AppConfig { export interface AppConfig {
baseUrl: string; baseUrl: string;
authBaseUrl: string; authBaseUrl: string;
resources: ResourceConfig[]; resources: ResourceConfig[];
enums: Record<string, string[]>;
profile?: { profile?: {
resource: string; resource: string;
extraFields?: Record<string, any>; extraFields?: Record<string, any>;

View File

@@ -1,14 +1,13 @@
export interface EnumOption { /**
key: string; * This file contains application-specific overrides and configuration
value: string; * for the generic Admin Panel.
} */
export interface FieldOverride { export interface FieldOverride {
displayField?: string | string[]; displayField?: string | string[];
display?: boolean; display?: boolean;
formatter?: (value: any) => string; formatter?: (value: any) => string;
filterType?: "autocomplete" | "multiselect" | "number-range" | "date-range"; filterType?: "autocomplete" | "multiselect" | "number-range" | "date-range";
enumLabels?: Record<string, string>;
} }
export interface ResourceOverride { export interface ResourceOverride {
@@ -19,5 +18,4 @@ export interface ResourceOverride {
mode?: "server" | "client"; mode?: "server" | "client";
fields?: string[]; fields?: string[];
}; };
enumOption?: EnumOption;
} }

View File

@@ -36,26 +36,6 @@ function mapOpenApiType(prop: any): FieldType {
/** /**
* Recursively converts OpenAPI schemas to ResourceField map * Recursively converts OpenAPI schemas to ResourceField map
*/ */
function mergeProperties(schema: any): { properties: Record<string, any>; required: string[] } {
let properties: Record<string, any> = {};
let required: string[] = [];
if (schema.allOf) {
for (const sub of schema.allOf) {
const merged = mergeProperties(sub);
properties = { ...properties, ...merged.properties };
required = [...required, ...merged.required];
}
}
if (schema.properties) {
properties = { ...properties, ...schema.properties };
}
if (schema.required) {
required = [...required, ...schema.required];
}
return { properties, required };
}
function parseSchemaFields( function parseSchemaFields(
schema: any, schema: any,
resourceName: string, resourceName: string,
@@ -63,19 +43,12 @@ function parseSchemaFields(
configuration: Record<string, any> = {} configuration: Record<string, any> = {}
): Record<string, ResourceField> { ): Record<string, ResourceField> {
const fields: Record<string, ResourceField> = {}; const fields: Record<string, ResourceField> = {};
const { properties, required } = mergeProperties(schema); const properties = schema.properties || {};
const required = schema.required || [];
const overrides = configuration[resourceName]?.fields || {}; const overrides = configuration[resourceName]?.fields || {};
for (const [key, prop] of Object.entries(properties) as [string, any]) { for (const [key, prop] of Object.entries(properties) as [string, any]) {
// Resolve oneOf/anyOf by merging all branch properties const type = mapOpenApiType(prop);
let resolvedProp = prop;
if (prop.oneOf || prop.anyOf) {
const branches = prop.oneOf || prop.anyOf;
const merged = mergeProperties({ allOf: branches });
resolvedProp = { ...prop, type: 'object', properties: merged.properties, required: merged.required };
}
const type = mapOpenApiType(resolvedProp);
const override = overrides[key]; const override = overrides[key];
// Explicitly skip 'id' as it's the primary key and handled elsewhere // Explicitly skip 'id' as it's the primary key and handled elsewhere
@@ -84,12 +57,12 @@ function parseSchemaFields(
fields[key] = { fields[key] = {
type, type,
label: label:
resolvedProp.title || prop.title ||
key.charAt(0).toUpperCase() + key.slice(1).replace(/_/g, " "), key.charAt(0).toUpperCase() + key.slice(1).replace(/_/g, " "),
required: required.includes(key), required: required.includes(key),
options: resolvedProp.enum, options: prop.enum,
readOnly: readOnly:
resolvedProp.readOnly || prop.readOnly ||
key === "created_at" || key === "created_at" ||
key === "updated_at", key === "updated_at",
...override, ...override,
@@ -98,35 +71,20 @@ function parseSchemaFields(
// STRICT RELATION DETECTION // STRICT RELATION DETECTION
// A field is a relation ONLY if its schema object (or items schema) // A field is a relation ONLY if its schema object (or items schema)
// exactly matches a schema that is defined as a resource. // exactly matches a schema that is defined as a resource.
let targetSchema = resolvedProp; let targetSchema = prop;
if (type === "array" && resolvedProp.items) { if (type === "array" && prop.items) {
targetSchema = resolvedProp.items; targetSchema = prop.items;
} }
// Check if this schema object is registered as a resource // Check if this schema object is registered as a resource
const relation = schemaToResourceMap.get(targetSchema); const relation = schemaToResourceMap.get(targetSchema);
if (relation) { if (relation) {
fields[key].relation = relation; fields[key].relation = relation;
// Propagate enumOption from target resource config, or derive from target schema
const explicitEnumOption = configuration[relation]?.enumOption;
if (explicitEnumOption) {
fields[key].enumOption = explicitEnumOption;
} else {
const targetProps = targetSchema.properties || {};
const valueField = Object.entries(targetProps).find(
([name, p]: [string, any]) => name !== 'id' && p.type === 'string'
)?.[0];
fields[key].enumOption = {
key: 'id',
value: valueField ?? 'id',
};
}
} }
// Recursively parse nested objects (only if not a relation) // Recursively parse nested objects (only if not a relation)
if (fields[key].type === "object" && resolvedProp.properties && !relation) { if (fields[key].type === "object" && prop.properties && !relation) {
fields[key].schema = parseSchemaFields(resolvedProp, resourceName, schemaToResourceMap, configuration); fields[key].schema = parseSchemaFields(prop, resourceName, schemaToResourceMap, configuration);
} }
} }
@@ -214,16 +172,6 @@ export async function loadConfigFromOpenApi(baseUrl: string, configuration: Reco
}); });
} }
// Collect standalone enum schemas (e.g. FetchRequestStatus, AccountType, etc.)
const enums: Record<string, string[]> = {};
if (api.components?.schemas) {
for (const [name, schema] of Object.entries(api.components.schemas) as [string, any]) {
if (schema.enum) {
enums[name] = schema.enum;
}
}
}
// @ts-ignore // @ts-ignore
const serverBaseUrl = import.meta.env.VITE_API_BASE_URL || (api.servers?.[0]?.url ?? "") const serverBaseUrl = import.meta.env.VITE_API_BASE_URL || (api.servers?.[0]?.url ?? "")
// @ts-ignore // @ts-ignore
@@ -232,7 +180,6 @@ export async function loadConfigFromOpenApi(baseUrl: string, configuration: Reco
baseUrl: serverBaseUrl, baseUrl: serverBaseUrl,
authBaseUrl: authBaseUrl, authBaseUrl: authBaseUrl,
resources, resources,
enums,
profile: profileConfiguration, profile: profileConfiguration,
}; };
} }

View File

@@ -1,33 +0,0 @@
import { ResourceField, SelectOption } from "../types/config";
export function resolveTemplate(template: string, item: any): string {
if (/\{(\w+)\}/.test(template)) {
return template.replace(/\{(\w+)\}/g, (_, field: string) => String(item[field] ?? ''));
}
return String(item[template] ?? '');
}
export function getFieldOptions(field: ResourceField, relationData?: any[]): SelectOption[] {
if (field.type === 'enum') {
return (field.options ?? []).map(opt => ({
key: opt,
value: field.enumLabels?.[opt] ?? opt,
}));
}
if (field.relation) {
const data = relationData ?? [];
const enumOption = field.enumOption ?? { key: 'id', value: 'name' };
return data.map(item => ({
key: String(item[enumOption.key] ?? ''),
value: resolveTemplate(enumOption.value, item),
}));
}
return [];
}
export function toGridValueOptions(options: SelectOption[]): { value: string; label: string }[] {
return options.map(opt => ({ value: opt.key, label: opt.value }));
}

View File

@@ -15,7 +15,6 @@ import {
StepIcon, StepIcon,
LinearProgress, LinearProgress,
IconButton, IconButton,
Snackbar,
} from "@mui/material"; } from "@mui/material";
import ArrowBackIcon from "@mui/icons-material/ArrowBack"; import ArrowBackIcon from "@mui/icons-material/ArrowBack";
import ReplayIcon from "@mui/icons-material/Replay"; import ReplayIcon from "@mui/icons-material/Replay";
@@ -23,8 +22,6 @@ import CheckCircleIcon from "@mui/icons-material/CheckCircle";
import ErrorIcon from "@mui/icons-material/Error"; import ErrorIcon from "@mui/icons-material/Error";
import WarningAmberIcon from "@mui/icons-material/WarningAmber"; import WarningAmberIcon from "@mui/icons-material/WarningAmber";
import PlayArrowIcon from "@mui/icons-material/PlayArrow"; import PlayArrowIcon from "@mui/icons-material/PlayArrow";
import RemoveCircleOutlineIcon from "@mui/icons-material/RemoveCircleOutline";
import FiberManualRecordIcon from "@mui/icons-material/FiberManualRecord";
import { import {
useFetchRequest, useFetchRequest,
useUpdateFetchRequest, useUpdateFetchRequest,
@@ -34,9 +31,8 @@ import {
import type { import type {
FetchRequestStatus, FetchRequestStatus,
SSEEvent, SSEEvent,
ProgressMessage,
} from "./features/fetch-requests"; } from "./features/fetch-requests";
import { RETRY_MAX, formatApiError } from "./features/fetch-requests"; import { RETRY_MAX } from "./features/fetch-requests";
import { useConfig } from "../react-openapi"; import { useConfig } from "../react-openapi";
const statusColors: Record<FetchRequestStatus, "default" | "primary" | "warning" | "info" | "success" | "error"> = { const statusColors: Record<FetchRequestStatus, "default" | "primary" | "warning" | "info" | "success" | "error"> = {
@@ -59,78 +55,26 @@ const statusIcons: Record<FetchRequestStatus, React.ReactNode> = {
failed: <ErrorIcon sx={{ fontSize: 16 }} />, failed: <ErrorIcon sx={{ fontSize: 16 }} />,
}; };
function computeProgressPercent( const stepLabels = ["Load Content", "Extract", "Enrich", "Save"];
status: FetchRequestStatus,
liveCount: number,
seenSteps: Set<string>,
stepStats: Record<string, number>,
txnBlockCount: number,
txnDictCount: number,
): number {
if (status === "pending") return 0;
if (status === "completed") return 100;
let pct = 0; function statusToActiveStep(status: FetchRequestStatus): number {
switch (status) {
if (seenSteps.has("raw_lines") || seenSteps.has("txn_blocks")) pct += 10; case "pending": return -1;
case "processing": return 0;
if (txnBlockCount > 0) { case "paused": return 1;
const current = Math.max(liveCount, stepStats.txn_dicts ?? 0); case "raw_expenses_done": return 2;
pct += Math.min(1, current / txnBlockCount) * 20; case "enriched_done": return 3;
case "completed": return 4;
case "failed": return 0;
default: return -1;
} }
if (txnDictCount > 0) {
pct += Math.min(1, (stepStats.enrich_count ?? 0) / txnDictCount) * 50;
pct += Math.min(1, (stepStats.save_count ?? 0) / txnDictCount) * 20;
}
return Math.round(Math.min(100, pct));
}
const stepLabels = ["Extract", "Raw Expense", "Enrich", "Save"];
function computeActiveStep(status: FetchRequestStatus, seenSteps: Set<string>): number {
if (status === "completed") return stepLabels.length;
if (seenSteps.has("save_expenses/completed") || seenSteps.has("complete/completed")) return stepLabels.length;
if (seenSteps.has("save_expenses") || seenSteps.has("complete")) return 3;
if (seenSteps.has("enrich/completed")) return 3;
if (seenSteps.has("enrich")) return 2;
if (seenSteps.has("txn_dicts/completed") || status === "raw_expenses_done") return 2;
if (seenSteps.has("txn_dicts")) return 1;
if (seenSteps.has("txn_blocks/completed")) return 1;
if (seenSteps.has("raw_lines") || seenSteps.has("txn_blocks")) return 0;
if (status === "processing" || status === "paused") return 0;
return -1;
}
function formatProgressMessage(msg: ProgressMessage): string {
if (msg.lines !== undefined) return `${msg.lines} lines`;
if (msg.blocks !== undefined) return `${msg.blocks} blocks`;
if (msg.count !== undefined && msg.unit) return `${msg.count} ${msg.unit}`;
if (msg.count !== undefined) return `${msg.count} items`;
if (msg.raw_ocr_line) return `"${msg.raw_ocr_line.slice(0, 60)}${msg.raw_ocr_line.length > 60 ? "…" : ""}"`;
if (msg.error) return msg.error.slice(0, 80);
return "";
} }
function sseIcon(status: SSEEvent["status"]) { function sseIcon(status: SSEEvent["status"]) {
switch (status) { switch (status) {
case "started": return <CircularProgress size={14} />; case "started": return <CircularProgress size={14} />;
case "completed": return <CheckCircleIcon sx={{ fontSize: 16, color: "success.main" }} />; case "completed": return <CheckCircleIcon sx={{ fontSize: 16, color: "success.main" }} />;
case "failed": return <ErrorIcon sx={{ fontSize: 16, color: "error.main" }} />;
case "skipped": return <RemoveCircleOutlineIcon sx={{ fontSize: 16, color: "text.disabled" }} />;
case "paused": return <WarningAmberIcon sx={{ fontSize: 16, color: "warning.main" }} />; case "paused": return <WarningAmberIcon sx={{ fontSize: 16, color: "warning.main" }} />;
case "progress": return (
<FiberManualRecordIcon
sx={{ fontSize: 14, color: "info.main" }}
/>
);
} }
} }
@@ -153,45 +97,37 @@ export default function FetchRequestDetail() {
const resolveMutation = useResolveAmbiguity(); const resolveMutation = useResolveAmbiguity();
const { data: ambiguities, refetch: refetchAmbiguities } = useFetchRequestAmbiguities(id!); const { data: ambiguities, refetch: refetchAmbiguities } = useFetchRequestAmbiguities(id!);
const [sseEvents, setSseEvents] = React.useState<SSEEvent[]>([]);
const [sseConnected, setSseConnected] = React.useState(false);
const [liveParsedCount, setLiveParsedCount] = React.useState<number | undefined>(undefined);
const [stepStats, setStepStats] = React.useState<Record<string, number>>({});
const [failNotif, setFailNotif] = React.useState<string | null>(null);
const sseRef = React.useRef<EventSource | null>(null);
const feedRef = React.useRef<HTMLDivElement>(null);
const txnBlockCount = React.useMemo(() => {
const blocks = (fetchRequest as any)?.source?.txn_blocks;
if (!blocks) return 0;
return Object.values(blocks).reduce(
(sum: number, list: any) => sum + (Array.isArray(list) ? list.length : 0),
0,
);
}, [fetchRequest]);
const stepMessages = React.useMemo(() => { const stepMessages = React.useMemo(() => {
const msgs: Record<number, string> = {}; const msgs: Record<number, string> = {};
const source = (fetchRequest as any)?.source; const source = (fetchRequest as any)?.source;
const rawLineCount = stepStats.raw_lines ?? (source?.raw_lines?.length ?? 0); if (source?.raw_lines?.length)
if (rawLineCount) msgs[0] = `${rawLineCount}`; msgs[0] = `${source.raw_lines.length} raw lines`;
const sourceDictCount = source?.txn_dict_count ?? source?.txn_dicts_count ?? 0; const blocks = source?.txn_blocks ?? {};
const dictLive = liveParsedCount ?? stepStats.txn_dicts ?? 0; const dicts = source?.txn_dicts ?? [];
const dictCurrent = Math.max(dictLive, sourceDictCount); const blockCount = typeof blocks === "object"
if (dictCurrent && txnBlockCount) msgs[1] = `${dictCurrent}/${txnBlockCount}`; ? Object.keys(blocks).length : Array.isArray(blocks) ? blocks.length : 0;
else if (dictCurrent) msgs[1] = `${dictCurrent}`; if (blockCount || dicts.length) {
const parts: string[] = [];
if (blockCount) parts.push(`${blockCount} blocks`);
if (dicts.length) parts.push(`${dicts.length} dicts`);
msgs[1] = parts.join(" · ");
}
const txnDictDenom = stepStats.txn_dicts ?? sourceDictCount; if (["enriched_done", "completed"].includes((fetchRequest as any)?.status))
if (stepStats.enrich_count && txnDictDenom) msgs[2] = `${stepStats.enrich_count}/${txnDictDenom}`; msgs[2] = "done";
else if (stepStats.enrich_count) msgs[2] = `${stepStats.enrich_count}`; if ((fetchRequest as any)?.status === "completed")
msgs[3] = (fetchRequest as any)?.completed_at
if (stepStats.save_count && txnDictDenom) msgs[3] = `${stepStats.save_count}/${txnDictDenom}`; ? new Date((fetchRequest as any).completed_at).toLocaleString() : "done";
else if (stepStats.save_count) msgs[3] = `${stepStats.save_count}`;
return msgs; return msgs;
}, [fetchRequest, stepStats, liveParsedCount, txnBlockCount]); }, [fetchRequest]);
const [sseEvents, setSseEvents] = React.useState<SSEEvent[]>([]);
const [sseConnected, setSseConnected] = React.useState(false);
const sseRef = React.useRef<EventSource | null>(null);
const feedRef = React.useRef<HTMLDivElement>(null);
React.useEffect(() => { React.useEffect(() => {
if (!id || !config?.baseUrl) return; if (!id || !config?.baseUrl) return;
@@ -205,36 +141,10 @@ export default function FetchRequestDetail() {
try { try {
const parsed: SSEEvent = JSON.parse(event.data); const parsed: SSEEvent = JSON.parse(event.data);
setSseEvents((prev) => [...prev, parsed]); setSseEvents((prev) => [...prev, parsed]);
if (parsed.status === "progress" && parsed.message.count !== undefined) {
if (parsed.step === "txn_dicts") setLiveParsedCount(parsed.message.count);
if (parsed.step === "enrich") setStepStats((prev) => ({ ...prev, enrich_count: parsed.message.count! }));
if (parsed.step === "save_expenses") setStepStats((prev) => ({ ...prev, save_count: parsed.message.count! }));
}
if (parsed.status === "completed" && parsed.message.count !== undefined) {
const stats: Record<string, number> = {};
if (parsed.step === "raw_lines" && parsed.message.lines !== undefined) stats.raw_lines = parsed.message.lines;
if (parsed.step === "txn_blocks" && parsed.message.blocks !== undefined) stats.txn_blocks = parsed.message.blocks;
if (parsed.step === "txn_dicts") stats.txn_dicts = parsed.message.count;
if (parsed.step === "enrich") stats.enrich_count = parsed.message.count;
if (parsed.step === "save_expenses") stats.save_count = parsed.message.count;
if (Object.keys(stats).length) {
setStepStats((prev) => ({ ...prev, ...stats }));
}
}
if (parsed.status === "paused") { if (parsed.status === "paused") {
refetchRequest(); refetchRequest();
refetchAmbiguities(); refetchAmbiguities();
} }
if (parsed.status === "failed") {
setFailNotif(parsed.message.error || "Fetch request failed");
refetchRequest();
}
if (parsed.status === "completed" || parsed.step === "resume_extract") {
refetchRequest();
}
} catch { } catch {
// ignore malformed events // ignore malformed events
} }
@@ -252,74 +162,12 @@ export default function FetchRequestDetail() {
} }
}, [sseEvents]); }, [sseEvents]);
const displayEvents = React.useMemo(() => {
const progressSteps = new Set(["txn_dicts", "enrich", "save_expenses"]);
const lastProgressIdx: Record<string, number> = {};
for (let i = sseEvents.length - 1; i >= 0; i--) {
const e = sseEvents[i];
if (progressSteps.has(e.step) && e.status === "progress" && lastProgressIdx[e.step] === undefined) {
lastProgressIdx[e.step] = i;
}
}
const terminalStatuses = new Set(["completed", "skipped", "paused", "failed"]);
return sseEvents.filter((e, i) => {
if (progressSteps.has(e.step) && e.status === "progress") return i === lastProgressIdx[e.step];
if (e.status === "started") {
return !sseEvents.slice(i + 1).some(
(later) => later.step === e.step && terminalStatuses.has(later.status),
);
}
return true;
});
}, [sseEvents]);
const seenSteps = React.useMemo(() => {
const steps = new Set<string>();
for (const evt of sseEvents) {
steps.add(evt.step);
if (evt.status === "completed") steps.add(`${evt.step}/completed`);
if (evt.status === "failed") steps.add(`${evt.step}/failed`);
if (evt.status === "started") steps.add(`${evt.step}/started`);
if (evt.status === "progress") steps.add(`${evt.step}/progress`);
}
return steps;
}, [sseEvents]);
const displayParsedCount = React.useMemo(() => {
if (liveParsedCount && liveParsedCount > 0) return liveParsedCount;
const source = (fetchRequest as any)?.source;
const persistedCount = source?.txn_dict_count ?? source?.txn_dicts_count ?? 0;
if (persistedCount > 0) return persistedCount;
const dicts = source?.txn_dicts;
if (Array.isArray(dicts) && dicts.length > 0) return dicts.length;
return 0;
}, [liveParsedCount, fetchRequest]);
const txnDictCount = React.useMemo(() => {
const source = (fetchRequest as any)?.source;
if (stepStats.txn_dicts && stepStats.txn_dicts > 0) return stepStats.txn_dicts;
return source?.txn_dict_count ?? source?.txn_dicts_count ?? 0;
}, [fetchRequest, stepStats]);
const progressPercent = React.useMemo(
() => computeProgressPercent(
(fetchRequest as any)?.status as FetchRequestStatus ?? "pending",
displayParsedCount,
seenSteps,
stepStats,
txnBlockCount,
txnDictCount,
),
[fetchRequest, displayParsedCount, seenSteps, stepStats, txnBlockCount, txnDictCount],
);
const handleRetry = async () => { const handleRetry = async () => {
if (!id) return; if (!id) return;
try { try {
await updateMutation.mutateAsync({ id, data: { status: "pending" } }); await updateMutation.mutateAsync({ id, data: { status: "pending" } });
} catch (err: any) { } catch {
setFailNotif(formatApiError(err)); // handled by react query
} }
}; };
@@ -351,7 +199,7 @@ export default function FetchRequestDetail() {
} }
const req = fetchRequest as any; const req = fetchRequest as any;
const activeStep = computeActiveStep(req.status as FetchRequestStatus, seenSteps); const activeStep = statusToActiveStep(req.status);
const retryCount = req.retry_count ?? 0; const retryCount = req.retry_count ?? 0;
const isRetryExhausted = retryCount >= RETRY_MAX; const isRetryExhausted = retryCount >= RETRY_MAX;
const pendingAmbiguities = ambiguities?.filter((a: any) => a.status === "pending") ?? []; const pendingAmbiguities = ambiguities?.filter((a: any) => a.status === "pending") ?? [];
@@ -401,28 +249,6 @@ export default function FetchRequestDetail() {
)} )}
</Box> </Box>
<Box sx={{ mb: 2 }}>
<Box sx={{ display: "flex", alignItems: "center", gap: 2, mb: 0.5 }}>
<Typography variant="caption" color="text.secondary">
Overall Progress
</Typography>
{["processing", "paused"].includes(req.status) && displayParsedCount > 0 && (
<Typography variant="caption" fontWeight={600} color="info.main">
Validated: {displayParsedCount} transactions
</Typography>
)}
</Box>
<LinearProgress
variant="determinate"
value={progressPercent}
color={req.status === "failed" ? "error" : req.status === "completed" ? "success" : "primary"}
sx={{ borderRadius: 1, height: 8, transition: "width 0.3s ease" }}
/>
<Typography variant="caption" color="text.secondary" sx={{ mt: 0.25, display: "block" }}>
{progressPercent}%
</Typography>
</Box>
<Box sx={{ display: "flex", alignItems: "center", gap: 2 }}> <Box sx={{ display: "flex", alignItems: "center", gap: 2 }}>
<Box sx={{ flex: 1, maxWidth: 300 }}> <Box sx={{ flex: 1, maxWidth: 300 }}>
<Typography variant="caption" color="text.secondary"> <Typography variant="caption" color="text.secondary">
@@ -534,12 +360,12 @@ export default function FetchRequestDetail() {
gap: 1, gap: 1,
}} }}
> >
{displayEvents.length === 0 ? ( {sseEvents.length === 0 ? (
<Typography variant="body2" color="text.disabled" sx={{ textAlign: "center", py: 2 }}> <Typography variant="body2" color="text.disabled" sx={{ textAlign: "center", py: 2 }}>
Waiting for events... Waiting for events...
</Typography> </Typography>
) : ( ) : (
displayEvents.map((evt, i) => ( sseEvents.map((evt, i) => (
<Box <Box
key={i} key={i}
sx={{ sx={{
@@ -556,9 +382,9 @@ export default function FetchRequestDetail() {
<Typography variant="body2" fontWeight={600}> <Typography variant="body2" fontWeight={600}>
{evt.step.replace(/_/g, " ")} {evt.step.replace(/_/g, " ")}
</Typography> </Typography>
{evt.message && formatProgressMessage(evt.message) && ( {evt.message && (
<Typography variant="caption" color="text.secondary"> <Typography variant="caption" color="text.secondary">
{formatProgressMessage(evt.message)} {evt.message}
</Typography> </Typography>
)} )}
</Box> </Box>
@@ -571,22 +397,32 @@ export default function FetchRequestDetail() {
</Box> </Box>
</Paper> </Paper>
{hasAmbiguities && ( {(hasAmbiguities || req.status === "paused") && (
<Paper sx={{ p: 3, borderRadius: 4, mb: 3 }} variant="outlined"> <Paper sx={{ p: 3, borderRadius: 4, mb: 3 }} variant="outlined">
<Typography variant="subtitle1" fontWeight={600} gutterBottom> <Typography variant="subtitle1" fontWeight={600} gutterBottom>
Ambiguity Resolution Ambiguity Resolution
</Typography> </Typography>
{allResolved ? ( {ambiguitiesLoading ? (
<Box sx={{ display: "flex", alignItems: "center", gap: 1, py: 2 }}>
<CircularProgress size={16} />
<Typography variant="body2" color="text.secondary">Loading ambiguities...</Typography>
</Box>
) : allResolved ? (
<Alert severity="success" sx={{ mb: 2, borderRadius: 2 }}> <Alert severity="success" sx={{ mb: 2, borderRadius: 2 }}>
All ambiguities resolved pipeline will resume on next poll cycle All ambiguities resolved pipeline will resume on next poll cycle
</Alert> </Alert>
) : !hasAmbiguities ? (
<Alert severity="info" sx={{ mb: 2, borderRadius: 2 }}>
Pipeline paused no ambiguities found
</Alert>
) : ( ) : (
<Alert severity="warning" sx={{ mb: 2, borderRadius: 2 }}> <Alert severity="warning" sx={{ mb: 2, borderRadius: 2 }}>
Pipeline paused resolve ambiguities to continue Pipeline paused resolve ambiguities to continue
</Alert> </Alert>
)} )}
{hasAmbiguities && (
<Box sx={{ display: "flex", flexDirection: "column", gap: 2 }}> <Box sx={{ display: "flex", flexDirection: "column", gap: 2 }}>
{ambiguities.map((ambiguity: any) => { {ambiguities.map((ambiguity: any) => {
const isResolved = ambiguity.status === "resolved"; const isResolved = ambiguity.status === "resolved";
@@ -658,18 +494,9 @@ export default function FetchRequestDetail() {
); );
})} })}
</Box> </Box>
)}
</Paper> </Paper>
)} )}
<Snackbar
open={!!failNotif}
autoHideDuration={6000}
onClose={() => setFailNotif(null)}
anchorOrigin={{ vertical: "bottom", horizontal: "center" }}
>
<Alert severity="error" onClose={() => setFailNotif(null)} sx={{ borderRadius: 2 }}>
{failNotif}
</Alert>
</Snackbar>
</Container> </Container>
); );
} }

View File

@@ -55,7 +55,7 @@ import type {
FileSource, FileSource,
EmailSource, EmailSource,
} from "./features/fetch-requests"; } from "./features/fetch-requests";
import { RETRY_MAX, formatApiError } from "./features/fetch-requests"; import { RETRY_MAX } from "./features/fetch-requests";
import { useNavigate } from "react-router-dom"; import { useNavigate } from "react-router-dom";
import { useResourceByName, useConfig } from "../react-openapi"; import { useResourceByName, useConfig } from "../react-openapi";
@@ -129,7 +129,7 @@ export default function FetchRequests() {
const config = useConfig(); const config = useConfig();
const fetchRes = config?.resources.find((r: any) => r.name === "fetch-requests"); const fetchRes = config?.resources.find((r: any) => r.name === "fetch-requests");
const formatOptions: string[] = fetchRes?.fields?.source?.schema?.format?.options as string[] ?? []; const formatOptions: string[] = (fetchRes?.fields?.source?.schema?.format?.options as string[]) ?? ["axis", "icici_ocr"];
const createMutation = useCreateFetchRequest(); const createMutation = useCreateFetchRequest();
const updateMutation = useUpdateFetchRequest(); const updateMutation = useUpdateFetchRequest();
@@ -180,7 +180,7 @@ export default function FetchRequests() {
if (err?.response?.status === 409) { if (err?.response?.status === 409) {
setSnackbar({ message: "Duplicate — same fingerprint already exists", severity: "error" }); setSnackbar({ message: "Duplicate — same fingerprint already exists", severity: "error" });
} else { } else {
setSnackbar({ message: formatApiError(err) || "Failed to create fetch request", severity: "error" }); setSnackbar({ message: err?.response?.data?.detail || "Failed to create fetch request", severity: "error" });
} }
} }
}; };
@@ -345,7 +345,7 @@ export default function FetchRequests() {
input={<OutlinedInput label="Status" />} input={<OutlinedInput label="Status" />}
renderValue={(selected) => (selected as string[]).join(", ")} renderValue={(selected) => (selected as string[]).join(", ")}
> >
{(config?.enums?.FetchRequestStatus ?? []).map((s: string) => ( {["pending", "processing", "paused", "raw_expenses_done", "enriched_done", "completed", "failed"].map((s) => (
<MenuItem key={s} value={s}>{s.replace(/_/g, " ")}</MenuItem> <MenuItem key={s} value={s}>{s.replace(/_/g, " ")}</MenuItem>
))} ))}
</Select> </Select>

View File

@@ -13,8 +13,6 @@ export interface FileSource {
raw_lines?: string[]; raw_lines?: string[];
txn_blocks?: Record<string, any>; txn_blocks?: Record<string, any>;
txn_dicts?: Record<string, any>[]; txn_dicts?: Record<string, any>[];
txn_dict_count?: number;
txn_dicts_count?: number;
} }
export interface EmailSource { export interface EmailSource {
@@ -22,8 +20,6 @@ export interface EmailSource {
from_email?: string; from_email?: string;
subject?: string; subject?: string;
raw_terms?: string[]; raw_terms?: string[];
txn_dict_count?: number;
txn_dicts_count?: number;
} }
export interface FetchRequestCreate { export interface FetchRequestCreate {
@@ -84,27 +80,10 @@ export interface ResolveAmbiguityPayload {
}; };
} }
export type SSEEventStep =
| "load_content" | "raw_lines" | "txn_blocks" | "txn_dicts"
| "resume_extract" | "extract" | "paused" | "complete" | "enrich"
| "save_expenses" | "pipeline";
export type SSEEventStatus =
| "started" | "completed" | "skipped" | "paused" | "progress" | "failed";
export interface ProgressMessage {
lines?: number;
blocks?: number;
count?: number;
unit?: string;
raw_ocr_line?: string;
error?: string;
}
export interface SSEEvent { export interface SSEEvent {
step: SSEEventStep; step: string;
status: SSEEventStatus; status: "started" | "completed" | "paused";
message: ProgressMessage; message: string;
} }
export interface FetchRequestFilters { export interface FetchRequestFilters {
@@ -113,21 +92,4 @@ export interface FetchRequestFilters {
source_type?: "file" | "email"; source_type?: "file" | "email";
} }
export function formatApiError(err: any): string {
if (!err?.response) return err?.message || "Request failed";
const data = err.response.data;
const status = err.response.status;
if (status === 422 && Array.isArray(data?.detail)) {
return data.detail.map((d: any) => {
const field = d.loc?.filter((s: string) => s !== "body").pop() || "field";
if (d.type === "value_error.missing") return `Missing: ${field}`;
return `${field}: ${d.msg}`;
}).join("; ");
}
if (typeof data?.detail === "string") return data.detail;
return `Request failed (${status})`;
}
export const RETRY_MAX = 3; export const RETRY_MAX = 3;

View File

@@ -11,11 +11,8 @@ export type {
AmbiguityCandidate, AmbiguityCandidate,
ResolveAmbiguityPayload, ResolveAmbiguityPayload,
SSEEvent, SSEEvent,
SSEEventStep,
SSEEventStatus,
ProgressMessage,
} from "./fetch-requests.models"; } from "./fetch-requests.models";
export { RETRY_MAX, formatApiError } from "./fetch-requests.models"; export { RETRY_MAX } from "./fetch-requests.models";
export { export {
useFetchRequestsList, useFetchRequestsList,
useFetchRequest, useFetchRequest,

View File

@@ -50,18 +50,6 @@ export const configuration: Record<string, ResourceOverride> = {
} }
}, },
}, },
accounts: {
enumOption: {
key: 'id',
value: '{name} - XXXX{number}'
}
},
tags: {
enumOption: {
key: 'id',
value: '{icon} {name}'
}
},
}; };
export const profileConfiguration = { export const profileConfiguration = {