Architecting Pack: Sub-1.2s Multimodal AI OCR, Local-First Reactivity, and Zero-Error Parcel Lockers

┌────────────────────────────────────────────────────────────────────────┐
│                   Pack Mobile & Locker Frontend                        │
│                                                                        │
│  [Camera Shutter / Barcode Scanner]                                    │
│                 │                                                      │
│                 ▼                                                      │
│  [Image Optimization Engine (WebP / 1200px max, 85% Q)]                │
│                 │                                                      │
│                 ▼                                                      │
│  [Legend-State v3 Reactive Local-First Cache (IndexedDB / MMKV)]        │
└─────────────────┬──────────────────────────────────────────────────────┘
                  │ (Parallel Async Dispatches)
       ┌──────────┴──────────────────────────────┐
       ▼                                         ▼
┌───────────────────────────────┐     ┌──────────────────────────────────┐
│ Google Cloud Platform         │     │ Amazon Web Services (CDK)        │
│                               │     │                                  │
│ [Cloud Run Multimodal Worker] │     │ [API Gateway HTTP v2]            │
│               │               │     │                 │                │
│ [Gemini 2.5 Flash OCR Engine] │     │ [Rust / Node.js Micro-Lambdas]   │
│ (Sub-1.2s Extraction)         │     │                 │                │
│               │               │     │ [DynamoDB Single-Table Storage]  │
│ [Firestore Real-Time DB]      │     │ (Audit Log & Enterprise History) │
│ (Instant Mobile Push Sync)    │     │                 │                │
└───────────────────────────────┘     │ [EventBridge 48h Dwell Schedulers│
                                      │ & WhatsApp / SES Dispatchers]    │
                                      └──────────────────────────────────┘

// cloud-run/ocr/extractParcelMetadata.ts
import { GoogleGenAI, Type } from "@google/genai";

const ai = new GoogleGenAI({ apiKey: process.env.GEMINI_API_KEY });

const ParcelExtractionSchema = {
  type: Type.OBJECT,
  properties: {
    carrier: {
      type: Type.STRING,
      enum: ["Correios", "MercadoLivre", "Loggi", "Jadlog", "TotalExpress", "FedEx", "Amazon", "Other"],
      description: "Identified logistics courier or transport provider."
    },
    trackingCode: {
      type: Type.STRING,
      description: "Primary tracking code or package identifier."
    },
    recipientName: {
      type: Type.STRING,
      description: "Clean recipient / consignee name without salutations."
    },
    unitIdentifier: {
      type: Type.STRING,
      description: "Apartment, suite, house, or commercial unit number."
    },
    blockIdentifier: {
      type: Type.STRING,
      nullable: true,
      description: "Building tower, block, or section (e.g., 'Torre B', 'Bloco 2')."
    },
    confidenceScore: {
      type: Type.NUMBER,
      description: "Extraction confidence between 0.0 and 1.0."
    }
  },
  required: ["carrier", "trackingCode", "recipientName", "unitIdentifier", "confidenceScore"]
};

export async function processShippingLabel(imageBuffer: Buffer, mimeType: string) {
  const response = await ai.models.generateContent({
    model: "gemini-2.5-flash",
    contents: [
      {
        inlineData: {
          mimeType,
          data: imageBuffer.toString("base64")
        }
      },
      {
        text: `You are an expert logistics parcel intake parser. Analyze this package shipping label.
Extract the logistics carrier, primary tracking barcode/number, recipient full name, apartment/unit number, and block/tower if present.
Normalize recipient names into Title Case. Extract only verified information with high confidence.`
      }
    ],
    config: {
      responseMimeType: "application/json",
      responseSchema: ParcelExtractionSchema,
      temperature: 0.1, // Near-deterministic extraction
    }
  });

  return JSON.parse(response.text!);
}

// app/lib/state/parcel-intake.ts
import { observable } from "@legendapp/state";
import { persistObservable } from "@legendapp/state/persist";
import { ObservablePersistMMKV } from "@legendapp/state/persist-plugins/mmkv";

export interface LocalParcel {
  id: string;
  trackingCode: string;
  carrier: string;
  recipientName: string;
  unit: string;
  block?: string;
  status: "pending_upload" | "synced" | "notified" | "picked_up";
  capturedAt: number;
  localImagePath: string;
}

export const parcelStore$ = observable<{
  parcels: Record<string, LocalParcel>;
  activeIntakeCount: number;
}>({
  parcels: {},
  activeIntakeCount: 0,
});

// Configure instant sync to native encrypted MMKV storage
persistObservable(parcelStore$, {
  local: "pack_parcel_intake_v3",
  pluginLocal: ObservablePersistMMKV,
});

export function registerScannedParcel(parcel: Omit<LocalParcel, "status" | "capturedAt">) {
  const id = crypto.randomUUID();
  
  // 1. Instant local write (0ms UI lag)
  parcelStore$.parcels[id].set({
    ...parcel,
    id,
    status: "pending_upload",
    capturedAt: Date.now(),
  });
  
  // 2. Asynchronous background queue dispatches sync worker
  queueMicrotask(() => {
    dispatchBackgroundSync(id);
  });
}

[Parcel Intake Confirmed]
          │
          ▼
[Create EventBridge One-Time Schedule] ── (Target: arn:aws:scheduler:48h)
          │
          ├── (If picked up before 48h) ──► [Delete Schedule API]
          │
          └── (If still in locker at 48h) ──► [Trigger EscalateLambda]
                                                     │
                                                     ├──► [WhatsApp Interactive Reminder]
                                                     ├──► [Building Manager Daily Digest]
                                                     └──► [Apply Tiered Holding Fee]

// infrastructure/lib/dwell-time-scheduler.ts
import * as scheduler from "aws-cdk-lib/aws-scheduler";
import * as iam from "aws-cdk-lib/aws-iam";

export function createDwellReminderSchedule(
  scope: Construct,
  parcelId: string,
  dwellExecutionTime: string, // ISO-8601 UTC timestamp 48 hours in future
  targetLambdaArn: string,
  roleArn: string
) {
  return new scheduler.CfnSchedule(scope, `DwellReminder-${parcelId}`, {
    name: `dwell-reminder-${parcelId}`,
    scheduleExpression: `at(${dwellExecutionTime})`,
    flexibleTimeWindow: { mode: "OFF" },
    target: {
      arn: targetLambdaArn,
      roleArn: roleArn,
      input: JSON.stringify({
        action: "NOTIFY_OVERDUE_DWELL",
        parcelId,
      }),
    },
  });
}

// app/lib/security/handover-token.ts
import { hmac } from "@noble/hashes/hmac";
import { sha256 } from "@noble/hashes/sha256";

export function generateDynamicHandoverToken(parcelId: string, secretKey: Uint8Array): {
  pin: string;
  qrPayload: string;
} {
  const timestampWindow = Math.floor(Date.now() / 30000); // 30-second rotating window
  const message = new TextEncoder().encode(`${parcelId}:${timestampWindow}`);
  
  const hmacDigest = hmac(sha256, secretKey, message);
  
  // Extract 6-digit PIN from dynamic binary truncation
  const offset = hmacDigest[hmacDigest.length - 1] & 0x0f;
  const binaryCode =
    ((hmacDigest[offset] & 0x7f) << 24) |
    ((hmacDigest[offset + 1] & 0xff) << 16) |
    ((hmacDigest[offset + 2] & 0xff) << 8) |
    (hmacDigest[offset + 3] & 0xff);

  const pin = (binaryCode % 1000000).toString().padStart(6, "0");
  const qrPayload = JSON.stringify({
    p: parcelId,
    w: timestampWindow,
    s: Buffer.from(hmacDigest.slice(0, 16)).toString("hex"),
  });

  return { pin, qrPayload };
}