4ventures-client-portal/scripts/backfill_deal_pipelines.ts

#!/usr/bin/env npx tsx
/**
 * Backfill deals.pipeline_id / pipeline_name / source / name from Clientify
 * for one tenant so KPI Online filter + attribution work for MoM/YoY.
 *
 *   TENANT_SLUG=bps npx tsx scripts/backfill_deal_pipelines.ts
 */
import { config } from 'dotenv';
config({ path: '.env.local' });

const CFY = 'https://api.clientify.com/v1';
const url = process.env.NEXT_PUBLIC_SUPABASE_URL!;
const key = process.env.SUPABASE_SERVICE_ROLE_KEY!;

function envKey(slug: string) {
  const s = slug.toUpperCase().replace(/-/g, '_');
  return (
    process.env[`CLIENTIFY_API_KEY_${s}`] ||
    process.env.CLIENTIFY_API_KEY ||
    ''
  );
}

function isPaidAds(s: string) {
  const x = s.toLowerCase();
  return (
    /^(meta|facebook|instagram|google|google ads|google-ads|google_ads)/.test(
      x
    ) ||
    x.includes('facebook') ||
    x.includes('google ads')
  );
}

function normalizeSource(s: string | null | undefined): string | null {
  const x = (s || '').toLowerCase().trim();
  if (!x) return null;
  if (isPaidAds(x)) return x.includes('google') ? 'google' : 'meta';
  if (x.includes('blue parrot') || x.includes('woocommerce') || x === 'ecommerce')
    return 'ecommerce';
  if (x.includes('recomend') || x.includes('amigo')) return 'referral';
  if (
    x.includes('nivel') ||
    x.includes('prueba') ||
    x.startsWith('test') ||
    x === 'general' ||
    x === 'organic' ||
    x === 'unknown' ||
    x === 'import' ||
    x === 'manual'
  )
    return 'organic';
  if (x.startsWith('integrations')) return 'integrations-api';
  return x.slice(0, 80);
}

function inferSource(d: {
  deal_source?: string;
  contact_source?: string;
  tags?: string[];
  name?: string;
}): string | null {
  const fromDeal = d.deal_source || d.contact_source || '';
  const tags = (d.tags || []).join(' ').toLowerCase();
  if (isPaidAds(fromDeal)) return normalizeSource(fromDeal);
  if (/facebook|instagram|\bmeta\b|\bfb\b/.test(tags)) return 'meta';
  if (/google/.test(tags)) return 'google';
  if (fromDeal) return normalizeSource(fromDeal);
  if (/woocommerce|ecommerce/.test(tags) || /^order\s+\d+/i.test(d.name || ''))
    return 'ecommerce';
  return null;
}

async function main() {
  const slug = (process.env.TENANT_SLUG || 'bps').trim();
  const token = envKey(slug);
  if (!token || !url || !key) throw new Error('missing env');

  const tenants = await (
    await fetch(
      `${url}/rest/v1/tenants?slug=eq.${encodeURIComponent(slug)}&select=id,name`,
      { headers: { apikey: key, Authorization: `Bearer ${key}` } }
    )
  ).json();
  const tid = tenants[0]?.id as string;
  if (!tid) throw new Error(`tenant ${slug} not found`);
  console.log('tenant', tenants[0].name, tid);

  // Load warehouse won deals needing backfill (or all)
  const wh: {
    id: string;
    crm_deal_id: string;
    lead_id: string | null;
    pipeline_id: string | null;
  }[] = [];
  for (let offset = 0; offset < 100000; offset += 1000) {
    const chunk = await (
      await fetch(
        `${url}/rest/v1/deals?tenant_id=eq.${tid}&stage=eq.won&select=id,crm_deal_id,lead_id,pipeline_id&order=won_at.desc.nullslast&limit=1000&offset=${offset}`,
        { headers: { apikey: key, Authorization: `Bearer ${key}` } }
      )
    ).json();
    if (!Array.isArray(chunk) || !chunk.length) break;
    wh.push(...chunk);
    if (chunk.length < 1000) break;
  }
  const byCrm = new Map(wh.map((d) => [d.crm_deal_id.replace(/^clientify:/, ''), d]));
  console.log('warehouse won', wh.length);

  // Lead sources for attribution priority
  const leadSrc = new Map<string, string>();
  const leadIds = [...new Set(wh.map((d) => d.lead_id).filter(Boolean))] as string[];
  for (let i = 0; i < leadIds.length; i += 200) {
    const ids = leadIds.slice(i, i + 200);
    const rows = await (
      await fetch(
        `${url}/rest/v1/leads?id=in.(${ids.join(',')})&select=id,source`,
        { headers: { apikey: key, Authorization: `Bearer ${key}` } }
      )
    ).json();
    if (Array.isArray(rows)) {
      for (const r of rows) if (r.source) leadSrc.set(r.id, r.source);
    }
  }

  let page = 1;
  let patched = 0;
  let seen = 0;
  while (page <= 200) {
    const j = await (
      await fetch(
        `${CFY}/deals/?page_size=100&page=${page}&status=3&ordering=-actual_closed_date`,
        { headers: { Authorization: `Token ${token}` } }
      )
    ).json();
    const results = j.results || [];
    if (!results.length) break;

    for (const d of results) {
      seen++;
      const did = String(d.id);
      const row = byCrm.get(did);
      if (!row) continue;

      const pipeUrl = String(d.pipeline || '');
      const pipeline_id = pipeUrl.includes('/pipelines/')
        ? pipeUrl.replace(/\/$/, '').split('/').pop() || null
        : null;
      const pipeline_name = d.pipeline_desc || null;

      const fromLead = row.lead_id ? leadSrc.get(row.lead_id) : null;
      let source: string | null = null;
      if (fromLead && isPaidAds(fromLead)) source = normalizeSource(fromLead);
      else {
        const inferred = inferSource(d);
        if (inferred && isPaidAds(inferred)) source = inferred;
        else if (fromLead) source = normalizeSource(fromLead);
        else source = inferred;
      }
      if (
        !source &&
        (pipeline_id === '77547' || /online/i.test(pipeline_name || ''))
      ) {
        source = 'ecommerce';
      }

      const r = await fetch(`${url}/rest/v1/deals?id=eq.${row.id}`, {
        method: 'PATCH',
        headers: {
          apikey: key,
          Authorization: `Bearer ${key}`,
          'Content-Type': 'application/json',
          Prefer: 'return=minimal',
        },
        body: JSON.stringify({
          pipeline_id,
          pipeline_name,
          source,
          name: d.name || null,
        }),
      });
      if (r.ok) patched++;
    }

    const oldest = (results[results.length - 1].actual_closed_date || '').slice(
      0,
      10
    );
    console.log(`page ${page} seen=${seen} patched=${patched} oldest=${oldest}`);
    if (!j.next) break;
    // Stop if we've gone past all warehouse deals' likely range
    if (oldest && oldest < '2024-01-01') break;
    page++;
  }

  console.log('DONE', { seen, patched, warehouse: wh.length });
}

main().catch((e) => {
  console.error(e);
  process.exit(1);
});