Voice-Native Document Intelligence

// ============================================================================
// CALL 1: Get the raw response from Gemini (WITH METRICS)
// ============================================================================
async function getGeminiRawResponse(
  history: any[],
  query: string,
  context: string,
  isNarrationRequest: boolean,
  voiceMode: 'standard' | 'expressive' = 'standard',
  trafficType: string = 'user'
): Promise<string> {
  const startTime = Date.now();

  return tracer.trace('gemini.request', { resource: 'getGeminiRawResponse' }, async (span) => {
    span?.setTag('voice_mode', voiceMode);
    console.log('[Call1:getGeminiRawResponse] Getting raw response from Gemini');

    const model: GenerativeModel = getVertexAI().getGenerativeModel({
      model: modelName,
      generationConfig: {
        maxOutputTokens: 8192,
        temperature: 0.7,
      }
    });

    const validHistory: Content[] = history.map((h: any) => ({
      role: h.role,
      parts: h.parts || [{ text: h.text }]
    }));

    let systemPrompt: string;
    let textToRead = context;

    if (isNarrationRequest) {
      const extracted = extractSection(context, query);
      if (extracted) {
        textToRead = extracted;
        console.log('[Call1] Extracted chapter/section');
      }

      systemPrompt = `You are a professional audiobook narrator. Read the following text aloud EXACTLY as written, word-for-word. 

CRITICAL RULES:
1. Do NOT summarize.
2. Do NOT paraphrase.
3. Do NOT skip any words.
4. Output the spoken text ONLY.
5. Do NOT include narration notes, stage directions, or descriptions of tone (e.g., (softly), (muttering), (friendly)).
6. Do NOT include speaker labels.

TEXT TO READ:
${textToRead}`;
    } else {
      systemPrompt = `You are a voice assistant answering questions about a document. Answer based STRICTLY on the provided context. Do NOT use training data to fill gaps.

CONTEXT:
${context}`;
    }

    span?.setTag('llm.model', modelName);
    span?.setTag('llm.is_narration', isNarrationRequest);

    const chat = model.startChat({
      history: validHistory,
      systemInstruction: {
        role: 'system',
        parts: [{ text: systemPrompt }]
      }
    });

    const message = isNarrationRequest && extractSection(context, query)
      ? `Please begin reading.`
      : query;

    try {
      const result = await chat.sendMessage(message);
      const response = await result.response;
      const rawText = response.candidates?.[0].content.parts[0].text || '';

      const usage = response.usageMetadata;
      if (usage && usage.promptTokenCount && usage.candidatesTokenCount) {
        span?.setTag('llm.prompt_tokens', usage.promptTokenCount);
        span?.setTag('llm.completion_tokens', usage.candidatesTokenCount);
        span?.setTag('llm.total_tokens', usage.totalTokenCount);

        // 📊 RECORD METRICS
        MetricsCollector.recordTokens(
          usage.promptTokenCount,
          usage.candidatesTokenCount,
          voiceMode,
          trafficType
        );
        MetricsCollector.recordLLMCost(
          usage.promptTokenCount,
          usage.candidatesTokenCount,
          voiceMode,
          trafficType
        );
      }

      console.log('[Call1] Got raw response, length:', rawText.length);
      return rawText;
    } catch (error: any) {
      span?.setTag('error', true);
      span?.setTag('error.message', error.message);

      // 📊 RECORD ERROR METRIC
      MetricsCollector.recordError('call1_error', voiceMode, trafficType);

      throw error;
    }
  });
}

// ============================================================================
// CALL 2: Add emotion tags using few-shot learning (WITH METRICS)
// ============================================================================
async function addEmotionTagsWithFewShot(
  rawText: string,
  persona: string,
  contextHint?: string,
  voiceMode: 'standard' | 'expressive' = 'expressive',
  trafficType: string = 'user'
): Promise<string> {
  return tracer.trace('gemini.request', { resource: 'addEmotionTagsWithFewShot' }, async (span) => {
    span?.setTag('voice_mode', voiceMode);
    console.log('[Call2:addEmotionTagsWithFewShot] Starting with text length:', rawText.length);

    const model: GenerativeModel = getVertexAI().getGenerativeModel({
      model: modelName,
      generationConfig: {
        temperature: 0.2,
        topP: 0.7,
      }
    });

    const fewShotExamples = getFewShotExamples(persona);

    const history: Content[] = [];
    for (const example of fewShotExamples) {
      history.push({
        role: 'user',
        parts: [{ text: example.userInput }]
      });
      history.push({
        role: 'assistant',
        parts: [{ text: example.expectedOutput }]
      });
    }

    const systemPrompt = `SYSTEM: You add emotion tags to text ONLY. Nothing else.

TAGS ONLY: [excited] [nervous] [frustrated] [sorrowful] [calm] [sigh] [laughs] [gulps] [gasps] [whispers] [pauses] [hesitates] [wearily] [warmly] [playfully] [stunned] [intrigued] [reflectively] [passionately] [poetically]

YOUR JOB:
1. Take input text
2. Add [tag] BEFORE words - no parentheses, no narration notes
3. Keep EVERY word exactly the same
4. Return ONLY the tagged text

NEVER add: (notes), descriptions, or stage directions. 
ONLY add: [emotion_tag] tags in square brackets.`;

    span?.setTag('llm.model', modelName);
    span?.setTag('llm.persona', persona);

    const chat = model.startChat({
      history,
      systemInstruction: {
        role: 'system',
        parts: [{ text: systemPrompt }]
      }
    });

    const userMessage = `Add ONLY emotion tags [like this] to this text. Do NOT add parentheses or narration notes. PRESERVE ALL WORDS:

${rawText}`;

    console.log('[Call2] Sending to Gemini...');

    try {
      const result = await chat.sendMessage(userMessage);
      const response = await result.response;
      let taggedText = response.candidates?.[0].content.parts[0].text || rawText;

      const usage = response.usageMetadata;
      if (usage && usage.promptTokenCount && usage.candidatesTokenCount) {
        span?.setTag('llm.prompt_tokens', usage.promptTokenCount);
        span?.setTag('llm.completion_tokens', usage.candidatesTokenCount);
        span?.setTag('llm.total_tokens', usage.totalTokenCount);

        // 📊 RECORD METRICS
        MetricsCollector.recordTokens(
          usage.promptTokenCount,
          usage.candidatesTokenCount,
          voiceMode,
          trafficType
        );
        MetricsCollector.recordLLMCost(
          usage.promptTokenCount,
          usage.candidatesTokenCount,
          voiceMode,
          trafficType
        );
      }

      taggedText = taggedText.replace(/\([^)]*?\)/g, '').trim();

      console.log('[Call2] After cleanup, length:', taggedText.length);

      return taggedText;
    } catch (error: any) {
      span?.setTag('error', true);
      span?.setTag('error.message', error.message);

      // 📊 RECORD ERROR METRIC
      MetricsCollector.recordError('call2_error', voiceMode, trafficType);

      throw error;
    }
  });
}

// ============================================================================
// STREAMING RESPONSE (WITH METRICS)
// ============================================================================
export async function* getGeminiStream(
  history: Content[],
  query: string,
  context: string,
  isNarrationRequest: boolean = false,
  expressiveMode: boolean = false,
  persona: string = 'narrative',
  trafficType: string = 'user'
) {
  const startTime = Date.now();
  console.log('[getGeminiStream] 🚀 Starting two-call process', { expressiveMode, persona, trafficType });

  try {
    console.log('[getGeminiStream] Is narration request:', isNarrationRequest);

    // CALL 1: Get raw response
    console.log('[getGeminiStream] ========== CALL 1: RAW RESPONSE ==========');
    const voiceMode = expressiveMode ? 'expressive' : 'standard';

    const rawResponse = await getGeminiRawResponse(history, query, context, isNarrationRequest, voiceMode, trafficType);

    console.log('[getGeminiStream] CALL 1 OUTPUT (length:', rawResponse.length, ')');

    if (!rawResponse.trim()) {
      yield 'No response generated.';

      // 📊 RECORD METRICS
      const duration = Date.now() - startTime;

      return;
    }

    // CALL 2: Add emotion tags if expressive mode
    let finalResponse: string;
    if (expressiveMode) {
      console.log('[getGeminiStream] ========== CALL 2: ADD EMOTION TAGS ==========');

      finalResponse = await addEmotionTagsWithFewShot(rawResponse, persona, undefined, voiceMode, trafficType);

      console.log('[getGeminiStream] CALL 2 OUTPUT (length:', finalResponse.length, ')');
    } else {
      // Cleanup any hallucinations like (narrative tone) or [pauses] that might leak in Standard mode
      finalResponse = rawResponse.replace(/\([^)]*?\)/g, '').replace(/\[[^\]]*?\]/g, '').trim();
      console.log('[getGeminiStream] Standard mode - cleanup applied and Call 2 skipped');
    }

    console.log('[getGeminiStream] ✅ Complete, yielding final response');

    yield finalResponse;
  } catch (error) {
    console.error('[getGeminiStream] ❌ Fatal error:', error);

    // 📊 RECORD ERROR
    MetricsCollector.recordError('fatal_error', expressiveMode ? 'expressive' : 'standard', trafficType);

    if (error instanceof Error) {
      console.error('[getGeminiStream] Message:', error.message);
      console.error('[getGeminiStream] Stack:', error.stack);
    }

    yield `\n\nERROR: ${error instanceof Error ? error.message : 'Unknown error'}\n`;
    throw error;
  }
}

export async function POST(req: NextRequest) {
    try {
        console.log('=== Transcribe API Called ===');

        const formData = await req.formData();
        const audioFile = formData.get('audio') as File;

        if (!audioFile) {
            console.error('No audio file provided');
            return NextResponse.json(
                { error: 'No audio file provided' },
                { status: 400 }
            );
        }

        console.log(`📦 Received audio file: ${audioFile.name}, size: ${audioFile.size} bytes, type: ${audioFile.type}`);

        // Convert File to Blob
        const arrayBuffer = await audioFile.arrayBuffer();
        const audioBlob = new Blob([arrayBuffer], { type: audioFile.type || 'audio/webm' });

        console.log('🎙️ Calling ElevenLabs speechToText.convert()...');

        const transcription = await getClient().speechToText.convert({
            file: audioBlob,
            modelId: 'scribe_v1',
            tagAudioEvents: true,
            languageCode: 'eng',
            diarize: true,
        });

        console.log('✅ Transcription successful:', JSON.stringify(transcription, null, 2));

        // Handle different response structures
        let transcribedText = '';

        // Check various possible response structures
        if ((transcription as any).text) {
            transcribedText = (transcription as any).text;
        } else if ((transcription as any).transcription) {
            transcribedText = (transcription as any).transcription;
        } else if ((transcription as any).result) {
            transcribedText = (transcription as any).result;
        } else if ((transcription as any).chunks && Array.isArray((transcription as any).chunks)) {
            // If it's multichannel with chunks
            transcribedText = (transcription as any).chunks
                .map((chunk: any) => chunk.text || chunk.transcript || '')
                .join(' ');
        } else if (Array.isArray(transcription)) {
            // If it's an array response
            transcribedText = transcription
                .map((item: any) => item.text || item.transcript || '')
                .join(' ');
        }

        console.log('📝 Extracted text:', transcribedText);

        return NextResponse.json({
            text: transcribedText,
            success: true,
            transcription: transcription,
        });

    } catch (error) {
        console.error('❌ Transcription error:', error);

        // Log detailed error info
        if (error instanceof Error) {
            console.error('Error message:', error.message);
            console.error('Error stack:', error.stack);
        }

        const errorMessage = error instanceof Error ? error.message : String(error);

        return NextResponse.json(
            {
                error: 'Transcription failed',
                details: errorMessage,
            },
            { status: 500 }
        );
    }
}

export async function POST(req: NextRequest) {
    try {
        const {
            text,
            persona,
            expressiveMode = false,   // 🔑 mirrors Gemini pipeline
        }: {
            text: string;
            persona?: Persona;
            expressiveMode?: boolean;
        } = await req.json();

        if (!text) {
            return NextResponse.json({ error: 'Text is required' }, { status: 400 });
        }

        const voiceId = getVoiceIdForPersona(persona as Persona);

        // 🎭 Model selection
        const modelId = expressiveMode
            ? 'eleven_v3'
            : 'eleven_flash_v2_5';

        const inferredExpressive =
            expressiveMode || /\[[^\]]+\]/.test(text);

        const finalExpressiveMode = inferredExpressive;

        // 🧹 Strip emotion tags in standard mode
        const processedText = finalExpressiveMode
            ? text
            : text.replace(/\[[^\]]*?\]/g, '').trim();

        console.log(
            `🎙️ TTS Request | Persona: ${persona || 'narrative'} | ` +
            `Mode: ${expressiveMode ? 'expressive' : 'standard'} | ` +
            `Model: ${modelId} | VoiceID: ${voiceId}`
        );

        const audio = await getClient().textToSpeech.convert(voiceId, {
            text: processedText,
            modelId,
            outputFormat: 'mp3_44100_128',
        });

        return new NextResponse(audio as any, {
            headers: {
                'Content-Type': 'audio/mpeg',
            },
        });

    } catch (error: any) {
        console.error('❌ TTS Error:', {
            message: error.message,
            statusCode: error.statusCode,
            body: error.body,
        });

        return NextResponse.json(
            { error: error.message || 'TTS failed', details: error.body || '' },
            { status: error.statusCode || 500 }
        );
    }
}

datadogRum.init({
    applicationId,
    clientToken,
    site,
    service: 'voicedoc-agent',
    sessionSampleRate: 100,
    sessionReplaySampleRate: 20, // 100% for development/testing
    trackUserInteractions: true,
    trackResources: true,
    trackLongTasks: true,
})

// Start Session Replay Recording
datadogRum.startSessionReplayRecording();

// PATH 1: AGENTLESS (HTTPS API) - Best for Hackathons
if (DD_API_KEY) {
    const response = await fetch(
        `https://api.${DD_SITE}/api/v1/series?api_key=${DD_API_KEY}`,
        {
            method: 'POST',
            headers: { 'Content-Type': 'application/json' },
            body: JSON.stringify({ series })
        }
    );
}

npm run dev

python scripts\traffic-generator.py