Building AI-Powered Applications: LLM Integration, Streaming, RAG & Function Calling in Node.js

// app/api/chat/route.ts — Next.js streaming route
import OpenAI from "openai";

const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });

export async function POST(req: Request) {
  const { message, context } = await req.json();

  const stream = await openai.chat.completions.create({
    model: "gpt-4o-mini",  // Fast and cheap for most tasks
    messages: [
      {
        role: "system",
        content: "You are a helpful assistant for a tech blog. Be concise and accurate.",
      },
      { role: "user", content: message },
    ],
    stream: true,     // Enable streaming
    max_tokens: 1024,
    temperature: 0.7,
  });

  // Stream Server-Sent Events to the client
  const encoder = new TextEncoder();
  const readable = new ReadableStream({
    async start(controller) {
      for await (const chunk of stream) {
        const content = chunk.choices[0]?.delta?.content ?? "";
        if (content) {
          controller.enqueue(encoder.encode(`data: ${JSON.stringify({ content })}

`));
        }
      }
      controller.enqueue(encoder.encode("data: [DONE]

"));
      controller.close();
    },
  });

  return new Response(readable, {
    headers: {
      "Content-Type": "text/event-stream",
      "Cache-Control": "no-cache",
      "Connection": "keep-alive",
    },
  });
}

// Client-side streaming consumer
"use client";
import { useState } from "react";

export function AIChat() {
  const [response, setResponse] = useState("");
  const [isStreaming, setIsStreaming] = useState(false);

  async function sendMessage(message: string) {
    setResponse("");
    setIsStreaming(true);

    const res = await fetch("/api/chat", {
      method: "POST",
      headers: { "Content-Type": "application/json" },
      body: JSON.stringify({ message }),
    });

    const reader = res.body!.getReader();
    const decoder = new TextDecoder();

    while (true) {
      const { done, value } = await reader.read();
      if (done) break;

      const text = decoder.decode(value);
      const lines = text.split("
").filter((l) => l.startsWith("data: "));

      for (const line of lines) {
        const data = line.slice(6);
        if (data === "[DONE]") { setIsStreaming(false); break; }
        const { content } = JSON.parse(data);
        setResponse((prev) => prev + content);
      }
    }
  }

  return (
    <div>
      <div>{response}</div>
      {isStreaming && <span>●</span>}
    </div>
  );
}

                    User Question
                         │
              ┌──────────▼──────────┐
              │  Embed question     │  (OpenAI text-embedding-3-small)
              │  → 1536-dim vector  │
              └──────────┬──────────┘
                         │ Cosine similarity search
              ┌──────────▼──────────┐
              │  Vector Database    │  (Supabase pgvector / Pinecone)
              │  Find top-5 chunks  │
              └──────────┬──────────┘
                         │ Relevant context
              ┌──────────▼──────────┐
              │  LLM Prompt         │
              │  System: "Answer    │
              │  based on context:  │
              │  [chunk1, chunk2..]"│
              └──────────┬──────────┘
                         │
                    Grounded Answer

// Build your knowledge base — embed blog posts
import OpenAI from "openai";
import { createClient } from "@supabase/supabase-js";

const openai = new OpenAI();
const supabase = createClient(process.env.SUPABASE_URL!, process.env.SUPABASE_SERVICE_ROLE_KEY!);

async function embedAndStorePost(post: Post) {
  // Split post into chunks (512 tokens each)
  const chunks = splitIntoChunks(post.content, 512);

  for (const chunk of chunks) {
    const embedding = await openai.embeddings.create({
      model: "text-embedding-3-small",
      input: chunk,
    });

    await supabase.from("post_embeddings").insert({
      post_id: post.id,
      content: chunk,
      embedding: embedding.data[0].embedding, // 1536-dimensional vector
    });
  }
}

// At query time — retrieve relevant context
async function getRelevantContext(question: string, topK = 5): Promise<string> {
  const questionEmbedding = await openai.embeddings.create({
    model: "text-embedding-3-small",
    input: question,
  });

  // Vector similarity search via Supabase pgvector
  const { data } = await supabase.rpc("match_post_embeddings", {
    query_embedding: questionEmbedding.data[0].embedding,
    match_threshold: 0.78,
    match_count: topK,
  });

  return data.map((d: { content: string }) => d.content).join("

");
}

// Complete RAG pipeline
async function answerWithRAG(question: string): Promise<string> {
  const context = await getRelevantContext(question);

  const response = await openai.chat.completions.create({
    model: "gpt-4o-mini",
    messages: [
      {
        role: "system",
        content: `You are a tech blog assistant. Answer questions based ONLY on the following context. If the answer isn't in the context, say so.

Context:
${context}`,
      },
      { role: "user", content: question },
    ],
  });

  return response.choices[0].message.content ?? "";
}

// Extract structured data from natural language
const response = await openai.chat.completions.create({
  model: "gpt-4o",
  messages: [
    {
      role: "user",
      content: "Schedule a meeting with Alice and Bob next Tuesday at 3pm for 1 hour to discuss the Q4 roadmap",
    },
  ],
  tools: [
    {
      type: "function",
      function: {
        name: "create_calendar_event",
        description: "Create a new calendar event",
        parameters: {
          type: "object",
          properties: {
            title: { type: "string", description: "Event title" },
            attendees: {
              type: "array",
              items: { type: "string" },
              description: "List of attendee names",
            },
            datetime: {
              type: "string",
              format: "date-time",
              description: "ISO 8601 datetime",
            },
            duration_minutes: { type: "number" },
          },
          required: ["title", "attendees", "datetime", "duration_minutes"],
        },
      },
    },
  ],
  tool_choice: "required", // Force the model to call a function
});

const toolCall = response.choices[0].message.tool_calls?.[0];
if (toolCall?.function.name === "create_calendar_event") {
  const event = JSON.parse(toolCall.function.arguments);
  // event = { title: "Q4 Roadmap", attendees: ["Alice", "Bob"], datetime: "2026-09-02T15:00:00Z", duration_minutes: 60 }
  await createCalendarEvent(event);
}

// Estimate cost before calling the API
import { encoding_for_model } from "tiktoken";

function estimateCost(prompt: string, model = "gpt-4o-mini"): number {
  const enc = encoding_for_model(model);
  const tokens = enc.encode(prompt).length;
  enc.free();

  // gpt-4o-mini: $0.15/1M input tokens, $0.60/1M output tokens
  const inputCost = (tokens / 1_000_000) * 0.15;
  return inputCost;
}

// Cache LLM responses — identical prompts shouldn't hit the API
const responseCache = new Map<string, string>();

async function cachedCompletion(prompt: string): Promise<string> {
  const cacheKey = crypto.createHash("md5").update(prompt).digest("hex");

  if (responseCache.has(cacheKey)) {
    return responseCache.get(cacheKey)!;
  }

  const response = await openai.chat.completions.create({
    model: "gpt-4o-mini",
    messages: [{ role: "user", content: prompt }],
  });

  const result = response.choices[0].message.content ?? "";
  responseCache.set(cacheKey, result);
  return result;
}