Streaming Text

Learn how to use real-time streaming text generation for better user experience and faster response times.

Overview

Streaming text generation provides real-time text output as it's being generated by AI models. This creates a more engaging user experience similar to ChatGPT, where users can see text appearing progressively rather than waiting for the complete response. The system supports streaming for all major AI providers with proper error handling and connection management.

Streaming API Endpoint

Streaming Text Generation API

// POST /api/demo/gen-stream-text
interface StreamingTextRequest {
  prompt: string;
  provider: "openai" | "deepseek" | "openrouter" | "siliconflow";
  model?: string;
  max_tokens?: number;
  temperature?: number;
}

// Server-Sent Events (SSE) Response
interface StreamingResponse {
  type: "chunk" | "done" | "error";
  data?: string;
  error?: string;
  usage?: {
    prompt_tokens: number;
    completion_tokens: number;
    total_tokens: number;
  };
}

Server-Side Implementation

// src/app/api/demo/gen-stream-text/route.ts
import { openai } from "@ai-sdk/openai";
import { streamText } from "ai";

export async function POST(request: Request) {
  try {
    const { prompt, provider, model = "gpt-3.5-turbo" } = await request.json();
    
    const result = await streamText({
      model: openai(model),
      prompt,
      maxTokens: 1000,
      temperature: 0.7,
    });

    return result.toDataStreamResponse();
  } catch (error) {
    return new Response("Error generating text", { status: 500 });
  }
}

Client-Side Usage

Basic Streaming Implementation

// Client-side streaming text generation
const generateStreamingText = async (prompt: string, provider: string) => {
  const response = await fetch("/api/demo/gen-stream-text", {
    method: "POST",
    headers: {
      "Content-Type": "application/json",
    },
    body: JSON.stringify({
      prompt,
      provider,
      max_tokens: 1000,
      temperature: 0.7,
    }),
  });

  if (!response.ok) {
    throw new Error("Failed to start streaming");
  }

  const reader = response.body?.getReader();
  const decoder = new TextDecoder();
  let fullText = "";

  if (reader) {
    while (true) {
      const { done, value } = await reader.read();
      
      if (done) break;
      
      const chunk = decoder.decode(value);
      const lines = chunk.split('\n');
      
      for (const line of lines) {
        if (line.startsWith('data: ')) {
          const data = line.slice(6);
          
          if (data === '[DONE]') {
            console.log("Streaming completed");
            return fullText;
          }
          
          try {
            const parsed = JSON.parse(data);
            if (parsed.type === 'chunk' && parsed.data) {
              fullText += parsed.data;
              // Update UI with new text
              updateTextDisplay(fullText);
            }
          } catch (e) {
            // Handle parsing errors
          }
        }
      }
    }
  }
  
  return fullText;
};

React Hook for Streaming

// Custom hook for streaming text
import { useState, useCallback } from 'react';

export function useStreamingText() {
  const [text, setText] = useState('');
  const [isStreaming, setIsStreaming] = useState(false);
  const [error, setError] = useState<string | null>(null);

  const generateText = useCallback(async (prompt: string, provider: string) => {
    setIsStreaming(true);
    setError(null);
    setText('');

    try {
      const response = await fetch("/api/demo/gen-stream-text", {
        method: "POST",
        headers: {
          "Content-Type": "application/json",
        },
        body: JSON.stringify({ prompt, provider }),
      });

      if (!response.ok) {
        throw new Error("Failed to start streaming");
      }

      const reader = response.body?.getReader();
      const decoder = new TextDecoder();

      if (reader) {
        while (true) {
          const { done, value } = await reader.read();
          
          if (done) break;
          
          const chunk = decoder.decode(value);
          const lines = chunk.split('\n');
          
          for (const line of lines) {
            if (line.startsWith('data: ')) {
              const data = line.slice(6);
              
              if (data === '[DONE]') {
                setIsStreaming(false);
                return;
              }
              
              try {
                const parsed = JSON.parse(data);
                if (parsed.type === 'chunk' && parsed.data) {
                  setText(prev => prev + parsed.data);
                }
              } catch (e) {
                // Handle parsing errors
              }
            }
          }
        }
      }
    } catch (err) {
      setError(err instanceof Error ? err.message : 'Unknown error');
      setIsStreaming(false);
    }
  }, []);

  return { text, isStreaming, error, generateText };
}

React Component Example

// Streaming text component
import React from 'react';
import { useStreamingText } from '@/hooks/useStreamingText';

export function StreamingTextGenerator() {
  const { text, isStreaming, error, generateText } = useStreamingText();
  const [prompt, setPrompt] = useState('');

  const handleGenerate = async () => {
    if (prompt.trim()) {
      await generateText(prompt, 'openai');
    }
  };

  return (
    <div className="space-y-4">
      <div>
        <textarea
          value={prompt}
          onChange={(e) => setPrompt(e.target.value)}
          placeholder="Enter your prompt..."
          className="w-full p-3 border rounded-lg"
          rows={3}
        />
      </div>
      
      <button
        onClick={handleGenerate}
        disabled={isStreaming || !prompt.trim()}
        className="px-4 py-2 bg-blue-500 text-white rounded-lg disabled:opacity-50"
      >
        {isStreaming ? 'Generating...' : 'Generate Text'}
      </button>
      
      {error && (
        <div className="p-3 bg-red-100 text-red-700 rounded-lg">
          Error: {error}
        </div>
      )}
      
      <div className="p-4 bg-gray-50 rounded-lg min-h-[200px]">
        <div className="whitespace-pre-wrap">
          {text}
          {isStreaming && (
            <span className="animate-pulse">|</span>
          )}
        </div>
      </div>
    </div>
  );
}

Advanced Features

Abort Controller for Cancellation

// Cancel streaming requests
const abortController = new AbortController();

const generateText = async (prompt: string) => {
  try {
    const response = await fetch("/api/demo/gen-stream-text", {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
      },
      body: JSON.stringify({ prompt }),
      signal: abortController.signal, // Add abort signal
    });

    // Handle streaming...
  } catch (error) {
    if (error.name === 'AbortError') {
      console.log('Streaming cancelled');
    } else {
      console.error('Streaming error:', error);
    }
  }
};

// Cancel the request
const cancelGeneration = () => {
  abortController.abort();
};

Error Handling and Retry

// Robust error handling with retry
const generateWithRetry = async (prompt: string, maxRetries = 3) => {
  for (let attempt = 1; attempt <= maxRetries; attempt++) {
    try {
      return await generateStreamingText(prompt, 'openai');
    } catch (error) {
      console.error(`Attempt ${attempt} failed:`, error);
      
      if (attempt === maxRetries) {
        throw new Error(`Failed after ${maxRetries} attempts`);
      }
      
      // Wait before retry
      await new Promise(resolve => setTimeout(resolve, 1000 * attempt));
    }
  }
};

Performance Optimization

Connection Pooling

// Optimize connection reuse
const connectionPool = new Map();

const getConnection = (provider: string) => {
  if (!connectionPool.has(provider)) {
    connectionPool.set(provider, new AbortController());
  }
  return connectionPool.get(provider);
};

Buffer Management

// Efficient text buffering
class TextBuffer {
  private buffer: string = '';
  private updateCallback: (text: string) => void;

  constructor(updateCallback: (text: string) => void) {
    this.updateCallback = updateCallback;
  }

  append(chunk: string) {
    this.buffer += chunk;
    this.updateCallback(this.buffer);
  }

  clear() {
    this.buffer = '';
    this.updateCallback(this.buffer);
  }
}

File Locations

• src/app/api/demo/gen-stream-text/route.ts - Streaming API endpoint
• src/hooks/useStreamingText.ts - React hook for streaming
• src/components/streaming-text-generator.tsx - Streaming component
• src/services/streaming-text.ts - Streaming business logic

Common Tasks

Create Chat Interface

// Chat-like streaming interface
const ChatInterface = () => {
  const [messages, setMessages] = useState([]);
  const { text, isStreaming, generateText } = useStreamingText();

  const sendMessage = async (userMessage: string) => {
    // Add user message
    setMessages(prev => [...prev, { role: 'user', content: userMessage }]);
    
    // Generate AI response
    await generateText(userMessage, 'openai');
    
    // Add AI response when complete
    if (!isStreaming) {
      setMessages(prev => [...prev, { role: 'assistant', content: text }]);
    }
  };

  return (
    <div className="chat-interface">
      {messages.map((msg, index) => (
        <div key={index} className={`message ${msg.role}`}>
          {msg.content}
        </div>
      ))}
      {isStreaming && (
        <div className="message assistant">
          {text}
          <span className="cursor">|</span>
        </div>
      )}
    </div>
  );
};

Real-time Code Generation

// Stream code generation
const CodeGenerator = () => {
  const { text, isStreaming, generateText } = useStreamingText();

  const generateCode = async (description: string) => {
    const prompt = `Generate code for: ${description}`;
    await generateText(prompt, 'deepseek');
  };

  return (
    <div>
      <textarea
        value={text}
        readOnly
        className="w-full h-96 font-mono text-sm"
        placeholder="Generated code will appear here..."
      />
      {isStreaming && <div className="text-green-500">Generating...</div>}
    </div>
  );
};

Troubleshooting

Streaming stops unexpectedly

Problem: Text generation stops mid-stream

Solution:

1. Check network connection stability
2. Verify server-side error handling
3. Check for timeout issues
4. Monitor server logs for errors

Slow streaming performance

Problem: Text appears very slowly

Solution:

1. Check AI provider response times
2. Optimize prompt length
3. Consider using faster models
4. Check network latency

Connection errors

Problem: Frequent connection drops

Solution:

1. Implement retry logic
2. Add connection health checks
3. Use exponential backoff
4. Monitor error rates

Next Steps

• Text Generation - Basic text generation
• Text-to-Speech - Convert text to audio
• AI Generator - User interface for AI features