Package Exports

nestamai-ai-3d-renderer

Readme

@nestamai/ai-3d-renderer

A secure, AI-powered Three.js 3D rendering library with sandboxed execution for educational applications.

🚀 Features

🔒 Security First: Web Worker sandboxing with AST-based code validation
🤖 AI-Powered: Generate 3D scenes from natural language queries
⚡ Performance: OffscreenCanvas rendering with 60fps target
🛡️ Error-Proof: Comprehensive fallback system with safe default scenes
🎯 Educational: Perfect for interactive learning visualizations
📱 React Ready: Easy integration with React applications

📦 Installation

npm install @nestamai/ai-3d-renderer

🎯 Quick Start

Basic Usage

import React from 'react';
import { AI3DRenderer } from '@nestamai/ai-3d-renderer';

function App() {
  return (
    <div>
      <h1>AI 3D Visualizations</h1>
      <AI3DRenderer 
        query="Create a rotating cube with physics simulation"
        width={800}
        height={600}
        fallbackText="Generating your 3D visualization..."
        onError={(error) => console.error('Render error:', error)}
        onLoad={() => console.log('Scene loaded successfully')}
      />
    </div>
  );
}

Interactive Example with AI Integration

import React, { useState, useRef } from 'react';
import { AI3DRenderer } from '@nestamai/ai-3d-renderer';

// Define the ref type for accessing renderer methods
interface AI3DRendererRef {
  renderScene: (code: string) => Promise<void>;
}

export function InteractiveExample() {
  const [query, setQuery] = useState('Create a rotating cube with physics simulation');
  const [isLoading, setIsLoading] = useState(false);
  const [error, setError] = useState<string | null>(null);
  const rendererRef = useRef<AI3DRendererRef>(null);

  // Mock AI API call - replace with your actual AI service
  const callMockAI = async (query: string): Promise<string> => {
    // Simulate API delay
    await new Promise(resolve => setTimeout(resolve, 1000));
    
    // Return mock Three.js code based on query
    if (query.toLowerCase().includes('cube')) {
      return `
// Rotating cube scene
console.log('Loading rotating cube scene');

// Clear existing scene
while (scene.children.length > 0) {
  scene.remove(scene.children[0]);
}

// Create a rotating cube
const geometry = new THREE.BoxGeometry(1, 1, 1);
const material = new THREE.MeshStandardMaterial({ 
  color: 0x4CAF50,
  metalness: 0.1,
  roughness: 0.8
});
const cube = new THREE.Mesh(geometry, material);
scene.add(cube);

// Add lighting
const ambientLight = new THREE.AmbientLight(0x404040, 0.4);
scene.add(ambientLight);

const directionalLight = new THREE.DirectionalLight(0xffffff, 0.8);
directionalLight.position.set(1, 1, 1);
scene.add(directionalLight);

// Animate
function animate() {
  cube.rotation.x += 0.01;
  cube.rotation.y += 0.01;
  renderer.render(scene, camera);
  requestAnimationFrame(animate);
}

animate();
console.log('Rotating cube scene loaded');
`;
    } else if (query.toLowerCase().includes('particle')) {
      return `
// Particle system scene
console.log('Loading particle system scene');

// Clear existing scene
while (scene.children.length > 0) {
  scene.remove(scene.children[0]);
}

// Create particle system
const particleCount = 100;
const particles = new THREE.BufferGeometry();
const positions = new Float32Array(particleCount * 3);
const colors = new Float32Array(particleCount * 3);

for (let i = 0; i < particleCount; i++) {
  positions[i * 3] = (Math.random() - 0.5) * 10;
  positions[i * 3 + 1] = (Math.random() - 0.5) * 10;
  positions[i * 3 + 2] = (Math.random() - 0.5) * 10;
  
  const color = new THREE.Color();
  color.setHSL(Math.random(), 0.7, 0.6);
  colors[i * 3] = color.r;
  colors[i * 3 + 1] = color.g;
  colors[i * 3 + 2] = color.b;
}

particles.setAttribute('position', new THREE.BufferAttribute(positions, 3));
particles.setAttribute('color', new THREE.BufferAttribute(colors, 3));

const particleMaterial = new THREE.PointsMaterial({
  size: 0.1,
  vertexColors: true,
  transparent: true,
  opacity: 0.8,
});

const particleSystem = new THREE.Points(particles, particleMaterial);
scene.add(particleSystem);

// Add lighting
const ambientLight = new THREE.AmbientLight(0x000020, 0.2);
scene.add(ambientLight);

// Animate
let time = 0;
function animate() {
  time += 0.01;
  particleSystem.rotation.y = time * 0.1;
  renderer.render(scene, camera);
  requestAnimationFrame(animate);
}

animate();
console.log('Particle system scene loaded');
`;
    } else {
      // Default scene
      return `
// Default scene
console.log('Loading default scene');

// Clear existing scene
while (scene.children.length > 0) {
  scene.remove(scene.children[0]);
}

// Create a simple rotating cube
const geometry = new THREE.BoxGeometry(1, 1, 1);
const material = new THREE.MeshStandardMaterial({ color: 0x4CAF50 });
const cube = new THREE.Mesh(geometry, material);
scene.add(cube);

// Add lighting
const ambientLight = new THREE.AmbientLight(0x404040, 0.4);
scene.add(ambientLight);

const directionalLight = new THREE.DirectionalLight(0xffffff, 0.8);
directionalLight.position.set(1, 1, 1);
scene.add(directionalLight);

// Animate
function animate() {
  cube.rotation.x += 0.01;
  cube.rotation.y += 0.01;
  renderer.render(scene, camera);
  requestAnimationFrame(animate);
}

animate();
console.log('Default scene loaded');
`;
    }
  };

  const handleGenerateScene = async () => {
    if (!rendererRef.current) return;

    setIsLoading(true);
    setError(null);

    try {
      // Call your AI API (replace with actual AI service)
      const code = await callMockAI(query);
      
      // Pass the generated code to the renderer
      await rendererRef.current.renderScene(code);
      
    } catch (error) {
      console.error('Failed to generate scene:', error);
      setError(error instanceof Error ? error.message : 'Failed to generate scene');
    } finally {
      setIsLoading(false);
    }
  };

  return (
    <div style={{ padding: '20px', fontFamily: 'Arial, sans-serif' }}>
      <h1>AI 3D Renderer - Interactive Example</h1>
      
      <div style={{ marginBottom: '20px' }}>
        <label htmlFor="query-input" style={{ display: 'block', marginBottom: '8px' }}>
          Describe your 3D scene:
        </label>
        <div style={{ display: 'flex', gap: '10px' }}>
          <input
            id="query-input"
            type="text"
            value={query}
            onChange={(e) => setQuery(e.target.value)}
            style={{
              flex: 1,
              padding: '12px',
              fontSize: '16px',
              border: '1px solid #ccc',
              borderRadius: '4px',
            }}
            placeholder="e.g., Create a solar system with planets orbiting the sun"
            onKeyPress={(e) => e.key === 'Enter' && handleGenerateScene()}
          />
          <button
            onClick={handleGenerateScene}
            disabled={isLoading}
            style={{
              padding: '12px 24px',
              fontSize: '16px',
              backgroundColor: isLoading ? '#ccc' : '#4CAF50',
              color: 'white',
              border: 'none',
              borderRadius: '4px',
              cursor: isLoading ? 'not-allowed' : 'pointer',
            }}
          >
            {isLoading ? 'Generating...' : 'Generate Scene'}
          </button>
        </div>
      </div>

      {error && (
        <div style={{
          padding: '12px',
          backgroundColor: '#ffebee',
          color: '#c62828',
          borderRadius: '4px',
          marginBottom: '20px',
        }}>
          <strong>Error:</strong> {error}
        </div>
      )}

      <div style={{
        border: '1px solid #ddd',
        borderRadius: '8px',
        overflow: 'hidden',
        backgroundColor: '#f5f5f5',
      }}>
        <AI3DRenderer
          ref={rendererRef}
          width={800}
          height={600}
          fallbackText="Generating your 3D visualization..."
          onError={(error) => setError(error.message)}
          onLoad={() => {
            setError(null);
            setIsLoading(false);
          }}
          onInteraction={(type, data) => {
            console.log('Interaction:', type, data);
          }}
          style={{
            width: '100%',
            height: '600px',
          }}
        />
      </div>

      <div style={{ marginTop: '20px', fontSize: '14px', color: '#666' }}>
        <h3>Example Queries:</h3>
        <ul>
          <li>"Create a rotating cube" - Shows a spinning green cube</li>
          <li>"Generate a particle system" - Shows colorful floating particles</li>
          <li>"Any other query" - Shows a default rotating cube</li>
        </ul>
        
        <div style={{ marginTop: '20px', padding: '15px', backgroundColor: '#e3f2fd', borderRadius: '4px' }}>
          <h4>💡 Note:</h4>
          <p>This example uses mock AI responses. In a real application, you would:</p>
          <ol>
            <li>Replace <code>callMockAI()</code> with your actual AI service (Azure AI, OpenAI, etc.)</li>
            <li>Handle retries and error cases</li>
            <li>Add proper loading states and user feedback</li>
          </ol>
          <p>See <code>CLIENT_USAGE.md</code> for complete integration examples.</p>
        </div>
      </div>
    </div>
  );
}

📚 API Reference

AI3DRenderer Props

Prop	Type	Default	Description
`query`	`string`	-	Required. Natural language description of the 3D scene to generate
`width`	`number`	`800`	Canvas width in pixels
`height`	`number`	`600`	Canvas height in pixels
`fallbackText`	`string`	`"Generating visualization..."`	Loading message shown while generating scene
`onError`	`(error: Error) => void`	-	Callback fired when rendering fails
`onLoad`	`() => void`	-	Callback fired when scene loads successfully
`onInteraction`	`(type: string, data: any) => void`	-	Callback fired on user interactions
`className`	`string`	-	CSS class name for the container
`style`	`React.CSSProperties`	-	Inline styles for the container

useRenderer Hook

For more advanced usage, you can use the useRenderer hook directly:

import { useRenderer } from '@nestamai/ai-3d-renderer';

function CustomRenderer() {
  const {
    canvasRef,
    isLoading,
    error,
    isInitialized,
    fallbackActive,
    handleCanvasClick,
    renderScene,
  } = useRenderer({
    query: "Show me a solar system animation",
    width: 1000,
    height: 800,
  });

  return (
    <div>
      {isLoading && <div>Loading...</div>}
      {error && <div>Error: {error.message}</div>}
      {fallbackActive && <div>Using fallback scene</div>}
      <canvas
        ref={canvasRef}
        onClick={handleCanvasClick}
        width={1000}
        height={800}
      />
    </div>
  );
}

🔧 Configuration

AI Backend Integration

The library expects an AI backend that can generate scene configurations. You'll need to implement the AI API integration:

// Example AI API integration
async function generateSceneConfig(query: string): Promise<SceneConfig> {
  const response = await fetch('/api/ai/generate-scene', {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({ query }),
  });
  
  return response.json();
}

Scene Configuration Schema

The AI backend should return a scene configuration matching this schema:

interface SceneConfig {
  metadata: {
    title: string;
    description: string;
    version: string;
    author?: string;
  };
  settings: {
    backgroundColor: string;
    camera: {
      position: [number, number, number];
      target: [number, number, number];
      fov: number;
      near: number;
      far: number;
    };
    lighting: {
      ambient: {
        color: string;
        intensity: number;
      };
      directional?: {
        color: string;
        intensity: number;
        position: [number, number, number];
      };
    };
  };
  objects: SceneObject[];
  animations?: AnimationDefinition[];
  interactions?: InteractionDefinition[];
}

🛡️ Security

The library implements multiple layers of security:

1. Web Worker Sandboxing

All rendering code runs in a Web Worker
No access to DOM, window, or network APIs
Complete isolation from the main thread

2. AST-Based Code Validation

Static analysis of generated code
Deny-list of dangerous JavaScript constructs
Allow-list of safe Three.js APIs

3. Safe API Wrapper

Limited subset of Three.js functionality
No direct access to dangerous methods
Controlled execution environment

Blocked Operations

eval, Function, setTimeout, setInterval
window, document, localStorage, sessionStorage
fetch, XMLHttpRequest, WebSocket
import, require, module, exports
Direct console access (uses safe logger)

🎨 Fallback System

When AI generation fails, the library automatically falls back to safe, pre-built scenes:

Default: Simple rotating cube
Physics: Falling objects simulation
Geometric: Animated geometric patterns
Particle: Particle system visualization

🧪 Testing

# Run tests
npm test

# Run tests with coverage
npm run test:coverage

# Run tests in UI mode
npm run test:ui

🏗️ Development

# Install dependencies
npm install

# Start development server
npm run dev

# Build for production
npm run build

# Lint code
npm run lint

# Type checking
npm run type-check

📊 Performance

Target FPS: 60fps
Memory Usage: < 100MB
Load Time: < 2s
Worker Startup: < 500ms

🤝 Contributing

Fork the repository
Create a feature branch
Make your changes
Add tests
Submit a pull request

📄 License

MIT License - see LICENSE file for details.

🆘 Support

📧 Email: support@nestamai.com
🐛 Issues: GitHub Issues
📖 Docs: Documentation

🗺️ Roadmap

Physics engine integration (Cannon-es)
Scene serialization and persistence
AR/VR support (WebXR)
Real-time collaboration
Advanced animation system
Performance profiling tools

Built with ❤️ by the NestamAI team for educational applications.