useGlob

Fetches file and resource lists using glob patterns. The useGlob hook is designed for components that need to access lists of files such as images, videos, text files, or model checkpoints. It provides a flexible pattern-matching interface for querying file systems or remote storage.

Type Interface

export interface GlobQuery {
  prefix?: string;              // Base directory prefix (used as working directory for glob)
  glob?: string;                // Glob pattern for matching files
  filters?: Record<string, any>; // Additional filters for results
}

export interface GlobResult {
  prefix: string;               // Directory/prefix where file was found
  src: string;                  // Full path or URL to the resource
  [key: string]: any;          // Additional metadata (size, mtime, etc.)
}

export type FetchGlobFunction = <T extends GlobResult = GlobResult>(
  query: GlobQuery
) => Promise<T[]>;

export interface UseGlobResult<T extends GlobResult = GlobResult> {
  data: T[];
  loading: boolean;
  error: Error | null;
}

Reference Implementation

1. Define the fetch function

The fetch function handles the data retrieval logic:

async function fetchGlob(query: GlobQuery): Promise<GlobResult[]> {
  const response = await fetch("/api/glob", {
    method: "POST",
    body: JSON.stringify(query),
  });
  return await response.json();
}

2. Pass to GlobDataProvider

The provider builds the internal useGlob hook using your fetch function:

<GlobDataProvider fetchGlob={fetchGlob}>
  <YourComponents />
</GlobDataProvider>

3. Internal hook implementation

The provider internally creates the useGlob hook with loading and error states:

// Inside GlobDataProvider
function useGlob<T extends GlobResult = GlobResult>(query: GlobQuery) {
  const [data, setData] = useState<T[]>([]);
  const [loading, setLoading] = useState(true);
  const [error, setError] = useState<Error | null>(null);

  useEffect(() => {
    let cancelled = false;

    async function load() {
      try {
        setLoading(true);
        setError(null);
        const result = await fetchGlob<T>(query);
        if (!cancelled) setData(result);
      } catch (err) {
        if (!cancelled) setError(err as Error);
      } finally {
        if (!cancelled) setLoading(false);
      }
    }

    load();
    return () => { cancelled = true; };
  }, [query]);

  return { data, loading, error };
}

Key Features

• Glob pattern matching - "images/*.png", "videos/**/*.mp4"
• Prefix-based filtering - Specify base directory for searches
• Extensible metadata - Include file size, modification time, custom attributes
• Type-safe results - Generic type parameter for custom result types

Common Glob Patterns

• "*.png" - All PNG files in the current directory
• "**/*.mp4" - All MP4 files in current and subdirectories
• "screenshots/*.{png,jpg}" - PNG or JPG files in screenshots directory
• "model-*.pt" - Files matching pattern in current directory
• "logs/**/error.txt" - error.txt files in any subdirectory

Usage Examples

Image Gallery

function ImageGallery() {
  const useGlob = useGlobData();
  const { data, loading, error } = useGlob({
    prefix: "experiments/run-1",
    glob: "images/*.png"
  });

  if (loading) return <div>Loading...</div>;
  if (error) return <div>Error: {error.message}</div>;

  return (
    <div className="gallery">
      {data.map(item => (
        <img key={item.src} src={item.src} alt={item.prefix} />
      ))}
    </div>
  );
}

Video Player

function VideoPlayer() {
  const useGlob = useGlobData();
  const { data } = useGlob({
    prefix: "recordings",
    glob: "episode-*.mp4"
  });

  return (
    <select>
      {data.map(item => (
        <option key={item.src} value={item.src}>
          {item.src}
        </option>
      ))}
    </select>
  );
}

Log Viewer

function LogViewer() {
  const useGlob = useGlobData();
  const { data } = useGlob({
    prefix: "logs",
    glob: "**/*.txt",
    filters: { size_gt: 1024 } // Files larger than 1KB
  });

  return (
    <ul>
      {data.map(log => (
        <li key={log.src}>{log.src}</li>
      ))}
    </ul>
  );
}

Model Checkpoints

function CheckpointSelector() {
  const useGlob = useGlobData();
  const { data } = useGlob({
    prefix: "models/training-run-42",
    glob: "checkpoint-*.pt"
  });

  return (
    <select>
      {data.map(checkpoint => (
        <option key={checkpoint.src} value={checkpoint.src}>
          {checkpoint.src}
        </option>
      ))}
    </select>
  );
}

With Custom Metadata

interface ImageResult extends GlobResult {
  width: number;
  height: number;
  size: number;
  mtime: string;
}

function ImageBrowser() {
  const useGlob = useGlobData();
  const { data } = useGlob<ImageResult>({
    prefix: "screenshots",
    glob: "*.png"
  });

  return (
    <div>
      {data.map(img => (
        <div key={img.src}>
          <img src={img.src} alt={img.src} />
          <p>{img.width} × {img.height}</p>
          <p>Size: {img.size} bytes</p>
          <p>Modified: {img.mtime}</p>
        </div>
      ))}
    </div>
  );
}

Alternative Backend: File System (Node.js/Electron)

import { glob } from 'glob';
import fs from 'fs';
import path from 'path';

export function useGlob(query: GlobQuery) {
  const data = useMemo(() => {
    const basePath = query.prefix || '.';
    const pattern = query.glob || '*';

    // Use glob library to find matching files
    const files = glob.sync(pattern, { cwd: basePath });

    // Map to GlobResult format
    return files.map(file => {
      const fullPath = path.join(basePath, file);
      const stats = fs.statSync(fullPath);

      return {
        prefix: basePath,
        src: fullPath,
        size: stats.size,
        mtime: stats.mtime.toISOString(),
      };
    });
  }, [query]);

  return { data, loading: false, error: null };
}

Alternative Backend: S3/Cloud Storage

import { S3Client, ListObjectsV2Command } from '@aws-sdk/client-s3';

async function fetchGlob(query: GlobQuery): Promise<GlobResult[]> {
  const s3 = new S3Client({ region: 'us-west-2' });

  const command = new ListObjectsV2Command({
    Bucket: 'my-bucket',
    Prefix: query.prefix,
  });

  const response = await s3.send(command);

  return (response.Contents || []).map(obj => ({
    prefix: query.prefix || '',
    src: `https://my-bucket.s3.amazonaws.com/${obj.Key}`,
    size: obj.Size,
    mtime: obj.LastModified?.toISOString(),
  }));
}

Use Cases

• Image galleries - Screenshots, renderings, generated images
• Video playback - Episode recordings, training videos
• Log files - Debug logs, experiment logs, error logs
• Model checkpoints - Training checkpoints, saved models
• Document viewers - PDFs, text files, markdown files
• Asset browsers - Audio files, 3D models, configuration files

Related Documentation

• Data Hooks Overview - Data providers and hook system
• Data Query Examples - Query patterns and examples
• Example Data & Globs - Example datasets and glob patterns