obsidian-rate-limits

Obsidian Rate Limits

Overview

Manage file system operations and implement throttling to prevent performance issues in Obsidian plugins.

Prerequisites

• Understanding of async JavaScript
• Familiarity with Obsidian vault operations
• Knowledge of file system performance considerations

Key Concepts

Obsidian Operation Limits

Instructions

Step 1: Implement Async Queue

// src/utils/async-queue.ts
export class AsyncQueue {
  private queue: Array<() => Promise<void>> = [];
  private processing = false;
  private concurrency: number;
  private activeCount = 0;

  constructor(concurrency: number = 1) {
    this.concurrency = concurrency;
  }

  async add<T>(task: () => Promise<T>): Promise<T> {
    return new Promise((resolve, reject) => {
      this.queue.push(async () => {
        try {
          const result = await task();
          resolve(result);
        } catch (error) {
          reject(error);
        }
      });
      this.process();
    });
  }

  private async process(): Promise<void> {
    if (this.activeCount >= this.concurrency) return;

    const task = this.queue.shift();
    if (!task) return;

    this.activeCount++;
    try {
      await task();
    } finally {
      this.activeCount--;
      this.process();
    }
  }

  get pending(): number {
    return this.queue.length;
  }

  get active(): number {
    return this.activeCount;
  }
}

Step 2: Implement Rate Limiter

// src/utils/rate-limiter.ts
export class RateLimiter {
  private timestamps: number[] = [];
  private limit: number;
  private window: number; // milliseconds

  constructor(limit: number, windowMs: number) {
    this.limit = limit;
    this.window = windowMs;
  }

  async acquire(): Promise<void> {
    const now = Date.now();

    // Remove timestamps outside window
    this.timestamps = this.timestamps.filter(t => now - t < this.window);

    if (this.timestamps.length >= this.limit) {
      // Calculate wait time
      const oldestTimestamp = this.timestamps[0];
      const waitTime = this.window - (now - oldestTimestamp);
      await this.sleep(waitTime);
      return this.acquire();
    }

    this.timestamps.push(now);
  }

  private sleep(ms: number): Promise<void> {
    return new Promise(resolve => setTimeout(resolve, ms));
  }
}

// Usage:
const writeLimiter = new RateLimiter(10, 1000); // 10 per second

async function safeWriteFile(file: TFile, content: string) {
  await writeLimiter.acquire();
  await this.app.vault.modify(file, content);
}

Step 3: Batch Processing Pattern

// src/utils/batch-processor.ts
export interface BatchOptions {
  batchSize: number;
  delayBetweenBatches: number;
  onProgress?: (processed: number, total: number) => void;
}

export async function processBatches<T, R>(
  items: T[],
  processor: (item: T) => Promise<R>,
  options: BatchOptions
): Promise<R[]> {
  const results: R[] = [];
  const total = items.length;

  for (let i = 0; i < items.length; i += options.batchSize) {
    const batch = items.slice(i, i + options.batchSize);

    // Process batch in parallel
    const batchResults = await Promise.all(
      batch.map(item => processor(item))
    );
    results.push(...batchResults);

    // Report progress
    options.onProgress?.(Math.min(i + options.batchSize, total), total);

    // Delay between batches (except for last batch)
    if (i + options.batchSize < items.length) {
      await sleep(options.delayBetweenBatches);
    }
  }

  return results;
}

function sleep(ms: number): Promise<void> {
  return new Promise(resolve => setTimeout(resolve, ms));
}

// Usage:
const files = this.app.vault.getMarkdownFiles();

const results = await processBatches(
  files,
  async (file) => {
    const content = await this.app.vault.read(file);
    return { path: file.path, length: content.length };
  },
  {
    batchSize: 50,
    delayBetweenBatches: 100, // 100ms between batches
    onProgress: (processed, total) => {
      console.log(`Processed ${processed}/${total}`);
    },
  }
);

Step 4: Debounced Operations

// src/utils/debounce.ts
export function debounce<T extends (...args: any[]) => any>(
  func: T,
  wait: number,
  options: { leading?: boolean; trailing?: boolean; maxWait?: number } = {}
): (...args: Parameters<T>) => void {
  let timeout: NodeJS.Timeout | null = null;
  let lastCallTime: number | null = null;
  let lastInvokeTime = 0;
  let result: ReturnType<T>;

  const { leading = false, trailing = true, maxWait } = options;

  function invokeFunc(time: number, args: Parameters<T>) {
    lastInvokeTime = time;
    result = func(...args);
    return result;
  }

  return function (...args: Parameters<T>) {
    const time = Date.now();
    const isInvoking = shouldInvoke(time);

    lastCallTime = time;

    if (isInvoking) {
      if (!timeout && leading) {
        return invokeFunc(time, args);
      }
    }

    if (!timeout) {
      timeout = setTimeout(() => {
        timeout = null;
        if (trailing && lastCallTime) {
          invokeFunc(Date.now(), args);
        }
      }, wait);
    }

    // Handle maxWait
    if (maxWait !== undefined) {
      const timeSinceLastInvoke = time - lastInvokeTime;
      if (timeSinceLastInvoke >= maxWait) {
        invokeFunc(time, args);
      }
    }
  };

  function shouldInvoke(time: number): boolean {
    const timeSinceLastCall = lastCallTime ? time - lastCallTime : 0;
    return !lastCallTime || timeSinceLastCall >= wait;
  }
}

// Usage for search input:
const debouncedSearch = debounce(
  async (query: string) => {
    const results = await performSearch(query);
    updateUI(results);
  },
  300,
  { leading: false, trailing: true }
);

// Typing triggers search only after 300ms pause
inputEl.addEventListener('input', (e) => {
  debouncedSearch(e.target.value);
});

Step 5: Throttled Event Handling

// src/utils/throttle.ts
export function throttle<T extends (...args: any[]) => any>(
  func: T,
  limit: number
): (...args: Parameters<T>) => void {
  let inThrottle = false;
  let lastArgs: Parameters<T> | null = null;

  return function (...args: Parameters<T>) {
    if (!inThrottle) {
      func(...args);
      inThrottle = true;
      setTimeout(() => {
        inThrottle = false;
        if (lastArgs) {
          func(...lastArgs);
          lastArgs = null;
        }
      }, limit);
    } else {
      lastArgs = args;
    }
  };
}

// Usage for scroll handling:
const throttledOnScroll = throttle(() => {
  // Update something based on scroll position
  console.log('Scroll position:', window.scrollY);
}, 100);

window.addEventListener('scroll', throttledOnScroll);

Output

• Async queue for sequential operations
• Rate limiter for controlled throughput
• Batch processor for bulk operations
• Debounce for user input
• Throttle for frequent events

Error Handling

Examples

Progress Modal for Long Operations

async function processAllFiles(app: App, plugin: Plugin) {
  const files = app.vault.getMarkdownFiles();
  const progressModal = new ProgressModal(app);
  progressModal.open();

  try {
    await processBatches(
      files,
      async (file) => {
        // Process file
        return processFile(file);
      },
      {
        batchSize: 20,
        delayBetweenBatches: 50,
        onProgress: (processed, total) => {
          const percent = Math.round((processed / total) * 100);
          progressModal.setProgress(percent, `Processing ${processed}/${total} files`);
        },
      }
    );
    new Notice('Processing complete!');
  } finally {
    progressModal.close();
  }
}

Generator for Large File Sets

async function* iterateFilesWithPause(
  files: TFile[],
  pauseEvery: number = 100,
  pauseMs: number = 10
): AsyncGenerator<TFile> {
  for (let i = 0; i < files.length; i++) {
    yield files[i];

    // Pause periodically to allow UI updates
    if (i > 0 && i % pauseEvery === 0) {
      await new Promise(r => setTimeout(r, pauseMs));
    }
  }
}

// Usage:
for await (const file of iterateFilesWithPause(files)) {
  await processFile(file);
}

Resources

• Obsidian Performance Tips
• JavaScript Event Loop

Next Steps

For security practices, see obsidian-security-basics.