Native APIs Overview

┌─────────────────────────────────────┐
│     TypeScript API Layer            │
│  (src/native.ts)                    │
├─────────────────────────────────────┤
│     Bun FFI Layer                   │
│  (dlopen, ptr)                      │
├─────────────────────────────────────┤
│     Native Shared Library           │
│  (bun_nltk.so/.dll/.dylib)          │
├─────────────────────────────────────┤
│     Zig Core Implementation         │
│  - SIMD optimizations               │
│  - Memory-efficient algorithms      │
│  - Cross-platform support           │
└─────────────────────────────────────┘

const ext = process.platform === "win32" ? "dll" 
  : process.platform === "darwin" ? "dylib" 
  : "so";

const prebuiltLibPath = resolve(
  import.meta.dir,
  "..",
  "native",
  "prebuilt",
  `${process.platform}-${process.arch}`,
  `bun_nltk.${ext}`,
);

export BUN_NLTK_NATIVE_LIB=/path/to/custom/bun_nltk.so

import { countTokensAscii } from "bun_nltk";

// Will load from BUN_NLTK_NATIVE_LIB if set
const count = countTokensAscii("Hello world");

import { dlopen, ptr } from "bun:ffi";

const lib = dlopen(nativeLibPath, {
  bunnltk_count_tokens_ascii: {
    args: ["ptr", "usize"],
    returns: "u64",
  },
  bunnltk_tokenize_ascii: {
    args: ["ptr", "usize", "ptr", "ptr", "usize"],
    returns: "u64",
  },
  // ... more function definitions
});

function countTokensAscii(text: string): number {
  const bytes = new TextEncoder().encode(text);
  const value = lib.symbols.bunnltk_count_tokens_ascii(
    ptr(bytes),
    bytes.length
  );
  return Number(value);
}

function lastError(): number {
  return lib.symbols.bunnltk_last_error_code();
}

function assertNoNativeError(context: string): void {
  const code = lastError();
  if (code !== 0) {
    throw new Error(`native error code ${code} in ${context}`);
  }
}

const count = lib.symbols.bunnltk_count_unique_tokens_ascii(
  ptr(bytes),
  bytes.length
);
assertNoNativeError("countUniqueTokensAscii");

import { WasmNltk } from "bun_nltk";

// Fallback to WASM on unsupported platforms
const wasm = await WasmNltk.init();
const tokens = wasm.tokenizeAscii("Hello world");

import { nativeLibraryPath } from "bun_nltk";

const libPath = nativeLibraryPath();
console.log(`Native library loaded from: ${libPath}`);
// Native library loaded from: /path/to/native/prebuilt/linux-x64/bun_nltk.so

# Build for current platform
bun run build:zig

# Build prebuilt binaries (requires cross-compilation setup)
bun run build:prebuilt