GitHub - m96-chan/0xBitNet: Run BitNet b1.58 ternary LLMs with WebGPU — in browsers and native apps

import { BitNet } from "0xbitnet";

const model = await BitNet.load(
  "https://huggingface.co/microsoft/bitnet-b1.58-2B-4T-gguf/resolve/main/ggml-model-i2_s.gguf",
  { onProgress: (p) => console.log(`${p.phase}: ${(p.fraction * 100).toFixed(1)}%`) }
);

for await (const token of model.generate("The meaning of life is")) {
  process.stdout.write(token);
}

model.dispose();

use oxbitnet::BitNet;
use futures::StreamExt;

let mut model = BitNet::load("model.gguf", Default::default()).await?;

let mut stream = model.generate("Hello!", Default::default());
while let Some(token) = stream.next().await {
    print!("{token}");
}

model.dispose();

from oxbitnet import BitNet

model = BitNet.load_sync("model.gguf")

model.chat(
    [("system", "You are a helpful assistant."), ("user", "Hello!")],
    on_token=lambda t: print(t, end="", flush=True),
    temperature=0.7,
)

model.dispose()

import OxBitNet

let model = try await BitNet.load("model.gguf")

for try await token in model.chat([.user("Hello!")], options: .init(temperature: 0.7)) {
    print(token, terminator: "")
}

model.dispose()

import io.github.m96chan.oxbitnet.*;
import java.util.List;

try (BitNet model = BitNet.loadSync("model.gguf")) {
    model.chat(
        List.of(new ChatMessage("user", "Hello!")),
        token -> {
            System.out.print(token);
            return true;
        },
        new GenerateOptions().temperature(0.7f)
    );
}

#include "oxbitnet.h"

static int32_t on_token(const char *token, uintptr_t len, void *userdata) {
    fwrite(token, 1, len, stdout);
    return 0; /* 0 = continue, non-zero = stop */
}

int main(void) {
    OxBitNet *model = oxbitnet_load("model.gguf", NULL);

    OxBitNetChatMessage messages[] = {
        { .role = "user", .content = "Hello!" },
    };
    OxBitNetGenerateOptions opts = oxbitnet_default_generate_options();

    oxbitnet_chat(model, messages, 1, &opts, on_token, NULL);
    oxbitnet_free(model);
}

const messages = [
  { role: "system" as const, content: "You are a helpful assistant." },
  { role: "user" as const, content: "Explain quantum computing in one sentence." },
];

for await (const token of model.generate(messages, { maxTokens: 128, temperature: 0.7 })) {
  process.stdout.write(token);
}

cd examples/node-cli
npm install && npm start

cd packages/rust
cargo run --example chat --release

pip install oxbitnet
python packages/rust/crates/oxbitnet-python/examples/chat.py

cd packages/rust
cargo build -p oxbitnet-ffi --release
cd crates/oxbitnet-swift
swift run -Xlinker -L../../../../target/release Chat model.gguf "Hello!"

cd packages/rust
cargo build -p oxbitnet-java --release
cd crates/oxbitnet-java/examples
javac -cp ../java/src/main/java:. Chat.java
java -Djava.library.path=../../../../target/release -cp ../java/src/main/java:. Chat model.gguf "Hello!"

cd packages/rust
cargo build -p oxbitnet-ffi --release
gcc crates/oxbitnet-ffi/examples/chat.c -Icrates/oxbitnet-ffi -Ltarget/release -loxbitnet_ffi -o chat
LD_LIBRARY_PATH=target/release ./chat model.gguf "Hello!"

0xbitnet/
├── packages/
│   ├── core/               # WGSL kernels + TypeScript API (npm: 0xbitnet)
│   │   └── src/
│   │       ├── gpu/        # WebGPU device init, buffer pool
│   │       ├── model/      # GGUF parser, weight loader, config
│   │       ├── nn/         # Transformer layers, attention, BitLinear
│   │       ├── shaders/    # 12 WGSL compute shaders (shared with Rust)
│   │       └── tokenizer/  # BPE tokenizer, chat templates
│   └── rust/               # Rust + Python bindings
│       └── crates/
│           ├── oxbitnet/           # Rust library (crates.io: oxbitnet)
│           ├── oxbitnet-python/    # Python bindings via PyO3 (PyPI: oxbitnet)
│           ├── oxbitnet-swift/     # Swift bindings via C FFI (SPM package)
│           ├── oxbitnet-java/      # Java/JNI bindings (Android-ready)
│           └── oxbitnet-ffi/       # C FFI bindings (cdylib + staticlib)
├── examples/
│   ├── web-chat/           # Chat app demo (Vite)
│   ├── tl-dr-widget/       # Offline TL;DR widget demo (Vite)
│   └── node-cli/           # Node.js CLI using Dawn WebGPU bindings
└── docs/