GitHub - pnrajan/fastlangml: Fast Language Detection for Conversational Agents

"Bonjour!"       -> French (correct)
"Comment ca va?" -> French (correct)
"Bien"           -> Spanish? French? German? (WRONG)
"ok"             -> English? Universal? (AMBIGUOUS)
"thx"            -> Unknown (FAIL)

# Full installation with all backends (recommended)
pip install fastlangml[all]

# Minimal installation (fasttext only)
pip install fastlangml[fasttext]

# Pick specific backends
pip install fastlangml[fasttext,lingua]
pip install fastlangml[langdetect]

from fastlangml import detect

# Direct comparison
detect("Hello") == "en"      # True
detect("Bonjour") == "fr"    # True

# Use as string
print(detect("Hola"))        # "es"
f"Language: {detect('Ciao')}"  # "Language: it"

# Full details when needed
result = detect("Hello, how are you?")
result.lang        # "en"
result.confidence  # 0.95
result.reliable    # True

# 1. Single-shot - no setup needed
from fastlangml import detect
detect("Bonjour") == "fr"  # True

# 2. Context - optional, for conversations
from fastlangml import detect, ConversationContext
ctx = ConversationContext()
detect("Bonjour", context=ctx)  # tracks history in memory
detect("ok", context=ctx) == "fr"  # True (uses context)

# 3. Stores - optional, for persistence across requests
from fastlangml.context import DiskContextStore
store = DiskContextStore("./data")
with store.session("user-123") as ctx:
    detect("Bonjour", context=ctx)

from fastlangml import detect, ConversationContext

# Create a context to track the conversation
context = ConversationContext()

# French conversation
detect("Bonjour!", context=context)        # "fr" (clear)
detect("Comment ca va?", context=context)  # "fr" (clear)
detect("Bien", context=context)            # "fr" <- context helps!
detect("ok", context=context) == "fr"      # True <- continues French

# The context tracks that this is a French conversation,
# so ambiguous words resolve to French

context = ConversationContext(
    max_turns=2,       # Remember last 2 messages (default)
    decay_factor=0.8,  # Weight recent messages higher
)

from fastlangml import detect, ConversationContext

def route_message(messages: list[str]) -> str:
    """Route conversation to correct language queue."""
    context = ConversationContext()

    for msg in messages:
        result = detect(msg, context=context)

    # Return dominant language of conversation
    return context.dominant_language or "en"

# French customer conversation
messages = ["Bonjour", "J'ai un probleme", "ok", "merci"]
queue = route_message(messages)  # Returns "fr"

pip install fastlangml[diskcache]

from fastlangml import detect
from fastlangml.context import DiskContextStore

store = DiskContextStore("./contexts", ttl_seconds=3600)

# Context manager - auto-saves on exit
with store.session(session_id) as ctx:
    result = detect(text, context=ctx, auto_update=True)

pip install fastlangml[redis]

from fastlangml import detect
from fastlangml.context import RedisContextStore

store = RedisContextStore("redis://localhost:6379", ttl_seconds=1800)

with store.session(session_id) as ctx:
    result = detect(text, context=ctx, auto_update=True)

# Serialize to your own storage
data = ctx.to_dict()
save_to_database(session_id, data)

# Load back
data = load_from_database(session_id)
ctx = ConversationContext.from_dict(data)

# Or use lightweight history format
ctx = ConversationContext.from_history([("fr", 0.95), ("fr", 0.9)])

from fastlangml import FastLangDetector, DetectionConfig

# Configure ensemble with 3 backends
detector = FastLangDetector(
    config=DetectionConfig(
        backends=["fasttext", "lingua", "langdetect"],
        backend_weights={
            "fasttext": 0.5,   # Trust fasttext most
            "lingua": 0.35,    # Lingua for accuracy
            "langdetect": 0.15 # Langdetect as tiebreaker
        },
    )
)

result = detector.detect("Ciao, come stai?")
print(result)         # "it"
print(result.backend) # "ensemble"

from fastlangml import FastLangDetector, DetectionConfig

detector = FastLangDetector(
    config=DetectionConfig(
        voting_strategy="weighted",
        backend_weights={
            "fasttext": 0.6,
            "lingua": 0.4,
        },
    )
)

detector = FastLangDetector(
    config=DetectionConfig(voting_strategy="hard")
)

from fastlangml import ConsensusVoting, FastLangDetector, DetectionConfig

detector = FastLangDetector(
    config=DetectionConfig(
        custom_voting=ConsensusVoting(min_agreement=2)
    )
)

from fastlangml import FastLangDetector, HintDictionary

# Load default hints for chat/SMS
hints = HintDictionary.default_short_words()

detector = FastLangDetector(hints=hints)
detector.detect("thx")   # "en" (thanks)
detector.detect("mdr")   # "fr" (mort de rire = LOL)
detector.detect("jaja")  # "es" (Spanish laugh)

from fastlangml import FastLangDetector

detector = FastLangDetector()

# Add hints for your domain
detector.add_hint("asap", "en")
detector.add_hint("btw", "en")
detector.add_hint("stp", "fr")  # s'il te plait

# Hints override backend detection
detector.detect("asap")  # "en"

def detect(
    text: str,
    context: ConversationContext | None = None,
    mode: str = "default",
    auto_update: bool = True,
) -> DetectionResult

@dataclass
class DetectionResult:
    lang: str           # ISO 639-1 code ("en", "fr", "und")
    confidence: float   # 0.0 to 1.0
    reliable: bool      # True if confidence >= threshold
    reason: str | None  # Why "und" was returned (if applicable)
    script: str | None  # Detected script ("latin", "cyrillic", etc.)
    backend: str        # Which backend or "ensemble"
    candidates: list    # Top-k alternatives
    meta: dict          # Timing and debug info

# Can be used directly as string
detect("Hello") == "en"      # True (via __eq__)
print(detect("Bonjour"))     # "fr" (via __str__)
f"{detect('Hola')}"          # "es"

context = ConversationContext(
    max_turns: int = 2,      # Max messages to remember
    decay_factor: float = 0.8, # Recency weight (0.0-1.0)
)

# Properties
context.dominant_language    # Most common language in history
context.language_distribution # {lang: weighted_count}
context.last_turn           # Most recent turn

# Methods
context.get_context_boost(lang)  # Get boost score for a language
context.get_language_streak()    # (lang, streak_count)
context.clear()                  # Reset context

detector = FastLangDetector(
    config=DetectionConfig(...),
    hints=HintDictionary(),
)

# Methods
detector.detect(text, context=None, mode="default")
detector.detect_batch(texts, mode="default")
detector.add_hint(word, lang)
detector.remove_hint(word)
detector.set_languages(["en", "fr", "es"])  # Restrict output
detector.available_backends()

from fastlangml import DetectionConfig

config = DetectionConfig(
    # Backend configuration
    backends=["fasttext", "lingua"],
    backend_weights={"fasttext": 0.6, "lingua": 0.4},

    # Voting
    voting_strategy="weighted",  # or "hard", "soft", "consensus"
    custom_voting=None,          # VotingStrategy instance

    # Thresholds
    thresholds={
        "short": 0.5,   # Confidence threshold for short mode
        "default": 0.7,
        "long": 0.8,
    },
    min_text_length=1,

    # Features
    filter_proper_nouns=False,  # Enable heuristic-based proper noun filtering
    use_script_filter=True,

    # spaCy NER for proper noun filtering (requires pip install fastlangml[spacy])
    # filter = ProperNounFilter(use_spacy=True)
    # Filters: PERSON, ORG, GPE, LOC, FAC, NORP entities

    # Weights
    hint_weight=1.5,
    context_weight=0.3,
)

from fastlangml import backend, Backend
from fastlangml.backends.base import DetectionResult

@backend("my_detector", reliability=4)  # reliability: 1-5
class MyBackend(Backend):
    """Custom language detection backend."""

    @property
    def name(self) -> str:
        return "my_detector"

    @property
    def is_available(self) -> bool:
        # Check if required dependencies are installed
        return True

    def detect(self, text: str) -> DetectionResult:
        # Your detection logic here
        lang = "en"  # Replace with actual detection
        confidence = 0.95
        return DetectionResult(self.name, lang, confidence)

    def supported_languages(self) -> set[str]:
        return {"en", "fr", "de", "es"}

from fastlangml import FastLangDetector, DetectionConfig

detector = FastLangDetector(
    config=DetectionConfig(
        backends=["my_detector", "fasttext"],
    )
)

from fastlangml import register_backend, unregister_backend, list_registered_backends

# Register
register_backend("my_backend", MyBackend, reliability=4)

# List all registered
print(list_registered_backends())  # ["my_backend"]

# Unregister
unregister_backend("my_backend")

from fastlangml import VotingStrategy, FastLangDetector, DetectionConfig

class ConfidenceOnlyVoting(VotingStrategy):
    """Pick the language with highest individual confidence."""

    def vote(
        self,
        results: list,
        weights: dict[str, float] | None = None,
    ) -> dict[str, float]:
        if not results:
            return {}

        # Find max confidence per language
        scores = {}
        for r in results:
            lang = r.language
            if lang not in scores or r.confidence > scores[lang]:
                scores[lang] = r.confidence

        return scores

# Use custom voting
detector = FastLangDetector(
    config=DetectionConfig(
        custom_voting=ConfidenceOnlyVoting()
    )
)

from fastlangml import ConfusionResolver, LanguageSimilarity

# Resolve ambiguity between similar languages
resolver = ConfusionResolver()

# Check if languages are a known confused pair
pair = resolver.get_confused_pair({"es", "pt"})  # frozenset({"es", "pt"})

# Adjust scores based on discriminating features
scores = {"es": 0.45, "pt": 0.42}
adjusted = resolver.resolve("Eu tenho um problema", scores)
# Portuguese boosted due to "tenho" (have)

# Get discriminating features
es_features, pt_features = resolver.get_discriminating_features("es", "pt")
# es_features: ["pero", "cuando", "donde", ...]
# pt_features: ["mas", "quando", "onde", ...]

# Check language relationships
sim = LanguageSimilarity()
sim.are_related("es", "pt")  # True (Romance family)
sim.are_related("en", "zh")  # False (different families)
sim.get_related_languages("es")  # {"pt", "fr", "it", "ro", "ca", "gl"}

from fastlangml import CodeSwitchDetector

detector = CodeSwitchDetector()

# Detect code-switching
result = detector.detect("That's muy importante for the proyecto")

result.is_mixed            # True
result.primary_language    # "en"
result.secondary_languages # ["es"]
result.languages           # ["en", "es"]
result.language_distribution  # {"en": 0.6, "es": 0.4}

# Get language spans
for span in result.spans:
    print(f"{span.text}: {span.language} ({span.confidence:.2f})")
# "That's": en (0.85)
# "muy": es (0.92)
# "importante": es (0.95)
# "for": en (0.88)
# "the": en (0.90)
# "proyecto": es (0.94)

# Quick check
detector.is_code_switched("Hello world")  # False
detector.is_code_switched("Hola, how are you?")  # True

# Pattern-based detection
from fastlangml import detect_code_switching_pattern

pattern = detect_code_switching_pattern("That's muy bueno")
# ("en", "es") - matches Spanglish pattern

# Run accuracy benchmarks
pytest tests/test_benchmarks.py -v

# Quick performance check
python -c "
from fastlangml import detect
import time

start = time.perf_counter()
for i in range(100):
    detect(f'Hello world {i}')
print(f'100 detections: {(time.perf_counter()-start)*1000:.0f}ms')
"

# Good: Context-aware
context = ConversationContext()
for msg in conversation:
    result = detect(msg, context=context)

# Bad: Stateless detection
for msg in conversation:
    result = detect(msg)  # Loses valuable context

detect("ok", mode="short")         # More lenient
detect("Hello world", mode="default")
detect(long_paragraph, mode="long") # More strict

detector.add_hint("gg", "en")     # Gaming
detector.add_hint("lol", "en")
detector.add_hint("mdr", "fr")    # French LOL
detector.add_hint("kek", "en")    # Gaming laugh

# Only consider English and Spanish
detector.set_languages(["en", "es"])

result = detect("ok")
if result == "und":  # Can compare directly
    # Fallback to default or ask user
    lang = result.candidates[0].lang if result.candidates else "en"
    print(f"Low confidence: {result.reason}")

# Development: Fast single backend
detector = FastLangDetector(
    config=DetectionConfig(backends=["fasttext"])
)

# Production: Reliable ensemble
detector = FastLangDetector(
    config=DetectionConfig(
        backends=["fasttext", "lingua"],
        voting_strategy="weighted",
    )
)

# Good: Batch processing (parallelized)
results = detector.detect_batch(messages, mode="short")

# Bad: Sequential detection
results = [detector.detect(msg) for msg in messages]  # Slower

# Clone and setup
git clone https://github.com/pnrajan/fastlangml.git
cd fastlangml
poetry install --all-extras

# Development commands
make test         # Run tests
make lint         # Check code style
make format       # Format code with ruff
make fix          # Auto-fix linting issues
make typecheck    # Run type checker
make check        # Run all checks (format + lint + typecheck + test)