Full-Text Search Guide

# $ pip install turbopuffer
import turbopuffer
import os

tpuf = turbopuffer.Turbopuffer(
    # API tokens are created in the dashboard: https://turbopuffer.com/dashboard
    api_key=os.getenv("TURBOPUFFER_API_KEY"),
    region="gcp-us-central1", # choose best region: https://turbopuffer.com/docs/regions
)

ns = tpuf.namespace(f'fts-basic-example-py')
ns.write(
    upsert_rows=[
        {
            'id': 1,
            'content': 'turbopuffer is a fast search engine with FTS, filtering, and vector search support'
        },
        {
            'id': 2,
            'content': 'turbopuffer can store billions and billions of documents cheaper than any other search engine'
        },
        {
            'id': 3,
            'content': 'turbopuffer will support many more types of queries as it evolves. turbopuffer will only get faster.'
        }
    ],
    schema={
        'content': {
            'type': 'string',
            # Enable BM25 with default settings
            # For all config options, see https://turbopuffer.com/docs/write#schema
            'full_text_search': True
        }
    }
)

# Basic FTS search.
results = ns.query(
    rank_by=('content', 'BM25', 'turbopuffer'),
    limit=10,
    include_attributes=['content']
)
# [3, 1, 2] is the default BM25 ranking based on document length and
# term frequency
print(results)

# Simple phrase matching filter, to limit results to documents that contain the
# terms "search" and "engine"
results = ns.query(
    rank_by=('content', 'BM25', 'turbopuffer'),
    filters=('content', 'ContainsAllTokens', 'search engine'),
    limit=10,
    include_attributes=['content']
)
# [1, 2] (same as above, but without document 3)
print(results)

# To combine with vector search, see:
# https://turbopuffer.com/docs/hybrid-search

import turbopuffer

tpuf = turbopuffer.Turbopuffer(
    region='gcp-us-central1', # choose best region: https://turbopuffer.com/docs/regions
)

ns = tpuf.namespace(f'fts-advanced-example-py')

# Write some documents with a rich set of attributes.
ns.write(
    upsert_rows=[
        {
            'id': 1,
            'title': 'Getting Started with Python',
            'content': 'Learn Python basics including variables, functions, and classes',
            'tags': ['python', 'programming', 'beginner'],
            'language': 'en',
            'publish_date': 1709251200
        },
        {
            'id': 2,
            'title': 'Advanced TypeScript Tips',
            'content': 'Discover advanced TypeScript features and type system tricks',
            'tags': ['typescript', 'javascript', 'advanced'],
            'language': 'en',
            'publish_date': 1709337600
        },
        {
            'id': 3,
            'title': 'Python vs JavaScript',
            'content': 'Compare Python and JavaScript for web development',
            'tags': ['python', 'javascript', 'comparison'],
            'language': 'en',
            'publish_date': 1709424000
        }
    ],
    schema={
        'title': {
            'type': 'string',
            'full_text_search': {
                # See all FTS indexing options at
                # https://turbopuffer.com/docs/write#param-full_text_search
                'language': 'english',
                'stemming': True,
                'remove_stopwords': True,
                'case_sensitive': False
            }
        },
        'content': {
            'type': 'string',
            'full_text_search': {
                'language': 'english',
                'stemming': True,
                'remove_stopwords': True
            }
        },
        'tags': {
            'type': '[]string',
            'full_text_search': {
                'stemming': False,
                'remove_stopwords': False,
                'case_sensitive': True
            }
        }
    }
)

# Advanced FTS search.
# In this example, hits on `title` and `tags` are weighted / boosted higher than
# hits on `content`.
result = ns.query(
    # See all FTS query options at https://turbopuffer.com/docs/query
    rank_by=('Sum', (
        ('Product', 3, ('title', 'BM25', 'python beginner')),
        ('Product', 2, ('tags', 'BM25', 'python beginner')),
        ('content', 'BM25', 'python beginner')
    )),
    filters=('And', (
        ('publish_date', 'Gte', 1709251200),
        ('language', 'Eq', 'en'),
    )),
    limit=10,
    include_attributes=['title', 'content', 'tags']
)
print(result.rows)

# To combine with vector search, see:
# https://turbopuffer.com/docs/hybrid-search

import turbopuffer
from typing import List

tpuf = turbopuffer.Turbopuffer(
    region='gcp-us-central1', # choose best region: https://turbopuffer.com/docs/regions
)

# A simple word tokenizer that preserves hyphens instead of splitting on them.
def tokenize(text: str) -> List[str]:
    # Replace all characters besides alphanumeric and '-' with spaces.
    cleaned = ""
    for ch in text:
        if ch.isalnum() or ch in "-":
            cleaned += ch
        else:
            cleaned += str(" ")
    # Lowercase and split on spaces.
    return cleaned.lower().split()

# Write some sample data.
ns = tpuf.namespace(f'fts-custom-tokenization-example-py')
ns.write(
    upsert_rows=[
        {"id": 1, "content": tokenize("We hold these truths to be self-evident.")},
        {"id": 2, "content": tokenize("For my own self, it seemed evident.")},
    ],
    schema={
        'content': {
            'type': '[]string',
            'full_text_search': {'tokenizer': 'pre_tokenized_array'}
        }
    }
)

# Query for "self" and "evident".
results = ns.query(
    # Notice that the BM25 operator now expects a list of tokens, not a string.
    rank_by=('content', 'BM25', ['self', 'evident']),
    limit=10,
)
# Only document 2 is matched, because document 1 has the token "self-evident"
# but neither the token "self" nor "evident".
print(results)

# Query for "self-evident".
results = ns.query(
    rank_by=('content', 'BM25', ['self-evident']),
    limit=10,
)
# Now only document 1 is matched.
print(results)

# To accept string queries, simply apply the tokenizer to the query string
# before passing it to the `BM25` operator.
def query_string(query: str):
    return ns.query(
        rank_by=('content', 'BM25', tokenize(query)),
        limit=10,
    )

# $ pip install turbopuffer
import turbopuffer

tpuf = turbopuffer.Turbopuffer(
    region='gcp-us-central1', # choose best region: https://turbopuffer.com/docs/regions
)
ns = tpuf.namespace(f'fts-fuzzy-example-py')
ns.write(
    upsert_rows=[
        {'id': 1, 'company_name': 'turbopuffer'},
        {'id': 2, 'company_name': 'turbopufer inc'},
    ],
    schema={
        'company_name': {
            'type': 'string',
            'fuzzy': True,
            'glob': True,
        },
    },
)
result = ns.query(
    rank_by=('Sum', (
        ('Product', 3, ('company_name', 'Glob', '*turbopufer*')),
        ('company_name', 'Fuzzy', 'turbopufer', {
            'max_edit_distance': [
                {'min_query_chars': 6, 'distance': 1},
            ],
            "case_sensitive": False
        }),
    )),
    include_attributes=["company_name"],
    limit=10
)
print(result.rows)