API Reference

High Level API

High-level Python bindings for llama.cpp.

llama_cpp.Llama

High-level Python wrapper for a llama.cpp model.

Source code in llama_cpp/llama.py

class Llama:
    """High-level Python wrapper for a llama.cpp model."""

    __backend_initialized = False

    def __init__(
        self,
        model_path: str,
        *,
        # Model Params
        n_gpu_layers: int = 0,
        split_mode: int = llama_cpp.LLAMA_SPLIT_MODE_LAYER,
        main_gpu: int = 0,
        tensor_split: Optional[List[float]] = None,
        vocab_only: bool = False,
        use_mmap: bool = True,
        use_mlock: bool = False,
        kv_overrides: Optional[Dict[str, Union[bool, int, float, str]]] = None,
        # Context Params
        seed: int = llama_cpp.LLAMA_DEFAULT_SEED,
        n_ctx: int = 512,
        n_batch: int = 512,
        n_threads: Optional[int] = None,
        n_threads_batch: Optional[int] = None,
        rope_scaling_type: Optional[int] = llama_cpp.LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED,
        pooling_type: int = llama_cpp.LLAMA_POOLING_TYPE_UNSPECIFIED,
        rope_freq_base: float = 0.0,
        rope_freq_scale: float = 0.0,
        yarn_ext_factor: float = -1.0,
        yarn_attn_factor: float = 1.0,
        yarn_beta_fast: float = 32.0,
        yarn_beta_slow: float = 1.0,
        yarn_orig_ctx: int = 0,
        logits_all: bool = False,
        embedding: bool = False,
        offload_kqv: bool = True,
        # Sampling Params
        last_n_tokens_size: int = 64,
        # LoRA Params
        lora_base: Optional[str] = None,
        lora_scale: float = 1.0,
        lora_path: Optional[str] = None,
        # Backend Params
        numa: Union[bool, int] = False,
        # Chat Format Params
        chat_format: Optional[str] = None,
        chat_handler: Optional[llama_chat_format.LlamaChatCompletionHandler] = None,
        # Speculative Decoding
        draft_model: Optional[LlamaDraftModel] = None,
        # Tokenizer Override
        tokenizer: Optional[BaseLlamaTokenizer] = None,
        # KV cache quantization
        type_k: Optional[int] = None,
        type_v: Optional[int] = None,
        # Misc
        verbose: bool = True,
        # Extra Params
        **kwargs,  # type: ignore
    ):
        """Load a llama.cpp model from `model_path`.

        Examples:
            Basic usage

            >>> import llama_cpp
            >>> model = llama_cpp.Llama(
            ...     model_path="path/to/model",
            ... )
            >>> print(model("The quick brown fox jumps ", stop=["."])["choices"][0]["text"])
            the lazy dog

            Loading a chat model

            >>> import llama_cpp
            >>> model = llama_cpp.Llama(
            ...     model_path="path/to/model",
            ...     chat_format="llama-2",
            ... )
            >>> print(model.create_chat_completion(
            ...     messages=[{
            ...         "role": "user",
            ...         "content": "what is the meaning of life?"
            ...     }]
            ... ))

        Args:
            model_path: Path to the model.
            n_gpu_layers: Number of layers to offload to GPU (-ngl). If -1, all layers are offloaded.
            split_mode: How to split the model across GPUs. See llama_cpp.LLAMA_SPLIT_* for options.
            main_gpu: main_gpu interpretation depends on split_mode: LLAMA_SPLIT_NONE: the GPU that is used for the entire model. LLAMA_SPLIT_ROW: the GPU that is used for small tensors and intermediate results. LLAMA_SPLIT_LAYER: ignored
            tensor_split: How split tensors should be distributed across GPUs. If None, the model is not split.
            vocab_only: Only load the vocabulary no weights.
            use_mmap: Use mmap if possible.
            use_mlock: Force the system to keep the model in RAM.
            kv_overrides: Key-value overrides for the model.
            seed: RNG seed, -1 for random
            n_ctx: Text context, 0 = from model
            n_batch: Prompt processing maximum batch size
            n_threads: Number of threads to use for generation
            n_threads_batch: Number of threads to use for batch processing
            rope_scaling_type: RoPE scaling type, from `enum llama_rope_scaling_type`. ref: https://github.com/ggerganov/llama.cpp/pull/2054
            pooling_type: Pooling type, from `enum llama_pooling_type`.
            rope_freq_base: RoPE base frequency, 0 = from model
            rope_freq_scale: RoPE frequency scaling factor, 0 = from model
            yarn_ext_factor: YaRN extrapolation mix factor, negative = from model
            yarn_attn_factor: YaRN magnitude scaling factor
            yarn_beta_fast: YaRN low correction dim
            yarn_beta_slow: YaRN high correction dim
            yarn_orig_ctx: YaRN original context size
            logits_all: Return logits for all tokens, not just the last token. Must be True for completion to return logprobs.
            embedding: Embedding mode only.
            offload_kqv: Offload K, Q, V to GPU.
            last_n_tokens_size: Maximum number of tokens to keep in the last_n_tokens deque.
            lora_base: Optional path to base model, useful if using a quantized base model and you want to apply LoRA to an f16 model.
            lora_path: Path to a LoRA file to apply to the model.
            numa: numa policy
            chat_format: String specifying the chat format to use when calling create_chat_completion.
            chat_handler: Optional chat handler to use when calling create_chat_completion.
            draft_model: Optional draft model to use for speculative decoding.
            tokenizer: Optional tokenizer to override the default tokenizer from llama.cpp.
            verbose: Print verbose output to stderr.
            type_k: KV cache data type for K (default: f16)
            type_v: KV cache data type for V (default: f16)

        Raises:
            ValueError: If the model path does not exist.

        Returns:
            A Llama instance.
        """
        self.verbose = verbose

        set_verbose(verbose)

        if not Llama.__backend_initialized:
            with suppress_stdout_stderr(disable=verbose):
                llama_cpp.llama_backend_init()
            Llama.__backend_initialized = True

        if isinstance(numa, bool):
            self.numa = (
                llama_cpp.GGML_NUMA_STRATEGY_DISTRIBUTE
                if numa
                else llama_cpp.GGML_NUMA_STRATEGY_DISABLED
            )
        else:
            self.numa = numa

        if self.numa != llama_cpp.GGML_NUMA_STRATEGY_DISABLED:
            with suppress_stdout_stderr(disable=verbose):
                llama_cpp.llama_numa_init(self.numa)

        self.model_path = model_path

        # Model Params
        self.model_params = llama_cpp.llama_model_default_params()
        self.model_params.n_gpu_layers = (
            0x7FFFFFFF if n_gpu_layers == -1 else n_gpu_layers
        )  # 0x7FFFFFFF is INT32 max, will be auto set to all layers
        self.model_params.split_mode = split_mode
        self.model_params.main_gpu = main_gpu
        self.tensor_split = tensor_split
        self._c_tensor_split = None
        if self.tensor_split is not None:
            if len(self.tensor_split) > llama_cpp.LLAMA_MAX_DEVICES:
                raise ValueError(
                    f"Attempt to split tensors that exceed maximum supported devices. Current LLAMA_MAX_DEVICES={llama_cpp.LLAMA_MAX_DEVICES}"
                )
            # Type conversion and expand the list to the length of LLAMA_MAX_DEVICES
            FloatArray = ctypes.c_float * llama_cpp.LLAMA_MAX_DEVICES
            self._c_tensor_split = FloatArray(
                *tensor_split  # type: ignore
            )  # keep a reference to the array so it is not gc'd
            self.model_params.tensor_split = self._c_tensor_split
        self.model_params.vocab_only = vocab_only
        self.model_params.use_mmap = use_mmap if lora_path is None else False
        self.model_params.use_mlock = use_mlock

        # kv_overrides is the original python dict
        self.kv_overrides = kv_overrides
        if kv_overrides is not None:
            # _kv_overrides_array is a ctypes.Array of llama_model_kv_override Structs
            kvo_array_len = len(kv_overrides) + 1  # for sentinel element
            self._kv_overrides_array = (
                llama_cpp.llama_model_kv_override * kvo_array_len
            )()

            for i, (k, v) in enumerate(kv_overrides.items()):
                self._kv_overrides_array[i].key = k.encode("utf-8")
                if isinstance(v, bool):
                    self._kv_overrides_array[i].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_BOOL
                    self._kv_overrides_array[i].value.bool_value = v
                elif isinstance(v, int):
                    self._kv_overrides_array[i].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_INT
                    self._kv_overrides_array[i].value.int_value = v
                elif isinstance(v, float):
                    self._kv_overrides_array[i].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_FLOAT
                    self._kv_overrides_array[i].value.float_value = v
                elif isinstance(v, str): # type: ignore
                    v_bytes = v.encode("utf-8")
                    if len(v_bytes) > 128: # TODO: Make this a constant
                        raise ValueError(f"Value for {k} is too long: {v}")
                    v_bytes = v_bytes.ljust(128, b"\0")
                    self._kv_overrides_array[i].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_STR
                    self._kv_overrides_array[i].value.str_value[:128] = v_bytes
                else:
                    raise ValueError(f"Unknown value type for {k}: {v}")

            self._kv_overrides_array[-1].key = (
                b"\0"  # ensure sentinel element is zeroed
            )
            self.model_params.kv_overrides = self._kv_overrides_array

        self.n_batch = min(n_ctx, n_batch)  # ???
        self.n_threads = n_threads or max(multiprocessing.cpu_count() // 2, 1)
        self.n_threads_batch = n_threads_batch or multiprocessing.cpu_count()

        # Context Params
        self.context_params = llama_cpp.llama_context_default_params()
        self.context_params.seed = seed
        self.context_params.n_ctx = n_ctx
        self.context_params.n_batch = self.n_batch
        self.context_params.n_threads = self.n_threads
        self.context_params.n_threads_batch = self.n_threads_batch
        self.context_params.rope_scaling_type = (
            rope_scaling_type
            if rope_scaling_type is not None
            else llama_cpp.LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED
        )
        self.context_params.pooling_type = pooling_type
        self.context_params.rope_freq_base = (
            rope_freq_base if rope_freq_base != 0.0 else 0
        )
        self.context_params.rope_freq_scale = (
            rope_freq_scale if rope_freq_scale != 0.0 else 0
        )
        self.context_params.yarn_ext_factor = (
            yarn_ext_factor if yarn_ext_factor != 0.0 else 0
        )
        self.context_params.yarn_attn_factor = (
            yarn_attn_factor if yarn_attn_factor != 0.0 else 0
        )
        self.context_params.yarn_beta_fast = (
            yarn_beta_fast if yarn_beta_fast != 0.0 else 0
        )
        self.context_params.yarn_beta_slow = (
            yarn_beta_slow if yarn_beta_slow != 0.0 else 0
        )
        self.context_params.yarn_orig_ctx = yarn_orig_ctx if yarn_orig_ctx != 0 else 0
        self.context_params.logits_all = (
            logits_all if draft_model is None else True
        )  # Must be set to True for speculative decoding
        self.context_params.embeddings = embedding # TODO: Rename to embeddings
        self.context_params.offload_kqv = offload_kqv
        #  KV cache quantization
        if type_k is not None:
            self.context_params.type_k = type_k
        if type_v is not None:
            self.context_params.type_v = type_v
        # Sampling Params
        self.last_n_tokens_size = last_n_tokens_size

        self.cache: Optional[BaseLlamaCache] = None

        self.lora_base = lora_base
        self.lora_scale = lora_scale
        self.lora_path = lora_path

        if not os.path.exists(model_path):
            raise ValueError(f"Model path does not exist: {model_path}")

        self._model = _LlamaModel(
            path_model=self.model_path, params=self.model_params, verbose=self.verbose
        )

        # Override tokenizer
        self.tokenizer_ = tokenizer or LlamaTokenizer(self)

        # Set the default value for the context and correct the batch
        if n_ctx == 0:
            n_ctx = self._model.n_ctx_train()
            self.n_batch = min(n_ctx, n_batch)
            self.context_params.n_ctx = self._model.n_ctx_train()
            self.context_params.n_batch = self.n_batch

        self._ctx = _LlamaContext(
            model=self._model,
            params=self.context_params,
            verbose=self.verbose,
        )

        self._batch = _LlamaBatch(
            n_tokens=self.n_batch,
            embd=0,
            n_seq_max=self.context_params.n_ctx,
            verbose=self.verbose,
        )

        if self.lora_path:
            if self._model.apply_lora_from_file(
                self.lora_path,
                self.lora_scale,
                self.lora_base,
                self.n_threads,
            ):
                raise RuntimeError(
                    f"Failed to apply LoRA from lora path: {self.lora_path} to base path: {self.lora_base}"
                )

        if self.verbose:
            print(llama_cpp.llama_print_system_info().decode("utf-8"), file=sys.stderr)

        self.chat_format = chat_format
        self.chat_handler = chat_handler

        self.draft_model = draft_model

        self._n_vocab = self.n_vocab()
        self._n_ctx = self.n_ctx()

        self._token_nl = self.token_nl()
        self._token_eos = self.token_eos()

        self._candidates = _LlamaTokenDataArray(n_vocab=self._n_vocab)

        self.n_tokens = 0
        self.input_ids: npt.NDArray[np.intc] = np.ndarray((n_ctx,), dtype=np.intc)
        self.scores: npt.NDArray[np.single] = np.ndarray(
            (n_ctx, self._n_vocab), dtype=np.single
        )

        self._mirostat_mu = ctypes.c_float(
            2.0 * 5.0
        )  # TODO: Move this to sampling context

        try:
            self.metadata = self._model.metadata()
        except Exception as e:
            self.metadata = {}
            if self.verbose:
                print(f"Failed to load metadata: {e}", file=sys.stderr)

        if self.verbose:
            print(f"Model metadata: {self.metadata}", file=sys.stderr)

        if (
            self.chat_format is None
            and self.chat_handler is None
            and "tokenizer.chat_template" in self.metadata
        ):
            chat_format = llama_chat_format.guess_chat_format_from_gguf_metadata(
                self.metadata
            )

            if chat_format is not None:
                self.chat_format = chat_format
                if self.verbose:
                    print(f"Guessed chat format: {chat_format}", file=sys.stderr)
            else:
                template = self.metadata["tokenizer.chat_template"]
                try:
                    eos_token_id = int(self.metadata["tokenizer.ggml.eos_token_id"])
                except:
                    eos_token_id = self.token_eos()
                try:
                    bos_token_id = int(self.metadata["tokenizer.ggml.bos_token_id"])
                except:
                    bos_token_id = self.token_bos()

                eos_token = self._model.token_get_text(eos_token_id)
                bos_token = self._model.token_get_text(bos_token_id)

                if self.verbose:
                    print(f"Using gguf chat template: {template}", file=sys.stderr)
                    print(f"Using chat eos_token: {eos_token}", file=sys.stderr)
                    print(f"Using chat bos_token: {bos_token}", file=sys.stderr)

                self.chat_handler = llama_chat_format.Jinja2ChatFormatter(
                    template=template,
                    eos_token=eos_token,
                    bos_token=bos_token,
                    stop_token_ids=[eos_token_id],
                ).to_chat_handler()

        if self.chat_format is None and self.chat_handler is None:
            self.chat_format = "llama-2"
            if self.verbose:
                print(f"Using fallback chat format: {chat_format}", file=sys.stderr)

    @property
    def ctx(self) -> llama_cpp.llama_context_p:
        assert self._ctx.ctx is not None
        return self._ctx.ctx

    @property
    def model(self) -> llama_cpp.llama_model_p:
        assert self._model.model is not None
        return self._model.model

    @property
    def _input_ids(self) -> npt.NDArray[np.intc]:
        return self.input_ids[: self.n_tokens]

    @property
    def _scores(self) -> npt.NDArray[np.single]:
        return self.scores[: self.n_tokens, :]

    @property
    def eval_tokens(self) -> Deque[int]:
        return deque(self.input_ids[: self.n_tokens].tolist(), maxlen=self._n_ctx)

    @property
    def eval_logits(self) -> Deque[List[float]]:
        return deque(
            self.scores[: self.n_tokens, :].tolist(),
            maxlen=self._n_ctx if self.context_params.logits_all else 1,
        )

    def tokenize(
        self, text: bytes, add_bos: bool = True, special: bool = False
    ) -> List[int]:
        """Tokenize a string.

        Args:
            text: The utf-8 encoded string to tokenize.

        Raises:
            RuntimeError: If the tokenization failed.

        Returns:
            A list of tokens.
        """
        return self.tokenizer_.tokenize(text, add_bos, special)

    def detokenize(
        self, tokens: List[int], prev_tokens: Optional[List[int]] = None
    ) -> bytes:
        """Detokenize a list of tokens.

        Args:
            tokens: The list of tokens to detokenize.
            prev_tokens: The list of previous tokens. Offset mapping will be performed if provided

        Returns:
            The detokenized string.
        """
        return self.tokenizer_.detokenize(tokens, prev_tokens=prev_tokens)

    def set_cache(self, cache: Optional[BaseLlamaCache]):
        """Set the cache.

        Args:
            cache: The cache to set.
        """
        self.cache = cache

    def set_seed(self, seed: int):
        """Set the random seed.

        Args:
            seed: The random seed.
        """
        assert self._ctx.ctx is not None
        llama_cpp.llama_set_rng_seed(self._ctx.ctx, seed)

    def reset(self):
        """Reset the model state."""
        self.n_tokens = 0

    def eval(self, tokens: Sequence[int]):
        """Evaluate a list of tokens.

        Args:
            tokens: The list of tokens to evaluate.
        """
        assert self._ctx.ctx is not None
        assert self._batch.batch is not None
        self._ctx.kv_cache_seq_rm(-1, self.n_tokens, -1)
        for i in range(0, len(tokens), self.n_batch):
            batch = tokens[i : min(len(tokens), i + self.n_batch)]
            n_past = self.n_tokens
            n_tokens = len(batch)
            self._batch.set_batch(
                batch=batch, n_past=n_past, logits_all=self.context_params.logits_all
            )
            self._ctx.decode(self._batch)
            # Save tokens
            self.input_ids[n_past : n_past + n_tokens] = batch
            # Save logits
            if self.context_params.logits_all:
                rows = n_tokens
                cols = self._n_vocab
                logits = self._ctx.get_logits()[: rows * cols]
                self.scores[n_past : n_past + n_tokens, :].reshape(-1)[: :] = logits
            else:
                rows = 1
                cols = self._n_vocab
                logits = self._ctx.get_logits()[: rows * cols]
                self.scores[n_past + n_tokens - 1, :].reshape(-1)[: :] = logits
            # Update n_tokens
            self.n_tokens += n_tokens

    def sample(
        self,
        top_k: int = 40,
        top_p: float = 0.95,
        min_p: float = 0.05,
        typical_p: float = 1.0,
        temp: float = 0.80,
        repeat_penalty: float = 1.1,
        frequency_penalty: float = 0.0,
        presence_penalty: float = 0.0,
        tfs_z: float = 1.0,
        mirostat_mode: int = 0,
        mirostat_eta: float = 0.1,
        mirostat_tau: float = 5.0,
        penalize_nl: bool = True,
        logits_processor: Optional[LogitsProcessorList] = None,
        grammar: Optional[LlamaGrammar] = None,
        idx: Optional[int] = None,
    ):
        """Sample a token from the model.

        Args:
            top_k: The top-k sampling parameter.
            top_p: The top-p sampling parameter.
            temp: The temperature parameter.
            repeat_penalty: The repeat penalty parameter.

        Returns:
            The sampled token.
        """
        assert self._ctx is not None
        assert self.n_tokens > 0

        if idx is None:
            logits: npt.NDArray[np.single] = self._scores[-1, :]
        else:
            logits = self._scores[idx, :]

        if logits_processor is not None:
            logits[:] = (
                logits_processor(self._input_ids, logits)
                if idx is None
                else logits_processor(self._input_ids[: idx + 1], logits)
            )

        sampling_params = _LlamaSamplingParams(
            top_k=top_k,
            top_p=top_p,
            min_p=min_p,
            tfs_z=tfs_z,
            typical_p=typical_p,
            temp=temp,
            penalty_last_n=self.last_n_tokens_size,
            penalty_repeat=repeat_penalty,
            penalty_freq=frequency_penalty,
            penalty_present=presence_penalty,
            mirostat=mirostat_mode,
            mirostat_tau=mirostat_tau,
            mirostat_eta=mirostat_eta,
            penalize_nl=penalize_nl,
        )
        sampling_context = _LlamaSamplingContext(
            params=sampling_params,
            grammar=grammar,
        )
        sampling_context.prev = list(self.eval_tokens)
        id = sampling_context.sample(ctx_main=self._ctx, logits_array=logits)
        sampling_context.accept(
            ctx_main=self._ctx,
            id=id,
            apply_grammar=grammar is not None,
        )
        return id

    def generate(
        self,
        tokens: Sequence[int],
        top_k: int = 40,
        top_p: float = 0.95,
        min_p: float = 0.05,
        typical_p: float = 1.0,
        temp: float = 0.80,
        repeat_penalty: float = 1.1,
        reset: bool = True,
        frequency_penalty: float = 0.0,
        presence_penalty: float = 0.0,
        tfs_z: float = 1.0,
        mirostat_mode: int = 0,
        mirostat_tau: float = 5.0,
        mirostat_eta: float = 0.1,
        penalize_nl: bool = True,
        logits_processor: Optional[LogitsProcessorList] = None,
        stopping_criteria: Optional[StoppingCriteriaList] = None,
        grammar: Optional[LlamaGrammar] = None,
    ) -> Generator[int, Optional[Sequence[int]], None]:
        """Create a generator of tokens from a prompt.

        Examples:
            >>> llama = Llama("models/ggml-7b.bin")
            >>> tokens = llama.tokenize(b"Hello, world!")
            >>> for token in llama.generate(tokens, top_k=40, top_p=0.95, temp=1.0, repeat_penalty=1.1):
            ...     print(llama.detokenize([token]))

        Args:
            tokens: The prompt tokens.
            top_k: The top-k sampling parameter.
            top_p: The top-p sampling parameter.
            temp: The temperature parameter.
            repeat_penalty: The repeat penalty parameter.
            reset: Whether to reset the model state.

        Yields:
            The generated tokens.
        """
        # Reset mirostat sampling
        self._mirostat_mu = ctypes.c_float(2.0 * mirostat_tau)

        # Check for kv cache prefix match
        if reset and self.n_tokens > 0:
            longest_prefix = 0
            for a, b in zip(self._input_ids, tokens[:-1]):
                if a == b:
                    longest_prefix += 1
                else:
                    break
            if longest_prefix > 0:
                if self.verbose:
                    print("Llama.generate: prefix-match hit", file=sys.stderr)
                reset = False
                tokens = tokens[longest_prefix:]
                self.n_tokens = longest_prefix

        # Reset the model state
        if reset:
            self.reset()

        # Reset the grammar
        if grammar is not None:
            grammar.reset()

        sample_idx = self.n_tokens + len(tokens) - 1
        tokens = list(tokens)

        # Eval and sample
        while True:
            self.eval(tokens)
            while sample_idx < self.n_tokens:
                token = self.sample(
                    top_k=top_k,
                    top_p=top_p,
                    min_p=min_p,
                    typical_p=typical_p,
                    temp=temp,
                    repeat_penalty=repeat_penalty,
                    frequency_penalty=frequency_penalty,
                    presence_penalty=presence_penalty,
                    tfs_z=tfs_z,
                    mirostat_mode=mirostat_mode,
                    mirostat_tau=mirostat_tau,
                    mirostat_eta=mirostat_eta,
                    logits_processor=logits_processor,
                    grammar=grammar,
                    penalize_nl=penalize_nl,
                    idx=sample_idx,
                )

                sample_idx += 1
                if stopping_criteria is not None and stopping_criteria(
                    self._input_ids, self._scores[-1, :]
                ):
                    return
                tokens_or_none = yield token
                tokens.clear()
                tokens.append(token)
                if tokens_or_none is not None:
                    tokens.extend(tokens_or_none)

                if sample_idx < self.n_tokens and token != self._input_ids[sample_idx]:
                    self.n_tokens = sample_idx
                    self._ctx.kv_cache_seq_rm(-1, self.n_tokens, -1)
                    break

            if self.draft_model is not None:
                self.input_ids[self.n_tokens : self.n_tokens + len(tokens)] = tokens
                draft_tokens = self.draft_model(
                    self.input_ids[: self.n_tokens + len(tokens)]
                )
                tokens.extend(
                    draft_tokens.astype(int)[
                        : self._n_ctx - self.n_tokens - len(tokens)
                    ]
                )

    def create_embedding(
        self, input: Union[str, List[str]], model: Optional[str] = None
    ) -> CreateEmbeddingResponse:
        """Embed a string.

        Args:
            input: The utf-8 encoded string to embed.

        Returns:
            An embedding object.
        """
        assert self._model.model is not None
        model_name: str = model if model is not None else self.model_path

        input = input if isinstance(input, list) else [input]

        # get numeric embeddings
        embeds: Union[List[List[float]], List[List[List[float]]]]
        total_tokens: int
        embeds, total_tokens = self.embed(input, return_count=True)  # type: ignore

        # convert to CreateEmbeddingResponse
        data: List[Embedding] = [
            {
                "object": "embedding",
                "embedding": emb,
                "index": idx,
            }
            for idx, emb in enumerate(embeds)
        ]

        return {
            "object": "list",
            "data": data,
            "model": model_name,
            "usage": {
                "prompt_tokens": total_tokens,
                "total_tokens": total_tokens,
            },
        }

    def embed(
        self,
        input: Union[str, List[str]],
        normalize: bool = False,
        truncate: bool = True,
        return_count: bool = False,
    ):
        """Embed a string.

        Args:
            input: The utf-8 encoded string to embed.

        Returns:
            A list of embeddings
        """
        assert self._ctx.ctx is not None
        n_embd = self.n_embd()
        n_batch = self.n_batch

        # get pooling information
        pooling_type = self.pooling_type()
        logits_all = pooling_type == llama_cpp.LLAMA_POOLING_TYPE_NONE

        if self.context_params.embeddings == False:
            raise RuntimeError(
                "Llama model must be created with embedding=True to call this method"
            )

        if self.verbose:
            llama_cpp.llama_reset_timings(self._ctx.ctx)

        if isinstance(input, str):
            inputs = [input]
        else:
            inputs = input

        # reset batch
        self._batch.reset()

        # decode and fetch embeddings
        data: Union[List[List[float]], List[List[List[float]]]] = []

        def decode_batch(seq_sizes: List[int]):
            assert self._ctx.ctx is not None
            llama_cpp.llama_kv_cache_clear(self._ctx.ctx)
            self._ctx.decode(self._batch)
            self._batch.reset()

            # store embeddings
            if pooling_type == llama_cpp.LLAMA_POOLING_TYPE_NONE:
                pos: int = 0
                for i, size in enumerate(seq_sizes):
                    ptr = llama_cpp.llama_get_embeddings(self._ctx.ctx)
                    embedding: List[List[float]] = [
                        ptr[pos + j * n_embd : pos + (j + 1) * n_embd] for j in range(size)
                    ]
                    if normalize:
                        embedding = [_normalize_embedding(e) for e in embedding]
                    data.append(embedding)
                    pos += size
            else:
                for i in range(len(seq_sizes)):
                    ptr = llama_cpp.llama_get_embeddings_seq(self._ctx.ctx, i)
                    embedding: List[float] = ptr[:n_embd]
                    if normalize:
                        embedding = _normalize_embedding(embedding)
                    data.append(embedding)

        # init state
        total_tokens = 0
        s_batch = []
        t_batch = 0
        p_batch = 0

        # accumulate batches and encode
        for text in inputs:
            tokens = self.tokenize(text.encode("utf-8"))
            if truncate:
                tokens = tokens[:n_batch]

            n_tokens = len(tokens)
            total_tokens += n_tokens

            # check for overrun
            if n_tokens > n_batch:
                raise ValueError(
                    f"Requested tokens ({n_tokens}) exceed batch size of {n_batch}"
                )

            # time to eval batch
            if t_batch + n_tokens > n_batch:
                decode_batch(s_batch)
                s_batch = []
                t_batch = 0
                p_batch = 0

            # add to batch
            self._batch.add_sequence(tokens, p_batch, logits_all)

            # update batch stats
            s_batch.append(n_tokens)
            t_batch += n_tokens
            p_batch += 1

        # hanlde last batch
        decode_batch(s_batch)

        if self.verbose:
            llama_cpp.llama_print_timings(self._ctx.ctx)

        output = data[0] if isinstance(input, str) else data

        llama_cpp.llama_kv_cache_clear(self._ctx.ctx)
        self.reset()

        if return_count:
            return output, total_tokens
        else:
            return output

    def _create_completion(
        self,
        prompt: Union[str, List[int]],
        suffix: Optional[str] = None,
        max_tokens: Optional[int] = 16,
        temperature: float = 0.8,
        top_p: float = 0.95,
        min_p: float = 0.05,
        typical_p: float = 1.0,
        logprobs: Optional[int] = None,
        echo: bool = False,
        stop: Optional[Union[str, List[str]]] = [],
        frequency_penalty: float = 0.0,
        presence_penalty: float = 0.0,
        repeat_penalty: float = 1.1,
        top_k: int = 40,
        stream: bool = False,
        seed: Optional[int] = None,
        tfs_z: float = 1.0,
        mirostat_mode: int = 0,
        mirostat_tau: float = 5.0,
        mirostat_eta: float = 0.1,
        model: Optional[str] = None,
        stopping_criteria: Optional[StoppingCriteriaList] = None,
        logits_processor: Optional[LogitsProcessorList] = None,
        grammar: Optional[LlamaGrammar] = None,
        logit_bias: Optional[Dict[str, float]] = None,
    ) -> Union[
        Iterator[CreateCompletionResponse], Iterator[CreateCompletionStreamResponse]
    ]:
        assert self._ctx is not None
        assert suffix is None or suffix.__class__ is str

        completion_id: str = f"cmpl-{str(uuid.uuid4())}"
        created: int = int(time.time())
        # If prompt is empty, initialize completion with BOS token to avoid
        # detokenization including a space at the beginning of the completion
        completion_tokens: List[int] = [] if len(prompt) > 0 else [self.token_bos()]
        # Add blank space to start of prompt to match OG llama tokenizer
        prompt_tokens: List[int] = (
            (
                self.tokenize(prompt.encode("utf-8"), special=True)
                if prompt != ""
                else [self.token_bos()]
            )
            if isinstance(prompt, str)
            else prompt
        )
        text: bytes = b""
        returned_tokens: int = 0
        stop = (
            stop if isinstance(stop, list) else [stop] if isinstance(stop, str) else []
        )
        model_name: str = model if model is not None else self.model_path

        # NOTE: This likely doesn't work correctly for the first token in the prompt
        # because of the extra space added to the start of the prompt_tokens
        if logit_bias is not None:
            logit_bias_map = {int(k): float(v) for k, v in logit_bias.items()}

            def logit_bias_processor(
                input_ids: npt.NDArray[np.intc],
                scores: npt.NDArray[np.single],
            ) -> npt.NDArray[np.single]:
                new_scores = np.copy(
                    scores
                )  # Does it make sense to copy the whole array or can we just overwrite the original one?
                for input_id, score in logit_bias_map.items():
                    new_scores[input_id] = score + scores[input_id]
                return new_scores

            _logit_bias_processor = LogitsProcessorList([logit_bias_processor])
            if logits_processor is None:
                logits_processor = _logit_bias_processor
            else:
                logits_processor = logits_processor.extend(_logit_bias_processor)

        if self.verbose:
            self._ctx.reset_timings()

        if len(prompt_tokens) >= self._n_ctx:
            raise ValueError(
                f"Requested tokens ({len(prompt_tokens)}) exceed context window of {llama_cpp.llama_n_ctx(self.ctx)}"
            )

        if max_tokens is None or max_tokens <= 0:
            # Unlimited, depending on n_ctx.
            max_tokens = self._n_ctx - len(prompt_tokens)

        # Truncate max_tokens if requested tokens would exceed the context window
        max_tokens = (
            max_tokens
            if max_tokens + len(prompt_tokens) < self._n_ctx
            else (self._n_ctx - len(prompt_tokens))
        )

        if stop != []:
            stop_sequences = [s.encode("utf-8") for s in stop]
        else:
            stop_sequences = []

        if logprobs is not None and self.context_params.logits_all is False:
            raise ValueError(
                "logprobs is not supported for models created with logits_all=False"
            )

        if self.cache:
            try:
                cache_item = self.cache[prompt_tokens]
                cache_prefix_len = Llama.longest_token_prefix(
                    cache_item.input_ids.tolist(), prompt_tokens
                )
                eval_prefix_len = Llama.longest_token_prefix(
                    self._input_ids.tolist(), prompt_tokens
                )
                if cache_prefix_len > eval_prefix_len:
                    self.load_state(cache_item)
                    if self.verbose:
                        print("Llama._create_completion: cache hit", file=sys.stderr)
            except KeyError:
                if self.verbose:
                    print("Llama._create_completion: cache miss", file=sys.stderr)

        if seed is not None:
            self._ctx.set_rng_seed(seed)

        finish_reason = "length"
        multibyte_fix = 0
        for token in self.generate(
            prompt_tokens,
            top_k=top_k,
            top_p=top_p,
            min_p=min_p,
            typical_p=typical_p,
            temp=temperature,
            tfs_z=tfs_z,
            mirostat_mode=mirostat_mode,
            mirostat_tau=mirostat_tau,
            mirostat_eta=mirostat_eta,
            frequency_penalty=frequency_penalty,
            presence_penalty=presence_penalty,
            repeat_penalty=repeat_penalty,
            stopping_criteria=stopping_criteria,
            logits_processor=logits_processor,
            grammar=grammar,
        ):
            assert self._model.model is not None
            if llama_cpp.llama_token_is_eog(self._model.model, token):
                text = self.detokenize(completion_tokens, prev_tokens=prompt_tokens)
                finish_reason = "stop"
                break

            completion_tokens.append(token)

            all_text = self.detokenize(completion_tokens, prev_tokens=prompt_tokens)

            # Contains multi-byte UTF8
            for k, char in enumerate(all_text[-3:]):
                k = 3 - k
                for num, pattern in [(2, 192), (3, 224), (4, 240)]:
                    # Bitwise AND check
                    if num > k and pattern & char == pattern:
                        multibyte_fix = num - k

            # Stop incomplete bytes from passing
            if multibyte_fix > 0:
                multibyte_fix -= 1
                continue

            any_stop = [s for s in stop_sequences if s in all_text]
            if len(any_stop) > 0:
                first_stop = any_stop[0]
                text = all_text[: all_text.index(first_stop)]
                finish_reason = "stop"
                break

            if stream:
                remaining_tokens = completion_tokens[returned_tokens:]
                remaining_text = self.detokenize(remaining_tokens, prev_tokens=prompt_tokens + completion_tokens[:returned_tokens])
                remaining_length = len(remaining_text)

                # We want to avoid yielding any characters from
                # the generated text if they are part of a stop
                # sequence.
                first_stop_position = 0
                for s in stop_sequences:
                    for i in range(min(len(s), remaining_length), 0, -1):
                        if remaining_text.endswith(s[:i]):
                            if i > first_stop_position:
                                first_stop_position = i
                            break

                token_end_position = 0

                if logprobs is not None:
                    # not sure how to handle this branch when dealing
                    # with CJK output, so keep it unchanged
                    for token in remaining_tokens:
                        if token == self.token_bos():
                            continue
                        token_end_position += len(self.detokenize([token], prev_tokens=prompt_tokens + completion_tokens[:returned_tokens]))
                        # Check if stop sequence is in the token
                        if token_end_position > (
                            remaining_length - first_stop_position
                        ):
                            break
                        token_str = self.detokenize([token], prev_tokens=prompt_tokens + completion_tokens[:returned_tokens]).decode(
                            "utf-8", errors="ignore"
                        )
                        text_offset = len(prompt) + len(
                            self.detokenize(completion_tokens[:returned_tokens], prev_tokens=prompt_tokens + completion_tokens[:returned_tokens]).decode(
                                "utf-8", errors="ignore"
                            )
                        )
                        token_offset = len(prompt_tokens) + returned_tokens
                        logits = self._scores[token_offset - 1, :]
                        current_logprobs = Llama.logits_to_logprobs(logits).tolist()
                        sorted_logprobs = list(
                            sorted(
                                zip(current_logprobs, range(len(current_logprobs))),
                                reverse=True,
                            )
                        )
                        top_logprob = {
                            self.detokenize([i]).decode(
                                "utf-8", errors="ignore"
                            ): logprob
                            for logprob, i in sorted_logprobs[:logprobs]
                        }
                        top_logprob.update({token_str: current_logprobs[int(token)]})
                        logprobs_or_none = {
                            "tokens": [
                                self.detokenize([token], prev_tokens=prompt_tokens + completion_tokens[:returned_tokens]).decode(
                                    "utf-8", errors="ignore"
                                )
                            ],
                            "text_offset": [text_offset],
                            "token_logprobs": [current_logprobs[int(token)]],
                            "top_logprobs": [top_logprob],
                        }
                        returned_tokens += 1
                        yield {
                            "id": completion_id,
                            "object": "text_completion",
                            "created": created,
                            "model": model_name,
                            "choices": [
                                {
                                    "text": self.detokenize([token], prev_tokens=prompt_tokens + completion_tokens[:returned_tokens]).decode(
                                        "utf-8", errors="ignore"
                                    ),
                                    "index": 0,
                                    "logprobs": logprobs_or_none,
                                    "finish_reason": None,
                                }
                            ],
                        }
                else:
                    while len(remaining_tokens) > 0:
                        decode_success = False
                        for i in range(1, len(remaining_tokens) + 1):
                            try:
                                bs = self.detokenize(remaining_tokens[:i], prev_tokens=prompt_tokens + completion_tokens[:returned_tokens])
                                ts = bs.decode("utf-8")
                                decode_success = True
                                break
                            except UnicodeError:
                                pass
                        else:
                            break
                        if not decode_success:
                            # all remaining tokens cannot be decoded to a UTF-8 character
                            break
                        token_end_position += len(bs)
                        if token_end_position > (
                            remaining_length - first_stop_position
                        ):
                            break
                        remaining_tokens = remaining_tokens[i:]
                        returned_tokens += i

                        yield {
                            "id": completion_id,
                            "object": "text_completion",
                            "created": created,
                            "model": model_name,
                            "choices": [
                                {
                                    "text": ts,
                                    "index": 0,
                                    "logprobs": None,
                                    "finish_reason": None,
                                }
                            ],
                        }

            if len(completion_tokens) >= max_tokens:
                text = self.detokenize(completion_tokens, prev_tokens=prompt_tokens)
                finish_reason = "length"
                break

        if stopping_criteria is not None and stopping_criteria(
            self._input_ids, self._scores[-1, :]
        ):
            text = self.detokenize(completion_tokens, prev_tokens=prompt_tokens)
            finish_reason = "stop"

        if self.verbose:
            self._ctx.print_timings()

        if stream:
            remaining_tokens = completion_tokens[returned_tokens:]
            all_text = self.detokenize(remaining_tokens, prev_tokens=prompt_tokens + completion_tokens[:returned_tokens])
            any_stop = [s for s in stop_sequences if s in all_text]
            if len(any_stop) > 0:
                end = min(all_text.index(stop) for stop in any_stop)
            else:
                end = len(all_text)

            token_end_position = 0
            for token in remaining_tokens:
                token_end_position += len(self.detokenize([token], prev_tokens=prompt_tokens + completion_tokens[:returned_tokens]))

                logprobs_or_none: Optional[CompletionLogprobs] = None
                if logprobs is not None:
                    if token == self.token_bos():
                        continue
                    token_str = self.detokenize([token]).decode(
                        "utf-8", errors="ignore"
                    )
                    text_offset = len(prompt) + len(
                        self.detokenize(completion_tokens[:returned_tokens], prev_tokens=prompt_tokens + completion_tokens[:returned_tokens])
                    )
                    token_offset = len(prompt_tokens) + returned_tokens - 1
                    logits = self._scores[token_offset, :]
                    current_logprobs = Llama.logits_to_logprobs(logits).tolist()
                    sorted_logprobs = list(
                        sorted(
                            zip(current_logprobs, range(len(current_logprobs))),
                            reverse=True,
                        )
                    )
                    top_logprob = {
                        self.detokenize([i]).decode("utf-8", errors="ignore"): logprob
                        for logprob, i in sorted_logprobs[:logprobs]
                    }
                    top_logprob.update({token_str: current_logprobs[int(token)]})
                    logprobs_or_none = {
                        "tokens": [
                            self.detokenize([token]).decode("utf-8", errors="ignore")
                        ],
                        "text_offset": [text_offset],
                        "token_logprobs": [current_logprobs[int(token)]],
                        "top_logprobs": [top_logprob],
                    }

                if token_end_position >= end:
                    last_text = self.detokenize([token])
                    if token_end_position == end - 1:
                        break
                    returned_tokens += 1
                    yield {
                        "id": completion_id,
                        "object": "text_completion",
                        "created": created,
                        "model": model_name,
                        "choices": [
                            {
                                "text": last_text[
                                    : len(last_text) - (token_end_position - end)
                                ].decode("utf-8", errors="ignore"),
                                "index": 0,
                                "logprobs": logprobs_or_none,
                                "finish_reason": None,
                            }
                        ],
                    }
                    break
                returned_tokens += 1
                yield {
                    "id": completion_id,
                    "object": "text_completion",
                    "created": created,
                    "model": model_name,
                    "choices": [
                        {
                            "text": self.detokenize([token]).decode(
                                "utf-8", errors="ignore"
                            ),
                            "index": 0,
                            "logprobs": logprobs_or_none,
                            "finish_reason": None,
                        }
                    ],
                }
            yield {
                "id": completion_id,
                "object": "text_completion",
                "created": created,
                "model": model_name,
                "choices": [
                    {
                        "text": "",
                        "index": 0,
                        "logprobs": None,
                        "finish_reason": finish_reason,
                    }
                ],
            }
            if self.cache:
                if self.verbose:
                    print("Llama._create_completion: cache save", file=sys.stderr)
                self.cache[prompt_tokens + completion_tokens] = self.save_state()
                print("Llama._create_completion: cache saved", file=sys.stderr)
            return

        if self.cache:
            if self.verbose:
                print("Llama._create_completion: cache save", file=sys.stderr)
            self.cache[prompt_tokens + completion_tokens] = self.save_state()

        text_str = text.decode("utf-8", errors="ignore")

        if echo:
            text_str = prompt + text_str

        if suffix is not None:
            text_str = text_str + suffix

        logprobs_or_none: Optional[CompletionLogprobs] = None
        if logprobs is not None:
            text_offset = 0 if echo else len(prompt)
            token_offset = 0 if echo else len(prompt_tokens[1:])
            text_offsets: List[int] = []
            token_logprobs: List[Optional[float]] = []
            tokens: List[str] = []
            top_logprobs: List[Optional[Dict[str, float]]] = []

            if echo:
                # Remove leading BOS token
                all_tokens = prompt_tokens[1:] + completion_tokens
            else:
                all_tokens = completion_tokens

            all_token_strs = [
                self.detokenize([token], prev_tokens=all_tokens[:i]).decode("utf-8", errors="ignore")
                for i, token in enumerate(all_tokens)
            ]
            all_logprobs = Llama.logits_to_logprobs(self._scores)[token_offset:]
            # TODO: may be able to change this loop to use np.take_along_dim
            for idx, (token, token_str, logprobs_token) in enumerate(
                zip(all_tokens, all_token_strs, all_logprobs)
            ):
                if token == self.token_bos():
                    continue
                text_offsets.append(
                    text_offset
                    + len(
                        self.detokenize(all_tokens[:idx]).decode(
                            "utf-8", errors="ignore"
                        )
                    )
                )
                tokens.append(token_str)
                sorted_logprobs = list(
                    sorted(
                        zip(logprobs_token, range(len(logprobs_token))), reverse=True
                    )
                )
                token_logprobs.append(logprobs_token[int(token)])
                top_logprob: Optional[Dict[str, float]] = {
                    self.detokenize([i], prev_tokens=all_tokens[:idx]).decode("utf-8", errors="ignore"): logprob
                    for logprob, i in sorted_logprobs[:logprobs]
                }
                top_logprob.update({token_str: logprobs_token[int(token)]})
                top_logprobs.append(top_logprob)
            # Weird idosincracy of the OpenAI API where
            # token_logprobs and top_logprobs are null for
            # the first token.
            if echo and len(all_tokens) > 0:
                token_logprobs[0] = None
                top_logprobs[0] = None
            logprobs_or_none = {
                "tokens": tokens,
                "text_offset": text_offsets,
                "token_logprobs": token_logprobs,
                "top_logprobs": top_logprobs,
            }

        yield {
            "id": completion_id,
            "object": "text_completion",
            "created": created,
            "model": model_name,
            "choices": [
                {
                    "text": text_str,
                    "index": 0,
                    "logprobs": logprobs_or_none,
                    "finish_reason": finish_reason,
                }
            ],
            "usage": {
                "prompt_tokens": len(prompt_tokens),
                "completion_tokens": len(completion_tokens),
                "total_tokens": len(prompt_tokens) + len(completion_tokens),
            },
        }

    def create_completion(
        self,
        prompt: Union[str, List[int]],
        suffix: Optional[str] = None,
        max_tokens: Optional[int] = 16,
        temperature: float = 0.8,
        top_p: float = 0.95,
        min_p: float = 0.05,
        typical_p: float = 1.0,
        logprobs: Optional[int] = None,
        echo: bool = False,
        stop: Optional[Union[str, List[str]]] = [],
        frequency_penalty: float = 0.0,
        presence_penalty: float = 0.0,
        repeat_penalty: float = 1.1,
        top_k: int = 40,
        stream: bool = False,
        seed: Optional[int] = None,
        tfs_z: float = 1.0,
        mirostat_mode: int = 0,
        mirostat_tau: float = 5.0,
        mirostat_eta: float = 0.1,
        model: Optional[str] = None,
        stopping_criteria: Optional[StoppingCriteriaList] = None,
        logits_processor: Optional[LogitsProcessorList] = None,
        grammar: Optional[LlamaGrammar] = None,
        logit_bias: Optional[Dict[str, float]] = None,
    ) -> Union[CreateCompletionResponse, Iterator[CreateCompletionStreamResponse]]:
        """Generate text from a prompt.

        Args:
            prompt: The prompt to generate text from.
            suffix: A suffix to append to the generated text. If None, no suffix is appended.
            max_tokens: The maximum number of tokens to generate. If max_tokens <= 0 or None, the maximum number of tokens to generate is unlimited and depends on n_ctx.
            temperature: The temperature to use for sampling.
            top_p: The top-p value to use for nucleus sampling. Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751
            min_p: The min-p value to use for minimum p sampling. Minimum P sampling as described in https://github.com/ggerganov/llama.cpp/pull/3841
            typical_p: The typical-p value to use for sampling. Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666.
            logprobs: The number of logprobs to return. If None, no logprobs are returned.
            echo: Whether to echo the prompt.
            stop: A list of strings to stop generation when encountered.
            frequency_penalty: The penalty to apply to tokens based on their frequency in the prompt.
            presence_penalty: The penalty to apply to tokens based on their presence in the prompt.
            repeat_penalty: The penalty to apply to repeated tokens.
            top_k: The top-k value to use for sampling. Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751
            stream: Whether to stream the results.
            seed: The seed to use for sampling.
            tfs_z: The tail-free sampling parameter. Tail Free Sampling described in https://www.trentonbricken.com/Tail-Free-Sampling/.
            mirostat_mode: The mirostat sampling mode.
            mirostat_tau: The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text.
            mirostat_eta: The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates.
            model: The name to use for the model in the completion object.
            stopping_criteria: A list of stopping criteria to use.
            logits_processor: A list of logits processors to use.
            grammar: A grammar to use for constrained sampling.
            logit_bias: A logit bias to use.

        Raises:
            ValueError: If the requested tokens exceed the context window.
            RuntimeError: If the prompt fails to tokenize or the model fails to evaluate the prompt.

        Returns:
            Response object containing the generated text.
        """
        completion_or_chunks = self._create_completion(
            prompt=prompt,
            suffix=suffix,
            max_tokens=-1 if max_tokens is None else max_tokens,
            temperature=temperature,
            top_p=top_p,
            min_p=min_p,
            typical_p=typical_p,
            logprobs=logprobs,
            echo=echo,
            stop=stop,
            frequency_penalty=frequency_penalty,
            presence_penalty=presence_penalty,
            repeat_penalty=repeat_penalty,
            top_k=top_k,
            stream=stream,
            seed=seed,
            tfs_z=tfs_z,
            mirostat_mode=mirostat_mode,
            mirostat_tau=mirostat_tau,
            mirostat_eta=mirostat_eta,
            model=model,
            stopping_criteria=stopping_criteria,
            logits_processor=logits_processor,
            grammar=grammar,
            logit_bias=logit_bias,
        )
        if stream:
            chunks: Iterator[CreateCompletionStreamResponse] = completion_or_chunks
            return chunks
        completion: Completion = next(completion_or_chunks)  # type: ignore
        return completion

    def __call__(
        self,
        prompt: str,
        suffix: Optional[str] = None,
        max_tokens: Optional[int] = 16,
        temperature: float = 0.8,
        top_p: float = 0.95,
        min_p: float = 0.05,
        typical_p: float = 1.0,
        logprobs: Optional[int] = None,
        echo: bool = False,
        stop: Optional[Union[str, List[str]]] = [],
        frequency_penalty: float = 0.0,
        presence_penalty: float = 0.0,
        repeat_penalty: float = 1.1,
        top_k: int = 40,
        stream: bool = False,
        seed: Optional[int] = None,
        tfs_z: float = 1.0,
        mirostat_mode: int = 0,
        mirostat_tau: float = 5.0,
        mirostat_eta: float = 0.1,
        model: Optional[str] = None,
        stopping_criteria: Optional[StoppingCriteriaList] = None,
        logits_processor: Optional[LogitsProcessorList] = None,
        grammar: Optional[LlamaGrammar] = None,
        logit_bias: Optional[Dict[str, float]] = None,
    ) -> Union[CreateCompletionResponse, Iterator[CreateCompletionStreamResponse]]:
        """Generate text from a prompt.

        Args:
            prompt: The prompt to generate text from.
            suffix: A suffix to append to the generated text. If None, no suffix is appended.
            max_tokens: The maximum number of tokens to generate. If max_tokens <= 0 or None, the maximum number of tokens to generate is unlimited and depends on n_ctx.
            temperature: The temperature to use for sampling.
            top_p: The top-p value to use for nucleus sampling. Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751
            min_p: The min-p value to use for minimum p sampling. Minimum P sampling as described in https://github.com/ggerganov/llama.cpp/pull/3841
            typical_p: The typical-p value to use for sampling. Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666.
            logprobs: The number of logprobs to return. If None, no logprobs are returned.
            echo: Whether to echo the prompt.
            stop: A list of strings to stop generation when encountered.
            frequency_penalty: The penalty to apply to tokens based on their frequency in the prompt.
            presence_penalty: The penalty to apply to tokens based on their presence in the prompt.
            repeat_penalty: The penalty to apply to repeated tokens.
            top_k: The top-k value to use for sampling. Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751
            stream: Whether to stream the results.
            seed: The seed to use for sampling.
            tfs_z: The tail-free sampling parameter. Tail Free Sampling described in https://www.trentonbricken.com/Tail-Free-Sampling/.
            mirostat_mode: The mirostat sampling mode.
            mirostat_tau: The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text.
            mirostat_eta: The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates.
            model: The name to use for the model in the completion object.
            stopping_criteria: A list of stopping criteria to use.
            logits_processor: A list of logits processors to use.
            grammar: A grammar to use for constrained sampling.
            logit_bias: A logit bias to use.

        Raises:
            ValueError: If the requested tokens exceed the context window.
            RuntimeError: If the prompt fails to tokenize or the model fails to evaluate the prompt.

        Returns:
            Response object containing the generated text.
        """
        return self.create_completion(
            prompt=prompt,
            suffix=suffix,
            max_tokens=max_tokens,
            temperature=temperature,
            top_p=top_p,
            min_p=min_p,
            typical_p=typical_p,
            logprobs=logprobs,
            echo=echo,
            stop=stop,
            frequency_penalty=frequency_penalty,
            presence_penalty=presence_penalty,
            repeat_penalty=repeat_penalty,
            top_k=top_k,
            stream=stream,
            seed=seed,
            tfs_z=tfs_z,
            mirostat_mode=mirostat_mode,
            mirostat_tau=mirostat_tau,
            mirostat_eta=mirostat_eta,
            model=model,
            stopping_criteria=stopping_criteria,
            logits_processor=logits_processor,
            grammar=grammar,
            logit_bias=logit_bias,
        )

    def create_chat_completion(
        self,
        messages: List[ChatCompletionRequestMessage],
        functions: Optional[List[ChatCompletionFunction]] = None,
        function_call: Optional[ChatCompletionRequestFunctionCall] = None,
        tools: Optional[List[ChatCompletionTool]] = None,
        tool_choice: Optional[ChatCompletionToolChoiceOption] = None,
        temperature: float = 0.2,
        top_p: float = 0.95,
        top_k: int = 40,
        min_p: float = 0.05,
        typical_p: float = 1.0,
        stream: bool = False,
        stop: Optional[Union[str, List[str]]] = [],
        seed: Optional[int] = None,
        response_format: Optional[ChatCompletionRequestResponseFormat] = None,
        max_tokens: Optional[int] = None,
        presence_penalty: float = 0.0,
        frequency_penalty: float = 0.0,
        repeat_penalty: float = 1.1,
        tfs_z: float = 1.0,
        mirostat_mode: int = 0,
        mirostat_tau: float = 5.0,
        mirostat_eta: float = 0.1,
        model: Optional[str] = None,
        logits_processor: Optional[LogitsProcessorList] = None,
        grammar: Optional[LlamaGrammar] = None,
        logit_bias: Optional[Dict[str, float]] = None,
        logprobs: Optional[bool] = None,
        top_logprobs: Optional[int] = None,
    ) -> Union[
        CreateChatCompletionResponse, Iterator[CreateChatCompletionStreamResponse]
    ]:
        """Generate a chat completion from a list of messages.

        Args:
            messages: A list of messages to generate a response for.
            functions: A list of functions to use for the chat completion.
            function_call: A function call to use for the chat completion.
            tools: A list of tools to use for the chat completion.
            tool_choice: A tool choice to use for the chat completion.
            temperature: The temperature to use for sampling.
            top_p: The top-p value to use for nucleus sampling. Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751
            top_k: The top-k value to use for sampling. Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751
            min_p: The min-p value to use for minimum p sampling. Minimum P sampling as described in https://github.com/ggerganov/llama.cpp/pull/3841
            typical_p: The typical-p value to use for sampling. Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666.
            stream: Whether to stream the results.
            stop: A list of strings to stop generation when encountered.
            seed: The seed to use for sampling.
            response_format: The response format to use for the chat completion. Use { "type": "json_object" } to contstrain output to only valid json.
            max_tokens: The maximum number of tokens to generate. If max_tokens <= 0 or None, the maximum number of tokens to generate is unlimited and depends on n_ctx.
            presence_penalty: The penalty to apply to tokens based on their presence in the prompt.
            frequency_penalty: The penalty to apply to tokens based on their frequency in the prompt.
            repeat_penalty: The penalty to apply to repeated tokens.
            tfs_z: The tail-free sampling parameter.
            mirostat_mode: The mirostat sampling mode.
            mirostat_tau: The mirostat sampling tau parameter.
            mirostat_eta: The mirostat sampling eta parameter.
            model: The name to use for the model in the completion object.
            logits_processor: A list of logits processors to use.
            grammar: A grammar to use.
            logit_bias: A logit bias to use.

        Returns:
            Generated chat completion or a stream of chat completion chunks.
        """
        handler = self.chat_handler or llama_chat_format.get_chat_completion_handler(
            self.chat_format
        )
        return handler(
            llama=self,
            messages=messages,
            functions=functions,
            function_call=function_call,
            tools=tools,
            tool_choice=tool_choice,
            temperature=temperature,
            top_p=top_p,
            top_k=top_k,
            min_p=min_p,
            typical_p=typical_p,
            logprobs=logprobs,
            top_logprobs=top_logprobs,
            stream=stream,
            stop=stop,
            seed=seed,
            response_format=response_format,
            max_tokens=max_tokens,
            presence_penalty=presence_penalty,
            frequency_penalty=frequency_penalty,
            repeat_penalty=repeat_penalty,
            tfs_z=tfs_z,
            mirostat_mode=mirostat_mode,
            mirostat_tau=mirostat_tau,
            mirostat_eta=mirostat_eta,
            model=model,
            logits_processor=logits_processor,
            grammar=grammar,
            logit_bias=logit_bias,
        )

    def create_chat_completion_openai_v1(
        self,
        *args: Any,
        **kwargs: Any,
    ):
        """Generate a chat completion with return type based on the the OpenAI v1 API.

        OpenAI python package is required to use this method.

        You can install it with `pip install openai`.

        Args:
            *args: Positional arguments to pass to create_chat_completion.
            **kwargs: Keyword arguments to pass to create_chat_completion.

        Returns:
            Generated chat completion or a stream of chat completion chunks.
        """
        try:
            from openai.types.chat import ChatCompletion, ChatCompletionChunk

            stream = kwargs.get("stream", False)  # type: ignore
            assert isinstance(stream, bool)
            if stream:
                return (ChatCompletionChunk(**chunk) for chunk in self.create_chat_completion(*args, **kwargs))  # type: ignore
            else:
                return ChatCompletion(**self.create_chat_completion(*args, **kwargs))  # type: ignore
        except ImportError:
            raise ImportError(
                "To use create_chat_completion_openai_v1, you must install the openai package."
                "You can install it with `pip install openai`."
            )

    def __getstate__(self):
        return dict(
            model_path=self.model_path,
            # Model Params
            n_gpu_layers=self.model_params.n_gpu_layers,
            split_mode=self.model_params.split_mode,
            main_gpu=self.model_params.main_gpu,
            tensor_split=self.tensor_split,
            vocab_only=self.model_params.vocab_only,
            use_mmap=self.model_params.use_mmap,
            use_mlock=self.model_params.use_mlock,
            kv_overrides=self.kv_overrides,
            # Context Params
            seed=self.context_params.seed,
            n_ctx=self.context_params.n_ctx,
            n_batch=self.n_batch,
            n_threads=self.context_params.n_threads,
            n_threads_batch=self.context_params.n_threads_batch,
            rope_scaling_type=self.context_params.rope_scaling_type,
            pooling_type=self.context_params.pooling_type,
            rope_freq_base=self.context_params.rope_freq_base,
            rope_freq_scale=self.context_params.rope_freq_scale,
            yarn_ext_factor=self.context_params.yarn_ext_factor,
            yarn_attn_factor=self.context_params.yarn_attn_factor,
            yarn_beta_fast=self.context_params.yarn_beta_fast,
            yarn_beta_slow=self.context_params.yarn_beta_slow,
            yarn_orig_ctx=self.context_params.yarn_orig_ctx,
            logits_all=self.context_params.logits_all,
            embedding=self.context_params.embeddings,
            offload_kqv=self.context_params.offload_kqv,
            # Sampling Params
            last_n_tokens_size=self.last_n_tokens_size,
            # LoRA Params
            lora_base=self.lora_base,
            lora_scale=self.lora_scale,
            lora_path=self.lora_path,
            # Backend Params
            numa=self.numa,
            # Chat Format Params
            chat_format=self.chat_format,
            chat_handler=self.chat_handler,
            # Speculative Decidng
            draft_model=self.draft_model,
            # KV cache quantization
            type_k=self.context_params.type_k,
            type_v=self.context_params.type_v,
            # Misc
            verbose=self.verbose,
        )

    def __setstate__(self, state):
        self.__init__(**state)

    def save_state(self) -> LlamaState:
        assert self._ctx.ctx is not None
        if self.verbose:
            print("Llama.save_state: saving llama state", file=sys.stderr)
        state_size = llama_cpp.llama_get_state_size(self._ctx.ctx)
        if self.verbose:
            print(f"Llama.save_state: got state size: {state_size}", file=sys.stderr)
        llama_state = (ctypes.c_uint8 * int(state_size))()
        if self.verbose:
            print("Llama.save_state: allocated state", file=sys.stderr)
        n_bytes = llama_cpp.llama_copy_state_data(self._ctx.ctx, llama_state)
        if self.verbose:
            print(f"Llama.save_state: copied llama state: {n_bytes}", file=sys.stderr)
        if int(n_bytes) > int(state_size):
            raise RuntimeError("Failed to copy llama state data")
        llama_state_compact = (ctypes.c_uint8 * int(n_bytes))()
        llama_cpp.ctypes.memmove(llama_state_compact, llama_state, int(n_bytes))
        if self.verbose:
            print(
                f"Llama.save_state: saving {n_bytes} bytes of llama state",
                file=sys.stderr,
            )
        return LlamaState(
            scores=self._scores.copy(),
            input_ids=self.input_ids.copy(),
            n_tokens=self.n_tokens,
            llama_state=bytes(llama_state_compact),
            llama_state_size=n_bytes,
        )

    def load_state(self, state: LlamaState) -> None:
        assert self._ctx.ctx is not None
        # Only filling in up to `n_tokens` and then zero-ing out the rest
        self.scores[: state.n_tokens, :] = state.scores.copy()
        self.scores[state.n_tokens :, :] = 0.0
        self.input_ids = state.input_ids.copy()
        self.n_tokens = state.n_tokens
        state_size = state.llama_state_size
        LLamaStateArrayType = ctypes.c_uint8 * state_size
        llama_state = LLamaStateArrayType.from_buffer_copy(state.llama_state)

        if llama_cpp.llama_set_state_data(self._ctx.ctx, llama_state) != state_size:
            raise RuntimeError("Failed to set llama state data")

    def n_ctx(self) -> int:
        """Return the context window size."""
        return self._ctx.n_ctx()

    def n_embd(self) -> int:
        """Return the embedding size."""
        return self._model.n_embd()

    def n_vocab(self) -> int:
        """Return the vocabulary size."""
        return self._model.n_vocab()

    def tokenizer(self) -> LlamaTokenizer:
        """Return the llama tokenizer for this model."""
        return LlamaTokenizer(self)

    def token_eos(self) -> int:
        """Return the end-of-sequence token."""
        return self._model.token_eos()

    def token_bos(self) -> int:
        """Return the beginning-of-sequence token."""
        return self._model.token_bos()

    def token_nl(self) -> int:
        """Return the newline token."""
        return self._model.token_nl()

    def pooling_type(self) -> str:
        """Return the pooling type."""
        return self._ctx.pooling_type()

    @staticmethod
    def logits_to_logprobs(
        logits: Union[npt.NDArray[np.single], List], axis: int = -1
    ) -> npt.NDArray[np.single]:
        # https://docs.scipy.org/doc/scipy/reference/generated/scipy.special.log_softmax.html
        logits_maxs: np.ndarray = np.amax(logits, axis=axis, keepdims=True)
        if logits_maxs.ndim > 0:
            logits_maxs[~np.isfinite(logits_maxs)] = 0
        elif not np.isfinite(logits_maxs):
            logits_maxs = 0
        subtract_maxs = np.subtract(logits, logits_maxs, dtype=np.single)
        exp = np.exp(subtract_maxs)
        # Suppress warnings about log of zero
        with np.errstate(divide="ignore"):
            summed = np.sum(exp, axis=axis, keepdims=True)
            out = np.log(summed)
        return subtract_maxs - out

    @staticmethod
    def longest_token_prefix(a: Sequence[int], b: Sequence[int]):
        longest_prefix = 0
        for _a, _b in zip(a, b):
            if _a == _b:
                longest_prefix += 1
            else:
                break
        return longest_prefix

    @classmethod
    def from_pretrained(
        cls,
        repo_id: str,
        filename: Optional[str],
        local_dir: Optional[Union[str, os.PathLike[str]]] = None,
        local_dir_use_symlinks: Union[bool, Literal["auto"]] = "auto",
        cache_dir: Optional[Union[str, os.PathLike[str]]] = None,
        **kwargs: Any,
    ) -> "Llama":
        """Create a Llama model from a pretrained model name or path.
        This method requires the huggingface-hub package.
        You can install it with `pip install huggingface-hub`.

        Args:
            repo_id: The model repo id.
            filename: A filename or glob pattern to match the model file in the repo.
            local_dir: The local directory to save the model to.
            local_dir_use_symlinks: Whether to use symlinks when downloading the model.
            **kwargs: Additional keyword arguments to pass to the Llama constructor.

        Returns:
            A Llama model."""
        try:
            from huggingface_hub import hf_hub_download, HfFileSystem
            from huggingface_hub.utils import validate_repo_id
        except ImportError:
            raise ImportError(
                "Llama.from_pretrained requires the huggingface-hub package. "
                "You can install it with `pip install huggingface-hub`."
            )

        validate_repo_id(repo_id)

        hffs = HfFileSystem()

        files = [
            file["name"] if isinstance(file, dict) else file
            for file in hffs.ls(repo_id)
        ]

        # split each file into repo_id, subfolder, filename
        file_list: List[str] = []
        for file in files:
            rel_path = Path(file).relative_to(repo_id)
            file_list.append(str(rel_path))

        matching_files = [file for file in file_list if fnmatch.fnmatch(file, filename)]  # type: ignore

        if len(matching_files) == 0:
            raise ValueError(
                f"No file found in {repo_id} that match {filename}\n\n"
                f"Available Files:\n{json.dumps(file_list)}"
            )

        if len(matching_files) > 1:
            raise ValueError(
                f"Multiple files found in {repo_id} matching {filename}\n\n"
                f"Available Files:\n{json.dumps(files)}"
            )

        (matching_file,) = matching_files

        subfolder = str(Path(matching_file).parent)
        filename = Path(matching_file).name

        # download the file
        hf_hub_download(
            repo_id=repo_id,
            filename=filename,
            subfolder=subfolder,
            local_dir=local_dir,
            local_dir_use_symlinks=local_dir_use_symlinks,
            cache_dir=cache_dir,
        )

        if local_dir is None:
            model_path = hf_hub_download(
                repo_id=repo_id,
                filename=filename,
                subfolder=subfolder,
                local_dir=local_dir,
                local_dir_use_symlinks=local_dir_use_symlinks,
                cache_dir=cache_dir,
                local_files_only=True,
            )
        else:
            model_path = os.path.join(local_dir, filename)

        return cls(
            model_path=model_path,
            **kwargs,
        )

__init__(model_path, *, n_gpu_layers=0, split_mode=llama_cpp.LLAMA_SPLIT_MODE_LAYER, main_gpu=0, tensor_split=None, vocab_only=False, use_mmap=True, use_mlock=False, kv_overrides=None, seed=llama_cpp.LLAMA_DEFAULT_SEED, n_ctx=512, n_batch=512, n_threads=None, n_threads_batch=None, rope_scaling_type=llama_cpp.LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED, pooling_type=llama_cpp.LLAMA_POOLING_TYPE_UNSPECIFIED, rope_freq_base=0.0, rope_freq_scale=0.0, yarn_ext_factor=-1.0, yarn_attn_factor=1.0, yarn_beta_fast=32.0, yarn_beta_slow=1.0, yarn_orig_ctx=0, logits_all=False, embedding=False, offload_kqv=True, last_n_tokens_size=64, lora_base=None, lora_scale=1.0, lora_path=None, numa=False, chat_format=None, chat_handler=None, draft_model=None, tokenizer=None, type_k=None, type_v=None, verbose=True, **kwargs)

Load a llama.cpp model from model_path.

Examples:

Basic usage

>>> import llama_cpp
>>> model = llama_cpp.Llama(
...     model_path="path/to/model",
... )
>>> print(model("The quick brown fox jumps ", stop=["."])["choices"][0]["text"])
the lazy dog

Loading a chat model

>>> import llama_cpp
>>> model = llama_cpp.Llama(
...     model_path="path/to/model",
...     chat_format="llama-2",
... )
>>> print(model.create_chat_completion(
...     messages=[{
...         "role": "user",
...         "content": "what is the meaning of life?"
...     }]
... ))

Parameters:

• model_path (str) –

  Path to the model.

• n_gpu_layers (int, default: 0 ) –

  Number of layers to offload to GPU (-ngl). If -1, all layers are offloaded.

• split_mode (int, default: LLAMA_SPLIT_MODE_LAYER ) –

  How to split the model across GPUs. See llama_cpp.LLAMA_SPLIT_* for options.

• main_gpu (int, default: 0 ) –

  main_gpu interpretation depends on split_mode: LLAMA_SPLIT_NONE: the GPU that is used for the entire model. LLAMA_SPLIT_ROW: the GPU that is used for small tensors and intermediate results. LLAMA_SPLIT_LAYER: ignored

• tensor_split (Optional[List[float]], default: None ) –

  How split tensors should be distributed across GPUs. If None, the model is not split.

• vocab_only (bool, default: False ) –

  Only load the vocabulary no weights.

• use_mmap (bool, default: True ) –

  Use mmap if possible.

• use_mlock (bool, default: False ) –

  Force the system to keep the model in RAM.

• kv_overrides (Optional[Dict[str, Union[bool, int, float, str]]], default: None ) –

  Key-value overrides for the model.

• seed (int, default: LLAMA_DEFAULT_SEED ) –

  RNG seed, -1 for random

• n_ctx (int, default: 512 ) –

  Text context, 0 = from model

• n_batch (int, default: 512 ) –

  Prompt processing maximum batch size

• n_threads (Optional[int], default: None ) –

  Number of threads to use for generation

• n_threads_batch (Optional[int], default: None ) –

  Number of threads to use for batch processing

• rope_scaling_type (Optional[int], default: LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED ) –

  RoPE scaling type, from enum llama_rope_scaling_type. ref: https://github.com/ggerganov/llama.cpp/pull/2054

• pooling_type (int, default: LLAMA_POOLING_TYPE_UNSPECIFIED ) –

  Pooling type, from enum llama_pooling_type.

• rope_freq_base (float, default: 0.0 ) –

  RoPE base frequency, 0 = from model

• rope_freq_scale (float, default: 0.0 ) –

  RoPE frequency scaling factor, 0 = from model

• yarn_ext_factor (float, default: -1.0 ) –

  YaRN extrapolation mix factor, negative = from model

• yarn_attn_factor (float, default: 1.0 ) –

  YaRN magnitude scaling factor

• yarn_beta_fast (float, default: 32.0 ) –

  YaRN low correction dim

• yarn_beta_slow (float, default: 1.0 ) –

  YaRN high correction dim

• yarn_orig_ctx (int, default: 0 ) –

  YaRN original context size

• logits_all (bool, default: False ) –

  Return logits for all tokens, not just the last token. Must be True for completion to return logprobs.

• embedding (bool, default: False ) –

  Embedding mode only.

• offload_kqv (bool, default: True ) –

  Offload K, Q, V to GPU.

• last_n_tokens_size (int, default: 64 ) –

  Maximum number of tokens to keep in the last_n_tokens deque.

• lora_base (Optional[str], default: None ) –

  Optional path to base model, useful if using a quantized base model and you want to apply LoRA to an f16 model.

• lora_path (Optional[str], default: None ) –

  Path to a LoRA file to apply to the model.

• numa (Union[bool, int], default: False ) –

  numa policy

• chat_format (Optional[str], default: None ) –

  String specifying the chat format to use when calling create_chat_completion.

• chat_handler (Optional[LlamaChatCompletionHandler], default: None ) –

  Optional chat handler to use when calling create_chat_completion.

• draft_model (Optional[LlamaDraftModel], default: None ) –

  Optional draft model to use for speculative decoding.

• tokenizer (Optional[BaseLlamaTokenizer], default: None ) –

  Optional tokenizer to override the default tokenizer from llama.cpp.

• verbose (bool, default: True ) –

  Print verbose output to stderr.

• type_k (Optional[int], default: None ) –

  KV cache data type for K (default: f16)

• type_v (Optional[int], default: None ) –

  KV cache data type for V (default: f16)

Raises:

• ValueError –

  If the model path does not exist.

Returns:

• –

  A Llama instance.

Source code in llama_cpp/llama.py

def __init__(
    self,
    model_path: str,
    *,
    # Model Params
    n_gpu_layers: int = 0,
    split_mode: int = llama_cpp.LLAMA_SPLIT_MODE_LAYER,
    main_gpu: int = 0,
    tensor_split: Optional[List[float]] = None,
    vocab_only: bool = False,
    use_mmap: bool = True,
    use_mlock: bool = False,
    kv_overrides: Optional[Dict[str, Union[bool, int, float, str]]] = None,
    # Context Params
    seed: int = llama_cpp.LLAMA_DEFAULT_SEED,
    n_ctx: int = 512,
    n_batch: int = 512,
    n_threads: Optional[int] = None,
    n_threads_batch: Optional[int] = None,
    rope_scaling_type: Optional[int] = llama_cpp.LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED,
    pooling_type: int = llama_cpp.LLAMA_POOLING_TYPE_UNSPECIFIED,
    rope_freq_base: float = 0.0,
    rope_freq_scale: float = 0.0,
    yarn_ext_factor: float = -1.0,
    yarn_attn_factor: float = 1.0,
    yarn_beta_fast: float = 32.0,
    yarn_beta_slow: float = 1.0,
    yarn_orig_ctx: int = 0,
    logits_all: bool = False,
    embedding: bool = False,
    offload_kqv: bool = True,
    # Sampling Params
    last_n_tokens_size: int = 64,
    # LoRA Params
    lora_base: Optional[str] = None,
    lora_scale: float = 1.0,
    lora_path: Optional[str] = None,
    # Backend Params
    numa: Union[bool, int] = False,
    # Chat Format Params
    chat_format: Optional[str] = None,
    chat_handler: Optional[llama_chat_format.LlamaChatCompletionHandler] = None,
    # Speculative Decoding
    draft_model: Optional[LlamaDraftModel] = None,
    # Tokenizer Override
    tokenizer: Optional[BaseLlamaTokenizer] = None,
    # KV cache quantization
    type_k: Optional[int] = None,
    type_v: Optional[int] = None,
    # Misc
    verbose: bool = True,
    # Extra Params
    **kwargs,  # type: ignore
):
    """Load a llama.cpp model from `model_path`.

    Examples:
        Basic usage

        >>> import llama_cpp
        >>> model = llama_cpp.Llama(
        ...     model_path="path/to/model",
        ... )
        >>> print(model("The quick brown fox jumps ", stop=["."])["choices"][0]["text"])
        the lazy dog

        Loading a chat model

        >>> import llama_cpp
        >>> model = llama_cpp.Llama(
        ...     model_path="path/to/model",
        ...     chat_format="llama-2",
        ... )
        >>> print(model.create_chat_completion(
        ...     messages=[{
        ...         "role": "user",
        ...         "content": "what is the meaning of life?"
        ...     }]
        ... ))

    Args:
        model_path: Path to the model.
        n_gpu_layers: Number of layers to offload to GPU (-ngl). If -1, all layers are offloaded.
        split_mode: How to split the model across GPUs. See llama_cpp.LLAMA_SPLIT_* for options.
        main_gpu: main_gpu interpretation depends on split_mode: LLAMA_SPLIT_NONE: the GPU that is used for the entire model. LLAMA_SPLIT_ROW: the GPU that is used for small tensors and intermediate results. LLAMA_SPLIT_LAYER: ignored
        tensor_split: How split tensors should be distributed across GPUs. If None, the model is not split.
        vocab_only: Only load the vocabulary no weights.
        use_mmap: Use mmap if possible.
        use_mlock: Force the system to keep the model in RAM.
        kv_overrides: Key-value overrides for the model.
        seed: RNG seed, -1 for random
        n_ctx: Text context, 0 = from model
        n_batch: Prompt processing maximum batch size
        n_threads: Number of threads to use for generation
        n_threads_batch: Number of threads to use for batch processing
        rope_scaling_type: RoPE scaling type, from `enum llama_rope_scaling_type`. ref: https://github.com/ggerganov/llama.cpp/pull/2054
        pooling_type: Pooling type, from `enum llama_pooling_type`.
        rope_freq_base: RoPE base frequency, 0 = from model
        rope_freq_scale: RoPE frequency scaling factor, 0 = from model
        yarn_ext_factor: YaRN extrapolation mix factor, negative = from model
        yarn_attn_factor: YaRN magnitude scaling factor
        yarn_beta_fast: YaRN low correction dim
        yarn_beta_slow: YaRN high correction dim
        yarn_orig_ctx: YaRN original context size
        logits_all: Return logits for all tokens, not just the last token. Must be True for completion to return logprobs.
        embedding: Embedding mode only.
        offload_kqv: Offload K, Q, V to GPU.
        last_n_tokens_size: Maximum number of tokens to keep in the last_n_tokens deque.
        lora_base: Optional path to base model, useful if using a quantized base model and you want to apply LoRA to an f16 model.
        lora_path: Path to a LoRA file to apply to the model.
        numa: numa policy
        chat_format: String specifying the chat format to use when calling create_chat_completion.
        chat_handler: Optional chat handler to use when calling create_chat_completion.
        draft_model: Optional draft model to use for speculative decoding.
        tokenizer: Optional tokenizer to override the default tokenizer from llama.cpp.
        verbose: Print verbose output to stderr.
        type_k: KV cache data type for K (default: f16)
        type_v: KV cache data type for V (default: f16)

    Raises:
        ValueError: If the model path does not exist.

    Returns:
        A Llama instance.
    """
    self.verbose = verbose

    set_verbose(verbose)

    if not Llama.__backend_initialized:
        with suppress_stdout_stderr(disable=verbose):
            llama_cpp.llama_backend_init()
        Llama.__backend_initialized = True

    if isinstance(numa, bool):
        self.numa = (
            llama_cpp.GGML_NUMA_STRATEGY_DISTRIBUTE
            if numa
            else llama_cpp.GGML_NUMA_STRATEGY_DISABLED
        )
    else:
        self.numa = numa

    if self.numa != llama_cpp.GGML_NUMA_STRATEGY_DISABLED:
        with suppress_stdout_stderr(disable=verbose):
            llama_cpp.llama_numa_init(self.numa)

    self.model_path = model_path

    # Model Params
    self.model_params = llama_cpp.llama_model_default_params()
    self.model_params.n_gpu_layers = (
        0x7FFFFFFF if n_gpu_layers == -1 else n_gpu_layers
    )  # 0x7FFFFFFF is INT32 max, will be auto set to all layers
    self.model_params.split_mode = split_mode
    self.model_params.main_gpu = main_gpu
    self.tensor_split = tensor_split
    self._c_tensor_split = None
    if self.tensor_split is not None:
        if len(self.tensor_split) > llama_cpp.LLAMA_MAX_DEVICES:
            raise ValueError(
                f"Attempt to split tensors that exceed maximum supported devices. Current LLAMA_MAX_DEVICES={llama_cpp.LLAMA_MAX_DEVICES}"
            )
        # Type conversion and expand the list to the length of LLAMA_MAX_DEVICES
        FloatArray = ctypes.c_float * llama_cpp.LLAMA_MAX_DEVICES
        self._c_tensor_split = FloatArray(
            *tensor_split  # type: ignore
        )  # keep a reference to the array so it is not gc'd
        self.model_params.tensor_split = self._c_tensor_split
    self.model_params.vocab_only = vocab_only
    self.model_params.use_mmap = use_mmap if lora_path is None else False
    self.model_params.use_mlock = use_mlock

    # kv_overrides is the original python dict
    self.kv_overrides = kv_overrides
    if kv_overrides is not None:
        # _kv_overrides_array is a ctypes.Array of llama_model_kv_override Structs
        kvo_array_len = len(kv_overrides) + 1  # for sentinel element
        self._kv_overrides_array = (
            llama_cpp.llama_model_kv_override * kvo_array_len
        )()

        for i, (k, v) in enumerate(kv_overrides.items()):
            self._kv_overrides_array[i].key = k.encode("utf-8")
            if isinstance(v, bool):
                self._kv_overrides_array[i].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_BOOL
                self._kv_overrides_array[i].value.bool_value = v
            elif isinstance(v, int):
                self._kv_overrides_array[i].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_INT
                self._kv_overrides_array[i].value.int_value = v
            elif isinstance(v, float):
                self._kv_overrides_array[i].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_FLOAT
                self._kv_overrides_array[i].value.float_value = v
            elif isinstance(v, str): # type: ignore
                v_bytes = v.encode("utf-8")
                if len(v_bytes) > 128: # TODO: Make this a constant
                    raise ValueError(f"Value for {k} is too long: {v}")
                v_bytes = v_bytes.ljust(128, b"\0")
                self._kv_overrides_array[i].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_STR
                self._kv_overrides_array[i].value.str_value[:128] = v_bytes
            else:
                raise ValueError(f"Unknown value type for {k}: {v}")

        self._kv_overrides_array[-1].key = (
            b"\0"  # ensure sentinel element is zeroed
        )
        self.model_params.kv_overrides = self._kv_overrides_array

    self.n_batch = min(n_ctx, n_batch)  # ???
    self.n_threads = n_threads or max(multiprocessing.cpu_count() // 2, 1)
    self.n_threads_batch = n_threads_batch or multiprocessing.cpu_count()

    # Context Params
    self.context_params = llama_cpp.llama_context_default_params()
    self.context_params.seed = seed
    self.context_params.n_ctx = n_ctx
    self.context_params.n_batch = self.n_batch
    self.context_params.n_threads = self.n_threads
    self.context_params.n_threads_batch = self.n_threads_batch
    self.context_params.rope_scaling_type = (
        rope_scaling_type
        if rope_scaling_type is not None
        else llama_cpp.LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED
    )
    self.context_params.pooling_type = pooling_type
    self.context_params.rope_freq_base = (
        rope_freq_base if rope_freq_base != 0.0 else 0
    )
    self.context_params.rope_freq_scale = (
        rope_freq_scale if rope_freq_scale != 0.0 else 0
    )
    self.context_params.yarn_ext_factor = (
        yarn_ext_factor if yarn_ext_factor != 0.0 else 0
    )
    self.context_params.yarn_attn_factor = (
        yarn_attn_factor if yarn_attn_factor != 0.0 else 0
    )
    self.context_params.yarn_beta_fast = (
        yarn_beta_fast if yarn_beta_fast != 0.0 else 0
    )
    self.context_params.yarn_beta_slow = (
        yarn_beta_slow if yarn_beta_slow != 0.0 else 0
    )
    self.context_params.yarn_orig_ctx = yarn_orig_ctx if yarn_orig_ctx != 0 else 0
    self.context_params.logits_all = (
        logits_all if draft_model is None else True
    )  # Must be set to True for speculative decoding
    self.context_params.embeddings = embedding # TODO: Rename to embeddings
    self.context_params.offload_kqv = offload_kqv
    #  KV cache quantization
    if type_k is not None:
        self.context_params.type_k = type_k
    if type_v is not None:
        self.context_params.type_v = type_v
    # Sampling Params
    self.last_n_tokens_size = last_n_tokens_size

    self.cache: Optional[BaseLlamaCache] = None

    self.lora_base = lora_base
    self.lora_scale = lora_scale
    self.lora_path = lora_path

    if not os.path.exists(model_path):
        raise ValueError(f"Model path does not exist: {model_path}")

    self._model = _LlamaModel(
        path_model=self.model_path, params=self.model_params, verbose=self.verbose
    )

    # Override tokenizer
    self.tokenizer_ = tokenizer or LlamaTokenizer(self)

    # Set the default value for the context and correct the batch
    if n_ctx == 0:
        n_ctx = self._model.n_ctx_train()
        self.n_batch = min(n_ctx, n_batch)
        self.context_params.n_ctx = self._model.n_ctx_train()
        self.context_params.n_batch = self.n_batch

    self._ctx = _LlamaContext(
        model=self._model,
        params=self.context_params,
        verbose=self.verbose,
    )

    self._batch = _LlamaBatch(
        n_tokens=self.n_batch,
        embd=0,
        n_seq_max=self.context_params.n_ctx,
        verbose=self.verbose,
    )

    if self.lora_path:
        if self._model.apply_lora_from_file(
            self.lora_path,
            self.lora_scale,
            self.lora_base,
            self.n_threads,
        ):
            raise RuntimeError(
                f"Failed to apply LoRA from lora path: {self.lora_path} to base path: {self.lora_base}"
            )

    if self.verbose:
        print(llama_cpp.llama_print_system_info().decode("utf-8"), file=sys.stderr)

    self.chat_format = chat_format
    self.chat_handler = chat_handler

    self.draft_model = draft_model

    self._n_vocab = self.n_vocab()
    self._n_ctx = self.n_ctx()

    self._token_nl = self.token_nl()
    self._token_eos = self.token_eos()

    self._candidates = _LlamaTokenDataArray(n_vocab=self._n_vocab)

    self.n_tokens = 0
    self.input_ids: npt.NDArray[np.intc] = np.ndarray((n_ctx,), dtype=np.intc)
    self.scores: npt.NDArray[np.single] = np.ndarray(
        (n_ctx, self._n_vocab), dtype=np.single
    )

    self._mirostat_mu = ctypes.c_float(
        2.0 * 5.0
    )  # TODO: Move this to sampling context

    try:
        self.metadata = self._model.metadata()
    except Exception as e:
        self.metadata = {}
        if self.verbose:
            print(f"Failed to load metadata: {e}", file=sys.stderr)

    if self.verbose:
        print(f"Model metadata: {self.metadata}", file=sys.stderr)

    if (
        self.chat_format is None
        and self.chat_handler is None
        and "tokenizer.chat_template" in self.metadata
    ):
        chat_format = llama_chat_format.guess_chat_format_from_gguf_metadata(
            self.metadata
        )

        if chat_format is not None:
            self.chat_format = chat_format
            if self.verbose:
                print(f"Guessed chat format: {chat_format}", file=sys.stderr)
        else:
            template = self.metadata["tokenizer.chat_template"]
            try:
                eos_token_id = int(self.metadata["tokenizer.ggml.eos_token_id"])
            except:
                eos_token_id = self.token_eos()
            try:
                bos_token_id = int(self.metadata["tokenizer.ggml.bos_token_id"])
            except:
                bos_token_id = self.token_bos()

            eos_token = self._model.token_get_text(eos_token_id)
            bos_token = self._model.token_get_text(bos_token_id)

            if self.verbose:
                print(f"Using gguf chat template: {template}", file=sys.stderr)
                print(f"Using chat eos_token: {eos_token}", file=sys.stderr)
                print(f"Using chat bos_token: {bos_token}", file=sys.stderr)

            self.chat_handler = llama_chat_format.Jinja2ChatFormatter(
                template=template,
                eos_token=eos_token,
                bos_token=bos_token,
                stop_token_ids=[eos_token_id],
            ).to_chat_handler()

    if self.chat_format is None and self.chat_handler is None:
        self.chat_format = "llama-2"
        if self.verbose:
            print(f"Using fallback chat format: {chat_format}", file=sys.stderr)

tokenize(text, add_bos=True, special=False)

Tokenize a string.

Parameters:

• text (bytes) –

  The utf-8 encoded string to tokenize.

Raises:

• RuntimeError –

  If the tokenization failed.

Returns:

• List[int] –

  A list of tokens.

Source code in llama_cpp/llama.py

def tokenize(
    self, text: bytes, add_bos: bool = True, special: bool = False
) -> List[int]:
    """Tokenize a string.

    Args:
        text: The utf-8 encoded string to tokenize.

    Raises:
        RuntimeError: If the tokenization failed.

    Returns:
        A list of tokens.
    """
    return self.tokenizer_.tokenize(text, add_bos, special)

detokenize(tokens, prev_tokens=None)

Detokenize a list of tokens.

Parameters:

• tokens (List[int]) –

  The list of tokens to detokenize.

• prev_tokens (Optional[List[int]], default: None ) –

  The list of previous tokens. Offset mapping will be performed if provided

Returns:

• bytes –

  The detokenized string.

Source code in llama_cpp/llama.py

def detokenize(
    self, tokens: List[int], prev_tokens: Optional[List[int]] = None
) -> bytes:
    """Detokenize a list of tokens.

    Args:
        tokens: The list of tokens to detokenize.
        prev_tokens: The list of previous tokens. Offset mapping will be performed if provided

    Returns:
        The detokenized string.
    """
    return self.tokenizer_.detokenize(tokens, prev_tokens=prev_tokens)

reset()

Reset the model state.

Source code in llama_cpp/llama.py

def reset(self):
    """Reset the model state."""
    self.n_tokens = 0

eval(tokens)

Evaluate a list of tokens.

Parameters:

• tokens (Sequence[int]) –

  The list of tokens to evaluate.

Source code in llama_cpp/llama.py

def eval(self, tokens: Sequence[int]):
    """Evaluate a list of tokens.

    Args:
        tokens: The list of tokens to evaluate.
    """
    assert self._ctx.ctx is not None
    assert self._batch.batch is not None
    self._ctx.kv_cache_seq_rm(-1, self.n_tokens, -1)
    for i in range(0, len(tokens), self.n_batch):
        batch = tokens[i : min(len(tokens), i + self.n_batch)]
        n_past = self.n_tokens
        n_tokens = len(batch)
        self._batch.set_batch(
            batch=batch, n_past=n_past, logits_all=self.context_params.logits_all
        )
        self._ctx.decode(self._batch)
        # Save tokens
        self.input_ids[n_past : n_past + n_tokens] = batch
        # Save logits
        if self.context_params.logits_all:
            rows = n_tokens
            cols = self._n_vocab
            logits = self._ctx.get_logits()[: rows * cols]
            self.scores[n_past : n_past + n_tokens, :].reshape(-1)[: :] = logits
        else:
            rows = 1
            cols = self._n_vocab
            logits = self._ctx.get_logits()[: rows * cols]
            self.scores[n_past + n_tokens - 1, :].reshape(-1)[: :] = logits
        # Update n_tokens
        self.n_tokens += n_tokens

sample(top_k=40, top_p=0.95, min_p=0.05, typical_p=1.0, temp=0.8, repeat_penalty=1.1, frequency_penalty=0.0, presence_penalty=0.0, tfs_z=1.0, mirostat_mode=0, mirostat_eta=0.1, mirostat_tau=5.0, penalize_nl=True, logits_processor=None, grammar=None, idx=None)

Sample a token from the model.

Parameters:

• top_k (int, default: 40 ) –

  The top-k sampling parameter.

• top_p (float, default: 0.95 ) –

  The top-p sampling parameter.

• temp (float, default: 0.8 ) –

  The temperature parameter.

• repeat_penalty (float, default: 1.1 ) –

  The repeat penalty parameter.

Returns:

• –

  The sampled token.

Source code in llama_cpp/llama.py

def sample(
    self,
    top_k: int = 40,
    top_p: float = 0.95,
    min_p: float = 0.05,
    typical_p: float = 1.0,
    temp: float = 0.80,
    repeat_penalty: float = 1.1,
    frequency_penalty: float = 0.0,
    presence_penalty: float = 0.0,
    tfs_z: float = 1.0,
    mirostat_mode: int = 0,
    mirostat_eta: float = 0.1,
    mirostat_tau: float = 5.0,
    penalize_nl: bool = True,
    logits_processor: Optional[LogitsProcessorList] = None,
    grammar: Optional[LlamaGrammar] = None,
    idx: Optional[int] = None,
):
    """Sample a token from the model.

    Args:
        top_k: The top-k sampling parameter.
        top_p: The top-p sampling parameter.
        temp: The temperature parameter.
        repeat_penalty: The repeat penalty parameter.

    Returns:
        The sampled token.
    """
    assert self._ctx is not None
    assert self.n_tokens > 0

    if idx is None:
        logits: npt.NDArray[np.single] = self._scores[-1, :]
    else:
        logits = self._scores[idx, :]

    if logits_processor is not None:
        logits[:] = (
            logits_processor(self._input_ids, logits)
            if idx is None
            else logits_processor(self._input_ids[: idx + 1], logits)
        )

    sampling_params = _LlamaSamplingParams(
        top_k=top_k,
        top_p=top_p,
        min_p=min_p,
        tfs_z=tfs_z,
        typical_p=typical_p,
        temp=temp,
        penalty_last_n=self.last_n_tokens_size,
        penalty_repeat=repeat_penalty,
        penalty_freq=frequency_penalty,
        penalty_present=presence_penalty,
        mirostat=mirostat_mode,
        mirostat_tau=mirostat_tau,
        mirostat_eta=mirostat_eta,
        penalize_nl=penalize_nl,
    )
    sampling_context = _LlamaSamplingContext(
        params=sampling_params,
        grammar=grammar,
    )
    sampling_context.prev = list(self.eval_tokens)
    id = sampling_context.sample(ctx_main=self._ctx, logits_array=logits)
    sampling_context.accept(
        ctx_main=self._ctx,
        id=id,
        apply_grammar=grammar is not None,
    )
    return id

generate(tokens, top_k=40, top_p=0.95, min_p=0.05, typical_p=1.0, temp=0.8, repeat_penalty=1.1, reset=True, frequency_penalty=0.0, presence_penalty=0.0, tfs_z=1.0, mirostat_mode=0, mirostat_tau=5.0, mirostat_eta=0.1, penalize_nl=True, logits_processor=None, stopping_criteria=None, grammar=None)

Create a generator of tokens from a prompt.

Examples:

>>> llama = Llama("models/ggml-7b.bin")
>>> tokens = llama.tokenize(b"Hello, world!")
>>> for token in llama.generate(tokens, top_k=40, top_p=0.95, temp=1.0, repeat_penalty=1.1):
...     print(llama.detokenize([token]))

Parameters:

• tokens (Sequence[int]) –

  The prompt tokens.

• top_k (int, default: 40 ) –

  The top-k sampling parameter.

• top_p (float, default: 0.95 ) –

  The top-p sampling parameter.

• temp (float, default: 0.8 ) –

  The temperature parameter.

• repeat_penalty (float, default: 1.1 ) –

  The repeat penalty parameter.

• reset (bool, default: True ) –

  Whether to reset the model state.

Yields:

• int –

  The generated tokens.

Source code in llama_cpp/llama.py

def generate(
    self,
    tokens: Sequence[int],
    top_k: int = 40,
    top_p: float = 0.95,
    min_p: float = 0.05,
    typical_p: float = 1.0,
    temp: float = 0.80,
    repeat_penalty: float = 1.1,
    reset: bool = True,
    frequency_penalty: float = 0.0,
    presence_penalty: float = 0.0,
    tfs_z: float = 1.0,
    mirostat_mode: int = 0,
    mirostat_tau: float = 5.0,
    mirostat_eta: float = 0.1,
    penalize_nl: bool = True,
    logits_processor: Optional[LogitsProcessorList] = None,
    stopping_criteria: Optional[StoppingCriteriaList] = None,
    grammar: Optional[LlamaGrammar] = None,
) -> Generator[int, Optional[Sequence[int]], None]:
    """Create a generator of tokens from a prompt.

    Examples:
        >>> llama = Llama("models/ggml-7b.bin")
        >>> tokens = llama.tokenize(b"Hello, world!")
        >>> for token in llama.generate(tokens, top_k=40, top_p=0.95, temp=1.0, repeat_penalty=1.1):
        ...     print(llama.detokenize([token]))

    Args:
        tokens: The prompt tokens.
        top_k: The top-k sampling parameter.
        top_p: The top-p sampling parameter.
        temp: The temperature parameter.
        repeat_penalty: The repeat penalty parameter.
        reset: Whether to reset the model state.

    Yields:
        The generated tokens.
    """
    # Reset mirostat sampling
    self._mirostat_mu = ctypes.c_float(2.0 * mirostat_tau)

    # Check for kv cache prefix match
    if reset and self.n_tokens > 0:
        longest_prefix = 0
        for a, b in zip(self._input_ids, tokens[:-1]):
            if a == b:
                longest_prefix += 1
            else:
                break
        if longest_prefix > 0:
            if self.verbose:
                print("Llama.generate: prefix-match hit", file=sys.stderr)
            reset = False
            tokens = tokens[longest_prefix:]
            self.n_tokens = longest_prefix

    # Reset the model state
    if reset:
        self.reset()

    # Reset the grammar
    if grammar is not None:
        grammar.reset()

    sample_idx = self.n_tokens + len(tokens) - 1
    tokens = list(tokens)

    # Eval and sample
    while True:
        self.eval(tokens)
        while sample_idx < self.n_tokens:
            token = self.sample(
                top_k=top_k,
                top_p=top_p,
                min_p=min_p,
                typical_p=typical_p,
                temp=temp,
                repeat_penalty=repeat_penalty,
                frequency_penalty=frequency_penalty,
                presence_penalty=presence_penalty,
                tfs_z=tfs_z,
                mirostat_mode=mirostat_mode,
                mirostat_tau=mirostat_tau,
                mirostat_eta=mirostat_eta,
                logits_processor=logits_processor,
                grammar=grammar,
                penalize_nl=penalize_nl,
                idx=sample_idx,
            )

            sample_idx += 1
            if stopping_criteria is not None and stopping_criteria(
                self._input_ids, self._scores[-1, :]
            ):
                return
            tokens_or_none = yield token
            tokens.clear()
            tokens.append(token)
            if tokens_or_none is not None:
                tokens.extend(tokens_or_none)

            if sample_idx < self.n_tokens and token != self._input_ids[sample_idx]:
                self.n_tokens = sample_idx
                self._ctx.kv_cache_seq_rm(-1, self.n_tokens, -1)
                break

        if self.draft_model is not None:
            self.input_ids[self.n_tokens : self.n_tokens + len(tokens)] = tokens
            draft_tokens = self.draft_model(
                self.input_ids[: self.n_tokens + len(tokens)]
            )
            tokens.extend(
                draft_tokens.astype(int)[
                    : self._n_ctx - self.n_tokens - len(tokens)
                ]
            )

create_embedding(input, model=None)

Embed a string.

Parameters:

• input (Union[str, List[str]]) –

  The utf-8 encoded string to embed.

Returns:

• CreateEmbeddingResponse –

  An embedding object.

Source code in llama_cpp/llama.py

def create_embedding(
    self, input: Union[str, List[str]], model: Optional[str] = None
) -> CreateEmbeddingResponse:
    """Embed a string.

    Args:
        input: The utf-8 encoded string to embed.

    Returns:
        An embedding object.
    """
    assert self._model.model is not None
    model_name: str = model if model is not None else self.model_path

    input = input if isinstance(input, list) else [input]

    # get numeric embeddings
    embeds: Union[List[List[float]], List[List[List[float]]]]
    total_tokens: int
    embeds, total_tokens = self.embed(input, return_count=True)  # type: ignore

    # convert to CreateEmbeddingResponse
    data: List[Embedding] = [
        {
            "object": "embedding",
            "embedding": emb,
            "index": idx,
        }
        for idx, emb in enumerate(embeds)
    ]

    return {
        "object": "list",
        "data": data,
        "model": model_name,
        "usage": {
            "prompt_tokens": total_tokens,
            "total_tokens": total_tokens,
        },
    }

embed(input, normalize=False, truncate=True, return_count=False)

Embed a string.

Parameters:

• input (Union[str, List[str]]) –

  The utf-8 encoded string to embed.

Returns:

• –

  A list of embeddings

Source code in llama_cpp/llama.py

def embed(
    self,
    input: Union[str, List[str]],
    normalize: bool = False,
    truncate: bool = True,
    return_count: bool = False,
):
    """Embed a string.

    Args:
        input: The utf-8 encoded string to embed.

    Returns:
        A list of embeddings
    """
    assert self._ctx.ctx is not None
    n_embd = self.n_embd()
    n_batch = self.n_batch

    # get pooling information
    pooling_type = self.pooling_type()
    logits_all = pooling_type == llama_cpp.LLAMA_POOLING_TYPE_NONE

    if self.context_params.embeddings == False:
        raise RuntimeError(
            "Llama model must be created with embedding=True to call this method"
        )

    if self.verbose:
        llama_cpp.llama_reset_timings(self._ctx.ctx)

    if isinstance(input, str):
        inputs = [input]
    else:
        inputs = input

    # reset batch
    self._batch.reset()

    # decode and fetch embeddings
    data: Union[List[List[float]], List[List[List[float]]]] = []

    def decode_batch(seq_sizes: List[int]):
        assert self._ctx.ctx is not None
        llama_cpp.llama_kv_cache_clear(self._ctx.ctx)
        self._ctx.decode(self._batch)
        self._batch.reset()

        # store embeddings
        if pooling_type == llama_cpp.LLAMA_POOLING_TYPE_NONE:
            pos: int = 0
            for i, size in enumerate(seq_sizes):
                ptr = llama_cpp.llama_get_embeddings(self._ctx.ctx)
                embedding: List[List[float]] = [
                    ptr[pos + j * n_embd : pos + (j + 1) * n_embd] for j in range(size)
                ]
                if normalize:
                    embedding = [_normalize_embedding(e) for e in embedding]
                data.append(embedding)
                pos += size
        else:
            for i in range(len(seq_sizes)):
                ptr = llama_cpp.llama_get_embeddings_seq(self._ctx.ctx, i)
                embedding: List[float] = ptr[:n_embd]
                if normalize:
                    embedding = _normalize_embedding(embedding)
                data.append(embedding)

    # init state
    total_tokens = 0
    s_batch = []
    t_batch = 0
    p_batch = 0

    # accumulate batches and encode
    for text in inputs:
        tokens = self.tokenize(text.encode("utf-8"))
        if truncate:
            tokens = tokens[:n_batch]

        n_tokens = len(tokens)
        total_tokens += n_tokens

        # check for overrun
        if n_tokens > n_batch:
            raise ValueError(
                f"Requested tokens ({n_tokens}) exceed batch size of {n_batch}"
            )

        # time to eval batch
        if t_batch + n_tokens > n_batch:
            decode_batch(s_batch)
            s_batch = []
            t_batch = 0
            p_batch = 0

        # add to batch
        self._batch.add_sequence(tokens, p_batch, logits_all)

        # update batch stats
        s_batch.append(n_tokens)
        t_batch += n_tokens
        p_batch += 1

    # hanlde last batch
    decode_batch(s_batch)

    if self.verbose:
        llama_cpp.llama_print_timings(self._ctx.ctx)

    output = data[0] if isinstance(input, str) else data

    llama_cpp.llama_kv_cache_clear(self._ctx.ctx)
    self.reset()

    if return_count:
        return output, total_tokens
    else:
        return output

create_completion(prompt, suffix=None, max_tokens=16, temperature=0.8, top_p=0.95, min_p=0.05, typical_p=1.0, logprobs=None, echo=False, stop=[], frequency_penalty=0.0, presence_penalty=0.0, repeat_penalty=1.1, top_k=40, stream=False, seed=None, tfs_z=1.0, mirostat_mode=0, mirostat_tau=5.0, mirostat_eta=0.1, model=None, stopping_criteria=None, logits_processor=None, grammar=None, logit_bias=None)

Generate text from a prompt.

Parameters:

• prompt (Union[str, List[int]]) –

  The prompt to generate text from.

• suffix (Optional[str], default: None ) –

  A suffix to append to the generated text. If None, no suffix is appended.

• max_tokens (Optional[int], default: 16 ) –

  The maximum number of tokens to generate. If max_tokens <= 0 or None, the maximum number of tokens to generate is unlimited and depends on n_ctx.

• temperature (float, default: 0.8 ) –

  The temperature to use for sampling.

• top_p (float, default: 0.95 ) –

  The top-p value to use for nucleus sampling. Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751

• min_p (float, default: 0.05 ) –

  The min-p value to use for minimum p sampling. Minimum P sampling as described in https://github.com/ggerganov/llama.cpp/pull/3841

• typical_p (float, default: 1.0 ) –

  The typical-p value to use for sampling. Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666.

• logprobs (Optional[int], default: None ) –

  The number of logprobs to return. If None, no logprobs are returned.

• echo (bool, default: False ) –

  Whether to echo the prompt.

• stop (Optional[Union[str, List[str]]], default: [] ) –

  A list of strings to stop generation when encountered.

• frequency_penalty (float, default: 0.0 ) –

  The penalty to apply to tokens based on their frequency in the prompt.

• presence_penalty (float, default: 0.0 ) –

  The penalty to apply to tokens based on their presence in the prompt.

• repeat_penalty (float, default: 1.1 ) –

  The penalty to apply to repeated tokens.

• top_k (int, default: 40 ) –

  The top-k value to use for sampling. Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751

• stream (bool, default: False ) –

  Whether to stream the results.

• seed (Optional[int], default: None ) –

  The seed to use for sampling.

• tfs_z (float, default: 1.0 ) –

  The tail-free sampling parameter. Tail Free Sampling described in https://www.trentonbricken.com/Tail-Free-Sampling/.

• mirostat_mode (int, default: 0 ) –

  The mirostat sampling mode.

• mirostat_tau (float, default: 5.0 ) –

  The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text.

• mirostat_eta (float, default: 0.1 ) –

  The learning rate used to update mu based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause mu to be updated more quickly, while a smaller learning rate will result in slower updates.

• model (Optional[str], default: None ) –

  The name to use for the model in the completion object.

• stopping_criteria (Optional[StoppingCriteriaList], default: None ) –

  A list of stopping criteria to use.

• logits_processor (Optional[LogitsProcessorList], default: None ) –

  A list of logits processors to use.

• grammar (Optional[LlamaGrammar], default: None ) –

  A grammar to use for constrained sampling.

• logit_bias (Optional[Dict[str, float]], default: None ) –

  A logit bias to use.

Raises:

• ValueError –

  If the requested tokens exceed the context window.

• RuntimeError –

  If the prompt fails to tokenize or the model fails to evaluate the prompt.

Returns:

• Union[CreateCompletionResponse, Iterator[CreateCompletionStreamResponse]] –

  Response object containing the generated text.

Source code in llama_cpp/llama.py

def create_completion(
    self,
    prompt: Union[str, List[int]],
    suffix: Optional[str] = None,
    max_tokens: Optional[int] = 16,
    temperature: float = 0.8,
    top_p: float = 0.95,
    min_p: float = 0.05,
    typical_p: float = 1.0,
    logprobs: Optional[int] = None,
    echo: bool = False,
    stop: Optional[Union[str, List[str]]] = [],
    frequency_penalty: float = 0.0,
    presence_penalty: float = 0.0,
    repeat_penalty: float = 1.1,
    top_k: int = 40,
    stream: bool = False,
    seed: Optional[int] = None,
    tfs_z: float = 1.0,
    mirostat_mode: int = 0,
    mirostat_tau: float = 5.0,
    mirostat_eta: float = 0.1,
    model: Optional[str] = None,
    stopping_criteria: Optional[StoppingCriteriaList] = None,
    logits_processor: Optional[LogitsProcessorList] = None,
    grammar: Optional[LlamaGrammar] = None,
    logit_bias: Optional[Dict[str, float]] = None,
) -> Union[CreateCompletionResponse, Iterator[CreateCompletionStreamResponse]]:
    """Generate text from a prompt.

    Args:
        prompt: The prompt to generate text from.
        suffix: A suffix to append to the generated text. If None, no suffix is appended.
        max_tokens: The maximum number of tokens to generate. If max_tokens <= 0 or None, the maximum number of tokens to generate is unlimited and depends on n_ctx.
        temperature: The temperature to use for sampling.
        top_p: The top-p value to use for nucleus sampling. Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751
        min_p: The min-p value to use for minimum p sampling. Minimum P sampling as described in https://github.com/ggerganov/llama.cpp/pull/3841
        typical_p: The typical-p value to use for sampling. Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666.
        logprobs: The number of logprobs to return. If None, no logprobs are returned.
        echo: Whether to echo the prompt.
        stop: A list of strings to stop generation when encountered.
        frequency_penalty: The penalty to apply to tokens based on their frequency in the prompt.
        presence_penalty: The penalty to apply to tokens based on their presence in the prompt.
        repeat_penalty: The penalty to apply to repeated tokens.
        top_k: The top-k value to use for sampling. Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751
        stream: Whether to stream the results.
        seed: The seed to use for sampling.
        tfs_z: The tail-free sampling parameter. Tail Free Sampling described in https://www.trentonbricken.com/Tail-Free-Sampling/.
        mirostat_mode: The mirostat sampling mode.
        mirostat_tau: The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text.
        mirostat_eta: The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates.
        model: The name to use for the model in the completion object.
        stopping_criteria: A list of stopping criteria to use.
        logits_processor: A list of logits processors to use.
        grammar: A grammar to use for constrained sampling.
        logit_bias: A logit bias to use.

    Raises:
        ValueError: If the requested tokens exceed the context window.
        RuntimeError: If the prompt fails to tokenize or the model fails to evaluate the prompt.

    Returns:
        Response object containing the generated text.
    """
    completion_or_chunks = self._create_completion(
        prompt=prompt,
        suffix=suffix,
        max_tokens=-1 if max_tokens is None else max_tokens,
        temperature=temperature,
        top_p=top_p,
        min_p=min_p,
        typical_p=typical_p,
        logprobs=logprobs,
        echo=echo,
        stop=stop,
        frequency_penalty=frequency_penalty,
        presence_penalty=presence_penalty,
        repeat_penalty=repeat_penalty,
        top_k=top_k,
        stream=stream,
        seed=seed,
        tfs_z=tfs_z,
        mirostat_mode=mirostat_mode,
        mirostat_tau=mirostat_tau,
        mirostat_eta=mirostat_eta,
        model=model,
        stopping_criteria=stopping_criteria,
        logits_processor=logits_processor,
        grammar=grammar,
        logit_bias=logit_bias,
    )
    if stream:
        chunks: Iterator[CreateCompletionStreamResponse] = completion_or_chunks
        return chunks
    completion: Completion = next(completion_or_chunks)  # type: ignore
    return completion

__call__(prompt, suffix=None, max_tokens=16, temperature=0.8, top_p=0.95, min_p=0.05, typical_p=1.0, logprobs=None, echo=False, stop=[], frequency_penalty=0.0, presence_penalty=0.0, repeat_penalty=1.1, top_k=40, stream=False, seed=None, tfs_z=1.0, mirostat_mode=0, mirostat_tau=5.0, mirostat_eta=0.1, model=None, stopping_criteria=None, logits_processor=None, grammar=None, logit_bias=None)

Generate text from a prompt.

Parameters:

• prompt (str) –

  The prompt to generate text from.

• suffix (Optional[str], default: None ) –

  A suffix to append to the generated text. If None, no suffix is appended.

• max_tokens (Optional[int], default: 16 ) –

  The maximum number of tokens to generate. If max_tokens <= 0 or None, the maximum number of tokens to generate is unlimited and depends on n_ctx.

• temperature (float, default: 0.8 ) –

  The temperature to use for sampling.

• top_p (float, default: 0.95 ) –

  The top-p value to use for nucleus sampling. Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751

• min_p (float, default: 0.05 ) –

  The min-p value to use for minimum p sampling. Minimum P sampling as described in https://github.com/ggerganov/llama.cpp/pull/3841

• typical_p (float, default: 1.0 ) –

  The typical-p value to use for sampling. Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666.

• logprobs (Optional[int], default: None ) –

  The number of logprobs to return. If None, no logprobs are returned.

• echo (bool, default: False ) –

  Whether to echo the prompt.

• stop (Optional[Union[str, List[str]]], default: [] ) –

  A list of strings to stop generation when encountered.

• frequency_penalty (float, default: 0.0 ) –

  The penalty to apply to tokens based on their frequency in the prompt.

• presence_penalty (float, default: 0.0 ) –

  The penalty to apply to tokens based on their presence in the prompt.

• repeat_penalty (float, default: 1.1 ) –

  The penalty to apply to repeated tokens.

• top_k (int, default: 40 ) –

  The top-k value to use for sampling. Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751

• stream (bool, default: False ) –

  Whether to stream the results.

• seed (Optional[int], default: None ) –

  The seed to use for sampling.

• tfs_z (float, default: 1.0 ) –

  The tail-free sampling parameter. Tail Free Sampling described in https://www.trentonbricken.com/Tail-Free-Sampling/.

• mirostat_mode (int, default: 0 ) –

  The mirostat sampling mode.

• mirostat_tau (float, default: 5.0 ) –

  The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text.

• mirostat_eta (float, default: 0.1 ) –

  The learning rate used to update mu based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause mu to be updated more quickly, while a smaller learning rate will result in slower updates.

• model (Optional[str], default: None ) –

  The name to use for the model in the completion object.

• stopping_criteria (Optional[StoppingCriteriaList], default: None ) –

  A list of stopping criteria to use.

• logits_processor (Optional[LogitsProcessorList], default: None ) –

  A list of logits processors to use.

• grammar (Optional[LlamaGrammar], default: None ) –

  A grammar to use for constrained sampling.

• logit_bias (Optional[Dict[str, float]], default: None ) –

  A logit bias to use.

Raises:

• ValueError –

  If the requested tokens exceed the context window.

• RuntimeError –

  If the prompt fails to tokenize or the model fails to evaluate the prompt.

Returns:

• Union[CreateCompletionResponse, Iterator[CreateCompletionStreamResponse]] –

  Response object containing the generated text.

Source code in llama_cpp/llama.py

def __call__(
    self,
    prompt: str,
    suffix: Optional[str] = None,
    max_tokens: Optional[int] = 16,
    temperature: float = 0.8,
    top_p: float = 0.95,
    min_p: float = 0.05,
    typical_p: float = 1.0,
    logprobs: Optional[int] = None,
    echo: bool = False,
    stop: Optional[Union[str, List[str]]] = [],
    frequency_penalty: float = 0.0,
    presence_penalty: float = 0.0,
    repeat_penalty: float = 1.1,
    top_k: int = 40,
    stream: bool = False,
    seed: Optional[int] = None,
    tfs_z: float = 1.0,
    mirostat_mode: int = 0,
    mirostat_tau: float = 5.0,
    mirostat_eta: float = 0.1,
    model: Optional[str] = None,
    stopping_criteria: Optional[StoppingCriteriaList] = None,
    logits_processor: Optional[LogitsProcessorList] = None,
    grammar: Optional[LlamaGrammar] = None,
    logit_bias: Optional[Dict[str, float]] = None,
) -> Union[CreateCompletionResponse, Iterator[CreateCompletionStreamResponse]]:
    """Generate text from a prompt.

    Args:
        prompt: The prompt to generate text from.
        suffix: A suffix to append to the generated text. If None, no suffix is appended.
        max_tokens: The maximum number of tokens to generate. If max_tokens <= 0 or None, the maximum number of tokens to generate is unlimited and depends on n_ctx.
        temperature: The temperature to use for sampling.
        top_p: The top-p value to use for nucleus sampling. Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751
        min_p: The min-p value to use for minimum p sampling. Minimum P sampling as described in https://github.com/ggerganov/llama.cpp/pull/3841
        typical_p: The typical-p value to use for sampling. Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666.
        logprobs: The number of logprobs to return. If None, no logprobs are returned.
        echo: Whether to echo the prompt.
        stop: A list of strings to stop generation when encountered.
        frequency_penalty: The penalty to apply to tokens based on their frequency in the prompt.
        presence_penalty: The penalty to apply to tokens based on their presence in the prompt.
        repeat_penalty: The penalty to apply to repeated tokens.
        top_k: The top-k value to use for sampling. Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751
        stream: Whether to stream the results.
        seed: The seed to use for sampling.
        tfs_z: The tail-free sampling parameter. Tail Free Sampling described in https://www.trentonbricken.com/Tail-Free-Sampling/.
        mirostat_mode: The mirostat sampling mode.
        mirostat_tau: The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text.
        mirostat_eta: The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates.
        model: The name to use for the model in the completion object.
        stopping_criteria: A list of stopping criteria to use.
        logits_processor: A list of logits processors to use.
        grammar: A grammar to use for constrained sampling.
        logit_bias: A logit bias to use.

    Raises:
        ValueError: If the requested tokens exceed the context window.
        RuntimeError: If the prompt fails to tokenize or the model fails to evaluate the prompt.

    Returns:
        Response object containing the generated text.
    """
    return self.create_completion(
        prompt=prompt,
        suffix=suffix,
        max_tokens=max_tokens,
        temperature=temperature,
        top_p=top_p,
        min_p=min_p,
        typical_p=typical_p,
        logprobs=logprobs,
        echo=echo,
        stop=stop,
        frequency_penalty=frequency_penalty,
        presence_penalty=presence_penalty,
        repeat_penalty=repeat_penalty,
        top_k=top_k,
        stream=stream,
        seed=seed,
        tfs_z=tfs_z,
        mirostat_mode=mirostat_mode,
        mirostat_tau=mirostat_tau,
        mirostat_eta=mirostat_eta,
        model=model,
        stopping_criteria=stopping_criteria,
        logits_processor=logits_processor,
        grammar=grammar,
        logit_bias=logit_bias,
    )

create_chat_completion(messages, functions=None, function_call=None, tools=None, tool_choice=None, temperature=0.2, top_p=0.95, top_k=40, min_p=0.05, typical_p=1.0, stream=False, stop=[], seed=None, response_format=None, max_tokens=None, presence_penalty=0.0, frequency_penalty=0.0, repeat_penalty=1.1, tfs_z=1.0, mirostat_mode=0, mirostat_tau=5.0, mirostat_eta=0.1, model=None, logits_processor=None, grammar=None, logit_bias=None, logprobs=None, top_logprobs=None)

Generate a chat completion from a list of messages.

Parameters:

• messages (List[ChatCompletionRequestMessage]) –

  A list of messages to generate a response for.

• functions (Optional[List[ChatCompletionFunction]], default: None ) –

  A list of functions to use for the chat completion.

• function_call (Optional[ChatCompletionRequestFunctionCall], default: None ) –

  A function call to use for the chat completion.

• tools (Optional[List[ChatCompletionTool]], default: None ) –

  A list of tools to use for the chat completion.

• tool_choice (Optional[ChatCompletionToolChoiceOption], default: None ) –

  A tool choice to use for the chat completion.

• temperature (float, default: 0.2 ) –

  The temperature to use for sampling.

• top_p (float, default: 0.95 ) –

  The top-p value to use for nucleus sampling. Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751

• top_k (int, default: 40 ) –

  The top-k value to use for sampling. Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751

• min_p (float, default: 0.05 ) –

  The min-p value to use for minimum p sampling. Minimum P sampling as described in https://github.com/ggerganov/llama.cpp/pull/3841

• typical_p (float, default: 1.0 ) –

  The typical-p value to use for sampling. Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666.

• stream (bool, default: False ) –

  Whether to stream the results.

• stop (Optional[Union[str, List[str]]], default: [] ) –

  A list of strings to stop generation when encountered.

• seed (Optional[int], default: None ) –

  The seed to use for sampling.

• response_format (Optional[ChatCompletionRequestResponseFormat], default: None ) –

  The response format to use for the chat completion. Use { "type": "json_object" } to contstrain output to only valid json.

• max_tokens (Optional[int], default: None ) –

  The maximum number of tokens to generate. If max_tokens <= 0 or None, the maximum number of tokens to generate is unlimited and depends on n_ctx.

• presence_penalty (float, default: 0.0 ) –

  The penalty to apply to tokens based on their presence in the prompt.

• frequency_penalty (float, default: 0.0 ) –

  The penalty to apply to tokens based on their frequency in the prompt.

• repeat_penalty (float, default: 1.1 ) –

  The penalty to apply to repeated tokens.

• tfs_z (float, default: 1.0 ) –

  The tail-free sampling parameter.

• mirostat_mode (int, default: 0 ) –

  The mirostat sampling mode.

• mirostat_tau (float, default: 5.0 ) –

  The mirostat sampling tau parameter.

• mirostat_eta (float, default: 0.1 ) –

  The mirostat sampling eta parameter.

• model (Optional[str], default: None ) –

  The name to use for the model in the completion object.

• logits_processor (Optional[LogitsProcessorList], default: None ) –

  A list of logits processors to use.

• grammar (Optional[LlamaGrammar], default: None ) –

  A grammar to use.

• logit_bias (Optional[Dict[str, float]], default: None ) –

  A logit bias to use.

Returns:

• Union[CreateChatCompletionResponse, Iterator[CreateChatCompletionStreamResponse]] –

  Generated chat completion or a stream of chat completion chunks.

Source code in llama_cpp/llama.py

def create_chat_completion(
    self,
    messages: List[ChatCompletionRequestMessage],
    functions: Optional[List[ChatCompletionFunction]] = None,
    function_call: Optional[ChatCompletionRequestFunctionCall] = None,
    tools: Optional[List[ChatCompletionTool]] = None,
    tool_choice: Optional[ChatCompletionToolChoiceOption] = None,
    temperature: float = 0.2,
    top_p: float = 0.95,
    top_k: int = 40,
    min_p: float = 0.05,
    typical_p: float = 1.0,
    stream: bool = False,
    stop: Optional[Union[str, List[str]]] = [],
    seed: Optional[int] = None,
    response_format: Optional[ChatCompletionRequestResponseFormat] = None,
    max_tokens: Optional[int] = None,
    presence_penalty: float = 0.0,
    frequency_penalty: float = 0.0,
    repeat_penalty: float = 1.1,
    tfs_z: float = 1.0,
    mirostat_mode: int = 0,
    mirostat_tau: float = 5.0,
    mirostat_eta: float = 0.1,
    model: Optional[str] = None,
    logits_processor: Optional[LogitsProcessorList] = None,
    grammar: Optional[LlamaGrammar] = None,
    logit_bias: Optional[Dict[str, float]] = None,
    logprobs: Optional[bool] = None,
    top_logprobs: Optional[int] = None,
) -> Union[
    CreateChatCompletionResponse, Iterator[CreateChatCompletionStreamResponse]
]:
    """Generate a chat completion from a list of messages.

    Args:
        messages: A list of messages to generate a response for.
        functions: A list of functions to use for the chat completion.
        function_call: A function call to use for the chat completion.
        tools: A list of tools to use for the chat completion.
        tool_choice: A tool choice to use for the chat completion.
        temperature: The temperature to use for sampling.
        top_p: The top-p value to use for nucleus sampling. Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751
        top_k: The top-k value to use for sampling. Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751
        min_p: The min-p value to use for minimum p sampling. Minimum P sampling as described in https://github.com/ggerganov/llama.cpp/pull/3841
        typical_p: The typical-p value to use for sampling. Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666.
        stream: Whether to stream the results.
        stop: A list of strings to stop generation when encountered.
        seed: The seed to use for sampling.
        response_format: The response format to use for the chat completion. Use { "type": "json_object" } to contstrain output to only valid json.
        max_tokens: The maximum number of tokens to generate. If max_tokens <= 0 or None, the maximum number of tokens to generate is unlimited and depends on n_ctx.
        presence_penalty: The penalty to apply to tokens based on their presence in the prompt.
        frequency_penalty: The penalty to apply to tokens based on their frequency in the prompt.
        repeat_penalty: The penalty to apply to repeated tokens.
        tfs_z: The tail-free sampling parameter.
        mirostat_mode: The mirostat sampling mode.
        mirostat_tau: The mirostat sampling tau parameter.
        mirostat_eta: The mirostat sampling eta parameter.
        model: The name to use for the model in the completion object.
        logits_processor: A list of logits processors to use.
        grammar: A grammar to use.
        logit_bias: A logit bias to use.

    Returns:
        Generated chat completion or a stream of chat completion chunks.
    """
    handler = self.chat_handler or llama_chat_format.get_chat_completion_handler(
        self.chat_format
    )
    return handler(
        llama=self,
        messages=messages,
        functions=functions,
        function_call=function_call,
        tools=tools,
        tool_choice=tool_choice,
        temperature=temperature,
        top_p=top_p,
        top_k=top_k,
        min_p=min_p,
        typical_p=typical_p,
        logprobs=logprobs,
        top_logprobs=top_logprobs,
        stream=stream,
        stop=stop,
        seed=seed,
        response_format=response_format,
        max_tokens=max_tokens,
        presence_penalty=presence_penalty,
        frequency_penalty=frequency_penalty,
        repeat_penalty=repeat_penalty,
        tfs_z=tfs_z,
        mirostat_mode=mirostat_mode,
        mirostat_tau=mirostat_tau,
        mirostat_eta=mirostat_eta,
        model=model,
        logits_processor=logits_processor,
        grammar=grammar,
        logit_bias=logit_bias,
    )

create_chat_completion_openai_v1(*args, **kwargs)

Generate a chat completion with return type based on the the OpenAI v1 API.

OpenAI python package is required to use this method.

You can install it with pip install openai.

Parameters:

• *args (Any, default: () ) –

  Positional arguments to pass to create_chat_completion.

• **kwargs (Any, default: {} ) –

  Keyword arguments to pass to create_chat_completion.

Returns:

• –

  Generated chat completion or a stream of chat completion chunks.

Source code in llama_cpp/llama.py

def create_chat_completion_openai_v1(
    self,
    *args: Any,
    **kwargs: Any,
):
    """Generate a chat completion with return type based on the the OpenAI v1 API.

    OpenAI python package is required to use this method.

    You can install it with `pip install openai`.

    Args:
        *args: Positional arguments to pass to create_chat_completion.
        **kwargs: Keyword arguments to pass to create_chat_completion.

    Returns:
        Generated chat completion or a stream of chat completion chunks.
    """
    try:
        from openai.types.chat import ChatCompletion, ChatCompletionChunk

        stream = kwargs.get("stream", False)  # type: ignore
        assert isinstance(stream, bool)
        if stream:
            return (ChatCompletionChunk(**chunk) for chunk in self.create_chat_completion(*args, **kwargs))  # type: ignore
        else:
            return ChatCompletion(**self.create_chat_completion(*args, **kwargs))  # type: ignore
    except ImportError:
        raise ImportError(
            "To use create_chat_completion_openai_v1, you must install the openai package."
            "You can install it with `pip install openai`."
        )

set_cache(cache)

Set the cache.

Parameters:

• cache (Optional[BaseLlamaCache]) –

  The cache to set.

Source code in llama_cpp/llama.py

def set_cache(self, cache: Optional[BaseLlamaCache]):
    """Set the cache.

    Args:
        cache: The cache to set.
    """
    self.cache = cache

save_state()

Source code in llama_cpp/llama.py

def save_state(self) -> LlamaState:
    assert self._ctx.ctx is not None
    if self.verbose:
        print("Llama.save_state: saving llama state", file=sys.stderr)
    state_size = llama_cpp.llama_get_state_size(self._ctx.ctx)
    if self.verbose:
        print(f"Llama.save_state: got state size: {state_size}", file=sys.stderr)
    llama_state = (ctypes.c_uint8 * int(state_size))()
    if self.verbose:
        print("Llama.save_state: allocated state", file=sys.stderr)
    n_bytes = llama_cpp.llama_copy_state_data(self._ctx.ctx, llama_state)
    if self.verbose:
        print(f"Llama.save_state: copied llama state: {n_bytes}", file=sys.stderr)
    if int(n_bytes) > int(state_size):
        raise RuntimeError("Failed to copy llama state data")
    llama_state_compact = (ctypes.c_uint8 * int(n_bytes))()
    llama_cpp.ctypes.memmove(llama_state_compact, llama_state, int(n_bytes))
    if self.verbose:
        print(
            f"Llama.save_state: saving {n_bytes} bytes of llama state",
            file=sys.stderr,
        )
    return LlamaState(
        scores=self._scores.copy(),
        input_ids=self.input_ids.copy(),
        n_tokens=self.n_tokens,
        llama_state=bytes(llama_state_compact),
        llama_state_size=n_bytes,
    )

load_state(state)

Source code in llama_cpp/llama.py

def load_state(self, state: LlamaState) -> None:
    assert self._ctx.ctx is not None
    # Only filling in up to `n_tokens` and then zero-ing out the rest
    self.scores[: state.n_tokens, :] = state.scores.copy()
    self.scores[state.n_tokens :, :] = 0.0
    self.input_ids = state.input_ids.copy()
    self.n_tokens = state.n_tokens
    state_size = state.llama_state_size
    LLamaStateArrayType = ctypes.c_uint8 * state_size
    llama_state = LLamaStateArrayType.from_buffer_copy(state.llama_state)

    if llama_cpp.llama_set_state_data(self._ctx.ctx, llama_state) != state_size:
        raise RuntimeError("Failed to set llama state data")

token_bos()

Return the beginning-of-sequence token.

Source code in llama_cpp/llama.py

def token_bos(self) -> int:
    """Return the beginning-of-sequence token."""
    return self._model.token_bos()

token_eos()

Return the end-of-sequence token.

Source code in llama_cpp/llama.py

def token_eos(self) -> int:
    """Return the end-of-sequence token."""
    return self._model.token_eos()

from_pretrained(repo_id, filename, local_dir=None, local_dir_use_symlinks='auto', cache_dir=None, **kwargs) classmethod

Create a Llama model from a pretrained model name or path. This method requires the huggingface-hub package. You can install it with pip install huggingface-hub.

Parameters:

• repo_id (str) –

  The model repo id.

• filename (Optional[str]) –

  A filename or glob pattern to match the model file in the repo.

• local_dir (Optional[Union[str, PathLike[str]]], default: None ) –

  The local directory to save the model to.

• local_dir_use_symlinks (Union[bool, Literal['auto']], default: 'auto' ) –

  Whether to use symlinks when downloading the model.

• **kwargs (Any, default: {} ) –

  Additional keyword arguments to pass to the Llama constructor.

Returns:

• 'Llama' –

  A Llama model.

Source code in llama_cpp/llama.py

@classmethod
def from_pretrained(
    cls,
    repo_id: str,
    filename: Optional[str],
    local_dir: Optional[Union[str, os.PathLike[str]]] = None,
    local_dir_use_symlinks: Union[bool, Literal["auto"]] = "auto",
    cache_dir: Optional[Union[str, os.PathLike[str]]] = None,
    **kwargs: Any,
) -> "Llama":
    """Create a Llama model from a pretrained model name or path.
    This method requires the huggingface-hub package.
    You can install it with `pip install huggingface-hub`.

    Args:
        repo_id: The model repo id.
        filename: A filename or glob pattern to match the model file in the repo.
        local_dir: The local directory to save the model to.
        local_dir_use_symlinks: Whether to use symlinks when downloading the model.
        **kwargs: Additional keyword arguments to pass to the Llama constructor.

    Returns:
        A Llama model."""
    try:
        from huggingface_hub import hf_hub_download, HfFileSystem
        from huggingface_hub.utils import validate_repo_id
    except ImportError:
        raise ImportError(
            "Llama.from_pretrained requires the huggingface-hub package. "
            "You can install it with `pip install huggingface-hub`."
        )

    validate_repo_id(repo_id)

    hffs = HfFileSystem()

    files = [
        file["name"] if isinstance(file, dict) else file
        for file in hffs.ls(repo_id)
    ]

    # split each file into repo_id, subfolder, filename
    file_list: List[str] = []
    for file in files:
        rel_path = Path(file).relative_to(repo_id)
        file_list.append(str(rel_path))

    matching_files = [file for file in file_list if fnmatch.fnmatch(file, filename)]  # type: ignore

    if len(matching_files) == 0:
        raise ValueError(
            f"No file found in {repo_id} that match {filename}\n\n"
            f"Available Files:\n{json.dumps(file_list)}"
        )

    if len(matching_files) > 1:
        raise ValueError(
            f"Multiple files found in {repo_id} matching {filename}\n\n"
            f"Available Files:\n{json.dumps(files)}"
        )

    (matching_file,) = matching_files

    subfolder = str(Path(matching_file).parent)
    filename = Path(matching_file).name

    # download the file
    hf_hub_download(
        repo_id=repo_id,
        filename=filename,
        subfolder=subfolder,
        local_dir=local_dir,
        local_dir_use_symlinks=local_dir_use_symlinks,
        cache_dir=cache_dir,
    )

    if local_dir is None:
        model_path = hf_hub_download(
            repo_id=repo_id,
            filename=filename,
            subfolder=subfolder,
            local_dir=local_dir,
            local_dir_use_symlinks=local_dir_use_symlinks,
            cache_dir=cache_dir,
            local_files_only=True,
        )
    else:
        model_path = os.path.join(local_dir, filename)

    return cls(
        model_path=model_path,
        **kwargs,
    )

llama_cpp.LlamaGrammar

Keeps reference counts of all the arguments, so that they are not garbage collected by Python.

Source code in llama_cpp/llama_grammar.py

class LlamaGrammar:
    """Keeps reference counts of all the arguments, so that they are not
    garbage collected by Python."""

    def __del__(self) -> None:
        """Free the grammar pointer when the object is deleted."""
        if self.grammar is not None:
            llama_cpp.llama_grammar_free(self.grammar)
            self.grammar = None

    def __init__(
        self,
        parsed_grammar: "parse_state",
    ) -> None:
        """Initialize the grammar pointer from the parsed state."""
        self._grammar_rules = (
            parsed_grammar.c_rules()
        )  # type: std.vector[std.vector[LlamaGrammarElement]]
        self._n_rules = self._grammar_rules.size()  # type: int
        self._start_rule_index = parsed_grammar.symbol_ids.at("root")  # type: int
        self.init()

    @classmethod
    def from_string(cls, grammar: str, verbose: bool = True) -> "LlamaGrammar":
        """Convert a GBNF grammar to a Llama grammar."""
        parsed_grammar = parse(const_char_p(grammar))  # type: parse_state
        if parsed_grammar.rules.empty():
            raise ValueError(
                f"{cls.from_string.__name__}: error parsing grammar file: parsed_grammar.rules is empty"
            )
        if verbose:
            print(f"{cls.from_string.__name__} grammar:", file=sys.stderr)
            print_grammar(sys.stderr, parsed_grammar)
            print(file=sys.stderr)
        return cls(parsed_grammar)

    @classmethod
    def from_json_schema(
        cls,
        json_schema: str,
        verbose: bool = True,
    ) -> "LlamaGrammar":
        """Convert a JSON schema to a Llama grammar."""
        return cls.from_string(json_schema_to_gbnf(json_schema), verbose=verbose)

    @classmethod
    def from_file(cls, file: Union[str, Path], verbose: bool = True) -> "LlamaGrammar":
        try:
            with open(file) as f:
                grammar = f.read()
        except Exception as err:
            raise Exception(
                f"{cls.from_file.__name__}: error reading grammar file: {err}"
            )

        if grammar:
            return cls.from_string(grammar, verbose=verbose)

        raise ValueError(
            f"{cls.from_file.__name__}: error parsing grammar file: params_grammer is empty"
        )

    def init(self) -> None:
        # Step 1: Convert LlamaGrammarElement to llama_grammar_element
        self._element_lists = [
            [
                llama_grammar_element(c_int(elem.type.value), c_uint32(elem.value))
                for elem in subvector
            ]
            for subvector in self._grammar_rules
        ]  # type: List[List[llama_grammar_element]]

        # Step 2: Convert each list to llama_grammar_element array and get pointer
        self._element_arrays = [
            (llama_grammar_element * len(sublist))(*sublist)
            for sublist in self._element_lists
        ]  # type: List[Array[llama_grammar_element]]

        # Step 3: Get pointer of each array
        self._element_array_pointers = [
            cast(subarray, llama_grammar_element_p) for subarray in self._element_arrays
        ]  # type: List[llama_grammar_element_p]

        # Step 4: Make array of these pointers and get its pointer
        self._rules = (llama_grammar_element_p * len(self._element_array_pointers))(
            *self._element_array_pointers
        )
        self.grammar = llama_cpp.llama_grammar_init(
            self._rules, c_size_t(self._n_rules), c_size_t(self._start_rule_index)
        )

    def reset(self) -> None:
        if self.grammar is not None:
            llama_cpp.llama_grammar_free(self.grammar)
        self.init()

from_string(grammar, verbose=True) classmethod

Convert a GBNF grammar to a Llama grammar.

Source code in llama_cpp/llama_grammar.py

@classmethod
def from_string(cls, grammar: str, verbose: bool = True) -> "LlamaGrammar":
    """Convert a GBNF grammar to a Llama grammar."""
    parsed_grammar = parse(const_char_p(grammar))  # type: parse_state
    if parsed_grammar.rules.empty():
        raise ValueError(
            f"{cls.from_string.__name__}: error parsing grammar file: parsed_grammar.rules is empty"
        )
    if verbose:
        print(f"{cls.from_string.__name__} grammar:", file=sys.stderr)
        print_grammar(sys.stderr, parsed_grammar)
        print(file=sys.stderr)
    return cls(parsed_grammar)

from_json_schema(json_schema, verbose=True) classmethod

Convert a JSON schema to a Llama grammar.

Source code in llama_cpp/llama_grammar.py

@classmethod
def from_json_schema(
    cls,
    json_schema: str,
    verbose: bool = True,
) -> "LlamaGrammar":
    """Convert a JSON schema to a Llama grammar."""
    return cls.from_string(json_schema_to_gbnf(json_schema), verbose=verbose)

llama_cpp.LlamaCache = LlamaRAMCache module-attribute

llama_cpp.LlamaState

Source code in llama_cpp/llama.py

class LlamaState:
    def __init__(
        self,
        input_ids: npt.NDArray[np.intc],
        scores: npt.NDArray[np.single],
        n_tokens: int,
        llama_state: bytes,
        llama_state_size: int,
    ):
        self.input_ids = input_ids
        self.scores = scores
        self.n_tokens = n_tokens
        self.llama_state = llama_state
        self.llama_state_size = llama_state_size

llama_cpp.LogitsProcessor = Callable[[npt.NDArray[np.intc], npt.NDArray[np.single]], npt.NDArray[np.single]] module-attribute

llama_cpp.LogitsProcessorList

Bases: List[LogitsProcessor]

Source code in llama_cpp/llama.py

class LogitsProcessorList(List[LogitsProcessor]):
    def __call__(
        self, input_ids: npt.NDArray[np.intc], scores: npt.NDArray[np.single]
    ) -> npt.NDArray[np.single]:
        for processor in self:
            scores = processor(input_ids, scores)
        return scores

llama_cpp.StoppingCriteria = Callable[[npt.NDArray[np.intc], npt.NDArray[np.single]], bool] module-attribute

llama_cpp.StoppingCriteriaList

Bases: List[StoppingCriteria]

Source code in llama_cpp/llama.py

class StoppingCriteriaList(List[StoppingCriteria]):
    def __call__(
        self, input_ids: npt.NDArray[np.intc], logits: npt.NDArray[np.single]
    ) -> bool:
        return any([stopping_criteria(input_ids, logits) for stopping_criteria in self])

Low Level API

Low-level Python bindings for llama.cpp using Python's ctypes library.

llama_cpp.llama_cpp

llama_model_p = NewType('llama_model_p', int) module-attribute

llama_model_p_ctypes = ctypes.c_void_p module-attribute

llama_context_p = NewType('llama_context_p', int) module-attribute

llama_context_p_ctypes = ctypes.c_void_p module-attribute

llama_pos = ctypes.c_int32 module-attribute

llama_token = ctypes.c_int32 module-attribute

llama_token_p = ctypes.POINTER(llama_token) module-attribute

llama_seq_id = ctypes.c_int32 module-attribute

llama_token_data

Bases: Structure

Used to store token data

Attributes:

• id (llama_token) –

  token id

• logit (float) –

  log-odds of the token

• p (float) –

  probability of the token

Source code in llama_cpp/llama_cpp.py

class llama_token_data(ctypes.Structure):
    """Used to store token data

    Attributes:
        id (llama_token): token id
        logit (float): log-odds of the token
        p (float): probability of the token"""

    if TYPE_CHECKING:
        id: llama_token
        logit: float
        p: float

    _fields_ = [
        ("id", llama_token),
        ("logit", ctypes.c_float),
        ("p", ctypes.c_float),
    ]

llama_token_data_p = ctypes.POINTER(llama_token_data) module-attribute

llama_token_data_array

Bases: Structure

Used to sample tokens given logits

Attributes:

• data (Array[llama_token_data]) –

  token data

• size (int) –

  size of the array

• sorted (bool) –

  whether the array is sorted

Source code in llama_cpp/llama_cpp.py

class llama_token_data_array(ctypes.Structure):
    """Used to sample tokens given logits

    Attributes:
        data (ctypes.Array[llama_token_data]): token data
        size (int): size of the array
        sorted (bool): whether the array is sorted"""

    if TYPE_CHECKING:
        data: CtypesArray[llama_token_data]
        size: int
        sorted: bool

    _fields_ = [
        ("data", llama_token_data_p),
        ("size", ctypes.c_size_t),
        ("sorted", ctypes.c_bool),
    ]

llama_token_data_array_p = ctypes.POINTER(llama_token_data_array) module-attribute

llama_progress_callback = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.c_float, ctypes.c_void_p) module-attribute

llama_batch

Bases: Structure

Input data for llama_decode

A llama_batch object can contain input about one or many sequences

The provided arrays (i.e. token, embd, pos, etc.) must have size of n_tokens

Attributes:

• n_tokens (int) –

  number of tokens

• token (Array[llama_token]) –

  the token ids of the input (used when embd is NULL)

• embd (Array[c_float]) –

  token embeddings (i.e. float vector of size n_embd) (used when token is NULL)

• pos (Array[Array[llama_pos]]) –

  the positions of the respective token in the sequence

• seq_id (Array[Array[llama_seq_id]]) –

  the sequence to which the respective token belongs

• logits (Array[c_int8]) –

  if zero, the logits for the respective token will not be output

Source code in llama_cpp/llama_cpp.py

class llama_batch(ctypes.Structure):
    """Input data for llama_decode

    A llama_batch object can contain input about one or many sequences

    The provided arrays (i.e. token, embd, pos, etc.) must have size of n_tokens

    Attributes:
        n_tokens (int): number of tokens
        token (ctypes.Array[llama_token]): the token ids of the input (used when embd is NULL)
        embd (ctypes.Array[ctypes.ctypes.c_float]): token embeddings (i.e. float vector of size n_embd) (used when token is NULL)
        pos (ctypes.Array[ctypes.Array[llama_pos]]): the positions of the respective token in the sequence
        seq_id (ctypes.Array[ctypes.Array[llama_seq_id]]): the sequence to which the respective token belongs
        logits (ctypes.Array[ctypes.ctypes.c_int8]): if zero, the logits for the respective token will not be output
    """

    if TYPE_CHECKING:
        n_tokens: int
        token: CtypesArray[llama_token]
        embd: CtypesArray[ctypes.c_float]
        pos: CtypesArray[CtypesArray[llama_pos]]
        n_seq_id: CtypesArray[ctypes.c_int]
        seq_id: CtypesArray[CtypesArray[llama_seq_id]]
        logits: CtypesArray[ctypes.c_int8]

    _fields_ = [
        ("n_tokens", ctypes.c_int32),
        ("token", ctypes.POINTER(llama_token)),
        ("embd", ctypes.POINTER(ctypes.c_float)),
        ("pos", ctypes.POINTER(llama_pos)),
        ("n_seq_id", ctypes.POINTER(ctypes.c_int32)),
        ("seq_id", ctypes.POINTER(ctypes.POINTER(llama_seq_id))),
        ("logits", ctypes.POINTER(ctypes.c_int8)),
        ("all_pos_0", llama_pos),
        ("all_pos_1", llama_pos),
        ("all_seq_id", llama_seq_id),
    ]

llama_model_kv_override_value

Bases: Union

Source code in llama_cpp/llama_cpp.py

class llama_model_kv_override_value(ctypes.Union):
    _fields_ = [
        ("int_value", ctypes.c_int64),
        ("float_value", ctypes.c_double),
        ("bool_value", ctypes.c_bool),
        ("str_value", ctypes.c_char * 128),
    ]

    if TYPE_CHECKING:
        int_value: int
        float_value: float
        bool_value: bool
        str_value: bytes

llama_model_kv_override

Bases: Structure

Source code in llama_cpp/llama_cpp.py

class llama_model_kv_override(ctypes.Structure):
    _fields_ = [
        ("tag", ctypes.c_int),
        ("key", ctypes.c_char * 128),
        ("value", llama_model_kv_override_value),
    ]

    if TYPE_CHECKING:
        tag: int
        key: bytes
        value: Union[int, float, bool, bytes]

llama_model_params

Bases: Structure

Parameters for llama_model

Attributes:

• n_gpu_layers (int) –

  number of layers to store in VRAM

• split_mode (int) –

  how to split the model across multiple GPUs

• main_gpu (int) –

  the GPU that is used for the entire model. main_gpu interpretation depends on split_mode: LLAMA_SPLIT_NONE: the GPU that is used for the entire model LLAMA_SPLIT_ROW: the GPU that is used for small tensors and intermediate results LLAMA_SPLIT_LAYER: ignored

• tensor_split (Array[c_float]) –

  proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices()

• progress_callback (llama_progress_callback) –

  called with a progress value between 0.0 and 1.0. Pass NULL to disable. If the provided progress_callback returns true, model loading continues. If it returns false, model loading is immediately aborted.

• progress_callback_user_data (c_void_p) –

  context pointer passed to the progress callback

• kv_overrides (Array[llama_model_kv_override]) –

  override key-value pairs of the model meta data

• vocab_only (bool) –

  only load the vocabulary, no weights

• use_mmap (bool) –

  use mmap if possible

• use_mlock (bool) –

  force system to keep model in RAM

• check_tensors (bool) –

  validate model tensor data

Source code in llama_cpp/llama_cpp.py

class llama_model_params(ctypes.Structure):
    """Parameters for llama_model

    Attributes:
        n_gpu_layers (int): number of layers to store in VRAM
        split_mode (int): how to split the model across multiple GPUs
        main_gpu (int): the GPU that is used for the entire model. main_gpu interpretation depends on split_mode: LLAMA_SPLIT_NONE: the GPU that is used for the entire model LLAMA_SPLIT_ROW: the GPU that is used for small tensors and intermediate results LLAMA_SPLIT_LAYER: ignored
        tensor_split (ctypes.Array[ctypes.ctypes.c_float]): proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices()
        progress_callback (llama_progress_callback): called with a progress value between 0.0 and 1.0. Pass NULL to disable. If the provided progress_callback returns true, model loading continues. If it returns false, model loading is immediately aborted.
        progress_callback_user_data (ctypes.ctypes.c_void_p): context pointer passed to the progress callback
        kv_overrides (ctypes.Array[llama_model_kv_override]): override key-value pairs of the model meta data
        vocab_only (bool): only load the vocabulary, no weights
        use_mmap (bool): use mmap if possible
        use_mlock (bool): force system to keep model in RAM
        check_tensors (bool): validate model tensor data"""

    if TYPE_CHECKING:
        n_gpu_layers: int
        split_mode: int
        main_gpu: int
        tensor_split: CtypesArray[ctypes.c_float]
        progress_callback: Callable[[float, ctypes.c_void_p], bool]
        progress_callback_user_data: ctypes.c_void_p
        kv_overrides: CtypesArray[llama_model_kv_override]
        vocab_only: bool
        use_mmap: bool
        use_mlock: bool
        check_tensors: bool

    _fields_ = [
        ("n_gpu_layers", ctypes.c_int32),
        ("split_mode", ctypes.c_int),
        ("main_gpu", ctypes.c_int32),
        ("tensor_split", ctypes.POINTER(ctypes.c_float)),
        ("progress_callback", llama_progress_callback),
        ("progress_callback_user_data", ctypes.c_void_p),
        ("kv_overrides", ctypes.POINTER(llama_model_kv_override)),
        ("vocab_only", ctypes.c_bool),
        ("use_mmap", ctypes.c_bool),
        ("use_mlock", ctypes.c_bool),
        ("check_tensors", ctypes.c_bool),
    ]

llama_context_params

Bases: Structure

Parameters for llama_context

Attributes:

• seed (int) –

  RNG seed, -1 for random

• n_ctx (int) –

  text context, 0 = from model

• n_batch (int) –

  logical maximum batch size that can be submitted to llama_decode

• n_ubatch (int) –

  physical maximum batch size

• n_seq_max (int) –

  max number of sequences (i.e. distinct states for recurrent models)

• n_threads (int) –

  number of threads to use for generation

• n_threads_batch (int) –

  number of threads to use for batch processing

• rope_scaling_type (int) –

  RoPE scaling type, from enum llama_rope_scaling_type

• pooling_type (int) –

  whether to pool (sum) embedding results by sequence id (ignored if no pooling layer)

• rope_freq_base (float) –

  RoPE base frequency, 0 = from model

• rope_freq_scale (float) –

  RoPE frequency scaling factor, 0 = from model

• yarn_ext_factor (float) –

  YaRN extrapolation mix factor, negative = from model

• yarn_attn_factor (float) –

  YaRN magnitude scaling factor

• yarn_beta_fast (float) –

  YaRN low correction dim

• yarn_beta_slow (float) –

  YaRN high correction dim

• yarn_orig_ctx (int) –

  YaRN original context size

• defrag_thold (float) –

  defragment the KV cache if holes/size > thold, < 0 disabled (default)

• cb_eval (ggml_backend_sched_eval_callback) –

  callback for scheduling eval

• cb_eval_user_data (c_void_p) –

  user data for cb_eval

• type_k (int) –

  data type for K cache

• type_v (int) –

  data type for V cache

• logits_all (bool) –

  the llama_eval() call computes all logits, not just the last one (DEPRECATED - set llama_batch.logits instead)

• embeddings (bool) –

  if true, extract embeddings (together with logits)

• offload_kqv (bool) –

  whether to offload the KQV ops (including the KV cache) to GPU

• abort_callback (ggml_abort_callback) –

  abort callback if it returns true, execution of llama_decode() will be aborted

• abort_callback_data (c_void_p) –

  data for abort_callback

Source code in llama_cpp/llama_cpp.py

class llama_context_params(ctypes.Structure):
    """Parameters for llama_context

    Attributes:
        seed (int): RNG seed, -1 for random
        n_ctx (int): text context, 0 = from model
        n_batch (int): logical maximum batch size that can be submitted to llama_decode
        n_ubatch (int): physical maximum batch size
        n_seq_max (int): max number of sequences (i.e. distinct states for recurrent models)
        n_threads (int): number of threads to use for generation
        n_threads_batch (int): number of threads to use for batch processing
        rope_scaling_type (int): RoPE scaling type, from `enum llama_rope_scaling_type`
        pooling_type (int): whether to pool (sum) embedding results by sequence id (ignored if no pooling layer)
        rope_freq_base (float): RoPE base frequency, 0 = from model
        rope_freq_scale (float): RoPE frequency scaling factor, 0 = from model
        yarn_ext_factor (float): YaRN extrapolation mix factor, negative = from model
        yarn_attn_factor (float): YaRN magnitude scaling factor
        yarn_beta_fast (float): YaRN low correction dim
        yarn_beta_slow (float): YaRN high correction dim
        yarn_orig_ctx (int): YaRN original context size
        defrag_thold (float): defragment the KV cache if holes/size > thold, < 0 disabled (default)
        cb_eval (ggml_backend_sched_eval_callback): callback for scheduling eval
        cb_eval_user_data (ctypes.ctypes.c_void_p): user data for cb_eval
        type_k (int): data type for K cache
        type_v (int): data type for V cache
        logits_all (bool): the llama_eval() call computes all logits, not just the last one (DEPRECATED - set llama_batch.logits instead)
        embeddings (bool): if true, extract embeddings (together with logits)
        offload_kqv (bool): whether to offload the KQV ops (including the KV cache) to GPU
        abort_callback (ggml_abort_callback): abort callback if it returns true, execution of llama_decode() will be aborted
        abort_callback_data (ctypes.ctypes.c_void_p): data for abort_callback
    """

    if TYPE_CHECKING:
        seed: int
        n_ctx: int
        n_batch: int
        n_ubatch: int
        n_seq_max: int
        n_threads: int
        n_threads_batch: int
        rope_scaling_type: int
        pooling_type: int
        rope_freq_base: float
        rope_freq_scale: float
        yarn_ext_factor: float
        yarn_attn_factor: float
        yarn_beta_fast: float
        yarn_beta_slow: float
        yarn_orig_ctx: int
        defrag_thold: float
        cb_eval: Callable[[ctypes.c_void_p, bool], bool]
        cb_eval_user_data: ctypes.c_void_p
        type_k: int
        type_v: int
        logits_all: bool
        embeddings: bool
        offload_kqv: bool
        abort_callback: Callable[[ctypes.c_void_p], bool]
        abort_callback_data: ctypes.c_void_p

    _fields_ = [
        ("seed", ctypes.c_uint32),
        ("n_ctx", ctypes.c_uint32),
        ("n_batch", ctypes.c_uint32),
        ("n_ubatch", ctypes.c_uint32),
        ("n_seq_max", ctypes.c_uint32),
        ("n_threads", ctypes.c_uint32),
        ("n_threads_batch", ctypes.c_uint32),
        ("rope_scaling_type", ctypes.c_int),
        ("pooling_type", ctypes.c_int),
        ("rope_freq_base", ctypes.c_float),
        ("rope_freq_scale", ctypes.c_float),
        ("yarn_ext_factor", ctypes.c_float),
        ("yarn_attn_factor", ctypes.c_float),
        ("yarn_beta_fast", ctypes.c_float),
        ("yarn_beta_slow", ctypes.c_float),
        ("yarn_orig_ctx", ctypes.c_uint32),
        ("defrag_thold", ctypes.c_float),
        ("cb_eval", ggml_backend_sched_eval_callback),
        ("cb_eval_user_data", ctypes.c_void_p),
        ("type_k", ctypes.c_int),
        ("type_v", ctypes.c_int),
        ("logits_all", ctypes.c_bool),
        ("embeddings", ctypes.c_bool),
        ("offload_kqv", ctypes.c_bool),
        ("abort_callback", ggml_abort_callback),
        ("abort_callback_data", ctypes.c_void_p),
    ]

llama_log_callback = ctypes.CFUNCTYPE(None, ctypes.c_int, ctypes.c_char_p, ctypes.c_void_p) module-attribute

Signature for logging events Note that text includes the new line character at the end for most events. If your logging mechanism cannot handle that, check if the last character is ' ' and strip it if it exists. It might not exist for progress report where '.' is output repeatedly.

llama_model_quantize_params

Bases: Structure

Parameters for llama_model_quantize

Attributes:

• nthread (int) –

  number of threads to use for quantizing, if <=0 will use std::hardware_concurrency()

• ftype (int) –

  quantize to this llama_ftype

• output_tensor_type (int) –

  output tensor type

• token_embedding_type (int) –

  itoken embeddings tensor type

• allow_requantize (bool) –

  allow quantizing non-f32/f16 tensors

• quantize_output_tensor (bool) –

  quantize output.weight

• only_copy (bool) –

  only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored

• pure (bool) –

  quantize all tensors to the default type

• keep_split (bool) –

  quantize to the same number of shards

• imatrix (c_void_p) –

  pointer to importance matrix data

• kv_overrides (c_void_p) –

  pointer to vector containing overrides

Source code in llama_cpp/llama_cpp.py

class llama_model_quantize_params(ctypes.Structure):
    """Parameters for llama_model_quantize

    Attributes:
        nthread (int): number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency()
        ftype (int): quantize to this llama_ftype
        output_tensor_type (int): output tensor type
        token_embedding_type (int): itoken embeddings tensor type
        allow_requantize (bool): allow quantizing non-f32/f16 tensors
        quantize_output_tensor (bool): quantize output.weight
        only_copy (bool): only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored
        pure (bool): quantize all tensors to the default type
        keep_split (bool): quantize to the same number of shards
        imatrix (ctypes.c_void_p): pointer to importance matrix data
        kv_overrides (ctypes.c_void_p): pointer to vector containing overrides
    """

    if TYPE_CHECKING:
        nthread: int
        ftype: int
        output_tensor_type: int
        token_embedding_type: int
        allow_requantize: bool
        quantize_output_tensor: bool
        only_copy: bool
        pure: bool
        keep_split: bool
        imatrix: ctypes.c_void_p
        kv_overrides: ctypes.c_void_p

    _fields_ = [
        ("nthread", ctypes.c_int32),
        ("ftype", ctypes.c_int),
        ("output_tensor_type", ctypes.c_int),
        ("token_embedding_type", ctypes.c_int),
        ("allow_requantize", ctypes.c_bool),
        ("quantize_output_tensor", ctypes.c_bool),
        ("only_copy", ctypes.c_bool),
        ("pure", ctypes.c_bool),
        ("keep_split", ctypes.c_bool),
        ("imatrix", ctypes.c_void_p),
        ("kv_overrides", ctypes.c_void_p),
    ]

llama_grammar_p = ctypes.c_void_p module-attribute

llama_grammar_element

Bases: Structure

Source code in llama_cpp/llama_cpp.py

class llama_grammar_element(ctypes.Structure):
    if TYPE_CHECKING:
        type: int
        value: int

    _fields_ = [
        ("type", ctypes.c_int),
        ("value", ctypes.c_uint32),
    ]

llama_grammar_element_p = ctypes.POINTER(llama_grammar_element) module-attribute

llama_timings

Bases: Structure

Source code in llama_cpp/llama_cpp.py

class llama_timings(ctypes.Structure):
    if TYPE_CHECKING:
        t_start_ms: float
        t_end_ms: float
        t_load_ms: float
        t_sample_ms: float
        t_p_eval_ms: float
        t_eval_ms: float
        n_sample: int
        n_p_eval: int
        n_eval: int

    _fields_ = [
        ("t_start_ms", ctypes.c_double),
        ("t_end_ms", ctypes.c_double),
        ("t_load_ms", ctypes.c_double),
        ("t_sample_ms", ctypes.c_double),
        ("t_p_eval_ms", ctypes.c_double),
        ("t_eval_ms", ctypes.c_double),
        ("n_sample", ctypes.c_int32),
        ("n_p_eval", ctypes.c_int32),
        ("n_eval", ctypes.c_int32),
    ]

llama_chat_message

Bases: Structure

Source code in llama_cpp/llama_cpp.py

class llama_chat_message(ctypes.Structure):
    _fields_ = [
        ("role", ctypes.c_char_p),
        ("content", ctypes.c_char_p),
    ]

llama_model_default_params()

Get default parameters for llama_model

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_model_default_params",
    [],
    llama_model_params,
)
def llama_model_default_params() -> llama_model_params:
    """Get default parameters for llama_model"""
    ...

llama_context_default_params()

Get default parameters for llama_context

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_context_default_params",
    [],
    llama_context_params,
)
def llama_context_default_params() -> llama_context_params:
    """Get default parameters for llama_context"""
    ...

llama_model_quantize_default_params()

Get default parameters for llama_model_quantize

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_model_quantize_default_params",
    [],
    llama_model_quantize_params,
)
def llama_model_quantize_default_params() -> llama_model_quantize_params:
    """Get default parameters for llama_model_quantize"""
    ...

llama_backend_init()

Initialize the llama + ggml backend If numa is true, use NUMA optimizations Call once at the start of the program

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_backend_init",
    [],
    None,
)
def llama_backend_init():
    """Initialize the llama + ggml backend
    If numa is true, use NUMA optimizations
    Call once at the start of the program"""
    ...

llama_numa_init(numa)

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_numa_init",
    [ctypes.c_int],
    None,
)
def llama_numa_init(numa: int, /): ...

llama_backend_free()

Call once at the end of the program - currently only used for MPI

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_backend_free",
    [],
    None,
)
def llama_backend_free():
    """Call once at the end of the program - currently only used for MPI"""
    ...

llama_load_model_from_file(path_model, params)

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_load_model_from_file",
    [ctypes.c_char_p, llama_model_params],
    llama_model_p_ctypes,
)
def llama_load_model_from_file(
    path_model: bytes, params: llama_model_params, /
) -> Optional[llama_model_p]: ...

llama_free_model(model)

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_free_model",
    [llama_model_p_ctypes],
    None,
)
def llama_free_model(model: llama_model_p, /): ...

llama_new_context_with_model(model, params)

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_new_context_with_model",
    [llama_model_p_ctypes, llama_context_params],
    llama_context_p_ctypes,
)
def llama_new_context_with_model(
    model: llama_model_p, params: llama_context_params, /
) -> Optional[llama_context_p]: ...

llama_free(ctx)

Frees all allocated memory

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_free",
    [llama_context_p_ctypes],
    None,
)
def llama_free(ctx: llama_context_p, /):
    """Frees all allocated memory"""
    ...

llama_time_us()

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_time_us",
    [],
    ctypes.c_int64,
)
def llama_time_us() -> int: ...

llama_max_devices()

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_max_devices", [], ctypes.c_size_t)
def llama_max_devices() -> int: ...

llama_supports_mmap()

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_supports_mmap", [], ctypes.c_bool)
def llama_supports_mmap() -> bool: ...

llama_supports_mlock()

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_supports_mlock", [], ctypes.c_bool)
def llama_supports_mlock() -> bool: ...

llama_supports_gpu_offload()

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_supports_gpu_offload", [], ctypes.c_bool)
def llama_supports_gpu_offload() -> bool: ...

llama_get_model(ctx)

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_get_model", [llama_context_p_ctypes], llama_model_p_ctypes)
def llama_get_model(ctx: llama_context_p, /) -> Optional[llama_model_p]: ...

llama_n_ctx(ctx)

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_n_ctx", [llama_context_p_ctypes], ctypes.c_uint32)
def llama_n_ctx(ctx: llama_context_p, /) -> int: ...

llama_n_batch(ctx)

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_n_batch", [llama_context_p_ctypes], ctypes.c_uint32)
def llama_n_batch(ctx: llama_context_p, /) -> int: ...

llama_n_ubatch(ctx)

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_n_ubatch", [llama_context_p_ctypes], ctypes.c_uint32)
def llama_n_ubatch(ctx: llama_context_p, /) -> int: ...

llama_n_seq_max(ctx)

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_n_seq_max", [llama_context_p_ctypes], ctypes.c_uint32)
def llama_n_seq_max(ctx: llama_context_p, /) -> int: ...

llama_pooling_type(ctx)

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_pooling_type", [llama_context_p_ctypes], ctypes.c_int)
def llama_pooling_type(ctx: llama_context_p, /) -> int: ...

llama_vocab_type(model)

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_vocab_type", [llama_model_p_ctypes], ctypes.c_int)
def llama_vocab_type(model: llama_model_p, /) -> int: ...

llama_rope_type(model)

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_rope_type", [llama_model_p_ctypes], ctypes.c_int)
def llama_rope_type(model: llama_model_p, /) -> int: ...

llama_n_vocab(model)

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_n_vocab", [llama_model_p_ctypes], ctypes.c_int32)
def llama_n_vocab(model: llama_model_p, /) -> int: ...

llama_n_ctx_train(model)

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_n_ctx_train", [llama_model_p_ctypes], ctypes.c_int32)
def llama_n_ctx_train(model: llama_model_p, /) -> int: ...

llama_n_embd(model)

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_n_embd", [llama_model_p_ctypes], ctypes.c_int32)
def llama_n_embd(model: llama_model_p, /) -> int: ...

llama_n_layer(model)

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_n_layer", [llama_model_p_ctypes], ctypes.c_int32)
def llama_n_layer(model: llama_model_p, /) -> int: ...

llama_rope_freq_scale_train(model)

Get the model's RoPE frequency scaling factor

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_rope_freq_scale_train", [llama_model_p_ctypes], ctypes.c_float)
def llama_rope_freq_scale_train(model: llama_model_p, /) -> float:
    """Get the model's RoPE frequency scaling factor"""
    ...

llama_model_meta_val_str(model, key, buf, buf_size)

Get metadata value as a string by key name

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_model_meta_val_str",
    [
        llama_model_p_ctypes,
        ctypes.c_char_p,
        ctypes.c_char_p,
        ctypes.c_size_t,
    ],
    ctypes.c_int32,
)
def llama_model_meta_val_str(
    model: llama_model_p,
    key: Union[ctypes.c_char_p, bytes],
    buf: bytes,
    buf_size: int,
    /,
) -> int:
    """Get metadata value as a string by key name"""
    ...

llama_model_meta_count(model)

Get the number of metadata key/value pairs

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_model_meta_count", [llama_model_p_ctypes], ctypes.c_int32)
def llama_model_meta_count(model: llama_model_p, /) -> int:
    """Get the number of metadata key/value pairs"""
    ...

llama_model_meta_key_by_index(model, i, buf, buf_size)

Get metadata key name by index

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_model_meta_key_by_index",
    [
        llama_model_p_ctypes,
        ctypes.c_int32,
        ctypes.c_char_p,
        ctypes.c_size_t,
    ],
    ctypes.c_int32,
)
def llama_model_meta_key_by_index(
    model: llama_model_p,
    i: Union[ctypes.c_int, int],
    buf: Union[bytes, CtypesArray[ctypes.c_char]],
    buf_size: int,
    /,
) -> int:
    """Get metadata key name by index"""
    ...

llama_model_meta_val_str_by_index(model, i, buf, buf_size)

Get metadata value as a string by index

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_model_meta_val_str_by_index",
    [
        llama_model_p_ctypes,
        ctypes.c_int32,
        ctypes.c_char_p,
        ctypes.c_size_t,
    ],
    ctypes.c_int32,
)
def llama_model_meta_val_str_by_index(
    model: llama_model_p,
    i: Union[ctypes.c_int, int],
    buf: Union[bytes, CtypesArray[ctypes.c_char]],
    buf_size: int,
    /,
) -> int:
    """Get metadata value as a string by index"""
    ...

llama_model_desc(model, buf, buf_size)

Get a string describing the model type

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_model_desc",
    [llama_model_p_ctypes, ctypes.c_char_p, ctypes.c_size_t],
    ctypes.c_int32,
)
def llama_model_desc(
    model: llama_model_p,
    buf: Union[bytes, CtypesArray[ctypes.c_char]],
    buf_size: Union[ctypes.c_size_t, int],
    /,
) -> int:
    """Get a string describing the model type"""
    ...

llama_model_size(model)

Returns the total size of all the tensors in the model in bytes

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_model_size", [llama_model_p_ctypes], ctypes.c_uint64)
def llama_model_size(model: llama_model_p, /) -> int:
    """Returns the total size of all the tensors in the model in bytes"""
    ...

llama_model_n_params(model)

Returns the total number of parameters in the model

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_model_n_params", [llama_model_p_ctypes], ctypes.c_uint64)
def llama_model_n_params(model: llama_model_p, /) -> int:
    """Returns the total number of parameters in the model"""
    ...

llama_get_model_tensor(model, name)

Get a llama model tensor

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_get_model_tensor", [llama_model_p_ctypes, ctypes.c_char_p], ctypes.c_void_p
)
def llama_get_model_tensor(
    model: llama_model_p, name: Union[ctypes.c_char_p, bytes], /
) -> ctypes.c_void_p:
    """Get a llama model tensor"""
    ...

llama_model_quantize(fname_inp, fname_out, params)

Returns 0 on success

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_model_quantize",
    [
        ctypes.c_char_p,
        ctypes.c_char_p,
        ctypes.POINTER(llama_model_quantize_params),
    ],
    ctypes.c_uint32,
)
def llama_model_quantize(
    fname_inp: bytes,
    fname_out: bytes,
    params: CtypesPointerOrRef[llama_model_quantize_params],
    /,
) -> int:
    """Returns 0 on success"""
    ...

llama_model_apply_lora_from_file(model, path_lora, scale, path_base_model, n_threads)

Apply a LoRA adapter to a loaded model path_base_model is the path to a higher quality model to use as a base for the layers modified by the adapter. Can be NULL to use the current loaded model. The model needs to be reloaded before applying a new adapter, otherwise the adapter will be applied on top of the previous one Returns 0 on success

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_model_apply_lora_from_file",
    [
        llama_model_p_ctypes,
        ctypes.c_char_p,
        ctypes.c_float,
        ctypes.c_char_p,
        ctypes.c_int32,
    ],
    ctypes.c_int32,
)
def llama_model_apply_lora_from_file(
    model: llama_model_p,
    path_lora: Union[ctypes.c_char_p, bytes],
    scale: Union[ctypes.c_float, float],
    path_base_model: Union[ctypes.c_char_p, bytes, None],
    n_threads: Union[ctypes.c_int32, int],
    /,
) -> int:
    """Apply a LoRA adapter to a loaded model
    path_base_model is the path to a higher quality model to use as a base for
    the layers modified by the adapter. Can be NULL to use the current loaded model.
    The model needs to be reloaded before applying a new adapter, otherwise the adapter
    will be applied on top of the previous one
    Returns 0 on success"""
    ...

llama_control_vector_apply(lctx, data, len, n_embd, il_start, il_end)

Apply a loaded control vector to a llama_context, or if data is NULL, clear the currently loaded vector. n_embd should be the size of a single layer's control, and data should point to an n_embd x n_layers buffer starting from layer 1. il_start and il_end are the layer range the vector should apply to (both inclusive) See llama_control_vector_load in common to load a control vector.

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_control_vector_apply",
    [
        llama_context_p_ctypes,
        ctypes.POINTER(ctypes.c_float),
        ctypes.c_size_t,
        ctypes.c_int32,
        ctypes.c_int32,
        ctypes.c_int32,
    ],
    ctypes.c_int32,
)
def llama_control_vector_apply(
    lctx: llama_context_p,
    data: CtypesPointerOrRef[ctypes.c_float],
    len: int,
    n_embd: int,
    il_start: int,
    il_end: int,
    /,
) -> int:
    """Apply a loaded control vector to a llama_context, or if data is NULL, clear
    the currently loaded vector.
    n_embd should be the size of a single layer's control, and data should point
    to an n_embd x n_layers buffer starting from layer 1.
    il_start and il_end are the layer range the vector should apply to (both inclusive)
    See llama_control_vector_load in common to load a control vector."""
    ...

llama_kv_cache_view_cell

Bases: Structure

Information associated with an individual cell in the KV cache view.

Attributes:

• pos (llama_pos) –

  The position for this cell. Takes KV cache shifts into account. May be negative if the cell is not populated.

Source code in llama_cpp/llama_cpp.py

class llama_kv_cache_view_cell(ctypes.Structure):
    """Information associated with an individual cell in the KV cache view.

    Attributes:
        pos (llama_pos): The position for this cell. Takes KV cache shifts into account.
            May be negative if the cell is not populated."""

    if TYPE_CHECKING:
        pos: llama_pos

    _fields_ = [("pos", llama_pos)]

llama_kv_cache_view

Bases: Structure

Source code in llama_cpp/llama_cpp.py

class llama_kv_cache_view(ctypes.Structure):
    if TYPE_CHECKING:
        n_cells: int
        n_max_seq: int
        token_count: int
        used_cells: int
        max_contiguous: int
        max_contiguous_idx: int
        cells: CtypesArray[llama_kv_cache_view_cell]
        cells_sequences: CtypesArray[llama_seq_id]

    _fields_ = [
        ("n_cells", ctypes.c_int32),
        ("n_max_seq", ctypes.c_int32),
        ("token_count", ctypes.c_int32),
        ("used_cells", ctypes.c_int32),
        ("max_contiguous", ctypes.c_int32),
        ("max_contiguous_idx", ctypes.c_int32),
        ("cells", ctypes.POINTER(llama_kv_cache_view_cell)),
        ("cells_sequences", ctypes.POINTER(llama_seq_id)),
    ]

llama_kv_cache_view_p = ctypes.POINTER(llama_kv_cache_view) module-attribute

llama_kv_cache_view_init(ctx, n_seq_max)

Create an empty KV cache view. (use only for debugging purposes)

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_kv_cache_view_init",
    [llama_context_p_ctypes, ctypes.c_int32],
    llama_kv_cache_view,
)
def llama_kv_cache_view_init(
    ctx: llama_context_p, n_seq_max: Union[ctypes.c_int32, int], /
) -> llama_kv_cache_view:
    """Create an empty KV cache view. (use only for debugging purposes)"""
    ...

llama_kv_cache_view_free(view)

Free a KV cache view. (use only for debugging purposes)

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_kv_cache_view_free", [llama_kv_cache_view_p], None)
def llama_kv_cache_view_free(view: "ctypes.pointer[llama_kv_cache_view]", /):  # type: ignore
    """Free a KV cache view. (use only for debugging purposes)"""
    ...

llama_kv_cache_view_update(ctx, view)

Update the KV cache view structure with the current state of the KV cache. (use only for debugging purposes)

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_kv_cache_view_update", [llama_context_p_ctypes, llama_kv_cache_view_p], None
)
def llama_kv_cache_view_update(ctx: llama_context_p, view: CtypesPointerOrRef[llama_kv_cache_view], /):  # type: ignore
    """Update the KV cache view structure with the current state of the KV cache. (use only for debugging purposes)"""
    ...

llama_get_kv_cache_token_count(ctx)

Returns the number of tokens in the KV cache (slow, use only for debug) If a KV cell has multiple sequences assigned to it, it will be counted multiple times

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_get_kv_cache_token_count", [llama_context_p_ctypes], ctypes.c_int32
)
def llama_get_kv_cache_token_count(ctx: llama_context_p, /) -> int:
    """Returns the number of tokens in the KV cache (slow, use only for debug)
    If a KV cell has multiple sequences assigned to it, it will be counted multiple times
    """
    ...

llama_get_kv_cache_used_cells(ctx)

Returns the number of used KV cells (i.e. have at least one sequence assigned to them)

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_get_kv_cache_used_cells", [llama_context_p_ctypes], ctypes.c_int32
)
def llama_get_kv_cache_used_cells(ctx: llama_context_p, /) -> int:
    """Returns the number of used KV cells (i.e. have at least one sequence assigned to them)"""
    ...

llama_kv_cache_clear(ctx)

Clear the KV cache

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_kv_cache_clear", [llama_context_p_ctypes], None)
def llama_kv_cache_clear(ctx: llama_context_p, /):
    """Clear the KV cache"""
    ...

llama_kv_cache_seq_rm(ctx, seq_id, p0, p1)

Removes all tokens that belong to the specified sequence and have positions in [p0, p1)

Returns false if a partial sequence cannot be removed. Removing a whole sequence never fails

seq_id < 0 : match any sequence p0 < 0 : [0, p1] p1 < 0 : [p0, inf)

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_kv_cache_seq_rm",
    [
        llama_context_p_ctypes,
        llama_seq_id,
        llama_pos,
        llama_pos,
    ],
    ctypes.c_bool,
)
def llama_kv_cache_seq_rm(
    ctx: llama_context_p,
    seq_id: Union[llama_seq_id, int],
    p0: Union[llama_pos, int],
    p1: Union[llama_pos, int],
    /,
) -> bool:
    """Removes all tokens that belong to the specified sequence and have positions in [p0, p1)

    Returns false if a partial sequence cannot be removed. Removing a whole sequence never fails

    seq_id < 0 : match any sequence
    p0 < 0     : [0,  p1]
    p1 < 0     : [p0, inf)"""
    ...

llama_kv_cache_seq_cp(ctx, seq_id_src, seq_id_dst, p0, p1)

Copy all tokens that belong to the specified sequence to another sequence Note that this does not allocate extra KV cache memory - it simply assigns the tokens to the new sequence p0 < 0 : [0, p1] p1 < 0 : [p0, inf)

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_kv_cache_seq_cp",
    [
        llama_context_p_ctypes,
        llama_seq_id,
        llama_seq_id,
        llama_pos,
        llama_pos,
    ],
    None,
)
def llama_kv_cache_seq_cp(
    ctx: llama_context_p,
    seq_id_src: Union[llama_seq_id, int],
    seq_id_dst: Union[llama_seq_id, int],
    p0: Union[llama_pos, int],
    p1: Union[llama_pos, int],
    /,
):
    """Copy all tokens that belong to the specified sequence to another sequence
    Note that this does not allocate extra KV cache memory - it simply assigns the tokens to the new sequence
    p0 < 0 : [0,  p1]
    p1 < 0 : [p0, inf)"""
    ...

llama_kv_cache_seq_keep(ctx, seq_id)

Removes all tokens that do not belong to the specified sequence

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_kv_cache_seq_keep", [llama_context_p_ctypes, llama_seq_id], None
)
def llama_kv_cache_seq_keep(ctx: llama_context_p, seq_id: Union[llama_seq_id, int], /):
    """Removes all tokens that do not belong to the specified sequence"""
    ...

llama_kv_cache_seq_add(ctx, seq_id, p0, p1, delta)

Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1) If the KV cache is RoPEd, the KV data is updated accordingly: - lazily on next llama_decode() - explicitly with llama_kv_cache_update() p0 < 0 : [0, p1] p1 < 0 : [p0, inf)

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_kv_cache_seq_add",
    [
        llama_context_p_ctypes,
        llama_seq_id,
        llama_pos,
        llama_pos,
        llama_pos,
    ],
    None,
)
def llama_kv_cache_seq_add(
    ctx: llama_context_p,
    seq_id: Union[llama_seq_id, int],
    p0: Union[llama_pos, int],
    p1: Union[llama_pos, int],
    delta: Union[llama_pos, int],
    /,
):
    """Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1)
    If the KV cache is RoPEd, the KV data is updated accordingly:
    - lazily on next llama_decode()
    - explicitly with llama_kv_cache_update()
    p0 < 0 : [0,  p1]
    p1 < 0 : [p0, inf)"""
    ...

llama_kv_cache_seq_div(ctx, seq_id, p0, p1, d)

Integer division of the positions by factor of d > 1 If the KV cache is RoPEd, the KV data is updated accordingly p0 < 0 : [0, p1] p1 < 0 : [p0, inf)

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_kv_cache_seq_div",
    [
        llama_context_p_ctypes,
        llama_seq_id,
        llama_pos,
        llama_pos,
        ctypes.c_int,
    ],
    None,
)
def llama_kv_cache_seq_div(
    ctx: llama_context_p,
    seq_id: Union[llama_seq_id, int],
    p0: Union[llama_pos, int],
    p1: Union[llama_pos, int],
    d: Union[ctypes.c_int, int],
    /,
):
    """Integer division of the positions by factor of `d > 1`
    If the KV cache is RoPEd, the KV data is updated accordingly
    p0 < 0 : [0,  p1]
    p1 < 0 : [p0, inf)"""
    ...

llama_kv_cache_defrag(ctx)

Defragment the KV cache This will be applied: - lazily on next llama_decode() - explicitly with llama_kv_cache_update()

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_kv_cache_defrag", [llama_context_p_ctypes], None)
def llama_kv_cache_defrag(ctx: llama_context_p, /):
    """Defragment the KV cache
    This will be applied:
    - lazily on next llama_decode()
    - explicitly with llama_kv_cache_update()"""
    ...

llama_kv_cache_update(ctx)

Apply the KV cache updates (such as K-shifts, defragmentation, etc.)

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_kv_cache_update", [llama_context_p_ctypes], None)
def llama_kv_cache_update(ctx: llama_context_p, /):
    """Apply the KV cache updates (such as K-shifts, defragmentation, etc.)"""
    ...

llama_state_get_size(ctx)

Returns the maximum size in bytes of the state (rng, logits, embedding and kv_cache) - will often be smaller after compacting tokens

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_state_get_size", [llama_context_p_ctypes], ctypes.c_size_t)
def llama_state_get_size(ctx: llama_context_p, /) -> int:
    """Returns the maximum size in bytes of the state (rng, logits, embedding
    and kv_cache) - will often be smaller after compacting tokens"""
    ...

llama_get_state_size(ctx)

Returns the maximum size in bytes of the state (rng, logits, embedding and kv_cache) - will often be smaller after compacting tokens

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_get_state_size", [llama_context_p_ctypes], ctypes.c_size_t)
def llama_get_state_size(ctx: llama_context_p, /) -> int:
    """Returns the maximum size in bytes of the state (rng, logits, embedding
    and kv_cache) - will often be smaller after compacting tokens"""
    ...

llama_state_get_data(ctx, dst)

Copies the state to the specified destination address. Destination needs to have allocated enough memory. Returns the number of bytes copied

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_state_get_data",
    [
        llama_context_p_ctypes,
        ctypes.POINTER(ctypes.c_uint8),
    ],
    ctypes.c_size_t,
)
def llama_state_get_data(
    ctx: llama_context_p, dst: CtypesArray[ctypes.c_uint8], /
) -> int:
    """Copies the state to the specified destination address.
    Destination needs to have allocated enough memory.
    Returns the number of bytes copied"""
    ...

llama_copy_state_data(ctx, dst)

Copies the state to the specified destination address. Destination needs to have allocated enough memory. Returns the number of bytes copied

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_copy_state_data",
    [
        llama_context_p_ctypes,
        ctypes.POINTER(ctypes.c_uint8),
    ],
    ctypes.c_size_t,
)
def llama_copy_state_data(
    ctx: llama_context_p, dst: CtypesArray[ctypes.c_uint8], /
) -> int:
    """Copies the state to the specified destination address.
    Destination needs to have allocated enough memory.
    Returns the number of bytes copied"""
    ...

llama_state_set_data(ctx, src)

Set the state reading from the specified address Returns the number of bytes read

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_state_set_data",
    [llama_context_p_ctypes, ctypes.POINTER(ctypes.c_uint8)],
    ctypes.c_size_t,
)
def llama_state_set_data(
    ctx: llama_context_p, src: CtypesArray[ctypes.c_uint8], /
) -> int:
    """Set the state reading from the specified address
    Returns the number of bytes read"""
    ...

llama_set_state_data(ctx, src)

Set the state reading from the specified address

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_set_state_data",
    [llama_context_p_ctypes, ctypes.POINTER(ctypes.c_uint8)],
    ctypes.c_size_t,
)
def llama_set_state_data(
    ctx: llama_context_p, src: CtypesArray[ctypes.c_uint8], /
) -> int:
    """Set the state reading from the specified address"""
    ...

llama_state_load_file(ctx, path_session, tokens_out, n_token_capacity, n_token_count_out)

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_state_load_file",
    [
        llama_context_p_ctypes,
        ctypes.c_char_p,
        llama_token_p,
        ctypes.c_size_t,
        ctypes.POINTER(ctypes.c_size_t),
    ],
    ctypes.c_bool,
)
def llama_state_load_file(
    ctx: llama_context_p,
    path_session: bytes,
    tokens_out: CtypesArray[llama_token],
    n_token_capacity: Union[ctypes.c_size_t, int],
    n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t],
    /,
) -> bool: ...

llama_load_session_file(ctx, path_session, tokens_out, n_token_capacity, n_token_count_out)

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_load_session_file",
    [
        llama_context_p_ctypes,
        ctypes.c_char_p,
        llama_token_p,
        ctypes.c_size_t,
        ctypes.POINTER(ctypes.c_size_t),
    ],
    ctypes.c_size_t,
)
def llama_load_session_file(
    ctx: llama_context_p,
    path_session: bytes,
    tokens_out: CtypesArray[llama_token],
    n_token_capacity: Union[ctypes.c_size_t, int],
    n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t],
    /,
) -> int: ...

llama_state_save_file(ctx, path_session, tokens, n_token_count)

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_state_save_file",
    [
        llama_context_p_ctypes,
        ctypes.c_char_p,
        llama_token_p,
        ctypes.c_size_t,
    ],
    ctypes.c_bool,
)
def llama_state_save_file(
    ctx: llama_context_p,
    path_session: bytes,
    tokens: CtypesArray[llama_token],
    n_token_count: Union[ctypes.c_size_t, int],
    /,
) -> bool: ...

llama_save_session_file(ctx, path_session, tokens, n_token_count)

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_save_session_file",
    [
        llama_context_p_ctypes,
        ctypes.c_char_p,
        llama_token_p,
        ctypes.c_size_t,
    ],
    ctypes.c_size_t,
)
def llama_save_session_file(
    ctx: llama_context_p,
    path_session: bytes,
    tokens: CtypesArray[llama_token],
    n_token_count: Union[ctypes.c_size_t, int],
    /,
) -> int: ...

llama_state_seq_get_size(ctx, seq_id)

Get the exact size needed to copy the KV cache of a single sequence

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_state_seq_get_size",
    [llama_context_p_ctypes, llama_seq_id],
    ctypes.c_size_t,
)
def llama_state_seq_get_size(ctx: llama_context_p, seq_id: llama_seq_id, /) -> int:
    """Get the exact size needed to copy the KV cache of a single sequence"""
    ...

llama_state_seq_get_data(ctx, dst, seq_id)

Copy the KV cache of a single sequence into the specified buffer

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_state_seq_get_data",
    [llama_context_p_ctypes, ctypes.POINTER(ctypes.c_uint8), llama_seq_id],
    ctypes.c_size_t,
)
def llama_state_seq_get_data(
    ctx: llama_context_p, dst: CtypesArray[ctypes.c_uint8], seq_id: llama_seq_id, /
) -> int:
    """Copy the KV cache of a single sequence into the specified buffer"""
    ...

llama_state_seq_set_data(ctx, src, dest_seq_id)

Copy the sequence data (originally copied with llama_state_seq_get_data) into the specified sequence

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_state_seq_set_data",
    [llama_context_p_ctypes, ctypes.POINTER(ctypes.c_uint8), llama_seq_id],
    ctypes.c_size_t,
)
def llama_state_seq_set_data(
    ctx: llama_context_p, src: CtypesArray[ctypes.c_uint8], dest_seq_id: llama_seq_id, /
) -> int:
    """Copy the sequence data (originally copied with `llama_state_seq_get_data`) into the specified sequence"""
    ...

llama_state_seq_save_file(ctx, filepath, seq_id, tokens, n_token_count)

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_state_seq_save_file",
    [
        llama_context_p_ctypes,
        ctypes.c_char_p,
        llama_seq_id,
        llama_token_p,
        ctypes.c_size_t,
    ],
    ctypes.c_size_t,
)
def llama_state_seq_save_file(
    ctx: llama_context_p,
    filepath: bytes,
    seq_id: llama_seq_id,
    tokens: CtypesArray[llama_token],
    n_token_count: Union[ctypes.c_size_t, int],
    /,
) -> int: ...

llama_state_seq_load_file(ctx, filepath, dest_seq_id, tokens_out, n_token_capacity, n_token_count_out)

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_state_seq_load_file",
    [
        llama_context_p_ctypes,
        ctypes.c_char_p,
        llama_seq_id,
        llama_token_p,
        ctypes.c_size_t,
        ctypes.POINTER(ctypes.c_size_t),
    ],
    ctypes.c_size_t,
)
def llama_state_seq_load_file(
    ctx: llama_context_p,
    filepath: bytes,
    dest_seq_id: llama_seq_id,
    tokens_out: CtypesArray[llama_token],
    n_token_capacity: Union[ctypes.c_size_t, int],
    n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t],
    /,
) -> int: ...

llama_batch_get_one(tokens, n_tokens, pos_0, seq_id)

Return batch for single sequence of tokens starting at pos_0

NOTE: this is a helper function to facilitate transition to the new batch API - avoid using it

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_batch_get_one",
    [
        llama_token_p,
        ctypes.c_int,
        llama_pos,
        llama_seq_id,
    ],
    llama_batch,
)
def llama_batch_get_one(
    tokens: CtypesArray[llama_token],
    n_tokens: Union[ctypes.c_int, int],
    pos_0: Union[llama_pos, int],
    seq_id: llama_seq_id,
    /,
) -> llama_batch:
    """Return batch for single sequence of tokens starting at pos_0

    NOTE: this is a helper function to facilitate transition to the new batch API - avoid using it
    """
    ...

llama_batch_init(n_tokens, embd, n_seq_max)

Allocates a batch of tokens on the heap that can hold a maximum of n_tokens Each token can be assigned up to n_seq_max sequence ids The batch has to be freed with llama_batch_free() If embd != 0, llama_batch.embd will be allocated with size of n_tokens * embd * sizeof(float) Otherwise, llama_batch.token will be allocated to store n_tokens llama_token The rest of the llama_batch members are allocated with size n_tokens All members are left uninitialized

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_batch_init", [ctypes.c_int32, ctypes.c_int32, ctypes.c_int32], llama_batch
)
def llama_batch_init(
    n_tokens: Union[ctypes.c_int32, int],
    embd: Union[ctypes.c_int32, int],
    n_seq_max: Union[ctypes.c_int32, int],
    /,
) -> llama_batch:
    """Allocates a batch of tokens on the heap that can hold a maximum of n_tokens
    Each token can be assigned up to n_seq_max sequence ids
    The batch has to be freed with llama_batch_free()
    If embd != 0, llama_batch.embd will be allocated with size of n_tokens * embd * sizeof(float)
    Otherwise, llama_batch.token will be allocated to store n_tokens llama_token
    The rest of the llama_batch members are allocated with size n_tokens
    All members are left uninitialized"""
    ...

llama_batch_free(batch)

Frees a batch of tokens allocated with llama_batch_init()

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_batch_free", [llama_batch], None)
def llama_batch_free(batch: llama_batch, /):
    """Frees a batch of tokens allocated with llama_batch_init()"""
    ...

llama_decode(ctx, batch)

Positive return values does not mean a fatal error, but rather a warning. 0 - success 1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context) < 0 - error

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_decode", [llama_context_p_ctypes, llama_batch], ctypes.c_int32)
def llama_decode(ctx: llama_context_p, batch: llama_batch, /) -> int:
    """Positive return values does not mean a fatal error, but rather a warning.
    0 - success
    1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context)
    < 0 - error"""
    ...

llama_set_n_threads(ctx, n_threads, n_threads_batch)

Set the number of threads used for decoding n_threads is the number of threads used for generation (single token) n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens)

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_set_n_threads",
    [
        llama_context_p_ctypes,
        ctypes.c_uint32,
        ctypes.c_uint32,
    ],
    None,
)
def llama_set_n_threads(
    ctx: llama_context_p,
    n_threads: Union[ctypes.c_uint32, int],
    n_threads_batch: Union[ctypes.c_uint32, int],
    /,
):
    """Set the number of threads used for decoding
    n_threads is the number of threads used for generation (single token)
    n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens)
    """
    ...

llama_set_causal_attn(ctx, causal_attn)

Set whether to use causal attention or not If set to true, the model will only attend to the past tokens

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_set_causal_attn", [llama_context_p_ctypes, ctypes.c_bool], None)
def llama_set_causal_attn(ctx: llama_context_p, causal_attn: bool, /):
    """Set whether to use causal attention or not
    If set to true, the model will only attend to the past tokens"""
    ...

llama_set_abort_callback(ctx, abort_callback, abort_callback_data)

Set abort callback

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_set_abort_callback",
    [llama_context_p_ctypes, ggml_abort_callback, ctypes.c_void_p],
    None,
)
def llama_set_abort_callback(
    ctx: llama_context_p,
    abort_callback: Callable[[ctypes.c_void_p], None],
    abort_callback_data: ctypes.c_void_p,
    /,
):
    """Set abort callback"""
    ...

llama_synchronize(ctx)

Wait until all computations are finished This is automatically done when using one of the functions below to obtain the computation results and is not necessary to call it explicitly in most cases

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_synchronize", [llama_context_p_ctypes], None)
def llama_synchronize(ctx: llama_context_p, /):
    """Wait until all computations are finished
    This is automatically done when using one of the functions below to obtain the computation results
    and is not necessary to call it explicitly in most cases"""
    ...

llama_get_logits(ctx)

Token logits obtained from the last call to llama_eval() The logits for the last token are stored in the last row Logits for which llama_batch.logits[i] == 0 are undefined Rows: n_tokens provided with llama_batch Cols: n_vocab

Returns:

• CtypesArray[c_float] –

  Pointer to the logits buffer of shape (n_tokens, n_vocab)

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_get_logits", [llama_context_p_ctypes], ctypes.POINTER(ctypes.c_float)
)
def llama_get_logits(ctx: llama_context_p, /) -> CtypesArray[ctypes.c_float]:
    """Token logits obtained from the last call to llama_eval()
    The logits for the last token are stored in the last row
    Logits for which llama_batch.logits[i] == 0 are undefined
    Rows: n_tokens provided with llama_batch
    Cols: n_vocab

    Returns:
        Pointer to the logits buffer of shape (n_tokens, n_vocab)"""
    ...

llama_get_logits_ith(ctx, i)

Logits for the ith token. Equivalent to: llama_get_logits(ctx) + i*n_vocab

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_get_logits_ith",
    [llama_context_p_ctypes, ctypes.c_int32],
    ctypes.POINTER(ctypes.c_float),
)
def llama_get_logits_ith(
    ctx: llama_context_p, i: Union[ctypes.c_int32, int], /
) -> CtypesArray[ctypes.c_float]:
    """Logits for the ith token. Equivalent to:
    llama_get_logits(ctx) + i*n_vocab"""
    ...

llama_get_embeddings(ctx)

Get the embeddings for the input shape: [n_embd] (1-dimensional)

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_get_embeddings", [llama_context_p_ctypes], ctypes.POINTER(ctypes.c_float)
)
def llama_get_embeddings(ctx: llama_context_p, /) -> CtypesArray[ctypes.c_float]:
    """Get the embeddings for the input
    shape: [n_embd] (1-dimensional)"""
    ...

llama_get_embeddings_ith(ctx, i)

Get the embeddings for the ith sequence llama_get_embeddings(ctx) + i*n_embd

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_get_embeddings_ith",
    [llama_context_p_ctypes, ctypes.c_int32],
    ctypes.POINTER(ctypes.c_float),
)
def llama_get_embeddings_ith(
    ctx: llama_context_p, i: Union[ctypes.c_int32, int], /
) -> CtypesArray[ctypes.c_float]:
    """Get the embeddings for the ith sequence
    llama_get_embeddings(ctx) + i*n_embd"""
    ...

llama_get_embeddings_seq(ctx, seq_id)

Get the embeddings for a sequence id Returns NULL if pooling_type is LLAMA_POOLING_TYPE_NONE shape: [n_embd] (1-dimensional)

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_get_embeddings_seq",
    [llama_context_p_ctypes, llama_seq_id],
    ctypes.POINTER(ctypes.c_float),
)
def llama_get_embeddings_seq(
    ctx: llama_context_p, seq_id: Union[llama_seq_id, int], /
) -> CtypesArray[ctypes.c_float]:
    """Get the embeddings for a sequence id
    Returns NULL if pooling_type is LLAMA_POOLING_TYPE_NONE
    shape: [n_embd] (1-dimensional)"""
    ...

llama_token_get_text(model, token)

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_token_get_text", [llama_model_p_ctypes, llama_token], ctypes.c_char_p
)
def llama_token_get_text(
    model: llama_model_p, token: Union[llama_token, int], /
) -> bytes: ...

llama_token_get_score(model, token)

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_token_get_score", [llama_model_p_ctypes, llama_token], ctypes.c_float
)
def llama_token_get_score(
    model: llama_model_p, token: Union[llama_token, int], /
) -> float: ...

llama_token_get_type(model, token)

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_token_get_type", [llama_model_p_ctypes, llama_token], ctypes.c_int
)
def llama_token_get_type(
    model: llama_model_p, token: Union[llama_token, int], /
) -> int: ...

llama_token_is_eog(model, token)

Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.)

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_token_is_eog", [llama_model_p_ctypes, llama_token], ctypes.c_bool
)
def llama_token_is_eog(model: llama_model_p, token: Union[llama_token, int], /) -> bool:
    """Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.)"""
    ...

llama_token_bos(model)

beginning-of-sentence

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_token_bos", [llama_model_p_ctypes], llama_token)
def llama_token_bos(model: llama_model_p, /) -> int:
    """beginning-of-sentence"""
    ...

llama_token_eos(model)

end-of-sentence

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_token_eos", [llama_model_p_ctypes], llama_token)
def llama_token_eos(model: llama_model_p, /) -> int:
    """end-of-sentence"""
    ...

llama_token_cls(model)

classification

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_token_cls", [llama_model_p_ctypes], llama_token)
def llama_token_cls(model: llama_model_p, /) -> int:
    """classification"""
    ...

llama_token_sep(model)

sentence separator

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_token_sep", [llama_model_p_ctypes], llama_token)
def llama_token_sep(model: llama_model_p, /) -> int:
    """sentence separator"""
    ...

llama_token_nl(model)

next-line

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_token_nl", [llama_model_p_ctypes], llama_token)
def llama_token_nl(model: llama_model_p, /) -> int:
    """next-line"""
    ...

llama_add_bos_token(model)

Returns -1 if unknown, 1 for true or 0 for false.

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_add_bos_token", [llama_model_p_ctypes], ctypes.c_int32)
def llama_add_bos_token(model: llama_model_p, /) -> int:
    """Returns -1 if unknown, 1 for true or 0 for false."""
    ...

llama_add_eos_token(model)

Returns -1 if unknown, 1 for true or 0 for false.

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_add_eos_token", [llama_model_p_ctypes], ctypes.c_int32)
def llama_add_eos_token(model: llama_model_p, /) -> int:
    """Returns -1 if unknown, 1 for true or 0 for false."""
    ...

llama_token_prefix(model)

codellama infill tokens

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_token_prefix", [llama_model_p_ctypes], llama_token)
def llama_token_prefix(model: llama_model_p) -> int:
    """codellama infill tokens"""
    ...

llama_token_middle(model)

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_token_middle", [llama_model_p_ctypes], llama_token)
def llama_token_middle(model: llama_model_p, /) -> int: ...

llama_token_suffix(model)

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_token_suffix", [llama_model_p_ctypes], llama_token)
def llama_token_suffix(model: llama_model_p, /) -> int: ...

llama_token_eot(model)

Source code in llama_cpp/llama_cpp.py

@ctypes_function("llama_token_eot", [llama_model_p_ctypes], llama_token)
def llama_token_eot(model: llama_model_p, /) -> int: ...

llama_tokenize(model, text, text_len, tokens, n_tokens_max, add_special, parse_special)

Convert the provided text into tokens.

Parameters:

• model (llama_model_p) –

  The model to use for tokenization.

• text (bytes) –

  The text to tokenize.

• text_len (Union[c_int, int]) –

  The length of the text.

• tokens (CtypesArray[llama_token]) –

  The tokens pointer must be large enough to hold the resulting tokens.

• n_max_tokens –

  The maximum number of tokens to return.

• add_special (Union[c_bool, bool]) –

  Allow tokenizing special and/or control tokens which otherwise are not exposed and treated as plaintext. Does not insert a leading space.

• parse_special (Union[c_bool, bool]) –

  Allow parsing special tokens.

Returns:

• int –

  Returns the number of tokens on success, no more than n_tokens_max

• int –

  Returns a negative number on failure - the number of tokens that would have been returned

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_tokenize",
    [
        llama_model_p_ctypes,
        ctypes.c_char_p,
        ctypes.c_int32,
        llama_token_p,
        ctypes.c_int32,
        ctypes.c_bool,
        ctypes.c_bool,
    ],
    ctypes.c_int32,
)
def llama_tokenize(
    model: llama_model_p,
    text: bytes,
    text_len: Union[ctypes.c_int, int],
    tokens: CtypesArray[llama_token],
    n_tokens_max: Union[ctypes.c_int, int],
    add_special: Union[ctypes.c_bool, bool],
    parse_special: Union[ctypes.c_bool, bool],
    /,
) -> int:
    """Convert the provided text into tokens.

    Args:
        model: The model to use for tokenization.
        text: The text to tokenize.
        text_len: The length of the text.
        tokens: The tokens pointer must be large enough to hold the resulting tokens.
        n_max_tokens: The maximum number of tokens to return.
        add_special: Allow tokenizing special and/or control tokens which otherwise are not exposed and treated as plaintext. Does not insert a leading space.
        parse_special: Allow parsing special tokens.

    Returns:
        Returns the number of tokens on success, no more than n_tokens_max
        Returns a negative number on failure - the number of tokens that would have been returned
    """
    ...

llama_token_to_piece(model, token, buf, length, special)

Token Id -> Piece. Uses the vocabulary in the provided context. Does not write null terminator to the buffer. User code is responsible to remove the leading whitespace of the first non-BOS token when decoding multiple tokens.

Parameters:

• model (llama_model_p) –

  The model to use for tokenization.

• token (Union[llama_token, int]) –

  The token to convert.

• buf (Union[c_char_p, bytes, CtypesArray[c_char]]) –

  The buffer to write the token to.

• length (Union[c_int, int]) –

  The length of the buffer.

• special (Union[c_bool, bool]) –

  If true, special tokens are rendered in the output.

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_token_to_piece",
    [
        llama_model_p_ctypes,
        llama_token,
        ctypes.c_char_p,
        ctypes.c_int32,
        ctypes.c_bool,
    ],
    ctypes.c_int32,
)
def llama_token_to_piece(
    model: llama_model_p,
    token: Union[llama_token, int],
    buf: Union[ctypes.c_char_p, bytes, CtypesArray[ctypes.c_char]],
    length: Union[ctypes.c_int, int],
    special: Union[ctypes.c_bool, bool],
    /,
) -> int:
    """Token Id -> Piece.
    Uses the vocabulary in the provided context.
    Does not write null terminator to the buffer.
    User code is responsible to remove the leading whitespace of the first non-BOS token when decoding multiple tokens.

    Args:
        model: The model to use for tokenization.
        token: The token to convert.
        buf: The buffer to write the token to.
        length: The length of the buffer.
        special: If true, special tokens are rendered in the output."""
    ...

llama_chat_apply_template(model, tmpl, chat, n_msg)

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_chat_apply_template",
    [
        ctypes.c_void_p,
        ctypes.c_char_p,
        ctypes.POINTER(llama_chat_message),
        ctypes.c_size_t,
    ],
    ctypes.c_int32,
)
def llama_chat_apply_template(
    model: llama_model_p,
    tmpl: bytes,
    chat: CtypesArray[llama_chat_message],
    n_msg: int,
    /,
) -> int: ...

llama_grammar_init(rules, n_rules, start_rule_index)

Initialize a grammar from a set of rules.

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_grammar_init",
    [
        ctypes.POINTER(llama_grammar_element_p),
        ctypes.c_size_t,
        ctypes.c_size_t,
    ],
    llama_grammar_p,
)
def llama_grammar_init(
    rules: CtypesArray[
        CtypesPointer[llama_grammar_element]
    ],  # NOTE: This might be wrong type sig
    n_rules: Union[ctypes.c_size_t, int],
    start_rule_index: Union[ctypes.c_size_t, int],
    /,
) -> llama_grammar_p:
    """Initialize a grammar from a set of rules."""
    ...

llama_grammar_free(grammar)

Free a grammar.

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_grammar_free",
    [llama_grammar_p],
    None,
)
def llama_grammar_free(grammar: llama_grammar_p, /):
    """Free a grammar."""
    ...

llama_grammar_copy(grammar)

Copy a grammar.

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_grammar_copy",
    [llama_grammar_p],
    llama_grammar_p,
)
def llama_grammar_copy(grammar: llama_grammar_p, /) -> llama_grammar_p:
    """Copy a grammar."""
    ...

llama_set_rng_seed(ctx, seed)

Sets the current rng seed.

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_set_rng_seed",
    [llama_context_p_ctypes, ctypes.c_uint32],
    None,
)
def llama_set_rng_seed(ctx: llama_context_p, seed: Union[ctypes.c_uint32, int], /):
    """Sets the current rng seed."""
    ...

llama_sample_repetition_penalties(ctx, candidates, last_tokens_data, penalty_last_n, penalty_repeat, penalty_freq, penalty_present)

Repetition penalty described in CTRL academic paper https://arxiv.org/abs/1909.05858, with negative logit fix. Frequency and presence penalties described in OpenAI API https://platform.openai.com/docs/api-reference/parameter-details.

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_sample_repetition_penalties",
    [
        llama_context_p_ctypes,
        llama_token_data_array_p,
        llama_token_p,
        ctypes.c_size_t,
        ctypes.c_float,
        ctypes.c_float,
        ctypes.c_float,
    ],
    None,
)
def llama_sample_repetition_penalties(
    ctx: llama_context_p,
    candidates: Union[
        CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array]
    ],
    last_tokens_data: CtypesArray[llama_token],
    penalty_last_n: Union[ctypes.c_size_t, int],
    penalty_repeat: Union[ctypes.c_float, float],
    penalty_freq: Union[ctypes.c_float, float],
    penalty_present: Union[ctypes.c_float, float],
    /,
):
    """Repetition penalty described in CTRL academic paper https://arxiv.org/abs/1909.05858, with negative logit fix.
    Frequency and presence penalties described in OpenAI API https://platform.openai.com/docs/api-reference/parameter-details.
    """
    ...

llama_sample_apply_guidance(ctx, logits, logits_guidance, scale)

Apply classifier-free guidance to the logits as described in academic paper "Stay on topic with Classifier-Free Guidance" https://arxiv.org/abs/2306.17806

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_sample_apply_guidance",
    [
        llama_context_p_ctypes,
        ctypes.POINTER(ctypes.c_float),
        ctypes.POINTER(ctypes.c_float),
        ctypes.c_float,
    ],
    None,
)
def llama_sample_apply_guidance(
    ctx: llama_context_p,
    logits: CtypesArray[ctypes.c_float],
    logits_guidance: CtypesArray[ctypes.c_float],
    scale: Union[ctypes.c_float, float],
    /,
):
    """Apply classifier-free guidance to the logits as described in academic paper "Stay on topic with Classifier-Free Guidance" https://arxiv.org/abs/2306.17806"""
    ...

llama_sample_softmax(ctx, candidates)

Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits.

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_sample_softmax",
    [llama_context_p_ctypes, llama_token_data_array_p],
    None,
)
def llama_sample_softmax(
    ctx: llama_context_p,
    candidates: Union[
        CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array]
    ],
    /,
):
    """Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits."""
    ...

llama_sample_top_k(ctx, candidates, k, min_keep)

Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_sample_top_k",
    [llama_context_p_ctypes, llama_token_data_array_p, ctypes.c_int32, ctypes.c_size_t],
    None,
)
def llama_sample_top_k(
    ctx: llama_context_p,
    candidates: Union[
        CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array]
    ],
    k: Union[ctypes.c_int, int],
    min_keep: Union[ctypes.c_size_t, int],
    /,
):
    """Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751"""
    ...

llama_sample_top_p(ctx, candidates, p, min_keep)

Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_sample_top_p",
    [llama_context_p_ctypes, llama_token_data_array_p, ctypes.c_float, ctypes.c_size_t],
    None,
)
def llama_sample_top_p(
    ctx: llama_context_p,
    candidates: Union[
        CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array]
    ],
    p: Union[ctypes.c_float, float],
    min_keep: Union[ctypes.c_size_t, int],
    /,
):
    """Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751"""
    ...

llama_sample_min_p(ctx, candidates, p, min_keep)

Minimum P sampling as described in https://github.com/ggerganov/llama.cpp/pull/3841

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_sample_min_p",
    [llama_context_p_ctypes, llama_token_data_array_p, ctypes.c_float, ctypes.c_size_t],
    None,
)
def llama_sample_min_p(
    ctx: llama_context_p,
    candidates: Union[
        CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array]
    ],
    p: Union[ctypes.c_float, float],
    min_keep: Union[ctypes.c_size_t, int],
    /,
):
    """Minimum P sampling as described in https://github.com/ggerganov/llama.cpp/pull/3841"""
    ...

llama_sample_tail_free(ctx, candidates, z, min_keep)

Tail Free Sampling described in https://www.trentonbricken.com/Tail-Free-Sampling/.

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_sample_tail_free",
    [llama_context_p_ctypes, llama_token_data_array_p, ctypes.c_float, ctypes.c_size_t],
    None,
)
def llama_sample_tail_free(
    ctx: llama_context_p,
    candidates: Union[
        CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array]
    ],
    z: Union[ctypes.c_float, float],
    min_keep: Union[ctypes.c_size_t, int],
    /,
):
    """Tail Free Sampling described in https://www.trentonbricken.com/Tail-Free-Sampling/."""
    ...

llama_sample_typical(ctx, candidates, p, min_keep)

Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666.

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_sample_typical",
    [llama_context_p_ctypes, llama_token_data_array_p, ctypes.c_float, ctypes.c_size_t],
    None,
)
def llama_sample_typical(
    ctx: llama_context_p,
    candidates: Union[
        CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array]
    ],
    p: Union[ctypes.c_float, float],
    min_keep: Union[ctypes.c_size_t, int],
    /,
):
    """Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666."""
    ...

llama_sample_entropy(ctx, candidates, min_temp, max_temp, exponent_val)

Dynamic temperature implementation described in the paper https://arxiv.org/abs/2309.02772.

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_sample_entropy",
    [
        llama_context_p_ctypes,
        llama_token_data_array_p,
        ctypes.c_float,
        ctypes.c_float,
        ctypes.c_float,
    ],
    None,
)
def llama_sample_entropy(
    ctx: llama_context_p,
    candidates: Union[
        CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array]
    ],
    min_temp: Union[ctypes.c_float, float],
    max_temp: Union[ctypes.c_float, float],
    exponent_val: Union[ctypes.c_float, float],
    /,
):
    """Dynamic temperature implementation described in the paper https://arxiv.org/abs/2309.02772."""
    ...

llama_sample_temp(ctx, candidates, temp)

Temperature sampling described in academic paper "Generating Long Sequences with Sparse Transformers" https://arxiv.org/abs/1904.10509

Parameters:

• candidates (Union[CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array]]) –

  A vector of llama_token_data containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text.

• temp (Union[c_float, float]) –

  The temperature value to use for the sampling. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text.

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_sample_temp",
    [llama_context_p_ctypes, llama_token_data_array_p, ctypes.c_float],
    None,
)
def llama_sample_temp(
    ctx: llama_context_p,
    candidates: Union[
        CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array]
    ],
    temp: Union[ctypes.c_float, float],
    /,
):
    """Temperature sampling described in academic paper "Generating Long Sequences with Sparse Transformers" https://arxiv.org/abs/1904.10509

    Parameters:
        candidates: A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text.
        temp: The temperature value to use for the sampling. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text.
    """
    ...

llama_sample_grammar(ctx, candidates, grammar)

Apply constraints from grammar

Parameters:

• candidates (Union[CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array]]) –

  A vector of llama_token_data containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text.

• grammar –

  A grammar object containing the rules and constraints to apply to the generated text.

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_sample_grammar",
    [llama_context_p_ctypes, llama_token_data_array_p, llama_grammar_p],
    None,
)
def llama_sample_grammar(
    ctx: llama_context_p,
    candidates: Union[
        CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array]
    ],
    grammar,  # type: llama_grammar_p
    /,
):
    """Apply constraints from grammar

    Parameters:
        candidates: A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text.
        grammar: A grammar object containing the rules and constraints to apply to the generated text.
    """
    ...

llama_sample_token_mirostat(ctx, candidates, tau, eta, m, mu)

Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words.

Parameters:

• candidates (Union[CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array]]) –

  A vector of llama_token_data containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text.

• tau (Union[c_float, float]) –

  The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text.

• eta (Union[c_float, float]) –

  The learning rate used to update mu based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause mu to be updated more quickly, while a smaller learning rate will result in slower updates.

• m (Union[c_int, int]) –

  The number of tokens considered in the estimation of s_hat. This is an arbitrary value that is used to calculate s_hat, which in turn helps to calculate the value of k. In the paper, they use m = 100, but you can experiment with different values to see how it affects the performance of the algorithm.

• mu (CtypesPointerOrRef[c_float]) –

  Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (2 * tau) and is updated in the algorithm based on the error between the target and observed surprisal.

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_sample_token_mirostat",
    [
        llama_context_p_ctypes,
        llama_token_data_array_p,
        ctypes.c_float,
        ctypes.c_float,
        ctypes.c_int32,
        ctypes.POINTER(ctypes.c_float),
    ],
    llama_token,
)
def llama_sample_token_mirostat(
    ctx: llama_context_p,
    candidates: Union[
        CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array]
    ],
    tau: Union[ctypes.c_float, float],
    eta: Union[ctypes.c_float, float],
    m: Union[ctypes.c_int, int],
    mu: CtypesPointerOrRef[ctypes.c_float],
    /,
) -> int:
    """Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words.

    Parameters:
        candidates: A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text.
        tau: The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text.
        eta: The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates.
        m: The number of tokens considered in the estimation of `s_hat`. This is an arbitrary value that is used to calculate `s_hat`, which in turn helps to calculate the value of `k`. In the paper, they use `m = 100`, but you can experiment with different values to see how it affects the performance of the algorithm.
        mu: Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal.
    """
    ...

llama_sample_token_mirostat_v2(ctx, candidates, tau, eta, mu)

Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words.

Parameters:

• candidates (Union[CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array]]) –

  A vector of llama_token_data containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text.

• tau (Union[c_float, float]) –

  The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text.

• eta (Union[c_float, float]) –

  The learning rate used to update mu based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause mu to be updated more quickly, while a smaller learning rate will result in slower updates.

• mu (CtypesPointerOrRef[c_float]) –

  Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (2 * tau) and is updated in the algorithm based on the error between the target and observed surprisal.

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_sample_token_mirostat_v2",
    [
        llama_context_p_ctypes,
        llama_token_data_array_p,
        ctypes.c_float,
        ctypes.c_float,
        ctypes.POINTER(ctypes.c_float),
    ],
    llama_token,
)
def llama_sample_token_mirostat_v2(
    ctx: llama_context_p,
    candidates: Union[
        CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array]
    ],
    tau: Union[ctypes.c_float, float],
    eta: Union[ctypes.c_float, float],
    mu: CtypesPointerOrRef[ctypes.c_float],
    /,
) -> int:
    """Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words.

    Parameters:
        candidates: A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text.
        tau: The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text.
        eta: The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates.
        mu: Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal.
    """
    ...

llama_sample_token_greedy(ctx, candidates)

Selects the token with the highest probability.

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_sample_token_greedy",
    [llama_context_p_ctypes, llama_token_data_array_p],
    llama_token,
)
def llama_sample_token_greedy(
    ctx: llama_context_p,
    candidates: Union[
        CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array]
    ],
    /,
) -> int:
    """Selects the token with the highest probability."""
    ...

llama_sample_token(ctx, candidates)

Randomly selects a token from the candidates based on their probabilities.

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_sample_token",
    [llama_context_p_ctypes, llama_token_data_array_p],
    llama_token,
)
def llama_sample_token(
    ctx: llama_context_p,
    candidates: Union[
        CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array]
    ],
    /,
) -> int:
    """Randomly selects a token from the candidates based on their probabilities."""
    ...

llama_grammar_accept_token(ctx, grammar, token)

Accepts the sampled token into the grammar

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_grammar_accept_token",
    [llama_context_p_ctypes, llama_grammar_p, llama_token],
    None,
)
def llama_grammar_accept_token(
    ctx: llama_context_p, grammar: llama_grammar_p, token: Union[llama_token, int], /
) -> None:
    """Accepts the sampled token into the grammar"""
    ...

llama_beam_view

Bases: Structure

Source code in llama_cpp/llama_cpp.py

class llama_beam_view(ctypes.Structure):
    if TYPE_CHECKING:
        tokens: CtypesArray[llama_token]
        n_tokens: int
        p: float
        eob: bool

    _fields_ = [
        ("tokens", llama_token_p),
        ("n_tokens", ctypes.c_size_t),
        ("p", ctypes.c_float),
        ("eob", ctypes.c_bool),
    ]

llama_beams_state

Bases: Structure

Source code in llama_cpp/llama_cpp.py

class llama_beams_state(ctypes.Structure):
    if TYPE_CHECKING:
        beam_views: CtypesArray[llama_beam_view]
        n_beams: int
        common_prefix_length: int
        last_call: bool

    _fields_ = [
        ("beam_views", ctypes.POINTER(llama_beam_view)),
        ("n_beams", ctypes.c_size_t),
        ("common_prefix_length", ctypes.c_size_t),
        ("last_call", ctypes.c_bool),
    ]

llama_beam_search_callback_fn_t = ctypes.CFUNCTYPE(None, ctypes.c_void_p, llama_beams_state) module-attribute

llama_beam_search(ctx, callback, callback_data, n_beams, n_past, n_predict)

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_beam_search",
    [
        llama_context_p_ctypes,
        llama_beam_search_callback_fn_t,
        ctypes.c_void_p,
        ctypes.c_size_t,
        ctypes.c_int32,
        ctypes.c_int32,
    ],
    None,
)
def llama_beam_search(
    ctx: llama_context_p,
    callback: CtypesFuncPointer,
    callback_data: ctypes.c_void_p,
    n_beams: Union[ctypes.c_size_t, int],
    n_past: Union[ctypes.c_int, int],
    n_predict: Union[ctypes.c_int, int],
    /,
): ...

llama_split_path(split_path, maxlen, path_prefix, split_no, split_count)

Build a split GGUF final path for this chunk.

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_split_path",
    [ctypes.c_char_p, ctypes.c_size_t, ctypes.c_char_p, ctypes.c_int, ctypes.c_int],
    ctypes.c_int,
)
def llama_split_path(
    split_path: bytes,
    maxlen: Union[ctypes.c_size_t, int],
    path_prefix: bytes,
    split_no: Union[ctypes.c_int, int],
    split_count: Union[ctypes.c_int, int],
    /,
) -> int:
    """Build a split GGUF final path for this chunk."""
    ...

llama_split_prefix(split_prefix, maxlen, split_path, split_no, split_count)

Extract the path prefix from the split_path if and only if the split_no and split_count match.

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_split_prefix",
    [ctypes.c_char_p, ctypes.c_size_t, ctypes.c_char_p, ctypes.c_int, ctypes.c_int],
    ctypes.c_int,
)
def llama_split_prefix(
    split_prefix: bytes,
    maxlen: Union[ctypes.c_size_t, int],
    split_path: bytes,
    split_no: Union[ctypes.c_int, int],
    split_count: Union[ctypes.c_int, int],
    /,
) -> int:
    """Extract the path prefix from the split_path if and only if the split_no and split_count match."""
    ...

llama_get_timings(ctx)

Get performance information

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_get_timings",
    [llama_context_p_ctypes],
    llama_timings,
)
def llama_get_timings(ctx: llama_context_p, /) -> llama_timings:
    """Get performance information"""
    ...

llama_print_timings(ctx)

Print performance information

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_print_timings",
    [llama_context_p_ctypes],
    None,
)
def llama_print_timings(ctx: llama_context_p, /):
    """Print performance information"""
    ...

llama_reset_timings(ctx)

Reset performance information

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_reset_timings",
    [llama_context_p_ctypes],
    None,
)
def llama_reset_timings(ctx: llama_context_p, /):
    """Reset performance information"""
    ...

llama_print_system_info()

Print system information

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_print_system_info",
    [],
    ctypes.c_char_p,
)
def llama_print_system_info() -> bytes:
    """Print system information"""
    ...

llama_log_set(log_callback, user_data)

Set callback for all future logging events.

If this is not called, or NULL is supplied, everything is output on stderr.

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_log_set",
    [ctypes.c_void_p, ctypes.c_void_p],
    None,
)
def llama_log_set(
    log_callback: Optional[CtypesFuncPointer],
    user_data: ctypes.c_void_p,
    /,
):
    """Set callback for all future logging events.

    If this is not called, or NULL is supplied, everything is output on stderr."""
    ...

llama_dump_timing_info_yaml(stream, ctx)

Source code in llama_cpp/llama_cpp.py

@ctypes_function(
    "llama_dump_timing_info_yaml",
    [ctypes.c_void_p, llama_context_p_ctypes],
    None,
)
def llama_dump_timing_info_yaml(stream: ctypes.c_void_p, ctx: llama_context_p, /): ...

LLAMA_MAX_DEVICES = _lib.llama_max_devices() module-attribute

LLAMA_DEFAULT_SEED = 4294967295 module-attribute

LLAMA_MAX_RNG_STATE = 64 * 1024 module-attribute

LLAMA_FILE_MAGIC_GGLA = 1734831201 module-attribute

LLAMA_FILE_MAGIC_GGSN = 1734833006 module-attribute

LLAMA_FILE_MAGIC_GGSQ = 1734833009 module-attribute

LLAMA_SESSION_MAGIC = LLAMA_FILE_MAGIC_GGSN module-attribute

LLAMA_SESSION_VERSION = 5 module-attribute

LLAMA_STATE_SEQ_MAGIC = LLAMA_FILE_MAGIC_GGSQ module-attribute

LLAMA_STATE_SEQ_VERSION = 1 module-attribute

LLAMA_VOCAB_TYPE_NONE = 0 module-attribute

For models without vocab

LLAMA_VOCAB_TYPE_SPM = 1 module-attribute

LLaMA tokenizer based on byte-level BPE with byte fallback

LLAMA_VOCAB_TYPE_BPE = 2 module-attribute

GPT-2 tokenizer based on byte-level BPE

LLAMA_VOCAB_TYPE_WPM = 3 module-attribute

BERT tokenizer based on WordPiece

LLAMA_ROPE_TYPE_NONE = -1 module-attribute

LLAMA_ROPE_TYPE_NORM = 0 module-attribute

LLAMA_ROPE_TYPE_NEOX = 2 module-attribute

LLAMA_ROPE_TYPE_GLM = 4 module-attribute

LLAMA_TOKEN_TYPE_UNDEFINED = 0 module-attribute

LLAMA_TOKEN_TYPE_NORMAL = 1 module-attribute

LLAMA_TOKEN_TYPE_UNKNOWN = 2 module-attribute

LLAMA_TOKEN_TYPE_CONTROL = 3 module-attribute

LLAMA_TOKEN_TYPE_USER_DEFINED = 4 module-attribute

LLAMA_TOKEN_TYPE_UNUSED = 5 module-attribute

LLAMA_TOKEN_TYPE_BYTE = 6 module-attribute

LLAMA_FTYPE_ALL_F32 = 0 module-attribute

LLAMA_FTYPE_MOSTLY_F16 = 1 module-attribute

LLAMA_FTYPE_MOSTLY_Q4_0 = 2 module-attribute

LLAMA_FTYPE_MOSTLY_Q4_1 = 3 module-attribute

LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4 module-attribute

LLAMA_FTYPE_MOSTLY_Q8_0 = 7 module-attribute

LLAMA_FTYPE_MOSTLY_Q5_0 = 8 module-attribute

LLAMA_FTYPE_MOSTLY_Q5_1 = 9 module-attribute

LLAMA_FTYPE_MOSTLY_Q2_K = 10 module-attribute

LLAMA_FTYPE_MOSTLY_Q3_K_S = 11 module-attribute

LLAMA_FTYPE_MOSTLY_Q3_K_M = 12 module-attribute

LLAMA_FTYPE_MOSTLY_Q3_K_L = 13 module-attribute

LLAMA_FTYPE_MOSTLY_Q4_K_S = 14 module-attribute

LLAMA_FTYPE_MOSTLY_Q4_K_M = 15 module-attribute

LLAMA_FTYPE_MOSTLY_Q5_K_S = 16 module-attribute

LLAMA_FTYPE_MOSTLY_Q5_K_M = 17 module-attribute

LLAMA_FTYPE_MOSTLY_Q6_K = 18 module-attribute

LLAMA_FTYPE_MOSTLY_IQ2_XXS = 19 module-attribute

LLAMA_FTYPE_MOSTLY_IQ2_XS = 20 module-attribute

LLAMA_FTYPE_MOSTLY_Q2_K_S = 21 module-attribute

LLAMA_FTYPE_MOSTLY_IQ3_XS = 22 module-attribute

LLAMA_FTYPE_MOSTLY_IQ3_XXS = 23 module-attribute

LLAMA_FTYPE_MOSTLY_IQ1_S = 24 module-attribute

LLAMA_FTYPE_MOSTLY_IQ4_NL = 25 module-attribute

LLAMA_FTYPE_MOSTLY_IQ3_S = 26 module-attribute

LLAMA_FTYPE_MOSTLY_IQ3_M = 27 module-attribute

LLAMA_FTYPE_MOSTLY_IQ2_S = 28 module-attribute

LLAMA_FTYPE_MOSTLY_IQ2_M = 29 module-attribute

LLAMA_FTYPE_MOSTLY_IQ4_XS = 30 module-attribute

LLAMA_FTYPE_GUESSED = 1024 module-attribute

LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED = -1 module-attribute

LLAMA_ROPE_SCALING_TYPE_NONE = 0 module-attribute

LLAMA_ROPE_SCALING_TYPE_LINEAR = 1 module-attribute

LLAMA_ROPE_SCALING_TYPE_YARN = 2 module-attribute

LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_YARN module-attribute

LLAMA_POOLING_TYPE_UNSPECIFIED = -1 module-attribute

LLAMA_POOLING_TYPE_NONE = 0 module-attribute

LLAMA_POOLING_TYPE_MEAN = 1 module-attribute

LLAMA_POOLING_TYPE_CLS = 2 module-attribute

LLAMA_SPLIT_MODE_NONE = 0 module-attribute

LLAMA_SPLIT_MODE_LAYER = 1 module-attribute

LLAMA_SPLIT_MODE_ROW = 2 module-attribute

LLAMA_KV_OVERRIDE_TYPE_INT = 0 module-attribute

LLAMA_KV_OVERRIDE_TYPE_FLOAT = 1 module-attribute

LLAMA_KV_OVERRIDE_TYPE_BOOL = 2 module-attribute

LLAMA_KV_OVERRIDE_TYPE_STR = 3 module-attribute

LLAMA_GRETYPE_END = 0 module-attribute

LLAMA_GRETYPE_ALT = 1 module-attribute

LLAMA_GRETYPE_RULE_REF = 2 module-attribute

LLAMA_GRETYPE_CHAR = 3 module-attribute

LLAMA_GRETYPE_CHAR_NOT = 4 module-attribute

LLAMA_GRETYPE_CHAR_RNG_UPPER = 5 module-attribute

LLAMA_GRETYPE_CHAR_ALT = 6 module-attribute

Misc

llama_cpp.llama_types

Types and request signatures for OpenAI compatibility

NOTE: These types may change to match the OpenAI OpenAPI specification.

Based on the OpenAI OpenAPI specification: https://github.com/openai/openai-openapi/blob/master/openapi.yaml

JsonType = Union[None, int, str, bool, List[Any], Dict[str, Any]] module-attribute

EmbeddingUsage

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class EmbeddingUsage(TypedDict):
    prompt_tokens: int
    total_tokens: int

prompt_tokens: int instance-attribute

total_tokens: int instance-attribute

Embedding

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class Embedding(TypedDict):
    index: int
    object: str
    embedding: Union[List[float], List[List[float]]]

index: int instance-attribute

object: str instance-attribute

embedding: Union[List[float], List[List[float]]] instance-attribute

CreateEmbeddingResponse

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class CreateEmbeddingResponse(TypedDict):
    object: Literal["list"]
    model: str
    data: List[Embedding]
    usage: EmbeddingUsage

object: Literal['list'] instance-attribute

model: str instance-attribute

data: List[Embedding] instance-attribute

usage: EmbeddingUsage instance-attribute

CompletionLogprobs

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class CompletionLogprobs(TypedDict):
    text_offset: List[int]
    token_logprobs: List[Optional[float]]
    tokens: List[str]
    top_logprobs: List[Optional[Dict[str, float]]]

text_offset: List[int] instance-attribute

token_logprobs: List[Optional[float]] instance-attribute

tokens: List[str] instance-attribute

top_logprobs: List[Optional[Dict[str, float]]] instance-attribute

CompletionChoice

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class CompletionChoice(TypedDict):
    text: str
    index: int
    logprobs: Optional[CompletionLogprobs]
    finish_reason: Optional[Literal["stop", "length"]]

text: str instance-attribute

index: int instance-attribute

logprobs: Optional[CompletionLogprobs] instance-attribute

finish_reason: Optional[Literal['stop', 'length']] instance-attribute

CompletionUsage

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class CompletionUsage(TypedDict):
    prompt_tokens: int
    completion_tokens: int
    total_tokens: int

prompt_tokens: int instance-attribute

completion_tokens: int instance-attribute

total_tokens: int instance-attribute

CreateCompletionResponse

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class CreateCompletionResponse(TypedDict):
    id: str
    object: Literal["text_completion"]
    created: int
    model: str
    choices: List[CompletionChoice]
    usage: NotRequired[CompletionUsage]

id: str instance-attribute

object: Literal['text_completion'] instance-attribute

created: int instance-attribute

model: str instance-attribute

choices: List[CompletionChoice] instance-attribute

usage: NotRequired[CompletionUsage] instance-attribute

ChatCompletionResponseFunctionCall

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class ChatCompletionResponseFunctionCall(TypedDict):
    name: str
    arguments: str

name: str instance-attribute

arguments: str instance-attribute

ChatCompletionResponseMessage

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class ChatCompletionResponseMessage(TypedDict):
    content: Optional[str]
    tool_calls: NotRequired["ChatCompletionMessageToolCalls"]
    role: Literal["assistant", "function"]  # NOTE: "function" may be incorrect here
    function_call: NotRequired[ChatCompletionResponseFunctionCall]  # DEPRECATED

content: Optional[str] instance-attribute

tool_calls: NotRequired[ChatCompletionMessageToolCalls] instance-attribute

role: Literal['assistant', 'function'] instance-attribute

function_call: NotRequired[ChatCompletionResponseFunctionCall] instance-attribute

ChatCompletionFunction

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class ChatCompletionFunction(TypedDict):
    name: str
    description: NotRequired[str]
    parameters: Dict[str, JsonType]  # TODO: make this more specific

name: str instance-attribute

description: NotRequired[str] instance-attribute

parameters: Dict[str, JsonType] instance-attribute

ChatCompletionResponseChoice

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class ChatCompletionResponseChoice(TypedDict):
    index: int
    message: "ChatCompletionResponseMessage"
    logprobs: Optional[CompletionLogprobs]
    finish_reason: Optional[str]

index: int instance-attribute

message: ChatCompletionResponseMessage instance-attribute

logprobs: Optional[CompletionLogprobs] instance-attribute

finish_reason: Optional[str] instance-attribute

CreateChatCompletionResponse

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class CreateChatCompletionResponse(TypedDict):
    id: str
    object: Literal["chat.completion"]
    created: int
    model: str
    choices: List["ChatCompletionResponseChoice"]
    usage: CompletionUsage

id: str instance-attribute

object: Literal['chat.completion'] instance-attribute

created: int instance-attribute

model: str instance-attribute

choices: List[ChatCompletionResponseChoice] instance-attribute

usage: CompletionUsage instance-attribute

ChatCompletionMessageToolCallChunkFunction

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class ChatCompletionMessageToolCallChunkFunction(TypedDict):
    name: Optional[str]
    arguments: str

name: Optional[str] instance-attribute

arguments: str instance-attribute

ChatCompletionMessageToolCallChunk

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class ChatCompletionMessageToolCallChunk(TypedDict):
    index: int
    id: NotRequired[str]
    type: Literal["function"]
    function: ChatCompletionMessageToolCallChunkFunction

index: int instance-attribute

id: NotRequired[str] instance-attribute

type: Literal['function'] instance-attribute

function: ChatCompletionMessageToolCallChunkFunction instance-attribute

ChatCompletionStreamResponseDeltaEmpty

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class ChatCompletionStreamResponseDeltaEmpty(TypedDict):
    pass

ChatCompletionStreamResponseDeltaFunctionCall

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class ChatCompletionStreamResponseDeltaFunctionCall(TypedDict):
    name: str
    arguments: str

name: str instance-attribute

arguments: str instance-attribute

ChatCompletionStreamResponseDelta

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class ChatCompletionStreamResponseDelta(TypedDict):
    content: NotRequired[Optional[str]]
    function_call: NotRequired[
        Optional[ChatCompletionStreamResponseDeltaFunctionCall]
    ]  # DEPRECATED
    tool_calls: NotRequired[Optional[List[ChatCompletionMessageToolCallChunk]]]
    role: NotRequired[Optional[Literal["system", "user", "assistant", "tool"]]]

content: NotRequired[Optional[str]] instance-attribute

function_call: NotRequired[Optional[ChatCompletionStreamResponseDeltaFunctionCall]] instance-attribute

tool_calls: NotRequired[Optional[List[ChatCompletionMessageToolCallChunk]]] instance-attribute

role: NotRequired[Optional[Literal['system', 'user', 'assistant', 'tool']]] instance-attribute

ChatCompletionStreamResponseChoice

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class ChatCompletionStreamResponseChoice(TypedDict):
    index: int
    delta: Union[
        ChatCompletionStreamResponseDelta, ChatCompletionStreamResponseDeltaEmpty
    ]
    finish_reason: Optional[Literal["stop", "length", "tool_calls", "function_call"]]
    logprobs: NotRequired[Optional[CompletionLogprobs]]

index: int instance-attribute

delta: Union[ChatCompletionStreamResponseDelta, ChatCompletionStreamResponseDeltaEmpty] instance-attribute

finish_reason: Optional[Literal['stop', 'length', 'tool_calls', 'function_call']] instance-attribute

logprobs: NotRequired[Optional[CompletionLogprobs]] instance-attribute

CreateChatCompletionStreamResponse

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class CreateChatCompletionStreamResponse(TypedDict):
    id: str
    model: str
    object: Literal["chat.completion.chunk"]
    created: int
    choices: List[ChatCompletionStreamResponseChoice]

id: str instance-attribute

model: str instance-attribute

object: Literal['chat.completion.chunk'] instance-attribute

created: int instance-attribute

choices: List[ChatCompletionStreamResponseChoice] instance-attribute

ChatCompletionFunctions

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class ChatCompletionFunctions(TypedDict):
    name: str
    description: NotRequired[str]
    parameters: Dict[str, JsonType]  # TODO: make this more specific

name: str instance-attribute

description: NotRequired[str] instance-attribute

parameters: Dict[str, JsonType] instance-attribute

ChatCompletionFunctionCallOption

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class ChatCompletionFunctionCallOption(TypedDict):
    name: str

name: str instance-attribute

ChatCompletionRequestResponseFormat

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class ChatCompletionRequestResponseFormat(TypedDict):
    type: Literal["text", "json_object"]
    schema: NotRequired[JsonType] # https://docs.endpoints.anyscale.com/guides/json_mode/

type: Literal['text', 'json_object'] instance-attribute

schema: NotRequired[JsonType] instance-attribute

ChatCompletionRequestMessageContentPartText

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class ChatCompletionRequestMessageContentPartText(TypedDict):
    type: Literal["text"]
    text: str

type: Literal['text'] instance-attribute

text: str instance-attribute

ChatCompletionRequestMessageContentPartImageImageUrl

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class ChatCompletionRequestMessageContentPartImageImageUrl(TypedDict):
    url: str
    detail: NotRequired[Literal["auto", "low", "high"]]

url: str instance-attribute

detail: NotRequired[Literal['auto', 'low', 'high']] instance-attribute

ChatCompletionRequestMessageContentPartImage

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class ChatCompletionRequestMessageContentPartImage(TypedDict):
    type: Literal["image_url"]
    image_url: Union[str, ChatCompletionRequestMessageContentPartImageImageUrl]

type: Literal['image_url'] instance-attribute

image_url: Union[str, ChatCompletionRequestMessageContentPartImageImageUrl] instance-attribute

ChatCompletionRequestMessageContentPart = Union[ChatCompletionRequestMessageContentPartText, ChatCompletionRequestMessageContentPartImage] module-attribute

ChatCompletionRequestSystemMessage

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class ChatCompletionRequestSystemMessage(TypedDict):
    role: Literal["system"]
    content: Optional[str]

role: Literal['system'] instance-attribute

content: Optional[str] instance-attribute

ChatCompletionRequestUserMessage

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class ChatCompletionRequestUserMessage(TypedDict):
    role: Literal["user"]
    content: Optional[Union[str, List[ChatCompletionRequestMessageContentPart]]]

role: Literal['user'] instance-attribute

content: Optional[Union[str, List[ChatCompletionRequestMessageContentPart]]] instance-attribute

ChatCompletionMessageToolCallFunction

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class ChatCompletionMessageToolCallFunction(TypedDict):
    name: str
    arguments: str

name: str instance-attribute

arguments: str instance-attribute

ChatCompletionMessageToolCall

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class ChatCompletionMessageToolCall(TypedDict):
    id: str
    type: Literal["function"]
    function: ChatCompletionMessageToolCallFunction

id: str instance-attribute

type: Literal['function'] instance-attribute

function: ChatCompletionMessageToolCallFunction instance-attribute

ChatCompletionMessageToolCalls = List[ChatCompletionMessageToolCall] module-attribute

ChatCompletionRequestAssistantMessageFunctionCall

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class ChatCompletionRequestAssistantMessageFunctionCall(TypedDict):
    name: str
    arguments: str

name: str instance-attribute

arguments: str instance-attribute

ChatCompletionRequestAssistantMessage

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class ChatCompletionRequestAssistantMessage(TypedDict):
    role: Literal["assistant"]
    content: Optional[str]
    tool_calls: NotRequired[ChatCompletionMessageToolCalls]
    function_call: NotRequired[
        ChatCompletionRequestAssistantMessageFunctionCall
    ]  # DEPRECATED

role: Literal['assistant'] instance-attribute

content: Optional[str] instance-attribute

tool_calls: NotRequired[ChatCompletionMessageToolCalls] instance-attribute

function_call: NotRequired[ChatCompletionRequestAssistantMessageFunctionCall] instance-attribute

ChatCompletionRequestToolMessage

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class ChatCompletionRequestToolMessage(TypedDict):
    role: Literal["tool"]
    content: Optional[str]
    tool_call_id: str

role: Literal['tool'] instance-attribute

content: Optional[str] instance-attribute

tool_call_id: str instance-attribute

ChatCompletionRequestFunctionMessage

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class ChatCompletionRequestFunctionMessage(TypedDict):
    role: Literal["function"]
    content: Optional[str]
    name: str

role: Literal['function'] instance-attribute

content: Optional[str] instance-attribute

name: str instance-attribute

ChatCompletionRequestMessage = Union[ChatCompletionRequestSystemMessage, ChatCompletionRequestUserMessage, ChatCompletionRequestAssistantMessage, ChatCompletionRequestUserMessage, ChatCompletionRequestToolMessage, ChatCompletionRequestFunctionMessage] module-attribute

ChatCompletionRequestFunctionCallOption

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class ChatCompletionRequestFunctionCallOption(TypedDict):
    name: str

name: str instance-attribute

ChatCompletionRequestFunctionCall = Union[Literal['none', 'auto'], ChatCompletionRequestFunctionCallOption] module-attribute

ChatCompletionFunctionParameters = Dict[str, JsonType] module-attribute

ChatCompletionToolFunction

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class ChatCompletionToolFunction(TypedDict):
    name: str
    description: NotRequired[str]
    parameters: ChatCompletionFunctionParameters

name: str instance-attribute

description: NotRequired[str] instance-attribute

parameters: ChatCompletionFunctionParameters instance-attribute

ChatCompletionTool

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class ChatCompletionTool(TypedDict):
    type: Literal["function"]
    function: ChatCompletionToolFunction

type: Literal['function'] instance-attribute

function: ChatCompletionToolFunction instance-attribute

ChatCompletionNamedToolChoiceFunction

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class ChatCompletionNamedToolChoiceFunction(TypedDict):
    name: str

name: str instance-attribute

ChatCompletionNamedToolChoice

Bases: TypedDict

Source code in llama_cpp/llama_types.py

class ChatCompletionNamedToolChoice(TypedDict):
    type: Literal["function"]
    function: ChatCompletionNamedToolChoiceFunction

type: Literal['function'] instance-attribute

function: ChatCompletionNamedToolChoiceFunction instance-attribute

ChatCompletionToolChoiceOption = Union[Literal['none', 'auto'], ChatCompletionNamedToolChoice] module-attribute

EmbeddingData = Embedding module-attribute

CompletionChunk = CreateCompletionResponse module-attribute

Completion = CreateCompletionResponse module-attribute

CreateCompletionStreamResponse = CreateCompletionResponse module-attribute

ChatCompletionMessage = ChatCompletionResponseMessage module-attribute

ChatCompletionChoice = ChatCompletionResponseChoice module-attribute

ChatCompletion = CreateChatCompletionResponse module-attribute

ChatCompletionChunkDeltaEmpty = ChatCompletionStreamResponseDeltaEmpty module-attribute

ChatCompletionChunkChoice = ChatCompletionStreamResponseChoice module-attribute

ChatCompletionChunkDelta = ChatCompletionStreamResponseDelta module-attribute

ChatCompletionChunk = CreateChatCompletionStreamResponse module-attribute

ChatCompletionStreamResponse = CreateChatCompletionStreamResponse module-attribute

ChatCompletionResponseFunction = ChatCompletionFunction module-attribute

ChatCompletionFunctionCall = ChatCompletionResponseFunctionCall module-attribute