Ollamex • OVERBRING Labs

An Elixir wrapper of ollama ’s REST API with a few niceties built-in, such as dealing with endless LLM (Large Language Model) repetitions through a timeout.

Ollamex is written based on the ollama REST API documentation for the following endpoints:

Installation

The package is available in Hex and can be installed by adding ollamex to your list of dependencies in mix.exs:

def deps do
  [
    {:ollamex, "~> 0.1.0"}
  ]
end

Documentation

The docs can be found at https://hexdocs.pm/ollamex .

Examples

API initialization

iex> api = Ollamex.API.new()
%Ollamex.API{
  uri: "http://localhost:11434/api",
  models: [
    %Ollamex.LLModel{
      name: "llama2:latest",
      digest: "78e26419b4469263f75331927a00a0284ef6544c1975b826b15abdaef17bb962",
      modified_at: "2024-01-09T22:24:14.925918123+02:00",
      size: 3826793677,
      details: %{
        "families" => ["llama"],
        "family" => "llama",
        "format" => "gguf",
        "parameter_size" => "7B",
        "quantization_level" => "Q4_0"
      },
      modelfile: nil,
      parameters: nil,
      template: nil
    },
    %Ollamex.LLModel{
      name: "mistral:latest",
      digest: "61e88e884507ba5e06c49b40e6226884b2a16e872382c2b44a42f2d119d804a5",
      modified_at: "2024-01-08T17:49:54.570542101+02:00",
      size: 4109865159,
      details: %{
        "families" => ["llama"],
        "family" => "llama",
        "format" => "gguf",
        "parameter_size" => "7B",
        "quantization_level" => "Q4_0"
      },
      modelfile: nil,
      parameters: nil,
      template: nil
    }
  ],
  timeout: 120000,
  errors: []
  }
  iex> Ollamex.API.list_models(api)
  ["llama2:latest", "mistral:latest"]

Generate a completion (`/generate` endpoint)

iex> p = %Ollamex.PromptRequest{model: "mistral:latest", prompt: "Explain using a simple paragraph like I'm 5 years old: Why is the sky not black like space?"}
%Ollamex.PromptRequest{
  model: "mistral:latest",
  prompt: "Explain using a simple paragraph like I'm 5 years old: Why is the sky not black like space?",
  raw: false,
  format: nil,
  stream: true,
  options: nil,
  images: []
}

iex> Ollamex.generate_with_timeout(p, api)
{:ok,
  %Ollamex.LLMResponse{
  context: [733, 16289, 28793, ...],
  created_at: "2024-01-10T19:23:12.943599755Z",
  done: true,
  eval_count: 100,
  eval_duration: 16850322000,
  model: "mistral:latest",
  prompt_eval_count: 33,
  prompt_eval_duration: 2865358000,
  response: " The sky isn't black like space because it has ...
  pretty colors, and nighttime with stars and the moon!",
  total_duration: 24862993618,
  message: nil,
  errors: nil
}}

Generate a chat completion

messages =
  []
  |> Ollamex.ChatMessage.append("user", "why is the sky blue?")
  |> Ollamex.ChatMessage.append("assistant", "due to rayleigh scattering!")
  |> Ollamex.ChatMessage.append("user", "how is that different to Mie scattering?")
  |> Enum.map(&Map.from_struct(&1))
iex>
[
%{content: "why is the sky blue?", images: [], role: "user"},
%{content: "due to rayleigh scattering!", images: [], role: "assistant"},
%{
  content: "how is that different to Mie scattering?",
  images: [],
  role: "user"
}
]

iex> cr = %Ollamex.ChatRequest{messages: messages, model: "llama2", stream: true}
%Ollamex.ChatRequest{
  model: "llama2",
  messages: [
    %{content: "why is the sky blue?", images: [], role: "user"},
    %{content: "due to rayleigh scattering!", images: [], role: "assistant"},
    %{
      content: "how is that different to Mie scattering?",
      images: [],
      role: "user"
    }
  ],
  format: nil,
  options: nil,
  template: nil,
  stream: true
}
iex> Ollamex.chat_with_timeout(cr, api)
{:ok,
 %Ollamex.LLMResponse{
  context: nil,
  created_at: "2024-01-10T19:29:05.771371091Z",
  done: true,
  eval_count: 515,
  eval_duration: 83246108000,
  model: "llama2",
  prompt_eval_count: 61,
  prompt_eval_duration: 7234332000,
  response: nil,
  total_duration: 95606709630,
  message: %{
    content: "Mie scattering is ... while Rayleigh scattering
    is responsible for the reddening of sunlight at sunrise
    and sunset.",
    role: "assistant"
  },
  errors: nil
}}