run-local-llm-server-part6 debian vmware on windows 10

 

docker pull ghcr.io/ggml-org/llama.cpp:server

 

Step 1: Download a model in GGUF format from Hugging Face.

# llama.cpp only works with GGUF

# Create a dedicated folder for models:
mkdir -p /datadocker/llama-cpp

cd /datadocker/llama-cpp

mkdir models

cd /datadocker/llama-cpp/models


https://huggingface.co/unsloth/gemma-4-E2B-it-GGUF/tree/main

https://huggingface.co/bartowski/google_gemma-4-E2B-it-GGUF/tree/main

google_gemma-4-E2B-it-Q4_0.gguf

wget https://huggingface.co/bartowski/google_gemma-4-E2B-it-GGUF/resolve/main/google_gemma-4-E2B-it-Q4_0.gguf?download=true

 

Step 2: Launch the model

cd /datadocker/llama-cpp

 

docker run -it --rm \
  --name llama \
  -v /datadocker/llama-cpp/models:/models \
  -p 8091:8080 \
  ghcr.io/ggml-org/llama.cpp:server \
  -m /models/google_gemma-4-E2B-it-Q4_0.gguf \
  --host 0.0.0.0 \
  --port 8080 \
  --threads 4 \
  --jinja