# 1. Install Docker Desktop for Apple Silicon (if not already installed)
#    https://docs.docker.com/desktop/mac/install/

# 2. Clone the OpenWebUI repo (the official one ships a docker‑compose file)
git clone https://github.com/open-webui/open-webui.git
cd open-webui

# 3. Edit docker-compose.yml to limit RAM for each service (see section 8)
#    (you can also keep the defaults – they already request ~2 GB per service)

# 4. Bring the stack up
docker compose up -d

# Clone the repo (arm64 works out‑of‑the‑box)
git clone https://github.com/ggerganov/llama.cpp
cd llama.cpp

# Build with Metal (GPU) and AVX2 (CPU fallback) – the Makefile detects M1/M2 automatically
make LLAMA_METAL=1

# Verify the binary works
./main -h

# Example: Phi‑3‑mini‑instruct (download the original .gguf from HuggingFace)
#   https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf

# Place the file in the llama.cpp folder, e.g. phi3-mini-gguf/phi3-mini-4bit.gguf

# If you have a .pth/.safetensors checkpoint, first convert:
python3 convert_hf_to_gguf.py \
    --model-dir /path/to/hf_repo \
    --outfile phi3-mini.gguf

# Quantise to 4‑bit (the default GGUF may already be 4‑bit; otherwise:)
./quantize phi3-mini.gguf phi3-mini-4bit.gguf q4_0

./server \
    -m ./phi3-mini-4bit.gguf \
    -c 4096 \
    -ngl 0 \   # number of GPU layers; 0 = all Metal (recommended on M2)
    -t 4       # number of CPU threads for the non‑GPU part

# Convert/quantise the embedding checkpoint the same way as above
# (the repo provides phi3-mini-embedding-gguf)

./server \
    -m ./phi3-mini-embedding-4bit.gguf \
    -c 2048 \
    -ngl 0 \
    -t 4 \
    --embedding

pip install "sentence-transformers[torch]"  # torch will use the Apple Silicon build
python - <<'PY'
from sentence_transformers import SentenceTransformer
model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
# Save for later reuse
model.save_pretrained("./miniLM-embed")
PY

# docker-compose.yml (add this service)
  qdrant:
    image: qdrant/qdrant:latest-arm64
    ports:
      - "6333:6333"
    volumes:
      - ./qdrant_storage:/qdrant/storage
    deploy:
      resources:
        limits:
          memory: 6G   # ~6 GB is enough for a few hundred thousand vectors

docker compose up -d qdrant

# rag_service.py
import os
import json
import httpx
from langchain.vectorstores import Qdrant
from langchain.embeddings import OpenAIEmbeddings   # we will point to our local embed API
from langchain.llms import OpenAI
from langchain.chains import RetrievalQA
from qdrant_client import QdrantClient

# ---------- CONFIG ----------
LLM_ENDPOINT = "http://127.0.0.1:8080/v1"          # llama.cpp server
EMBED_ENDPOINT = "http://127.0.0.1:8080/v1"        # same server, /embeddings
QDRANT_HOST = "localhost"
QDRANT_PORT = 6333
COLLECTION_NAME = "docs"
# ---------------------------

# 1️⃣ Embedding wrapper
class LocalEmbedding(OpenAIEmbeddings):
    def __init__(self):
        super().__init__(api_key="sk-no-key",  # dummy, not used
                         base_url=EMBED_ENDPOINT,
                         model="phi3-mini-embedding")

# 2️⃣ LLM wrapper
class LocalLLM(OpenAI):
    def __init__(self):
        super().__init__(api_key="sk-no-key",
                         base_url=LLM_ENDPOINT,
                         model="phi3-mini-instruct",
                         temperature=0.7,
                         max_tokens=1024)

# 3️⃣ Vector store
client = QdrantClient(host=QDRANT_HOST, port=QDRANT_PORT)
vectorstore = Qdrant(client=client,
                     collection_name=COLLECTION_NAME,
                     embeddings=LocalEmbedding())

# 4️⃣ RetrievalQA chain
qa = RetrievalQA.from_chain_type(
    llm=LocalLLM(),
    retriever=vectorstore.as_retriever(search_kwargs={"k": 4}),
    return_source_documents=True,
)

# ---------- Simple REPL ----------
if __name__ == "__main__":
    while True:
        query = input("\n❓ Question: ")
        if query.lower() in {"exit", "quit"}:
            break
        resp = qa({"query": query})
        print("\n🗣 Answer:", resp["result"])
        for doc in resp["source_documents"]:
            print("\n--- Source snippet ---")
            print(doc.page_content[:500], "...")

pip install langchain qdrant-client openai==1.12.0  # openai lib for the API wrapper
python rag_service.py

pip install coremltools onnxruntime

# image_tag_service.py
import base64, io
from fastapi import FastAPI, File, UploadFile
from PIL import Image
import torch
import clip  # pip install git+https://github.com/openai/CLIP.git

app = FastAPI()
device = "mps"   # Apple Silicon GPU via Metal Performance Shaders

model, preprocess = clip.load("ViT-B/32", device=device)

@app.post("/tag")
async def tag_image(file: UploadFile = File(...)):
    img = Image.open(io.BytesIO(await file.read())).convert("RGB")
    img_input = preprocess(img).unsqueeze(0).to(device)

    # Use a small set of candidate tags (you can load a larger list from a file)
    candidates = ["cat", "dog", "car", "tree", "person", "food", "building"]
    text_inputs = torch.cat([clip.tokenize(c) for c in candidates]).to(device)

    with torch.no_grad():
        image_features = model.encode_image(img_input)
        text_features = model.encode_text(text_inputs)

        # Cosine similarity
        sims = (image_features @ text_features.T).squeeze(0)
        topk = sims.topk(3).indices.tolist()
        tags = [candidates[i] for i in topk]

    return {"tags": tags}

uvicorn image_tag_service:app --host 0.0.0.0 --port 8200

version: "3.9"
services:
  open-webui:
    image: ghcr.io/open-webui/open-webui:latest
    ports: ["8080:8080"]
    environment:
      - OLLAMA_HOST=ollama   # not used if you point to llama.cpp
    deploy:
      resources:
        limits:
          memory: 4G
          cpus: "2.0"

  qdrant:
    image: qdrant/qdrant:latest-arm64
    ports: ["6333:6333"]
    deploy:
      resources:
        limits:
          memory: 6G
          cpus: "2.0"

  rag-service:
    build: ./rag_service   # Dockerfile that installs python + deps
    command: python rag_service.py
    deploy:
      resources:
        limits:
          memory: 2G
          cpus: "1.0"

  image-tag:
    image: python:3.11-slim
    command: uvicorn image_tag_service:app --host 0.0.0.0 --port 8200
    deploy:
      resources:
        limits:
          memory: 2G
          cpus: "1.0"

# 1️⃣ Install Docker Desktop (Apple Silicon) → already done

# 2️⃣ Clone OpenWebUI
git clone https://github.com/open-webui/open-webui.git && cd open-webui

# 3️⃣ Add the services (qdrant, rag, image‑tag) to docker‑compose.yml (see above)

# 4️⃣ Build llama.cpp with Metal
git clone https://github.com/ggerganov/llama.cpp && cd llama.cpp
make LLAMA_METAL=1
# download phi‑3‑mini GGUF (4‑bit) → place in ./models/
# start the server
./server -m ./models/phi3-mini-4bit.gguf -c 4096 -ngl 0 -t 4 &

# 5️⃣ Start the embedding server (same binary, add --embedding flag)
./server -m ./models/phi3-mini-embedding-4bit.gguf -c 2048 -ngl 0 -t 4 --embedding &

# 6️⃣ Build & run the rest of the stack
docker compose up -d   # will start open-webui, qdrant, rag-service, image-tag

# 7️⃣ Open a browser → http://localhost:8080
#    In Settings → Model → “Custom OpenAI API”:
#      Base URL: http://host.docker.internal:8080/v1
#      Model name: phi3-mini-instruct
#    (You can also add a “Tool” that calls the image‑tag endpoint.)

# 8️⃣ Test a RAG query:
#    - Upload a few documents → embed via the /embeddings endpoint → store in Qdrant.
#    - Ask a question → you should see a retrieved answer with source snippets.

# 9️⃣ Monitor memory:
htop   # you should see llama.cpp using ~4 GB, qdrant ~5‑6 GB, others < 2 GB each.