Hamhoosh/edge/orangepi_service/README.md

how ot run

sudo apt update sudo apt install -y build-essential python3-dev libatlas-base-dev cmake git mkdir -p daemon cd daemon uv init orangepi_service cd orangepi_service uv add install fastapi uvicorn chromadb sentence-transformers pypdf langchain-community python-multipart CMAKE_ARGS="-DLLAMA_METAL=OFF -DCMAKE_CXX_FLAGS=-march=armv8.2-a+fp16" FORCE_CMAKE=1 vi ~/.config/uv/uv.toml

[[index]]
url = "https://pypi.tuna.tsinghua.edu.cn/simple"
default = true

export MODEL_PATH="models/qwen-0.5b-q4_k_m.gguf" export CHROMA_DIR="./chroma_db" export UPLOAD_DIR="./uploaded_pdfs" mkdir -p models wget -O models/qwen-0.5b-q4_k_m.gguf https://huggingface.co/Qwen/Qwen1.5-0.5B-GGUF/resolve/main/qwen1_5-0_5b-q4_k_m.gguf uv run service.py

Set Static IP

ip a sudo nano /etc/netplan/orangepi-default.yaml

network:
  version: 2
  renderer: networkd
  ethernets:
    enp49s0:              # Replace 'eth0' with the name you found from 'ip a'
      dhcp4: no        # Disable DHCP client for IPv4
      addresses:
        - 192.168.1.101/24   # Your desired static IP and subnet mask
      routes:
        - to: default
          via: 192.168.1.1   # Your router's gateway address
      nameservers:
        addresses: [8.8.8.8, 8.8.4.4]  # DNS servers (e.g., Google DNS)

sudo netplan try sudo netplan apply sudo nano /etc/systemd/system/orangepi_rag.service

[Unit]
Description=Orange Pi RAG Service
After=network-online.target
Wants=network-online.target

[Service]
User=orangepi
Environment="HF_HUB_OFFLINE=1"
Environment="TRANSFORMERS_OFFLINE=1"
WorkingDirectory=/home/orangepi/daemon/orangepi_service
Environment="PATH=/home/orangepi/daemon/orangepi_service/.venv/bin"
ExecStart=/home/orangepi/daemon/orangepi_service/.venv/bin/python service.py
Restart=always
RestartSec=10

[Install]
WantedBy=multi-user.target

sudo systemctl daemon-reload sudo systemctl enable orangepi_rag.service sudo systemctl start orangepi_rag.service

Test API

curl http://0.0.0.0:8080/health curl -X POST http://0.0.0.0:8080/answer -H "Content-Type: application/json" -d '{"question": "What is the capital of France?"}'