Skip to content
Snippets Groups Projects
Select Git revision
  • 897b59438168ce2261e882b4418816036eaddd27
  • master default protected
2 results

batches.py

Blame
    • run.sh 308 B
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13 
    

    

    

    #!/bin/bash

PORT=8321
#MODEL=mistralai/Mistral-7B-Instruct-v0.1
#MODEL=lmsys/vicuna-13b-v1.3
MODEL=microsoft/phi-2
HOST=`hostname`
SEED=1234

. env.sh
echo Serving $MODEL on http://$HOST:$PORT
python -m vllm.entrypoints.api_server --model $MODEL --trust-remote-code --host 0.0.0.0 --port $PORT --seed $SEED