initial commit

README

+install requirements
+change run.sh to pick the model of your choice, and then run it to start vllm server
+run client with python dialog.py --host localhost

dialog.py

+"""Example Python client for vllm.entrypoints.api_server"""
+
+import argparse
+import json
+from typing import Iterable, List
+
+import requests
+
+
+def clear_line(n: int = 1) -> None:
+    LINE_UP = '\033[1A'
+    LINE_CLEAR = '\x1b[2K'
+    for _ in range(n):
+        print(LINE_UP, end=LINE_CLEAR, flush=True)
+
+
+def post_http_request(prompt: str,
+                      api_url: str,
+                      n: int = 1,
+                      stream: bool = False) -> requests.Response:
+    headers = {"User-Agent": "Test Client"}
+    pload = {
+        "prompt": prompt,
+        #"n": n,
+        #"use_beam_search": True,
+        "temperature": 0.0,
+        "max_tokens": 1024,
+        "stream": stream,
+    }
+    response = requests.post(api_url, headers=headers, json=pload, stream=True)
+    return response
+
+
+def get_streaming_response(response: requests.Response) -> Iterable[List[str]]:
+    for chunk in response.iter_lines(chunk_size=8192,
+                                     decode_unicode=False,
+                                     delimiter=b"\0"):
+        if chunk:
+            data = json.loads(chunk.decode("utf-8"))
+            output = data["text"]
+            yield output
+
+
+def get_response(response: requests.Response) -> List[str]:
+    data = json.loads(response.content)
+    output = data["text"]
+    return output
+
+
+context = {
+'Buyer': 'You are the seller in a transaction involving a precious stone. Your starting price is $1000. You will not lower the price under $500. Negociate the price so that the transation is successful.',
+'Seller': 'You are the buyer in a transaction involving a precious stone. You want to buy it at the lowest possible price and you cannot spend more than $400. Negociate the price so that the transation is successful.'
+}
+
+turns = [{'role': 'Buyer', 'message': 'How much for this stone?'}, {'role': 'Seller', 'message': 'This stone is $1000.'}]
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--host", type=str, default="diflives1")
+    parser.add_argument("--port", type=int, default=8321)
+    args = parser.parse_args()
+    api_url = f"http://{args.host}:{args.port}/generate"
+
+    print('\n'.join(turn['role'] + ': ' + turn['message'] for turn in turns))
+    roles = ['Buyer', 'Seller']
+    for i in range(10):
+        role = roles[i % 2]
+        prompt = context[role] + '\n' + '\n'.join(turn['role'] + ': ' + turn['message'] for turn in turns) + '\n' + role + ':'
+        response = post_http_request(prompt, api_url)
+        output = get_response(response)
+        message = output[0][len(prompt):].split('\n')[0].strip()
+        print(role + ': ' + message)
+        turns.append({'role': role, 'message': message})

requirements.txt

+vllm
+openai

run.sh

+#!/bin/bash
+
+PORT=8321
+#MODEL=mistralai/Mistral-7B-Instruct-v0.1
+#MODEL=lmsys/vicuna-13b-v1.3
+MODEL=microsoft/phi-2
+HOST=`hostname`
+SEED=1234
+
+. env.sh
+echo Serving $MODEL on http://$HOST:$PORT
+python -m vllm.entrypoints.api_server --model $MODEL --trust-remote-code --host 0.0.0.0 --port $PORT --seed $SEED