feat: implement inference server by using vllm

[zhuangqh]

Reason for Change:

Requirements

• added unit tests and e2e tests (if applicable).

Issue Fixed:

Notes for Reviewers:

[zhuangqh]

testing code

from openai import OpenAI

# Modify OpenAI's API key and API base to use vLLM's API server.
openai_api_key = "EMPTY"
openai_api_base = "http://localhost:5000/v1"

client = OpenAI(
    # defaults to os.environ.get("OPENAI_API_KEY")
    api_key=openai_api_key,
    base_url=openai_api_base,
)

models = client.models.list()
model = models.data[0].id

completion = client.chat.completions.create(
    model=model,
    messages=[
        {"role": "system", "content": "You are a helpful assistant."},
        {
            "role": "user",
            "content": "What is kubernetes?"
        }
    ],
    n=2,
)

print(completion.choices[0].message)

[zhuangqh]

link #608

[ishaansehgal99]

sweeet