Skip to content

Commit

Permalink
feat: Handle HF Remote API Call Format (#751)
Browse files Browse the repository at this point in the history
**Reason for Change**:
Remote HuggingFace API uses a particular format: 

```
curl 'https://api-inference.huggingface.co/models/HuggingFaceH4/zephyr-7b-beta/v1/chat/completions' \
-H 'Authorization: Bearer hf_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' \
-H 'Content-Type: application/json' \
--data '{
    "model": "HuggingFaceH4/zephyr-7b-beta",
    "messages": [
		{
			"role": "user",
			"content": "What is the capital of France?"
		}
	],
    "max_tokens": 500,
    "stream": false
}'
```

This needs to be adhered to in the code. 

Reference:
https://huggingface.co/HuggingFaceH4/zephyr-7b-beta?inference_api=true

Adding this code ensures we continue to support OAI, HF Remote and
Custom URL as our LLM backend
  • Loading branch information
ishaansehgal99 authored Dec 4, 2024
1 parent 0f9a11d commit e0f28f0
Showing 1 changed file with 9 additions and 0 deletions.
9 changes: 9 additions & 0 deletions presets/ragengine/inference/inference.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ def complete(self, prompt: str, **kwargs) -> CompletionResponse:
try:
if "openai" in INFERENCE_URL:
return self._openai_complete(prompt, **kwargs, **self.params)
elif "api-inference.huggingface" in INFERENCE_URL:
return self._huggingface_remote_complete(prompt, **kwargs, **self.params)
else:
return self._custom_api_complete(prompt, **kwargs, **self.params)
finally:
Expand All @@ -39,6 +41,13 @@ def _openai_complete(self, prompt: str, **kwargs: Any) -> CompletionResponse:
)
return llm.complete(prompt)

def _huggingface_remote_complete(self, prompt: str, **kwargs: Any) -> CompletionResponse:
headers = {"Authorization": f"Bearer {INFERENCE_ACCESS_SECRET}"}
data = {"messages": [{"role": "user", "content": prompt}]}
response = requests.post(INFERENCE_URL, json=data, headers=headers)
response_data = response.json()
return CompletionResponse(text=str(response_data))

def _custom_api_complete(self, prompt: str, **kwargs: Any) -> CompletionResponse:
headers = {"Authorization": f"Bearer {INFERENCE_ACCESS_SECRET}"}
data = {"prompt": prompt, **kwargs}
Expand Down

0 comments on commit e0f28f0

Please sign in to comment.