Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add support for NVIDIA inference for ElizaOS #2512

Merged
merged 6 commits into from
Jan 19, 2025
Merged

feat: add support for NVIDIA inference for ElizaOS #2512

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
4bb39ea
Adding support for NVIDIA inference for ElizaOS
AIFlowML Jan 19, 2025
50b1752
removed wrong image generation
AIFlowML Jan 19, 2025
f18eeb9
Fixed wrong indentation
AIFlowML Jan 19, 2025
0e71891
resolving conflicts
AIFlowML Jan 19, 2025
a5f3a31
Merge branch 'develop' into feat-nvidia-inference-modelprovider
wtfsayo Jan 19, 2025
0fc7c31
add missing changes
wtfsayo Jan 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions agent/src/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,6 @@ import { LensAgentClient } from "@elizaos/client-lens";
import { SlackClientInterface } from "@elizaos/client-slack";
import { TelegramClientInterface } from "@elizaos/client-telegram";
import { TwitterClientInterface } from "@elizaos/client-twitter";
import { FarcasterClientInterface } from "@elizaos/client-farcaster";
import { DirectClient } from "@elizaos/client-direct";
wtfsayo marked this conversation as resolved.
Show resolved Hide resolved
// import { ReclaimAdapter } from "@elizaos/plugin-reclaim";
import { PrimusAdapter } from "@elizaos/plugin-primus";
import { elizaCodeinPlugin, onchainJson } from "@elizaos/plugin-iq6900";
Expand Down Expand Up @@ -546,15 +544,17 @@ export function getTokenForProvider(
character.settings?.secrets?.HYPERBOLIC_API_KEY ||
settings.HYPERBOLIC_API_KEY
);

case ModelProviderName.VENICE:
return (
character.settings?.secrets?.VENICE_API_KEY ||
settings.VENICE_API_KEY
);
case ModelProviderName.ATOMA:
// Add Nvidia
case ModelProviderName.NVIDIA:
return (
character.settings?.secrets?.ATOMASDK_BEARER_AUTH ||
settings.ATOMASDK_BEARER_AUTH
wtfsayo marked this conversation as resolved.
Show resolved Hide resolved
character.settings?.secrets?.NVIDIA_API_KEY ||
settings.NVIDIA_API_KEY
);
case ModelProviderName.AKASH_CHAT_API:
return (
Expand Down Expand Up @@ -918,6 +918,7 @@ export async function createAgent(
getSecret(character, "FAL_API_KEY") ||
getSecret(character, "OPENAI_API_KEY") ||
getSecret(character, "VENICE_API_KEY") ||
getSecret(character, "NVIDIA_API_KEY") ||
getSecret(character, "NINETEEN_AI_API_KEY") ||
getSecret(character, "HEURIST_API_KEY") ||
getSecret(character, "LIVEPEER_GATEWAY_URL")
Expand Down
30 changes: 28 additions & 2 deletions packages/core/src/generation.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1157,6 +1157,32 @@ export async function generateText({
break;
}

case ModelProviderName.NVIDIA: {
elizaLogger.debug("Initializing NVIDIA model.");
const nvidia = createOpenAI({
apiKey: apiKey,
baseURL: endpoint,
});

const { text: nvidiaResponse } = await aiGenerateText({
model: nvidia.languageModel(model),
prompt: context,
system:
runtime.character.system ??
settings.SYSTEM_PROMPT ??
undefined,
tools: tools,
onStepFinish: onStepFinish,
temperature: temperature,
maxSteps: maxSteps,
maxTokens: max_response_length,
});

response = nvidiaResponse;
elizaLogger.debug("Received response from NVIDIA model.");
break;
}

case ModelProviderName.DEEPSEEK: {
elizaLogger.debug("Initializing Deepseek model.");
const serverUrl = models[provider].endpoint;
Expand Down Expand Up @@ -1615,8 +1641,8 @@ export const generateImage = async (
return runtime.getSetting("FAL_API_KEY");
case ModelProviderName.OPENAI:
return runtime.getSetting("OPENAI_API_KEY");
case ModelProviderName.VENICE:
return runtime.getSetting("VENICE_API_KEY");
case ModelProviderName.VENICE:
return runtime.getSetting("VENICE_API_KEY");
case ModelProviderName.LIVEPEER:
return runtime.getSetting("LIVEPEER_GATEWAY_URL");
default:
Expand Down
26 changes: 26 additions & 0 deletions packages/core/src/models.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -862,6 +862,32 @@ export const models: Models = {
},
},
},
[ModelProviderName.NVIDIA]: {
endpoint: "https://integrate.api.nvidia.com/v1",
model: {
[ModelClass.SMALL]: {
name: settings.SMALL_NVIDIA_MODEL || "meta/llama-3.2-3b-instruct",
stop: [],
maxInputTokens: 128000,
maxOutputTokens: 8192,
temperature: 0.6,
},
[ModelClass.MEDIUM]: {
name: settings.MEDIUM_NVIDIA_MODEL || "meta/llama-3.3-70b-instruct",
stop: [],
maxInputTokens: 128000,
maxOutputTokens: 8192,
temperature: 0.6,
},
[ModelClass.LARGE]: {
name: settings.LARGE_NVIDIA_MODEL || "meta/llama-3.1-405b-instruct",
stop: [],
maxInputTokens: 128000,
maxOutputTokens: 8192,
temperature: 0.6,
},
},
},
[ModelProviderName.NINETEEN_AI]: {
endpoint: "https://api.nineteen.ai/v1",
model: {
Expand Down
2 changes: 2 additions & 0 deletions packages/core/src/types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,7 @@ export type Models = {
[ModelProviderName.NANOGPT]: Model;
[ModelProviderName.HYPERBOLIC]: Model;
[ModelProviderName.VENICE]: Model;
[ModelProviderName.NVIDIA]: Model;
[ModelProviderName.NINETEEN_AI]: Model;
[ModelProviderName.AKASH_CHAT_API]: Model;
[ModelProviderName.LIVEPEER]: Model;
Expand Down Expand Up @@ -259,6 +260,7 @@ export enum ModelProviderName {
NANOGPT = "nanogpt",
HYPERBOLIC = "hyperbolic",
VENICE = "venice",
NVIDIA = "nvidia",
NINETEEN_AI = "nineteen_ai",
AKASH_CHAT_API = "akash_chat_api",
LIVEPEER = "livepeer",
Expand Down
Loading