jacob/katniss-ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: novita, nvidia nim, vercel ai gateway

Browse Source
This commit is contained in:
Jacob
2026-10-13 23:37:24 -04:00
parent a0d95d52e4
commit f4f657c219
6 changed files with 918 additions and 794 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/lib.rs
View File

@@ -54,9 +54,18 @@ pub struct AppArgs {
#[arg(long)]
pub mistral_api_key: Vec<SecretString>,
#[arg(long)]
pub novita_ai_api_key: Vec<SecretString>,
#[arg(long)]
pub nvidia_nim_api_key: Vec<SecretString>,
#[arg(long)]
pub openrouter_api_key: Vec<SecretString>,
#[arg(long)]
pub vercel_ai_gateway_api_key: Vec<SecretString>,
#[arg(long)]
pub z_ai_api_key: Vec<SecretString>,
}

96
src/server/chat/completions.rs
View File

@@ -94,26 +94,42 @@ impl Shortcut {
match self {
S::XS => vec![
(P::GoogleAiStudio, "gemini-3.5-flash-lite".into()),
(P::NovitaAi, "apodex/apodex-1.1-mini".into()),
(P::OpenRouter, "apodex/apodex-1.1-mini:free".into()),
(P::VercelAiGateway, "openai/gpt-5.4-nano".into()),
(P::HuggingFace, "Qwen/Qwen3.6-35B-A3B".into()),
(P::VercelAiGateway, "liquid/lfm2.5-24b-a2b".into()),
(P::GoogleAiStudio, "gemma-4-26b-a4b-it".into()),
(P::OpenRouter, "google/gemma-4-26b-a4b-it:free".into()),
(P::HuggingFace, "google/gemma-4-26B-A4B-it".into()),
(P::NvidiaNim, "poolside/laguna-xs-2.1".into()),
(P::OpenRouter, "poolside/laguna-xs-2.1:free".into()),
(P::ZAi, "glm-4.7-flash".into()),
(P::NvidiaNim, "google/diffusiongemma-26b-a4b-it".into()),
(P::OpenRouter, "cohere/north-mini-code:free".into()),
(P::NvidiaNim, "nvidia/nemotron-3.5-lightning-30b-a3b".into()),
(
P::VercelAiGateway,
"nvidia/nemotron-3.5-lightning-free".into(),
),
(P::OpenRouter, "nvidia/nemotron-3.5-lightning:free".into()),
(P::ZAi, "glm-4.6v-flash".into()),
(P::ZAi, "glm-4.5-flash".into()),
(P::OpenRouter, "poolside/laguna-xs.2:free".into()),
(
P::NvidiaNim,
"nvidia/nemotron-3-nano-omni-30b-a3b-reasoning".into(),
),
(
P::OpenRouter,
"nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free".into(),
),
(P::NvidiaNim, "openai/gpt-oss-20b".into()),
(P::CloudflareWorkersAi, "@cf/openai/gpt-oss-20b".into()),
(P::OpenRouter, "openai/gpt-oss-20b:free".into()),
(P::Groq, "openai/gpt-oss-20b".into()),
(P::HuggingFace, "openai/gpt-oss-20b".into()),
(P::NvidiaNim, "meta/llama-3.2-11b-vision-instruct".into()),
(P::OpenRouter, "liquid/lfm-2.5-2.6b:free".into()),
(P::Groq, "groq/compound-mini".into()),
(P::OpenRouter, "openrouter/free".into()),
@@ -124,8 +140,10 @@ impl Shortcut {
(P::OpenRouter, "qwen/qwen3.8-27b:free".into()),
(P::Groq, "qwen/qwen3.8-27b".into()),
(P::HuggingFace, "Qwen/Qwen3.8-27B".into()),
(P::NvidiaNim, "meta/muse-glimmer-30b".into()),
(P::HuggingFace, "meta-models/Muse-Glimmer-30B".into()),
(P::GoogleAiStudio, "gemma-4-31b-it".into()),
(P::NvidiaNim, "google/gemma-4-31b-it".into()),
(P::OpenRouter, "google/gemma-4-31b-it:free".into()),
(P::HuggingFace, "google/gemma-4-31B-it".into()),
(P::Mistral, "mistral-small-2603".into()),
@@ -135,52 +153,84 @@ impl Shortcut {
S::M => vec![
(P::GoogleAiStudio, "gemini-3.8-flash".into()),
(P::GoogleAiStudio, "gemini-3.7-flash".into()),
(P::VercelAiGateway, "alibaba/qwen3.7-flash".into()),
(P::VercelAiGateway, "alibaba/qwen3.8-omni-flash".into()),
(P::Mistral, "mistral-medium-3.5".into()),
(P::NovitaAi, "inclusionai/ling-3.0-flash-vl".into()),
(P::OpenRouter, "inclusionai/ling-3.0-flash-vl:free".into()),
(P::VercelAiGateway, "inclusionai/ling-3.0-flash-vl".into()),
(P::VercelAiGateway, "poolside/laguna-s-2.1-free".into()),
(P::OpenRouter, "poolside/laguna-s-2.1:free".into()),
(P::VercelAiGateway, "stepfun/step-3.7-flash".into()),
(P::NovitaAi, "inclusionai/ling-3.0-flash-fin".into()),
(P::OpenRouter, "inclusionai/ling-3.0-flash-fin:free".into()),
(P::VercelAiGateway, "inclusionai/ling-3.0-flash-fin".into()),
(P::NovitaAi, "inclusionai/ling-3.0-flash-sante".into()),
(
P::OpenRouter,
"inclusionai/ling-3.0-flash-sante:free".into(),
),
(
P::VercelAiGateway,
"inclusionai/ling-3.0-flash-sante".into(),
),
(P::VercelAiGateway, "mistral/mistral-large-3".into()),
(P::NvidiaNim, "nvidia/nemotron-3-super-120b-a12b".into()),
(
P::OpenRouter,
"nvidia/nemotron-3-super-120b-a12b:free".into(),
),
(P::Groq, "openai/gpt-oss-120b".into()),
(P::CloudflareWorkersAi, "@cf/openai/gpt-oss-120b".into()),
(P::OpenRouter, "openai/gpt-oss-120b:free".into()),
(P::Groq, "openai/gpt-oss-120b".into()),
(P::HuggingFace, "openai/gpt-oss-120b".into()),
(P::NvidiaNim, "meta/llama-3.2-90b-vision-instruct".into()),
],
S::L => vec![
(P::CloudflareWorkersAi, "@cf/zai-org/glm-5.3-flash".into()),
(P::NvidiaNim, "z-ai/glm-5-3-flash".into()),
(P::HuggingFace, "zai-org/GLM-5.3-Flash".into()),
(P::HuggingFace, "zai-org/GLM-5.3-Flash-BF16".into()),
(P::VercelAiGateway, "openai/gpt-5.2".into()),
(P::NvidiaNim, "deepseek-ai/deepseek-v4.1-flash".into()),
(P::HuggingFace, "deepseek-ai/DeepSeek-V4.1-Flash".into()),
(P::VercelAiGateway, "xiaomi/mimo-v2.6-flash".into()),
(P::HuggingFace, "XiaomiMiMo/MiMo-V2.6-Flash-RL".into()),
(P::VercelAiGateway, "inclusionai/ling-3.1-flash".into()),
(P::OpenRouter, "inclusionai/ling-3.1-flash".into()),
(P::VercelAiGateway, "xiaomi/mimo-v2.5".into()),
(P::OpenRouter, "dots-studio/dots-3-note-preview:free".into()),
(P::HuggingFace, "MiniMaxAI/MiniMax-M3".into()),
(P::OpenRouter, "thinkingmachines/inkling-small:free".into()),
(P::HuggingFace, "thinkingmachines/Inkling-Small".into()),
(P::NvidiaNim, "nvidia/nemotron-3-ultra-550b-a55b".into()),
],
S::XL => vec![
(P::NvidiaNim, "z-ai/glm-5-3".into()),
(P::HuggingFace, "zai-org/GLM-5.3".into()),
(P::CloudflareWorkersAi, "@cf/zai-org/glm-5.3".into()),
(P::Mistral, "zai-glm-5-3".into()),
(P::HuggingFace, "tencent/Hy4-preview".into()),
(P::HuggingFace, "XiaomiMiMo/MiMo-V2.6-Pro-RL".into()),
(P::Mistral, "mistral-large-4".into()),
(P::NvidiaNim, "moonshotai/kimi-k3".into()),
(P::HuggingFace, "moonshotai/Kimi-K3".into()),
(P::HuggingFace, "Qwen/Qwen3.8-2.4T-A95B".into()),
(P::VercelAiGateway, "xiaomi/mimo-v2.5-pro".into()),
(P::VercelAiGateway, "tencent/hy3".into()),
(P::OpenRouter, "thinkingmachines/inkling:free".into()),
(P::HuggingFace, "thinkingmachines/Inkling".into()),
(P::VercelAiGateway, "thinkingmachines/inkling".into()),
(P::GoogleAiStudio, "gemini-3.1-pro-preview".into()),
(P::VercelAiGateway, "moonshotai/kimi-k2.7-code".into()),
],
S::Experimental => vec![(P::OpenRouter, "space-bunny-alpha".into())],
S::Experimental => vec![
(P::OpenRouter, "space-bunny-alpha".into()),
(P::VercelAiGateway, "stealth/glyph-cluster".into()),
],
}
}
}
@@ -329,6 +379,30 @@ pub async fn create_chat_completion(
.build()
})
.collect(),
Provider::NvidiaNim => state
.app_args
.nvidia_nim_api_key
.iter()
.cloned()
.map(|api_key| {
Upstream::builder()
.url("https://integrate.api.nvidia.com/v1/chat/completions".parse().expect("TODO"))
.api_key(api_key)
.build()
})
.collect(),
Provider::NovitaAi => state
.app_args
.novita_ai_api_key
.iter()
.cloned()
.map(|api_key| {
Upstream::builder()
.url("https://api.novita.ai/openai/v1/chat/completions".parse().expect("TODO"))
.api_key(api_key)
.build()
})
.collect(),
Provider::OpenRouter => state
.app_args
.openrouter_api_key
@@ -341,6 +415,18 @@ pub async fn create_chat_completion(
.build()
})
.collect(),
Provider::VercelAiGateway => state
.app_args
.vercel_ai_gateway_api_key
.iter()
.cloned()
.map(|api_key| {
Upstream::builder()
.url("https://ai-gateway.vercel.sh/v1/chat/completions".parse().expect("TODO"))
.api_key(api_key)
.build()
})
.collect(),
Provider::ZAi => state
.app_args
.z_ai_api_key
@@ -459,9 +545,7 @@ pub async fn create_chat_completion(
.build()
.expect("TODO");
let mut upstream_request = client
.post(upstream.url)
.json(&request);
let mut upstream_request = client.post(upstream.url).json(&request);
if let Some(api_key) = upstream.api_key {
let authorization =

9
src/server/decisions.rs
View File

@@ -224,8 +224,13 @@ pub async fn create_decision(
.into_iter()
.map(|(choice_id, probability)| {
openai::decisions::ScoreProbability {
value: choice_id.parse().expect("it should be a number, I think"),
label: legend.get(&choice_id).expect("it should exist, I think").clone(),
value: choice_id
.parse()
.expect("it should be a number, I think"),
label: legend
.get(&choice_id)
.expect("it should exist, I think")
.clone(),
probability,
}
})

35
src/server/embeddings.rs
View File

@@ -51,18 +51,15 @@ impl Shortcut {
use Shortcut as S;
match self {
S::CodestralEmbed2505 => vec![
(P::Mistral, "codestral-embed-2505".into())
],
S::CodestralEmbed2505 => vec![(P::Mistral, "codestral-embed-2505".into())],
S::Lfm2_5Embedding350M => {
vec![(P::OpenRouter, "liquid/lfm-2.5-embedding-350m:free".into())]
},
S::MistralEmbed2312 => vec![
(P::Mistral, "mistral-embed-2312".into())
],
S::Nemotron3Embed1B => {
vec![(P::OpenRouter, "nvidia/nemotron-3-embed-1b:free".into())]
}
S::MistralEmbed2312 => vec![(P::Mistral, "mistral-embed-2312".into())],
S::Nemotron3Embed1B => vec![
(P::NvidiaNim, "nvidia/nemotron-3-embed-1b".into()),
(P::OpenRouter, "nvidia/nemotron-3-embed-1b:free".into()),
],
}
}
}
@@ -175,6 +172,19 @@ pub async fn create_embeddings(
.build()
})
.collect(),
Provider::NvidiaNim => state
.app_args
.nvidia_nim_api_key
.iter()
.cloned()
.map(|api_key| {
Upstream::builder()
.url("https://integrate.api.nvidia.com/v1/embeddings".parse().expect("TODO"))
.api_key(api_key)
.build()
})
.collect(),
Provider::NovitaAi => unreachable!(),
Provider::OpenRouter => state
.app_args
.openrouter_api_key
@@ -187,6 +197,7 @@ pub async fn create_embeddings(
.build()
})
.collect(),
Provider::VercelAiGateway => unreachable!(),
Provider::ZAi => unreachable!(),
};
@@ -219,7 +230,6 @@ pub async fn create_embeddings(
user,
};
let client = reqwest::Client::builder()
.tls_certs_only(
webpki_root_certs::TLS_SERVER_ROOT_CERTS
@@ -229,9 +239,7 @@ pub async fn create_embeddings(
.build()
.expect("TODO");
let mut upstream_request = client
.post(upstream.url)
.json(&request);
let mut upstream_request = client.post(upstream.url).json(&request);
if let Some(api_key) = upstream.api_key {
let authorization =
@@ -264,7 +272,6 @@ pub async fn create_embeddings(
continue;
};
let response = match upstream_response.json().await {
Ok(response) => response,
Err(json_error) => {

3
src/server/mod.rs
View File

@@ -28,7 +28,10 @@ enum Provider {
HuggingFace,
LiquidAi,
Mistral,
NovitaAi,
NvidiaNim,
OpenRouter,
VercelAiGateway,
ZAi,
}

16
src/server/system_one.rs
View File

@@ -49,6 +49,8 @@ impl Shortcut {
(P::CloudflareWorkersAi, "clef-flash".into()),
(P::OpenRouter, "respan/span-01-lite:free".into()),
(P::OpenRouter, "respan/span-01-lite".into()),
(P::VercelAiGateway, "convaiinnovations/laya-free".into()),
(P::VercelAiGateway, "convaiinnovations/laya".into()),
]
}
S::M => {
@@ -132,6 +134,8 @@ pub async fn create_system_one(
})
.collect(),
Provider::Mistral => unreachable!(),
Provider::NovitaAi => unreachable!(),
Provider::NvidiaNim => unreachable!(),
Provider::OpenRouter => state
.app_args
.openrouter_api_key
@@ -144,6 +148,18 @@ pub async fn create_system_one(
.build()
})
.collect(),
Provider::VercelAiGateway => state
.app_args
.vercel_ai_gateway_api_key
.iter()
.cloned()
.map(|api_key| {
Upstream::builder()
.url("https://ai-gateway.vercel.sh/typesafe/v1/systemone".parse().expect("TODO"))
.api_key(api_key)
.build()
})
.collect(),
Provider::ZAi => unreachable!(),
};