feat: more models, make decisions a wrapper for system one
This commit is contained in:
@@ -1,25 +1,16 @@
|
||||
use crate::CloudflareCredentials;
|
||||
use crate::openai;
|
||||
use crate::server::AppState;
|
||||
use crate::server::Provider;
|
||||
use crate::server::Upstream;
|
||||
use crate::server::take_random_by_swapping;
|
||||
use axum::Json;
|
||||
use axum::extract::State;
|
||||
use axum::http::StatusCode;
|
||||
use axum::response::IntoResponse;
|
||||
use axum::response::Response;
|
||||
use axum_extra::routing::TypedPath;
|
||||
use ecow::EcoString;
|
||||
use headers::Header;
|
||||
use headers::authorization::Bearer;
|
||||
use mitsein::vec1::Vec1;
|
||||
use reqwest::Certificate;
|
||||
use secrecy::ExposeSecret;
|
||||
use serde::Deserialize;
|
||||
use serde_with::DeserializeFromStr;
|
||||
use serde_with::SerializeDisplay;
|
||||
use std::collections::VecDeque;
|
||||
use std::collections::HashMap;
|
||||
use std::hash::RandomState;
|
||||
use strum::EnumString;
|
||||
use strum::VariantArray;
|
||||
|
||||
@@ -31,20 +22,12 @@ pub struct V1DecisionsPath;
|
||||
Clone, Debug, EnumString, DeserializeFromStr, strum::Display, SerializeDisplay, VariantArray,
|
||||
)]
|
||||
#[strum(serialize_all = "lowercase")]
|
||||
pub enum Shortcut {}
|
||||
|
||||
impl Shortcut {
|
||||
fn suggestions(&self) -> Vec<(Provider, EcoString)> {
|
||||
use Provider as P;
|
||||
use Shortcut as S;
|
||||
|
||||
match self {
|
||||
_ => unreachable!()
|
||||
}
|
||||
}
|
||||
pub enum Shortcut {
|
||||
S,
|
||||
M,
|
||||
}
|
||||
|
||||
pub struct DecisionsResponse(openai::embeddings::CreateEmbeddingResponse<String>);
|
||||
pub struct DecisionsResponse(openai::decisions::DecisionResponse<String>);
|
||||
|
||||
impl IntoResponse for DecisionsResponse {
|
||||
fn into_response(self) -> Response {
|
||||
@@ -66,95 +49,175 @@ impl IntoResponse for DecisionsError {
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(state, request))]
|
||||
#[tracing::instrument(skip(app_state, request))]
|
||||
pub async fn create_decision(
|
||||
_: V1DecisionsPath,
|
||||
State(state): State<AppState>,
|
||||
State(app_state): State<AppState>,
|
||||
Json(request): Json<openai::decisions::DecisionRequest<Shortcut>>,
|
||||
) -> Result<DecisionsResponse, DecisionsError> {
|
||||
let shortcut = request.model.clone();
|
||||
let mut model_upstreams = VecDeque::new();
|
||||
use super::system_one;
|
||||
|
||||
for (provider, model) in shortcut.suggestions() {
|
||||
let upstreams: Vec<Upstream> = match provider {
|
||||
Provider::CloudflareWorkersAi => unreachable!(),
|
||||
Provider::GoogleAiStudio => unreachable!(),
|
||||
Provider::Groq => unreachable!(),
|
||||
Provider::HuggingFace => unreachable!(),
|
||||
Provider::LiquidAi => unreachable!(),
|
||||
Provider::Mistral => unreachable!(),
|
||||
Provider::OpenRouter => unreachable!(),
|
||||
Provider::ZAi => unreachable!(),
|
||||
};
|
||||
let openai::decisions::DecisionRequest {
|
||||
model,
|
||||
input,
|
||||
questions,
|
||||
} = request;
|
||||
|
||||
if let Ok(upstreams) = Vec1::try_from(upstreams) {
|
||||
model_upstreams.push_back((model, upstreams));
|
||||
}
|
||||
}
|
||||
let model = match model {
|
||||
Shortcut::S => system_one::Shortcut::S,
|
||||
Shortcut::M => system_one::Shortcut::M,
|
||||
};
|
||||
|
||||
while let Some((model, upstreams)) = model_upstreams.pop_front() {
|
||||
let (upstream, upstreams) = take_random_by_swapping(upstreams, &mut rand::rng());
|
||||
if let Some(upstreams) = upstreams {
|
||||
model_upstreams.push_back((model.clone(), upstreams));
|
||||
}
|
||||
let (state, images, files) = match input {
|
||||
openai::decisions::DecisionInput::Messages(messages) => {
|
||||
let mut images = vec![];
|
||||
let mut texts = vec![];
|
||||
|
||||
tracing::info!(?model, ?upstream);
|
||||
|
||||
let openai::decisions::DecisionRequest { model: _shortcut, input, questions } = request.clone();
|
||||
|
||||
let request = openai::decisions::DecisionRequest { model, input, questions };
|
||||
|
||||
let client = reqwest::Client::builder()
|
||||
.tls_certs_only(
|
||||
webpki_root_certs::TLS_SERVER_ROOT_CERTS
|
||||
.iter()
|
||||
.filter_map(|cert| Certificate::from_der(cert).ok()),
|
||||
)
|
||||
.build()
|
||||
.expect("TODO");
|
||||
|
||||
let mut upstream_request = client.post(upstream.url).json(&request);
|
||||
|
||||
if let Some(api_key) = upstream.api_key {
|
||||
let authorization =
|
||||
headers::Authorization::bearer(api_key.expose_secret()).expect("TODO");
|
||||
upstream_request = upstream_request.header(headers::Authorization::<Bearer>::name(), {
|
||||
let mut vec = vec![];
|
||||
|
||||
authorization.encode(&mut vec);
|
||||
|
||||
let [one] = vec.try_into().expect("TODO");
|
||||
|
||||
one
|
||||
});
|
||||
}
|
||||
|
||||
let upstream_response = match upstream_request.send().await {
|
||||
Ok(upstream_response) => upstream_response,
|
||||
Err(upstream_error) => {
|
||||
tracing::error!(?upstream_error);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
if let Err(upstream_error) = upstream_response.error_for_status_ref() {
|
||||
tracing::error!(?upstream_error);
|
||||
match upstream_response.text().await {
|
||||
Ok(body) => tracing::error!(?body),
|
||||
Err(body_error) => tracing::error!(?body_error),
|
||||
for openai::decisions::DecisionInputMessage { role, content } in messages {
|
||||
match content {
|
||||
openai::decisions::DecisionInputContent::Text(text) => texts.push(text),
|
||||
openai::decisions::DecisionInputContent::Parts(decision_input_parts) => {
|
||||
for decision_input_part in decision_input_parts {
|
||||
match decision_input_part {
|
||||
openai::decisions::DecisionInputPart::Image { image_url, detail } => images.push(image_url),
|
||||
openai::decisions::DecisionInputPart::Text { text } => texts.push(text),
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
};
|
||||
continue;
|
||||
};
|
||||
|
||||
let response = match upstream_response.json().await {
|
||||
Ok(response) => response,
|
||||
Err(json_error) => {
|
||||
tracing::error!(?json_error);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let text = texts.join("\n\n");
|
||||
|
||||
return Ok(DecisionsResponse(response));
|
||||
}
|
||||
Err(DecisionsError::NoneSucceeded)
|
||||
let files = images.clone();
|
||||
|
||||
(openai::system_one::State::Text(text), Some(images), Some(files))
|
||||
}
|
||||
openai::decisions::DecisionInput::Text(text) => (openai::system_one::State::Text(text), None, None),
|
||||
};
|
||||
|
||||
let questions = questions.into_iter().map(|question| match question {
|
||||
openai::decisions::DecisionQuestion::Predicate(predicate_question) => todo!(),
|
||||
openai::decisions::DecisionQuestion::Choice(choice_question) => todo!(),
|
||||
openai::decisions::DecisionQuestion::Score(score_question) => todo!(),
|
||||
} ).collect();
|
||||
|
||||
let request = openai::system_one::SystemOneRequest {
|
||||
model,
|
||||
state,
|
||||
files,
|
||||
images,
|
||||
questions,
|
||||
};
|
||||
|
||||
system_one::create_system_one(system_one::V1SystemOnePath, State(app_state), Json(request))
|
||||
.await
|
||||
.map(
|
||||
|system_one::SystemOneResponse(openai::system_one::SystemOneResponse {
|
||||
model,
|
||||
answers,
|
||||
usage,
|
||||
})| {
|
||||
let answers = answers
|
||||
.into_iter()
|
||||
.map(|(id, answer)| match answer {
|
||||
openai::system_one::Answer::Noul(openai::system_one::NoulAnswer {
|
||||
noul,
|
||||
}) => openai::decisions::DecisionAnswer::Predicate(
|
||||
openai::decisions::PredicateAnswer {
|
||||
name: Some(id),
|
||||
probability: noul,
|
||||
},
|
||||
),
|
||||
openai::system_one::Answer::Choice(openai::system_one::ChoiceAnswer {
|
||||
choice,
|
||||
probabilities,
|
||||
confidence,
|
||||
}) => openai::decisions::DecisionAnswer::Choice(
|
||||
openai::decisions::ChoiceAnswer {
|
||||
name: Some(id),
|
||||
choice: openai::decisions::Choice::String(choice),
|
||||
confidence,
|
||||
probabilities: probabilities
|
||||
.into_iter()
|
||||
.map(|(choice_id, probability)| {
|
||||
openai::decisions::ChoiceProbability {
|
||||
value: openai::decisions::Choice::String(choice_id),
|
||||
probability,
|
||||
}
|
||||
})
|
||||
.collect(),
|
||||
},
|
||||
),
|
||||
openai::system_one::Answer::Score(openai::system_one::ScoreAnswer {
|
||||
score,
|
||||
legend,
|
||||
probabilities,
|
||||
confidence,
|
||||
}) => openai::decisions::DecisionAnswer::Score(
|
||||
openai::decisions::ScoreAnswer {
|
||||
name: Some(id),
|
||||
score,
|
||||
confidence,
|
||||
probabilities: {
|
||||
let reverse_legend = HashMap::<_, _, RandomState>::from_iter(
|
||||
legend.into_iter().map(|(index, value)| {
|
||||
(
|
||||
value,
|
||||
index
|
||||
.parse::<i32>()
|
||||
.expect("this should be an integer, I believe"),
|
||||
)
|
||||
}),
|
||||
);
|
||||
|
||||
probabilities
|
||||
.into_iter()
|
||||
.map(|(choice_id, probability)| {
|
||||
openai::decisions::ScoreProbability {
|
||||
value: reverse_legend
|
||||
.get(&choice_id)
|
||||
.expect("it should exist, I think")
|
||||
.clone(),
|
||||
label: choice_id,
|
||||
probability,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
},
|
||||
},
|
||||
),
|
||||
})
|
||||
.collect();
|
||||
|
||||
let openai::system_one::Usage {
|
||||
input_tokens,
|
||||
output_tokens,
|
||||
} = usage;
|
||||
let input_tokens = input_tokens as i32;
|
||||
let output_tokens = output_tokens as i32;
|
||||
let total_tokens = input_tokens + output_tokens;
|
||||
let input_tokens_details = openai::decisions::TokenUsageDetails {
|
||||
cache_write_tokens: 0,
|
||||
cached_tokens: 0,
|
||||
};
|
||||
let output_tokens_details = openai::decisions::OutputTokenDetails {
|
||||
reasoning_tokens: 0,
|
||||
};
|
||||
let usage = openai::decisions::TokenUsage {
|
||||
input_tokens,
|
||||
input_tokens_details,
|
||||
output_tokens,
|
||||
output_tokens_details,
|
||||
total_tokens,
|
||||
};
|
||||
|
||||
DecisionsResponse(openai::decisions::DecisionResponse {
|
||||
model,
|
||||
answers,
|
||||
usage,
|
||||
})
|
||||
},
|
||||
)
|
||||
.map_err(|_| DecisionsError::NoneSucceeded)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user