From a3edbe0d9f1304bec10712bfb6a2dec893a6efaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81d=C3=A1m=20Ruman?= <469068@muni.cz> Date: Wed, 27 Nov 2024 10:00:55 +0100 Subject: [PATCH 1/5] add batch querying of services via pks --- .gitlab-ci.yml | 4 ++-- Cargo.toml | 2 +- Readme.md | 4 ++-- proc_macros/src/lib.rs | 1 - shared/src/lib.rs | 1 + shared/src/queries.rs | 31 +++++++++++++++++++++++++++++++ src/lib.rs | 11 +++++++++-- {api => web_api}/Cargo.toml | 2 +- {api => web_api}/src/main.rs | 0 9 files changed, 47 insertions(+), 9 deletions(-) rename {api => web_api}/Cargo.toml (97%) rename {api => web_api}/src/main.rs (100%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e65da00..f28454e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -37,7 +37,7 @@ test-job: variables: RELEASE_DIR: "bin/" - API: "${RELEASE_DIR}api" + API: "${RELEASE_DIR}web_api" MIGRATION: "${RELEASE_DIR}migration" POPULATE: "${RELEASE_DIR}populate" @@ -73,7 +73,7 @@ release-job: tag_name: '$CI_COMMIT_SHORT_SHA' assets: links: - - name: 'api' + - name: 'web_api' # Use variables to build a URL to access the artifacts # ${CI_PROJECT_URL} is the repository URL # ${BUILD_JOB_ID} is from the previous job, diff --git a/Cargo.toml b/Cargo.toml index a7352cc..4e28f7c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ publish=false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [workspace] -members = [".", "shared", "migration", "populate", "proc_macros", "api"] +members = [".", "shared", "migration", "populate", "proc_macros", "web_api"] [dependencies] shared = {path = "shared"} diff --git a/Readme.md b/Readme.md index 9d317e1..3fdf68b 100644 --- a/Readme.md +++ b/Readme.md @@ -11,7 +11,7 @@ This projects is the base ground for all `Facts` for **AI-Dojo**. ## Structure - * `api` contains an actix-web REST API for querying the database. + * `web_api` contains an actix-web REST API for querying the database. * `migrations` hold the database migration "scripts" and wrapper. * `populate` contains the logic to fill the databases. * `shared` hold entities used throughout all the tooling, and models for streaming query results. @@ -29,7 +29,7 @@ This projects is the base ground for all `Facts` for **AI-Dojo**. - `cargo build --release --workspace` to build the maintenance & api binaries. - `./target/release/migration<.exe>` to run database migrations. (Only do this on first startup.) - `./target/release/populate<.exe>` to populate the database. See the executables' options. The flag `--real` starts population by crawling online sources and databases (**not yet implemented**). The `--mock <scenario>` option populates the database artificially by the selected scenario. - - `./target/release.api --interface <all/localhost> &` to start the REST API. The interface should be chosen based on where users of the database will reside. + - `./target/release/web_api --interface <all/localhost> &` to start the REST API. The interface should be chosen based on where users of the database will reside. ## For developers diff --git a/proc_macros/src/lib.rs b/proc_macros/src/lib.rs index 20b7ded..cabbe6c 100644 --- a/proc_macros/src/lib.rs +++ b/proc_macros/src/lib.rs @@ -1,7 +1,6 @@ extern crate proc_macro; use proc_macro::TokenStream; - use quote::{format_ident, quote}; struct TypeDuo { diff --git a/shared/src/lib.rs b/shared/src/lib.rs index a3aebc1..6b28b48 100644 --- a/shared/src/lib.rs +++ b/shared/src/lib.rs @@ -8,6 +8,7 @@ pub mod mutations; pub mod queries; mod query_results; +pub use entity::prelude::ApplicabilityMatrixModel; pub use error::GringottsError; pub use query_results::{enabler::Enabler, service::Service}; pub use types::GringottsResult; diff --git a/shared/src/queries.rs b/shared/src/queries.rs index 8db6508..535fb78 100644 --- a/shared/src/queries.rs +++ b/shared/src/queries.rs @@ -53,6 +53,7 @@ impl Queries { impl_get_count!(ActionEnt::action); impl_stream_all_items!(ServiceEnt::ServiceModel::service); + impl_stream_all_items!(EnablerEnt::EnablerModel::enabler); pub async fn get_services_with_tag(tag: &str, conn: Arc<DatabaseConnection>) -> GringottsResult<Vec<ServiceModel>> { @@ -95,6 +96,36 @@ impl Queries { }) } + #[must_use] + pub fn services_by_pks(pks: &[i32], conn: Arc<DatabaseConnection>) -> BoxStream<'static, GringottsResult<Service>> { + let pks_copy = pks.to_vec(); + Box::pin(stream! { + let conn_owned = Arc::clone(&conn); + let mut db_stream = ServiceEnt::find() + .filter(services::Column::Id.is_in(pks_copy)) + .expr(Expr::cust("array_agg(service_tags.tag || '') as tags")) // hacky as hell, but works - must combine it with empty string, otherwise we gert type mismatches + .expr(Expr::cust("array_agg(service_impacts.impact || '') as impacts")) + .select_column(functional_data::Column::AutoElevation) + .select_column(functional_data::Column::ExecutableAccess) + .select_column(functional_data::Column::Ports) + .select_column(functional_data::Column::TargetSoftware) + .join(JoinType::Join, services::Relation::FunctionalData.def()) + .join(JoinType::Join, services::Relation::ServiceTags.def()) + .join(JoinType::Join, services::Relation::ServiceImpacts.def()) + .group_by(services::Column::Id) + .group_by(functional_data::Column::AutoElevation) + .group_by(functional_data::Column::ExecutableAccess) + .group_by(functional_data::Column::Ports) + .group_by(functional_data::Column::TargetSoftware) + .into_model::<Service>() + .stream(conn_owned.as_ref()).await?; + + while let Some(item) = db_stream.try_next().await? { + yield Ok(item) + } + }) + } + pub async fn get_service_for_pagan_by_pk( pk: ServiceHandle, conn: Arc<DatabaseConnection>, diff --git a/src/lib.rs b/src/lib.rs index 1244453..ea8efa8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,10 @@ #![allow(clippy::multiple_crate_versions)] -pub use shared::{entity::sea_orm_active_enums::*, Enabler, GringottsError, GringottsResult, Service, queries::Queries}; - +pub use shared::{ + entity::sea_orm_active_enums::*, + queries::Queries, + ApplicabilityMatrixModel, + Enabler, + GringottsError, + GringottsResult, + Service, +}; diff --git a/api/Cargo.toml b/web_api/Cargo.toml similarity index 97% rename from api/Cargo.toml rename to web_api/Cargo.toml index 2e57ede..522ddcf 100644 --- a/api/Cargo.toml +++ b/web_api/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "api" +name = "web_api" version = "0.1.0" edition = "2021" description = "REST API for the gringotts database." diff --git a/api/src/main.rs b/web_api/src/main.rs similarity index 100% rename from api/src/main.rs rename to web_api/src/main.rs -- GitLab From 65b96f5888cd5d0f29e8e00a7a5a0c3fd79a10f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81d=C3=A1m=20Ruman?= <469068@muni.cz> Date: Wed, 27 Nov 2024 10:09:33 +0100 Subject: [PATCH 2/5] bump seaorm version --- migration/Cargo.toml | 2 +- populate/Cargo.toml | 2 +- shared/Cargo.toml | 2 +- web_api/Cargo.toml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/migration/Cargo.toml b/migration/Cargo.toml index d4b518a..f217618 100644 --- a/migration/Cargo.toml +++ b/migration/Cargo.toml @@ -17,7 +17,7 @@ async-std = { version = "1", features = ["attributes", "tokio1"] } dotenvy = "0.15.7" [dependencies.sea-orm-migration] -version = "0.12.15" +version = "1.1.1" features = [ # Enable at least one `ASYNC_RUNTIME` and `DATABASE_DRIVER` feature if you want to run migration via CLI. # View the list of supported features at https://www.sea-ql.org/SeaORM/docs/install-and-config/database-and-async-runtime. diff --git a/populate/Cargo.toml b/populate/Cargo.toml index e52ebde..e89bebe 100644 --- a/populate/Cargo.toml +++ b/populate/Cargo.toml @@ -15,7 +15,7 @@ publish=false [dependencies] shared = { path = "../shared"} tokio = {version = "1.37.0", features = ["full"]} -sea-orm = {version="0.12.15", features = ["sqlx-postgres", "runtime-tokio-rustls", "macros", "with-json"]} +sea-orm = {version= "1.1.1", features = ["sqlx-postgres", "runtime-tokio-rustls", "macros", "with-json"]} dotenvy = "0.15.7" serde_json = { version = "1.0.116", features = [] } env_logger = "0.11.3" diff --git a/shared/Cargo.toml b/shared/Cargo.toml index 33ee567..5a91d3d 100644 --- a/shared/Cargo.toml +++ b/shared/Cargo.toml @@ -13,7 +13,7 @@ publish=false [dependencies] gringotts_macros = {path = "../proc_macros"} serde = { version = "1.0.197", features = ["derive"] } -sea-orm = { version = "1.0.1", features = ["sqlx-postgres", "macros", "with-json", "runtime-tokio-rustls"] } +sea-orm = { version = "1.1.1", features = ["sqlx-postgres", "macros", "with-json", "runtime-tokio-rustls"] } tokio = { version = "1.37.0", features = ["full"] } serde_json = "1.0.116" async-stream = "0.3.5" diff --git a/web_api/Cargo.toml b/web_api/Cargo.toml index 522ddcf..f4b35e0 100644 --- a/web_api/Cargo.toml +++ b/web_api/Cargo.toml @@ -14,7 +14,7 @@ shared = {path = "../shared"} futures = "0.3.30" clap = { version = "4.5.4", features = ["derive"] } log = "0.4.21" -sea-orm = { version = "0.12.15", features = ["sqlx-postgres", "macros", "with-json", "runtime-tokio-rustls"] } +sea-orm = { version = "1.1.1", features = ["sqlx-postgres", "macros", "with-json", "runtime-tokio-rustls"] } serde = { version = "1.0.197", features = ["derive"] } serde_json = "1.0.116" actix-web = "4.5.1" -- GitLab From 9addae45882bbbcbefb1f3657df7d46a9df36f4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81d=C3=A1m=20Ruman?= <469068@muni.cz> Date: Wed, 27 Nov 2024 10:13:59 +0100 Subject: [PATCH 3/5] change new stringlen type --- .../src/m20220101_000001_create_table_deployment_data.rs | 6 +++--- migration/src/m20220101_000009_create_table_enablers.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/migration/src/m20220101_000001_create_table_deployment_data.rs b/migration/src/m20220101_000001_create_table_deployment_data.rs index ed72659..76204b0 100644 --- a/migration/src/m20220101_000001_create_table_deployment_data.rs +++ b/migration/src/m20220101_000001_create_table_deployment_data.rs @@ -20,9 +20,9 @@ impl MigrationTrait for Migration { .auto_increment() .primary_key(), ) - .col(ColumnDef::new(DeploymentData::Vendor).array(ColumnType::String(Some(128)))) - .col(ColumnDef::new(DeploymentData::Source).array(ColumnType::String(Some(128)))) - .col(ColumnDef::new(DeploymentData::DockerInfo).array(ColumnType::String(Some(128)))) + .col(ColumnDef::new(DeploymentData::Vendor).array(ColumnType::String(StringLen::N(128)))) + .col(ColumnDef::new(DeploymentData::Source).array(ColumnType::String(StringLen::N(128)))) + .col(ColumnDef::new(DeploymentData::DockerInfo).array(ColumnType::String(StringLen::N(128)))) .col(ColumnDef::new(DeploymentData::Service).integer().not_null()) .to_owned(), ) diff --git a/migration/src/m20220101_000009_create_table_enablers.rs b/migration/src/m20220101_000009_create_table_enablers.rs index d6e69cb..cd12ec0 100644 --- a/migration/src/m20220101_000009_create_table_enablers.rs +++ b/migration/src/m20220101_000009_create_table_enablers.rs @@ -34,7 +34,7 @@ impl MigrationTrait for Migration { .col(ColumnDef::new(Enablers::Type).enumeration(EnablerType::Table, EnablerType::iter().skip(1))) .col(ColumnDef::new(Enablers::Details).json()) .col(ColumnDef::new(Enablers::MainCPE).string().not_null()) - .col(ColumnDef::new(Enablers::Config).array(ColumnType::String(Some(128)))) + .col(ColumnDef::new(Enablers::Config).array(ColumnType::String(StringLen::N(128)))) .to_owned(), ) .await -- GitLab From 3d4ca06a848300df8d6e399c14d6dbb31436990c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81d=C3=A1m=20Ruman?= <469068@muni.cz> Date: Wed, 27 Nov 2024 10:15:58 +0100 Subject: [PATCH 4/5] change Query usage --- web_api/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web_api/src/main.rs b/web_api/src/main.rs index bb8243c..1c2d805 100644 --- a/web_api/src/main.rs +++ b/web_api/src/main.rs @@ -52,7 +52,7 @@ async fn service_by_pk(pk_query: web::Query<PkQuery>, app_data: web::Data<Databa #[get("/services_all")] async fn services_all(app_data: web::Data<DatabaseConnection>) -> HttpResponse { - let stream = Queries::stream_all_service(app_data.into_inner().clone()).await; + let stream = Queries{}.stream_all_service(app_data.into_inner().clone()).await; let processed_stream = stream.map(|element: GringottsResult<ServiceModel>| serialize_model(element)); HttpResponse::Ok().content_type("jsonl").streaming(processed_stream) } -- GitLab From 7130dc93a9747a83eac416567089f60aa0942dc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81d=C3=A1m=20Ruman?= <469068@muni.cz> Date: Wed, 27 Nov 2024 10:19:52 +0100 Subject: [PATCH 5/5] clippy repair --- web_api/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web_api/src/main.rs b/web_api/src/main.rs index 1c2d805..16abeba 100644 --- a/web_api/src/main.rs +++ b/web_api/src/main.rs @@ -52,7 +52,7 @@ async fn service_by_pk(pk_query: web::Query<PkQuery>, app_data: web::Data<Databa #[get("/services_all")] async fn services_all(app_data: web::Data<DatabaseConnection>) -> HttpResponse { - let stream = Queries{}.stream_all_service(app_data.into_inner().clone()).await; + let stream = Queries {}.stream_all_service(app_data.into_inner().clone()).await; let processed_stream = stream.map(|element: GringottsResult<ServiceModel>| serialize_model(element)); HttpResponse::Ok().content_type("jsonl").streaming(processed_stream) } -- GitLab