From a3edbe0d9f1304bec10712bfb6a2dec893a6efaf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=81d=C3=A1m=20Ruman?= <469068@muni.cz>
Date: Wed, 27 Nov 2024 10:00:55 +0100
Subject: [PATCH 1/5] add batch querying of services via pks

---
 .gitlab-ci.yml               |  4 ++--
 Cargo.toml                   |  2 +-
 Readme.md                    |  4 ++--
 proc_macros/src/lib.rs       |  1 -
 shared/src/lib.rs            |  1 +
 shared/src/queries.rs        | 31 +++++++++++++++++++++++++++++++
 src/lib.rs                   | 11 +++++++++--
 {api => web_api}/Cargo.toml  |  2 +-
 {api => web_api}/src/main.rs |  0
 9 files changed, 47 insertions(+), 9 deletions(-)
 rename {api => web_api}/Cargo.toml (97%)
 rename {api => web_api}/src/main.rs (100%)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e65da00..f28454e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -37,7 +37,7 @@ test-job:
   
 variables:
   RELEASE_DIR: "bin/"
-  API: "${RELEASE_DIR}api"
+  API: "${RELEASE_DIR}web_api"
   MIGRATION: "${RELEASE_DIR}migration"
   POPULATE: "${RELEASE_DIR}populate"
 
@@ -73,7 +73,7 @@ release-job:
     tag_name: '$CI_COMMIT_SHORT_SHA'
     assets:
       links:
-        - name: 'api'
+        - name: 'web_api'
           # Use variables to build a URL to access the artifacts
           # ${CI_PROJECT_URL} is the repository URL
           # ${BUILD_JOB_ID} is from the previous job,
diff --git a/Cargo.toml b/Cargo.toml
index a7352cc..4e28f7c 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -10,7 +10,7 @@ publish=false
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 [workspace]
-members = [".", "shared", "migration", "populate", "proc_macros", "api"]
+members = [".", "shared", "migration", "populate", "proc_macros", "web_api"]
 
 [dependencies]
 shared = {path = "shared"}
diff --git a/Readme.md b/Readme.md
index 9d317e1..3fdf68b 100644
--- a/Readme.md
+++ b/Readme.md
@@ -11,7 +11,7 @@ This projects is the base ground for all `Facts` for **AI-Dojo**.
 
 ## Structure
 
- *  `api` contains an actix-web REST API for querying the database.
+ *  `web_api` contains an actix-web REST API for querying the database.
  *  `migrations` hold the database migration "scripts" and wrapper.
  *  `populate` contains the logic to fill the databases.
  *  `shared` hold entities used throughout all the tooling, and models for streaming query results.
@@ -29,7 +29,7 @@ This projects is the base ground for all `Facts` for **AI-Dojo**.
   - `cargo build --release --workspace` to build the maintenance & api binaries.
   - `./target/release/migration<.exe>` to run database migrations. (Only do this on first startup.)
   - `./target/release/populate<.exe>` to populate the database. See the executables' options. The flag `--real` starts population by crawling online sources and databases (**not yet implemented**). The `--mock <scenario>` option populates the database artificially by the selected scenario.
-  - `./target/release.api --interface <all/localhost> &` to start the REST API. The interface should be chosen based on where users of the database will reside. 
+  - `./target/release/web_api --interface <all/localhost> &` to start the REST API. The interface should be chosen based on where users of the database will reside. 
 
 ## For developers
 
diff --git a/proc_macros/src/lib.rs b/proc_macros/src/lib.rs
index 20b7ded..cabbe6c 100644
--- a/proc_macros/src/lib.rs
+++ b/proc_macros/src/lib.rs
@@ -1,7 +1,6 @@
 extern crate proc_macro;
 
 use proc_macro::TokenStream;
-
 use quote::{format_ident, quote};
 
 struct TypeDuo {
diff --git a/shared/src/lib.rs b/shared/src/lib.rs
index a3aebc1..6b28b48 100644
--- a/shared/src/lib.rs
+++ b/shared/src/lib.rs
@@ -8,6 +8,7 @@ pub mod mutations;
 pub mod queries;
 mod query_results;
 
+pub use entity::prelude::ApplicabilityMatrixModel;
 pub use error::GringottsError;
 pub use query_results::{enabler::Enabler, service::Service};
 pub use types::GringottsResult;
diff --git a/shared/src/queries.rs b/shared/src/queries.rs
index 8db6508..535fb78 100644
--- a/shared/src/queries.rs
+++ b/shared/src/queries.rs
@@ -53,6 +53,7 @@ impl Queries {
     impl_get_count!(ActionEnt::action);
 
     impl_stream_all_items!(ServiceEnt::ServiceModel::service);
+
     impl_stream_all_items!(EnablerEnt::EnablerModel::enabler);
 
     pub async fn get_services_with_tag(tag: &str, conn: Arc<DatabaseConnection>) -> GringottsResult<Vec<ServiceModel>> {
@@ -95,6 +96,36 @@ impl Queries {
         })
     }
 
+    #[must_use]
+    pub fn services_by_pks(pks: &[i32], conn: Arc<DatabaseConnection>) -> BoxStream<'static, GringottsResult<Service>> {
+        let pks_copy = pks.to_vec();
+        Box::pin(stream! {
+            let conn_owned = Arc::clone(&conn);
+            let mut db_stream = ServiceEnt::find()
+            .filter(services::Column::Id.is_in(pks_copy))
+            .expr(Expr::cust("array_agg(service_tags.tag || '') as tags")) // hacky as hell, but works - must combine it with empty string, otherwise we gert type mismatches
+            .expr(Expr::cust("array_agg(service_impacts.impact || '') as impacts"))
+            .select_column(functional_data::Column::AutoElevation)
+            .select_column(functional_data::Column::ExecutableAccess)
+            .select_column(functional_data::Column::Ports)
+            .select_column(functional_data::Column::TargetSoftware)
+            .join(JoinType::Join, services::Relation::FunctionalData.def())
+            .join(JoinType::Join, services::Relation::ServiceTags.def())
+            .join(JoinType::Join, services::Relation::ServiceImpacts.def())
+            .group_by(services::Column::Id)
+            .group_by(functional_data::Column::AutoElevation)
+            .group_by(functional_data::Column::ExecutableAccess)
+            .group_by(functional_data::Column::Ports)
+            .group_by(functional_data::Column::TargetSoftware)
+            .into_model::<Service>()
+            .stream(conn_owned.as_ref()).await?;
+
+            while let Some(item) = db_stream.try_next().await? {
+                yield Ok(item)
+            }
+        })
+    }
+
     pub async fn get_service_for_pagan_by_pk(
         pk: ServiceHandle,
         conn: Arc<DatabaseConnection>,
diff --git a/src/lib.rs b/src/lib.rs
index 1244453..ea8efa8 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,3 +1,10 @@
 #![allow(clippy::multiple_crate_versions)]
-pub use shared::{entity::sea_orm_active_enums::*, Enabler, GringottsError, GringottsResult, Service, queries::Queries};
-
+pub use shared::{
+    entity::sea_orm_active_enums::*,
+    queries::Queries,
+    ApplicabilityMatrixModel,
+    Enabler,
+    GringottsError,
+    GringottsResult,
+    Service,
+};
diff --git a/api/Cargo.toml b/web_api/Cargo.toml
similarity index 97%
rename from api/Cargo.toml
rename to web_api/Cargo.toml
index 2e57ede..522ddcf 100644
--- a/api/Cargo.toml
+++ b/web_api/Cargo.toml
@@ -1,5 +1,5 @@
 [package]
-name = "api"
+name = "web_api"
 version = "0.1.0"
 edition = "2021"
 description = "REST API for the gringotts database."
diff --git a/api/src/main.rs b/web_api/src/main.rs
similarity index 100%
rename from api/src/main.rs
rename to web_api/src/main.rs
-- 
GitLab


From 65b96f5888cd5d0f29e8e00a7a5a0c3fd79a10f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=81d=C3=A1m=20Ruman?= <469068@muni.cz>
Date: Wed, 27 Nov 2024 10:09:33 +0100
Subject: [PATCH 2/5] bump seaorm version

---
 migration/Cargo.toml | 2 +-
 populate/Cargo.toml  | 2 +-
 shared/Cargo.toml    | 2 +-
 web_api/Cargo.toml   | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/migration/Cargo.toml b/migration/Cargo.toml
index d4b518a..f217618 100644
--- a/migration/Cargo.toml
+++ b/migration/Cargo.toml
@@ -17,7 +17,7 @@ async-std = { version = "1", features = ["attributes", "tokio1"] }
 dotenvy = "0.15.7"
 
 [dependencies.sea-orm-migration]
-version = "0.12.15"
+version = "1.1.1"
 features = [
   # Enable at least one `ASYNC_RUNTIME` and `DATABASE_DRIVER` feature if you want to run migration via CLI.
   # View the list of supported features at https://www.sea-ql.org/SeaORM/docs/install-and-config/database-and-async-runtime.
diff --git a/populate/Cargo.toml b/populate/Cargo.toml
index e52ebde..e89bebe 100644
--- a/populate/Cargo.toml
+++ b/populate/Cargo.toml
@@ -15,7 +15,7 @@ publish=false
 [dependencies]
 shared = { path = "../shared"}
 tokio = {version = "1.37.0", features = ["full"]}
-sea-orm = {version="0.12.15", features = ["sqlx-postgres", "runtime-tokio-rustls", "macros", "with-json"]}
+sea-orm = {version= "1.1.1", features = ["sqlx-postgres", "runtime-tokio-rustls", "macros", "with-json"]}
 dotenvy = "0.15.7"
 serde_json = { version = "1.0.116", features = [] }
 env_logger = "0.11.3"
diff --git a/shared/Cargo.toml b/shared/Cargo.toml
index 33ee567..5a91d3d 100644
--- a/shared/Cargo.toml
+++ b/shared/Cargo.toml
@@ -13,7 +13,7 @@ publish=false
 [dependencies]
 gringotts_macros = {path = "../proc_macros"}
 serde = { version = "1.0.197", features = ["derive"] }
-sea-orm = { version = "1.0.1", features = ["sqlx-postgres", "macros", "with-json", "runtime-tokio-rustls"] }
+sea-orm = { version = "1.1.1", features = ["sqlx-postgres", "macros", "with-json", "runtime-tokio-rustls"] }
 tokio = { version = "1.37.0", features = ["full"] }
 serde_json = "1.0.116"
 async-stream = "0.3.5"
diff --git a/web_api/Cargo.toml b/web_api/Cargo.toml
index 522ddcf..f4b35e0 100644
--- a/web_api/Cargo.toml
+++ b/web_api/Cargo.toml
@@ -14,7 +14,7 @@ shared = {path = "../shared"}
 futures = "0.3.30"
 clap = { version = "4.5.4", features = ["derive"] }
 log = "0.4.21"
-sea-orm = { version = "0.12.15", features = ["sqlx-postgres", "macros", "with-json", "runtime-tokio-rustls"] }
+sea-orm = { version = "1.1.1", features = ["sqlx-postgres", "macros", "with-json", "runtime-tokio-rustls"] }
 serde = { version = "1.0.197", features = ["derive"] }
 serde_json = "1.0.116"
 actix-web = "4.5.1"
-- 
GitLab


From 9addae45882bbbcbefb1f3657df7d46a9df36f4b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=81d=C3=A1m=20Ruman?= <469068@muni.cz>
Date: Wed, 27 Nov 2024 10:13:59 +0100
Subject: [PATCH 3/5] change new stringlen type

---
 .../src/m20220101_000001_create_table_deployment_data.rs    | 6 +++---
 migration/src/m20220101_000009_create_table_enablers.rs     | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/migration/src/m20220101_000001_create_table_deployment_data.rs b/migration/src/m20220101_000001_create_table_deployment_data.rs
index ed72659..76204b0 100644
--- a/migration/src/m20220101_000001_create_table_deployment_data.rs
+++ b/migration/src/m20220101_000001_create_table_deployment_data.rs
@@ -20,9 +20,9 @@ impl MigrationTrait for Migration {
                             .auto_increment()
                             .primary_key(),
                     )
-                    .col(ColumnDef::new(DeploymentData::Vendor).array(ColumnType::String(Some(128))))
-                    .col(ColumnDef::new(DeploymentData::Source).array(ColumnType::String(Some(128))))
-                    .col(ColumnDef::new(DeploymentData::DockerInfo).array(ColumnType::String(Some(128))))
+                    .col(ColumnDef::new(DeploymentData::Vendor).array(ColumnType::String(StringLen::N(128))))
+                    .col(ColumnDef::new(DeploymentData::Source).array(ColumnType::String(StringLen::N(128))))
+                    .col(ColumnDef::new(DeploymentData::DockerInfo).array(ColumnType::String(StringLen::N(128))))
                     .col(ColumnDef::new(DeploymentData::Service).integer().not_null())
                     .to_owned(),
             )
diff --git a/migration/src/m20220101_000009_create_table_enablers.rs b/migration/src/m20220101_000009_create_table_enablers.rs
index d6e69cb..cd12ec0 100644
--- a/migration/src/m20220101_000009_create_table_enablers.rs
+++ b/migration/src/m20220101_000009_create_table_enablers.rs
@@ -34,7 +34,7 @@ impl MigrationTrait for Migration {
                     .col(ColumnDef::new(Enablers::Type).enumeration(EnablerType::Table, EnablerType::iter().skip(1)))
                     .col(ColumnDef::new(Enablers::Details).json())
                     .col(ColumnDef::new(Enablers::MainCPE).string().not_null())
-                    .col(ColumnDef::new(Enablers::Config).array(ColumnType::String(Some(128))))
+                    .col(ColumnDef::new(Enablers::Config).array(ColumnType::String(StringLen::N(128))))
                     .to_owned(),
             )
             .await
-- 
GitLab


From 3d4ca06a848300df8d6e399c14d6dbb31436990c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=81d=C3=A1m=20Ruman?= <469068@muni.cz>
Date: Wed, 27 Nov 2024 10:15:58 +0100
Subject: [PATCH 4/5] change Query usage

---
 web_api/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/web_api/src/main.rs b/web_api/src/main.rs
index bb8243c..1c2d805 100644
--- a/web_api/src/main.rs
+++ b/web_api/src/main.rs
@@ -52,7 +52,7 @@ async fn service_by_pk(pk_query: web::Query<PkQuery>, app_data: web::Data<Databa
 
 #[get("/services_all")]
 async fn services_all(app_data: web::Data<DatabaseConnection>) -> HttpResponse {
-    let stream = Queries::stream_all_service(app_data.into_inner().clone()).await;
+    let stream = Queries{}.stream_all_service(app_data.into_inner().clone()).await;
     let processed_stream = stream.map(|element: GringottsResult<ServiceModel>| serialize_model(element));
     HttpResponse::Ok().content_type("jsonl").streaming(processed_stream)
 }
-- 
GitLab


From 7130dc93a9747a83eac416567089f60aa0942dc9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=81d=C3=A1m=20Ruman?= <469068@muni.cz>
Date: Wed, 27 Nov 2024 10:19:52 +0100
Subject: [PATCH 5/5] clippy repair

---
 web_api/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/web_api/src/main.rs b/web_api/src/main.rs
index 1c2d805..16abeba 100644
--- a/web_api/src/main.rs
+++ b/web_api/src/main.rs
@@ -52,7 +52,7 @@ async fn service_by_pk(pk_query: web::Query<PkQuery>, app_data: web::Data<Databa
 
 #[get("/services_all")]
 async fn services_all(app_data: web::Data<DatabaseConnection>) -> HttpResponse {
-    let stream = Queries{}.stream_all_service(app_data.into_inner().clone()).await;
+    let stream = Queries {}.stream_all_service(app_data.into_inner().clone()).await;
     let processed_stream = stream.map(|element: GringottsResult<ServiceModel>| serialize_model(element));
     HttpResponse::Ok().content_type("jsonl").streaming(processed_stream)
 }
-- 
GitLab