Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

6295 add crawl function for generating reports #6347

Merged
merged 6 commits into from
Jan 30, 2025
Merged

6295 add crawl function for generating reports #6347

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
349ca0a
Add nested report generation crawl
fergie-nz Jan 29, 2025
f55faef
Update error mapping in yarn install
fergie-nz Jan 29, 2025
a3345a9
Rename manifest ro report-manifest
fergie-nz Jan 29, 2025
4d8715d
Remove storage connection from report building
fergie-nz Jan 30, 2025
92063eb
Merge branch 'develop' into 6295-add-crawl-function-for-generating-re…
fergie-nz Jan 30, 2025
d1adb5e
Regenerate standard reports
fergie-nz Jan 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
220 changes: 13 additions & 207 deletions server/cli/src/cli.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,8 @@ use anyhow::anyhow;
use async_graphql::EmptySubscription;
use chrono::Utc;
use clap::{ArgAction, Parser};
use cli::RefreshDatesRepository;
use graphql::{Mutations, OperationalSchema, Queries};
use log::info;
use report_builder::{build::build_report_definition, BuildArgs};

use repository::{
get_storage_connection_manager, schema_from_row, test_db, ContextType, EqualFilter,
Expand All @@ -21,7 +19,7 @@ use service::{
plugin::validation::sign_plugin,
service_provider::{ServiceContext, ServiceProvider},
settings::Settings,
standard_reports::{ReportData, ReportsData, StandardReports},
standard_reports::{ReportsData, StandardReports},
sync::{
file_sync_driver::FileSyncDriver, settings::SyncSettings, sync_status::logger::SyncLogger,
synchroniser::integrate_and_translate_sync_buffer, synchroniser_driver::SynchroniserDriver,
Expand All @@ -30,9 +28,9 @@ use service::{
};
use simple_log::LogConfigBuilder;
use std::{
ffi::OsStr,
fs,
path::{Path, PathBuf},
process::{Command, Stdio},
sync::{Arc, RwLock},
};
use util::inline_init;
Expand All @@ -43,6 +41,8 @@ use backup::*;
mod plugins;
use plugins::*;

use cli::{generate_reports_recursive, RefreshDatesRepository};

const DATA_EXPORT_FOLDER: &str = "data";

/// omSupply remote server cli
Expand Down Expand Up @@ -395,144 +395,21 @@ async fn main() -> anyhow::Result<()> {
}
Action::SignPlugin { path, key, cert } => sign_plugin(&path, &key, &cert)?,
Action::BuildReports { path } => {
let connection_manager = get_storage_connection_manager(&settings.database);
let con = connection_manager.connection()?;
let base_reports_dir = match path.clone() {
Some(path) => path,
None => PathBuf::new().join("reports"),
};
let report_names: Vec<PathBuf> = fs::read_dir(base_reports_dir.clone())?
.filter_map(|r| r.ok())
.map(|e| e.path())
.filter(|p| p.is_dir())
.filter(|name| name != &Path::new("reports").join("generated"))
.map(|p| p)
.collect();

let mut reports_data = ReportsData { reports: vec![] };

for name_dir in report_names {
let report_versions: Vec<PathBuf> = fs::read_dir(&name_dir)?
.filter_map(|r| r.ok())
.map(|e| e.path())
.filter(|p| p.is_dir())
.map(|p| p)
.collect();

let parent = name_dir
.components()
.next()
.expect("failed to read report name directory");
let name = name_dir.strip_prefix(parent).unwrap();

for version_dir in report_versions {
// install esbuild depedencies
if let Err(e) = run_yarn_install(&version_dir) {
eprintln!(
"Failed to run yarn install in {}: {}",
version_dir.display(),
e
);
continue;
}
// read manifest file

let manifest_file =
fs::File::open(version_dir.join("manifest.json")).expect(&format!(
"manifest file should open read only in report {:?} {:?}",
name, version_dir
));

let manifest: Manifest =
serde_json::from_reader(manifest_file).expect(&format!(
"manifest json not formatted correctly {:?} {:?}",
name, version_dir
));
let code = manifest.code;

let version = manifest.version;
let id_version = str::replace(&version, ".", "_");

let context = manifest.context;
let report_name = manifest.name;
let is_custom = manifest.is_custom;
let id = format!("{code}_{id_version}_{is_custom}");
let sub_context = manifest.sub_context;
let arguments_path = manifest
.arguments
.clone()
.and_then(|a| a.schema)
.and_then(|schema| Some(version_dir.join(schema)));
let arguments_ui_path = manifest
.arguments
.and_then(|a| a.ui)
.and_then(|ui| Some(version_dir.join(ui)));
let graphql_query = manifest.queries.clone().and_then(|q| q.gql);
let sql_queries = manifest.queries.clone().and_then(|q| q.sql);
let convert_data = manifest
.convert_data
.and_then(|cd| Some(version_dir.join(cd)));
let custom_wasm_function = manifest.custom_wasm_function;
let query_default = manifest.query_default;

let args = BuildArgs {
dir: version_dir.join("src"),
output: Some(version_dir.join("generated").join("built_report.json")),
template: "template.html".to_string(),
header: manifest.header,
footer: manifest.footer,
query_gql: graphql_query,
query_default: query_default,
query_sql: sql_queries,
convert_data,
custom_wasm_function,
};

let report_definition = build_report_definition(&args)
.map_err(|_| anyhow!("Failed to build report {:?}", id))?;

let filter = ReportFilter::new().id(EqualFilter::equal_to(&id));
let existing_report =
ReportRepository::new(&con).query_by_filter(filter)?.pop();

let argument_schema_id = existing_report
.and_then(|r| r.argument_schema.as_ref().map(|r| r.id.clone()));

let form_schema_json = match (arguments_path, arguments_ui_path) {
(Some(_), None) | (None, Some(_)) => {
return Err(anyhow!(
"When arguments_path is specified arguments_ui_path must also be specified in report and vice versa {:?} {:?}", name, version_dir
))
}
(Some(arguments_path), Some(arguments_ui_path)) => {
Some(schema_from_row(FormSchemaRow {
id: argument_schema_id.unwrap_or(format!("for_report_{}", id)),
r#type: "reportArgument".to_string(),
json_schema: fs::read_to_string(arguments_path)?,
ui_schema: fs::read_to_string(arguments_ui_path)?,
})?)
}
(None, None) => None,
};

let report_data = ReportData {
id,
name: report_name,
r#type: repository::ReportType::OmSupply,
template: report_definition,
context,
sub_context,
argument_schema_id: form_schema_json.clone().map(|r| r.id.clone()),
comment: None,
is_custom,
version: version.to_string(),
code,
form_schema: form_schema_json,
};

reports_data.reports.push(report_data);
}
}
let ignore_paths = vec![OsStr::new("node_modules")];
let manifest_name = OsStr::new("report-manifest.json");

generate_reports_recursive(
&mut reports_data,
&ignore_paths,
manifest_name,
&base_reports_dir,
)?;

let output_name = if path.is_some() {
"reports.json"
Expand Down Expand Up @@ -659,74 +536,3 @@ fn export_paths(name: &str) -> (PathBuf, PathBuf, PathBuf) {

(export_folder, export_file_path, users_file_path)
}

fn run_yarn_install(directory: &PathBuf) -> Result<(), Box<dyn std::error::Error>> {
let convert_dir = directory.join("convert_data_js");

if !convert_dir.exists() {
info!(
"No conversion function for {}. Skipping esbuild install.",
convert_dir.display().to_string()
);
return Ok(());
}

let node_modules_path = convert_dir.join("node_modules");

if !node_modules_path.exists() {
let status = Command::new("yarn")
.args(["install", "--cwd"])
.arg(convert_dir)
.args(["--no-lockfile", "--check-files"])
.stdout(Stdio::inherit())
.stderr(Stdio::inherit())
.status()?;

if !status.success() {
info!("Error: `yarn install` failed");
return Err("Failed to run yarn install".into());
}
} else {
info!("Dependencies up to date");
}

Ok(())
}

#[derive(serde::Deserialize, Clone)]
pub struct Manifest {
pub is_custom: bool,
pub version: String,
pub code: String,
pub context: ContextType,
pub sub_context: Option<String>,
pub name: String,
pub header: Option<String>,
pub footer: Option<String>,
pub queries: Option<ManifestQueries>,
pub default_query: Option<String>,
pub arguments: Option<Arguments>,
pub test_arguments: Option<TestReportArguments>,
pub convert_data: Option<String>,
pub custom_wasm_function: Option<String>,
pub query_default: Option<String>,
}

#[derive(serde::Deserialize, Clone)]
pub struct ManifestQueries {
pub gql: Option<String>,
pub sql: Option<Vec<String>>,
}

#[derive(serde::Deserialize, Clone)]
pub struct Arguments {
pub schema: Option<String>,
pub ui: Option<String>,
}

#[derive(serde::Deserialize, Clone)]
pub struct TestReportArguments {
pub arguments: Option<String>,
pub reference_data: Option<String>,
pub data_id: Option<String>,
}
2 changes: 2 additions & 0 deletions server/cli/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
extern crate diesel;
mod refresh_dates;
pub use refresh_dates::*;
mod report_utils;
pub use report_utils::*;
Loading
Loading