From 9e580753e48b43ca5e676128f407b53f935291d6 Mon Sep 17 00:00:00 2001 From: lovasoa Date: Sat, 5 Oct 2024 22:31:29 +0200 Subject: [PATCH] Easier json handling in databases without a native json type. SQLPage now detects when you use a json function in SQLite or MariaDB to generate a column, and automatically converts the resulting string to a json object. This allows easily using components that take json parameters (like the new columns component) in MariaDB and SQLite. fixes #633 --- CHANGELOG.md | 1 + docker-compose.yml | 2 + src/webserver/database/execute_queries.rs | 32 +++++- src/webserver/database/sql.rs | 105 +++++++++++++++++- .../it_works_columns_component_json.sql | 6 + tests/upload_csv_test.sql | 7 +- 6 files changed, 148 insertions(+), 5 deletions(-) create mode 100644 tests/sql_test_files/it_works_columns_component_json.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index 45d00b45..bc4e1bd6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - **Fix**: the search feature in the shell component was not working when no menu item was defined. - Add support for encrypted Microsoft SQL Server connections. This finally allows connecting to databases that refuse clear-text connections, such as those hosted on Azure. + - Easier json handling in databases without a native json type. SQLPage now detects when you use a json function in SQLite or MariaDB to generate a column, and automatically converts the resulting string to a json object. This allows easily using components that take json parameters (like the new columns component) in MariaDB and SQLite. ## 0.29.0 (2024-09-25) - New columns component: `columns`. Useful to display a comparison between items, or large key figures to an user. diff --git a/docker-compose.yml b/docker-compose.yml index 2f3086a9..f6a753f4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,6 +31,7 @@ services: environment: MYSQL_ROOT_PASSWORD: Password123! MYSQL_DATABASE: sqlpage + mssql: profiles: ["mssql"] ports: ["1433:1433"] @@ -41,6 +42,7 @@ services: timeout: 3s retries: 10 start_period: 10s + mariadb: profiles: ["mariadb"] ports: ["3306:3306"] diff --git a/src/webserver/database/execute_queries.rs b/src/webserver/database/execute_queries.rs index 87876027..e6c2f5c6 100644 --- a/src/webserver/database/execute_queries.rs +++ b/src/webserver/database/execute_queries.rs @@ -4,6 +4,7 @@ use futures_util::StreamExt; use std::borrow::Cow; use std::collections::HashMap; use std::pin::Pin; +use serde_json::Value; use super::csv_import::run_csv_import; use super::sql::{ @@ -59,6 +60,7 @@ pub fn stream_query_results_with_conn<'a>( let is_err = elem.is_err(); let mut query_result = parse_single_sql_result(&stmt.query, elem); apply_delayed_functions(request, &stmt.delayed_functions, &mut query_result).await?; + apply_json_columns(&mut query_result, &stmt.json_columns); for i in parse_dynamic_rows(query_result) { yield i; } @@ -333,6 +335,34 @@ fn json_to_fn_param(json: serde_json::Value) -> Option> { } } +fn apply_json_columns(item: &mut DbItem, json_columns: &[String]) { + if let DbItem::Row(Value::Object(ref mut row)) = item { + for column in json_columns { + if let Some(value) = row.get_mut(column) { + if let Value::String(json_str) = value { + if let Ok(parsed_json) = serde_json::from_str(json_str) { + log::trace!("Parsed JSON column {column}: {parsed_json}"); + *value = parsed_json; + } else { + log::warn!("The column {column} contains invalid JSON: {json_str}"); + } + } else if let Value::Array(array) = value { + for item in array { + if let Value::String(json_str) = item { + if let Ok(parsed_json) = serde_json::from_str(json_str) { + log::trace!("Parsed JSON array item: {parsed_json}"); + *item = parsed_json; + } + } + } + } + } else { + log::warn!("The column {column} is missing from the result set, so it cannot be converted to JSON."); + } + } + } +} + pub struct StatementWithParams<'a> { sql: &'a str, arguments: AnyArguments<'a>, @@ -355,4 +385,4 @@ impl<'q> sqlx::Execute<'q, Any> for StatementWithParams<'q> { // Let sqlx create a prepared statement the first time it is executed, and then reuse it. true } -} +} \ No newline at end of file diff --git a/src/webserver/database/sql.rs b/src/webserver/database/sql.rs index 5eed05e3..b4a47420 100644 --- a/src/webserver/database/sql.rs +++ b/src/webserver/database/sql.rs @@ -9,7 +9,7 @@ use async_trait::async_trait; use sqlparser::ast::{ BinaryOperator, CastKind, CharacterLength, DataType, Expr, Function, FunctionArg, FunctionArgExpr, FunctionArgumentList, FunctionArguments, Ident, ObjectName, - OneOrManyWithParens, SelectItem, Statement, Value, VisitMut, VisitorMut, + OneOrManyWithParens, SelectItem, SetExpr, Statement, Value, VisitMut, VisitorMut, }; use sqlparser::dialect::{Dialect, MsSqlDialect, MySqlDialect, PostgreSqlDialect, SQLiteDialect}; use sqlparser::parser::{Parser, ParserError}; @@ -64,6 +64,9 @@ pub(super) struct StmtWithParams { /// Functions that are called on the result set after the query has been executed, /// and which can be passed the result of the query as an argument. pub delayed_functions: Vec, + /// Columns that are JSON columns, and which should be converted to JSON objects after the query is executed. + /// Only relevant for databases that do not have a native JSON type, and which return JSON values as text. + pub json_columns: Vec, } #[derive(Debug)] @@ -136,6 +139,7 @@ fn parse_single_statement( query, params, delayed_functions: Vec::new(), + json_columns: Vec::new(), }, }); } @@ -148,6 +152,7 @@ fn parse_single_statement( } let delayed_functions = extract_toplevel_functions(&mut stmt); remove_invalid_function_calls(&mut stmt, &mut params); + let json_columns = extract_json_columns(&stmt, db_kind); let query = format!( "{stmt}{semicolon}", semicolon = if semicolon { ";" } else { "" } @@ -157,6 +162,7 @@ fn parse_single_statement( query, params, delayed_functions, + json_columns, })) } @@ -771,6 +777,64 @@ fn sqlpage_func_name(func_name_parts: &[Ident]) -> &str { } } +fn extract_json_columns(stmt: &Statement, db_kind: AnyKind) -> Vec { + // Only extract JSON columns for databases without native JSON support + if matches!(db_kind, AnyKind::Postgres | AnyKind::Mssql) { + return Vec::new(); + } + + let mut json_columns = Vec::new(); + + if let Statement::Query(query) = stmt { + if let SetExpr::Select(select) = query.body.as_ref() { + for item in &select.projection { + if let SelectItem::ExprWithAlias { expr, alias } = item { + if is_json_function(expr) { + json_columns.push(alias.value.clone()); + log::trace!("Found JSON column: {alias}"); + } + } + } + } + } + + json_columns +} + +fn is_json_function(expr: &Expr) -> bool { + match expr { + Expr::Function(function) => { + if let [Ident { value, .. }] = function.name.0.as_slice() { + [ + "json_object", + "json_array", + "json_build_object", + "json_build_array", + "to_json", + "to_jsonb", + "json_agg", + "jsonb_agg", + "json_arrayagg", + "json_objectagg", + "json_group_array", + "json_group_object", + ] + .iter() + .any(|&func| value.eq_ignore_ascii_case(func)) + } else { + false + } + } + Expr::Cast { data_type, .. } => { + matches!(data_type, DataType::JSON | DataType::JSONB) + || (matches!(data_type, DataType::Custom(ObjectName(parts), _) if + (parts.len() == 1) + && (parts[0].value.eq_ignore_ascii_case("json")))) + } + _ => false, + } +} + #[cfg(test)] mod test { use super::super::sqlpage_functions::functions::SqlPageFunctionName; @@ -1131,4 +1195,43 @@ mod test { None ); } + + #[test] + fn test_extract_json_columns() { + let sql = r#" + WITH json_cte AS ( + SELECT json_build_object('a', x, 'b', y) AS cte_json + FROM generate_series(1, 3) x + JOIN generate_series(4, 6) y ON true + ) + SELECT + json_object('key', 'value') AS json_col1, + json_array(1, 2, 3) AS json_col2, + (SELECT json_build_object('nested', subq.val) + FROM (SELECT AVG(x) AS val FROM generate_series(1, 5) x) subq + ) AS json_col3, -- not supported because of the subquery + CASE + WHEN EXISTS (SELECT 1 FROM json_cte WHERE cte_json->>'a' = '2') + THEN to_json(ARRAY(SELECT cte_json FROM json_cte)) + ELSE json_build_array() + END AS json_col4, -- not supported because of the CASE + json_unknown_fn(regular_column) AS non_json_col, + CAST(json_col1 AS json) AS json_col6 + FROM some_table + CROSS JOIN json_cte + WHERE json_typeof(json_col1) = 'object' + "#; + + let stmt = parse_postgres_stmt(sql); + let json_columns = extract_json_columns(&stmt, AnyKind::Sqlite); + + assert_eq!( + json_columns, + vec![ + "json_col1".to_string(), + "json_col2".to_string(), + "json_col6".to_string() + ] + ); + } } diff --git a/tests/sql_test_files/it_works_columns_component_json.sql b/tests/sql_test_files/it_works_columns_component_json.sql new file mode 100644 index 00000000..cb67e501 --- /dev/null +++ b/tests/sql_test_files/it_works_columns_component_json.sql @@ -0,0 +1,6 @@ +select 'columns' as component; + +select + JSON_OBJECT('description', 'It works !') as item, + JSON_OBJECT('description', 'It works !') as item +; \ No newline at end of file diff --git a/tests/upload_csv_test.sql b/tests/upload_csv_test.sql index 2f7ecb9e..0caf6e66 100644 --- a/tests/upload_csv_test.sql +++ b/tests/upload_csv_test.sql @@ -1,5 +1,6 @@ -create table people(name text, age text); -copy people(name, age) from 'people_file' with (format csv, header true); +drop table if exists sqlpage_people_test_table; +create table sqlpage_people_test_table(name text, age text); +copy sqlpage_people_test_table(name, age) from 'people_file' with (format csv, header true); select 'text' as component, name || ' is ' || age || ' years old. ' as contents -from people; \ No newline at end of file +from sqlpage_people_test_table; \ No newline at end of file