From b3e818afcc36cef488e16e0abb4d85420d1b9828 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 4 Feb 2025 13:04:38 -0600 Subject: [PATCH 1/3] Add 'rawquery' --- CHANGELOG.md | 1 + parts/03-query-utils.sh | 4 ++++ parts/22-query.sh | 2 +- parts/70-look-for.sh | 1 + 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ae52250..a38427aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ - Added: - query tools-usage (like tools-usage-per-month but for a year or for whole history) by @lldelisle + - Return query results in raw (unaligned, tuples-only) format with 'rawquery' from @natefoo - Fixed: - query tools-usage-per-month when 'no_version' was used there were still one line per version. diff --git a/parts/03-query-utils.sh b/parts/03-query-utils.sh index b8686536..f4f95a20 100644 --- a/parts/03-query-utils.sh +++ b/parts/03-query-utils.sh @@ -16,6 +16,10 @@ query_tbl_wrapper() { fi } +query_raw() { + psql -At <<< "$1" +} + query_tsv() { psql <<-EOF COPY ($1) to STDOUT with CSV DELIMITER E'\t' diff --git a/parts/22-query.sh b/parts/22-query.sh index 91736974..bdbf80a1 100644 --- a/parts/22-query.sh +++ b/parts/22-query.sh @@ -4,7 +4,7 @@ registered_subcommands="$registered_subcommands query" _query_short_help="DB Queries" _query_long_help=" - 'query' can be exchanged with 'tsvquery' or 'csvquery' for tab- and comma-separated variants. + 'query' can be exchanged with 'tsvquery' or 'csvquery' for tab- and comma-separated variants. Additionally, 'rawquery' returns unaligned tuples only. In some cases 'iquery' is supported for InfluxDB compatible output. In all cases 'explainquery' will show you the query plan, in case you need to optimise or index data. 'explainjsonquery' is useful with PEV: http://tatiyants.com/pev/ " diff --git a/parts/70-look-for.sh b/parts/70-look-for.sh index 584a3cf7..8a33d97c 100644 --- a/parts/70-look-for.sh +++ b/parts/70-look-for.sh @@ -254,6 +254,7 @@ look_for() { # Run the queries case "$group_name" in + rawquery ) query_raw "$QUERY";; tsvquery ) query_tsv "$QUERY";; csvquery ) query_csv "$QUERY";; query ) query_tbl_wrapper "$QUERY";; From 1629e5470cbb01ddfe8710dff26680876f64cf27 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 4 Feb 2025 13:56:31 -0600 Subject: [PATCH 2/3] Add --escape option to `query aq` to convert bytea columns to ASCII --- CHANGELOG.md | 3 +++ parts/22-query.sh | 36 +++++++++++++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a38427aa..4b8677e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,10 @@ - Added: - query tools-usage (like tools-usage-per-month but for a year or for whole history) by @lldelisle + - query tool-memory-efficiency, evaluates the memory efficiency of recent jobs from @natefoo - Return query results in raw (unaligned, tuples-only) format with 'rawquery' from @natefoo +- Updated: + - query aq to add --escape option to encode bytea columns to ASCII from @natefoo - Fixed: - query tools-usage-per-month when 'no_version' was used there were still one line per version. diff --git a/parts/22-query.sh b/parts/22-query.sh index bdbf80a1..10431e15 100644 --- a/parts/22-query.sh +++ b/parts/22-query.sh @@ -4306,16 +4306,50 @@ query_data-origin-distribution-summary() { ##? [--human]: breakdown of data sour EOF } -query_aq() { ## <-|job_id [job_id [...]]>: Given a list of IDs from a table (e.g. 'job'), access a specific column from that table +query_aq() { ## [--escape]
<-|job_id [job_id [...]]>: Given a list of IDs from a table (e.g. 'job'), access a specific column from that table meta <<-EOF ADDED: 15 EOF handle_help "$@" <<-EOF + Select a column from a table matching the given id(s). + + The --escape option can be used to convert data stored in the escape format, such as the job destination + parameters: + + $ gxadmin query aq --escape job destination_params 42 43 + encode + ------------------------------------------------------------------------------------------------ + {"embed_metadata_in_job": false, "require_container": true, "singularity_enabled": true} + {"native_specification": "--partition=normal --nodes=1 --ntasks=2 --mem=8192 --time=24:00:00"} + (2 rows) + + This is particularly useful to combine with 'rawquery' and jq: + + $ gxadmin rawquery aq --escape job destination_params 42 43 | jq -s + [ + { + "embed_metadata_in_job": false, + "require_container": true, + "singularity_enabled": true + }, + { + "native_specification": "--partition=normal --nodes=1 --ntasks=2 --mem=8192 --time=24:00:00" + } + ] EOF + escape=false + if [[ "$1" == '--escape' ]]; then + escape=true; shift + fi + table=$1; shift column=$1; shift + if $escape; then + column="encode($column, 'escape')" + fi + if [[ "$1" == "-" ]]; then # read jobs from stdin ids=$(cat | paste -s -d' ') From 84125627254f0e1c3bbd84f7e968ba9b15bd49be Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 4 Feb 2025 13:56:59 -0600 Subject: [PATCH 3/3] Preserve command line order of results in `query aq` --- parts/22-query.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/parts/22-query.sh b/parts/22-query.sh index 10431e15..8ebcf792 100644 --- a/parts/22-query.sh +++ b/parts/22-query.sh @@ -4361,11 +4361,12 @@ query_aq() { ## [--escape]
<-|job_id [job_id [...]]>: Given a l # shellcheck disable=SC2068 ids_string=$(join_by ',' ${ids[@]}) + # unnest(...) stands in for WHERE id IN (...) while preserving the command line order read -r -d '' QUERY <<-EOF SELECT $column FROM $table - WHERE id in ($ids_string) + JOIN unnest('{$ids_string}'::int[]) WITH ORDINALITY t(id, ord) USING (id) EOF }