From 1b0b8f8c616356a211005f11a8960eac5300a57c Mon Sep 17 00:00:00 2001 From: Jacek Gebal Date: Sat, 15 Feb 2020 14:46:26 +0000 Subject: [PATCH 1/2] Refactored expectations on compound data With this change the reporting of differences is unified across: `unordered`, `ordered` and `join_by` comparison. --- .../data_values/ut_compound_data_helper.pkb | 131 +++++++++++------- .../expectations/test_expectation_anydata.pkb | 6 +- .../expectations/test_expectations_cursor.pkb | 22 ++- 3 files changed, 90 insertions(+), 69 deletions(-) diff --git a/source/expectations/data_values/ut_compound_data_helper.pkb b/source/expectations/data_values/ut_compound_data_helper.pkb index d7c0aab76..6a6b30a50 100644 --- a/source/expectations/data_values/ut_compound_data_helper.pkb +++ b/source/expectations/data_values/ut_compound_data_helper.pkb @@ -417,39 +417,78 @@ create or replace package body ut_compound_data_helper is l_join_xpath varchar2(32767) := ut_utils.to_xpath(a_join_by_list); l_results tt_row_diffs; l_sql varchar2(32767); + l_column_order varchar2(100); begin + if a_enforce_column_order then + l_column_order := 'col_no'; + else + l_column_order := 'col_name'; + end if; l_sql := q'[ - with exp as ( + with + exp_rows as( + select + exp_data_id, extract( exp_item_data, :column_path ) exp_item_data, item_no, diff_id + from ut_compound_data_diff_tmp + where diff_id = :diff_id + and exp_data_id = :self_guid + ), + act_rows as ( + select + act_data_id, extract( act_item_data, :column_path ) act_item_data, item_no, diff_id + from ut_compound_data_diff_tmp ucd + where diff_id = :diff_id + and ucd.act_data_id = :other_guid + ), + exp_cols as ( select - exp_item_data, exp_data_id, item_no rn, rownum col_no, pk_value, + exp_item_data, exp_data_id, item_no rn, rownum col_no, diff_id, s.column_value col, s.column_value.getRootElement() col_name, nvl(s.column_value.getclobval(),empty_clob()) col_val - from ( - select - exp_data_id, extract( ucd.exp_item_data, :column_path ) exp_item_data, item_no, - replace( extract( ucd.exp_item_data, :join_by ).getclobval(), chr(10) ) pk_value - from ut_compound_data_diff_tmp ucd - where diff_id = :diff_id - and ucd.exp_data_id = :self_guid - ) i, - table( xmlsequence( extract(i.exp_item_data,:extract_path) ) ) s + from exp_rows i, + table( xmlsequence( extract( i.exp_item_data, :extract_path ) ) ) s ), - act as ( + act_cols as ( select - act_item_data, act_data_id, item_no rn, rownum col_no, pk_value, + act_item_data, act_data_id, item_no rn, rownum col_no, diff_id, s.column_value col, s.column_value.getRootElement() col_name, nvl(s.column_value.getclobval(),empty_clob()) col_val + from act_rows i, + table( xmlsequence( extract( i.act_item_data, :extract_path ) ) ) s + ), + data_diff as ( + select rn, diff_type, xmlserialize(content data_item no indent) diffed_row, null pk_value, col_name, diff_id from ( - select - act_data_id, extract( ucd.act_item_data, :column_path ) act_item_data, item_no, - replace( extract( ucd.act_item_data, :join_by ).getclobval(), chr(10) ) pk_value - from ut_compound_data_diff_tmp ucd - where diff_id = :diff_id - and ucd.act_data_id = :other_guid - ) i, - table( xmlsequence( extract(i.act_item_data,:extract_path) ) ) s + select nvl(exp.rn, act.rn) rn, + exp.diff_id diff_id, + xmlagg(exp.col order by exp.col_no) exp_item, + xmlagg(act.col order by nvl(exp.col_no, act.col_no)) act_item, + max(nvl(exp.col_name,act.col_name)) col_name + from exp_cols exp + join act_cols act + on exp.rn = act.rn and exp.col_name = act.col_name + where dbms_lob.compare(exp.col_val, act.col_val) != 0 + group by nvl(exp.rn, act.rn), exp.diff_id + ) + unpivot ( data_item for diff_type in (exp_item as 'Expected:', act_item as 'Actual:') ) + ), + unordered_diff as ( + select rn, diff_type, xmlserialize(content data_item no indent) diffed_row, null pk_value, col_name, diff_id + from ( + select nvl(exp.rn, act.rn) rn, + exp.diff_id diff_id, + xmlagg(exp.col order by exp.col_name) exp_item, + xmlagg(act.col order by act.col_name) act_item, + max(nvl(exp.col_name,act.col_name)) col_name + from exp_cols exp + join act_cols act + on exp.rn = act.rn and exp.col_name = act.col_name + where dbms_lob.compare(exp.col_val, act.col_val) != 0 + group by nvl(exp.rn, act.rn), exp.diff_id + ) + unpivot ( data_item for diff_type in (exp_item as 'Expected:', act_item as 'Actual:') ) ) - select rn, diff_type, diffed_row, pk_value pk_value + select rn, diff_type, diffed_row, pk_value from ( select rn, diff_type, diffed_row, pk_value, case when diff_type = 'Actual:' then 1 else 2 end rnk, @@ -457,33 +496,20 @@ create or replace package body ut_compound_data_helper is col_name from ( ]' || case when a_unordered then q'[ - select rn, diff_type, xmlserialize(content data_item no indent) diffed_row, pk_value, col_name - from ( - select nvl(exp.rn, act.rn) rn, - nvl(exp.pk_value, act.pk_value) pk_value, - exp.col exp_item, - act.col act_item, - nvl(exp.col_name,act.col_name) col_name - from exp - join act - on exp.rn = act.rn and exp.col_name = act.col_name - where dbms_lob.compare(exp.col_val, act.col_val) != 0 - ) - unpivot ( data_item for diff_type in (exp_item as 'Expected:', act_item as 'Actual:') ) ]' + select u.rn, u.diff_type, u.diffed_row, + replace( + extract( case when i.exp_data_id is null then i.act_item_data else i.exp_item_data end, :join_by ).getclobval(), + chr(10) + ) pk_value, + u.col_name + from data_diff u + join ut_compound_data_diff_tmp i + on i.diff_id = u.diff_id + and i.item_no = u.rn]' else q'[ - select rn, diff_type, xmlserialize(content data_item no indent) diffed_row, null pk_value, col_name - from ( - select nvl(exp.rn, act.rn) rn, - xmlagg(exp.col order by exp.col_no) exp_item, - xmlagg(act.col order by act.col_no) act_item, - max(nvl(exp.col_name,act.col_name)) col_name - from exp exp - join act act - on exp.rn = act.rn and exp.col_name = act.col_name - where dbms_lob.compare(exp.col_val, act.col_val) != 0 - group by (exp.rn, act.rn) - ) - unpivot ( data_item for diff_type in (exp_item as 'Expected:', act_item as 'Actual:') ) ]' + select rn, diff_type, diffed_row, pk_value, col_name + from data_diff + where :join_by is null ]' end ||q'[ ) union all @@ -522,14 +548,13 @@ create or replace package body ut_compound_data_helper is case when final_order = 1 then to_char(rn) else col_name end, case when final_order = 1 then to_char(rnk) else col_name end ]' - else - null end; execute immediate l_sql bulk collect into l_results - using l_exp_extract_xpath, l_join_xpath, a_diff_id, a_expected_dataset_guid,a_extract_path, - l_act_extract_xpath, l_join_xpath, a_diff_id, a_actual_dataset_guid,a_extract_path, - l_join_xpath, l_join_xpath, a_diff_id; + using l_exp_extract_xpath, a_diff_id, a_expected_dataset_guid, + l_act_extract_xpath, a_diff_id, a_actual_dataset_guid, + a_extract_path, a_extract_path, + l_join_xpath, l_join_xpath, l_join_xpath, a_diff_id; return l_results; end; diff --git a/test/ut3_user/expectations/test_expectation_anydata.pkb b/test/ut3_user/expectations/test_expectation_anydata.pkb index f94b0efc0..ece5457a7 100644 --- a/test/ut3_user/expectations/test_expectation_anydata.pkb +++ b/test/ut3_user/expectations/test_expectation_anydata.pkb @@ -864,10 +864,8 @@ Rows: [ 60 differences, showing first 20 ] l_expected_message := q'[%Actual: ut3_tester_helper.test_dummy_object_list [ count = 2 ] was expected to equal: ut3_tester_helper.test_dummy_object_list [ count = 2 ] %Diff: %Rows: [ 3 differences ] -%PK 2 - Actual: Something 2 -%PK 2 - Actual: 2 -%PK 2 - Expected: Something 1 -%PK 2 - Expected: 1 +%PK 2 - Actual: Something 22 +%PK 2 - Expected: Something 11 %PK 1 - Extra: 1Something 11 %PK 4 - Missing: 4Something 22]'; l_actual_message := ut3_tester_helper.main_helper.get_failed_expectations(1); diff --git a/test/ut3_user/expectations/test_expectations_cursor.pkb b/test/ut3_user/expectations/test_expectations_cursor.pkb index 054c65e17..5a0b68f2f 100644 --- a/test/ut3_user/expectations/test_expectations_cursor.pkb +++ b/test/ut3_user/expectations/test_expectations_cursor.pkb @@ -371,7 +371,7 @@ create or replace package body test_expectations_cursor is l_expected_message := q'[Actual: refcursor [ count = 1 ] was expected to equal: refcursor [ count = 1 ] %Diff: %Rows: [ 1 differences ] -%Row No. 1 - Actual: 4030 +%Row No. 1 - Actual: 3040 %Row No. 1 - Expected: 34]'; l_actual_message := ut3_tester_helper.main_helper.get_failed_expectations(1); --Assert @@ -394,7 +394,7 @@ create or replace package body test_expectations_cursor is l_expected_message := q'[Actual: refcursor [ count = 1 ] was expected to equal: refcursor [ count = 1 ] %Diff: %Rows: [ 1 differences ] -%Row No. 1 - Actual: 4030 +%Row No. 1 - Actual: 3040 %Row No. 1 - Expected: 34]'; l_actual_message := ut3_tester_helper.main_helper.get_failed_expectations(1); --Assert @@ -860,9 +860,9 @@ Columns: Rows: [ 4 differences ] Row No. 1 - Actual: 25000 Row No. 1 - Expected: 10000 - Row No. 2 - Actual: TONYSTARK3100000 + Row No. 2 - Actual: 3TONYSTARK100000 Row No. 2 - Expected: 2LUKESKYWALKER1000 - Row No. 3 - Actual: JESSICAJONES42345 + Row No. 3 - Actual: 4JESSICAJONES2345 Row No. 3 - Expected: 3TONYSTARK100000 Row No. 4 - Extra: MLUKESKYWALKER21000]'; l_actual_message := ut3_tester_helper.main_helper.get_failed_expectations(1); @@ -896,9 +896,9 @@ Columns: Rows: [ 4 differences ] Row No. 1 - Actual: 25000 Row No. 1 - Expected: 10000 - Row No. 2 - Actual: TONYSTARK3100000 + Row No. 2 - Actual: 3TONYSTARK100000 Row No. 2 - Expected: 2LUKESKYWALKER1000 - Row No. 3 - Actual: JESSICAJONES42345 + Row No. 3 - Actual: 4JESSICAJONES2345 Row No. 3 - Expected: 3TONYSTARK100000 Row No. 4 - Extra: MLUKESKYWALKER21000]'; l_actual_message := ut3_tester_helper.main_helper.get_failed_expectations(1); @@ -1237,10 +1237,8 @@ Rows: [ 4 differences ] l_expected_message := q'[Actual: refcursor [ count = 1 ] was expected to equal: refcursor [ count = 1 ] %Diff: %Rows: [ 1 differences ] -%PK 1 - Actual: 30 -%PK 1 - Expected: 3 -%PK 1 - Actual: 40 -%PK 1 - Expected: 4]'; +%PK 1 - Actual: 3040 +%PK 1 - Expected: 34]'; l_actual_message := ut3_tester_helper.main_helper.get_failed_expectations(1); --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); @@ -2511,7 +2509,7 @@ Diff:% begin l_exp_message :='ORA-20218: SQL exception thrown when fetching data from cursor: ORA-01476: divisor is equal to zero -at "UT3$USER#.TEST_EXPECTATIONS_CURSOR%", line 2522 ut3.ut.expect(l_actual).to_equal(l_expected);% +at "UT3$USER#.TEST_EXPECTATIONS_CURSOR%", line % ut3.ut.expect(l_actual).to_equal(l_expected);% Check the query and data for errors.'; open l_actual for @@ -2536,7 +2534,7 @@ Check the query and data for errors.'; l_exp_message :='ORA-20218: SQL exception thrown when fetching data from cursor: ORA-01476: divisor is equal to zero -at "UT3$USER#.TEST_EXPECTATIONS_CURSOR%", line 2547 ut3.ut.expect(l_actual).to_equal(l_expected);% +at "UT3$USER#.TEST_EXPECTATIONS_CURSOR%", line % ut3.ut.expect(l_actual).to_equal(l_expected);% Check the query and data for errors.'; open l_expected for From 054c39fb522fd456aeb39545d395bb840296db7c Mon Sep 17 00:00:00 2001 From: Jacek Gebal Date: Sun, 16 Feb 2020 00:02:29 +0000 Subject: [PATCH 2/2] Fixed issues with new cursor-results compare query on 11g --- .../data_values/ut_compound_data_helper.pkb | 50 +++++++++---------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/source/expectations/data_values/ut_compound_data_helper.pkb b/source/expectations/data_values/ut_compound_data_helper.pkb index 6a6b30a50..69ba90811 100644 --- a/source/expectations/data_values/ut_compound_data_helper.pkb +++ b/source/expectations/data_values/ut_compound_data_helper.pkb @@ -119,8 +119,6 @@ create or replace package body ut_compound_data_helper is || case when a_order_enforced then q'[ --column position is not matching (both when excluded extra/missing columns as well as when they are included) or (a_pos_nn != e_pos_nn and exp_col_pos != act_col_pos)]' - else - null end ||q'[ order by exp_col_pos, act_col_pos]' bulk collect into l_results using a_expected, a_actual; @@ -426,34 +424,34 @@ create or replace package body ut_compound_data_helper is end if; l_sql := q'[ with - exp_rows as( - select - exp_data_id, extract( exp_item_data, :column_path ) exp_item_data, item_no, diff_id - from ut_compound_data_diff_tmp - where diff_id = :diff_id - and exp_data_id = :self_guid - ), - act_rows as ( - select - act_data_id, extract( act_item_data, :column_path ) act_item_data, item_no, diff_id - from ut_compound_data_diff_tmp ucd - where diff_id = :diff_id - and ucd.act_data_id = :other_guid - ), exp_cols as ( select - exp_item_data, exp_data_id, item_no rn, rownum col_no, diff_id, + i.exp_item_data, i.exp_data_id, i.item_no rn, rownum col_no, i.diff_id, s.column_value col, s.column_value.getRootElement() col_name, - nvl(s.column_value.getclobval(),empty_clob()) col_val - from exp_rows i, + nvl( s.column_value.getclobval(), empty_clob() ) col_val + from ( + select + ucd.exp_data_id, extract( ucd.exp_item_data, :column_path ) exp_item_data, + ucd.item_no, ucd.diff_id + from ut_compound_data_diff_tmp ucd + where ucd.diff_id = :diff_id + and ucd.exp_data_id = :self_guid + ) i, table( xmlsequence( extract( i.exp_item_data, :extract_path ) ) ) s ), act_cols as ( select - act_item_data, act_data_id, item_no rn, rownum col_no, diff_id, + i.act_item_data, i.act_data_id, i.item_no rn, rownum col_no, i.diff_id, s.column_value col, s.column_value.getRootElement() col_name, - nvl(s.column_value.getclobval(),empty_clob()) col_val - from act_rows i, + nvl( s.column_value.getclobval(), empty_clob() ) col_val + from ( + select + ucd.act_data_id, extract( ucd.act_item_data, :column_path ) act_item_data, + ucd.item_no, ucd.diff_id + from ut_compound_data_diff_tmp ucd + where ucd.diff_id = :diff_id + and ucd.act_data_id = :other_guid + ) i, table( xmlsequence( extract( i.act_item_data, :extract_path ) ) ) s ), data_diff as ( @@ -496,7 +494,8 @@ create or replace package body ut_compound_data_helper is col_name from ( ]' || case when a_unordered then q'[ - select u.rn, u.diff_type, u.diffed_row, + select /*+ no_unnest */ + u.rn, u.diff_type, u.diffed_row, replace( extract( case when i.exp_data_id is null then i.act_item_data else i.exp_item_data end, :join_by ).getclobval(), chr(10) @@ -551,9 +550,8 @@ create or replace package body ut_compound_data_helper is end; execute immediate l_sql bulk collect into l_results - using l_exp_extract_xpath, a_diff_id, a_expected_dataset_guid, - l_act_extract_xpath, a_diff_id, a_actual_dataset_guid, - a_extract_path, a_extract_path, + using l_exp_extract_xpath, a_diff_id, a_expected_dataset_guid, a_extract_path, + l_act_extract_xpath, a_diff_id, a_actual_dataset_guid, a_extract_path, l_join_xpath, l_join_xpath, l_join_xpath, a_diff_id; return l_results; end;