8000 Fix output precision limit for double values (issue #118) by vitcpp · Pull Request #119 · postgrespro/pgsphere · GitHub
[go: up one dir, main page]
Skip to content

Fix output precision limit for double values (issue #118) #119

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Mar 26, 2024
Merged

Fix output precision limit for double values (issue #118) #119

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b7e12f3
Fix output precision limit for double values (issue #118)
vitcpp Mar 14, 2024
c2116ae
Update upgrade script (reset_sphere_output_precision function)
vitcpp Mar 15, 2024
d9d5461
Add test for pgSphere output precision with different settings
vitcpp Mar 15, 2024
7b3b114
Add extra_float_digits = 2 for epochprop and bounding_box_gist tests
vitcpp Mar 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Add test for pgSphere output precision with different settings 
expected/output_precision.out   - PG 10-11
expected/output_precision_1.out - PG 12+
  • Loading branch information
@vitcpp
vitcpp committed Mar 15, 2024
commit d9d5461a477e3bb7af7321b0824c8fcf9c96b155
3 changes: 2 additions & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@ DATA_built = $(RELEASE_SQL) \
DOCS = README.pg_sphere COPYRIGHT.pg_sphere
TESTS = version tables points euler circle line ellipse poly path box \
index contains_ops contains_ops_compat bounding_box_gist gnomo \
epochprop contains overlaps spoint_brin sbox_brin selectivity knn
epochprop contains overlaps spoint_brin sbox_brin selectivity \
knn output_precision
REGRESS = init $(TESTS)

PG_CFLAGS += -DPGSPHERE_VERSION=$(PGSPHERE_VERSION)
Expand Down
212 changes: 212 additions & 0 deletions expected/output_precision.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,212 @@
--
-- Test default and custom output precisions for double values.
--
SELECT set_sphere_output( 'RAD' );
set_sphere_output
-------------------
SET RAD
(1 row)

--
-- Check default precision
--
SELECT '( 1h 2m 30s , +1d 2m 30s)'::spoint;
spoint
------------------------------------------
(0.272707695624114 , 0.0181805130416076)
(1 row)

--
-- Check option extra_float_digits
--
SET extra_float_digits TO -6;
SELECT '( 1h 2m 30s , +1d 2m 30s)'::spoint;
spoint
-----------------------------
(0.272707696 , 0.018180513)
(1 row)

SET extra_float_digits TO -2;
SELECT '( 1h 2m 30s , +1d 2m 30s)'::spoint;
spoint
--------------------------------------
(0.2727076956241 , 0.01818051304161)
(1 row)

SET extra_float_digits TO 0;
SELECT '( 1h 2m 30s , +1d 2m 30s)'::spoint;
spoint
------------------------------------------
(0.272707695624114 , 0.0181805130416076)
(1 row)

SET extra_float_digits TO 1;
SELECT '( 1h 2m 30s , +1d 2m 30s)'::spoint;
spoint
------------------------------------------
(0.272707695624114 , 0.0181805130416076)
(1 row)

SET extra_float_digits TO 2;
SELECT '( 1h 2m 30s , +1d 2m 30s)'::spoint;
spoint
----------------------------------------------
(0.27270769562411401 , 0.018180513041607602)
(1 row)

SET extra_float_digits TO 3;
SELECT '( 1h 2m 30s , +1d 2m 30s)'::spoint;
spoint
-----------------------------------------------
(0.27270769562411401 , 0.0181805130416076016)
(1 row)

SET extra_float_digits TO 6;
ERROR: 6 is outside the valid range for parameter "extra_float_digits" (-15 .. 3)
SELECT '( 1h 2m 30s , +1d 2m 30s)'::spoint;
spoint
-----------------------------------------------
(0.27270769562411401 , 0.0181805130416076016)
(1 row)

--
-- Check compatibility behaviour
--
SELECT set_sphere_output_precision(10);
set_sphere_output_precision
-----------------------------
SET 10
(1 row)

SELECT '( 1h 2m 30s , +1d 2m 30s)'::spoint;
spoint
--------------------------------
(0.2727076956 , 0.01818051304)
(1 row)

SELECT set_sphere_output_precision(12);
set_sphere_output_precision
-----------------------------
SET 12
(1 row)

SELECT '( 1h 2m 30s , +1d 2m 30s)'::spoint;
spoint
------------------------------------
(0.272707695624 , 0.0181805130416)
(1 row)

SELECT set_sphere_output_precision(15);
set_sphere_output_precision
-----------------------------
SET 15
(1 row)

SELECT '( 1h 2m 30s , +1d 2m 30s)'::spoint;
spoint
------------------------------------------
(0.272707695624114 , 0.0181805130416076)
(1 row)

SELECT set_sphere_output_precision(17);
set_sphere_output_precision
-----------------------------
SET 15
(1 row)

SELECT '( 1h 2m 30s , +1d 2m 30s)'::spoint;
spoint
------------------------------------------
(0.272707695624114 , 0.0181805130416076)
(1 row)

SELECT set_sphere_output_precision(20);
set_sphere_output_precision
-----------------------------
SET 15
(1 row)

SELECT '( 1h 2m 30s , +1d 2m 30s)'::spoint;
spoint
------------------------------------------
(0.272707695624114 , 0.0181805130416076)
(1 row)

SELECT set_sphere_output_precision(0);
set_sphere_output_precision
-----------------------------
SET 15
(1 row)

SELECT '( 1h 2m 30s , +1d 2m 30s)'::spoint;
spoint
------------------------------------------
(0.272707695624114 , 0.0181805130416076)
(1 row)

SELECT set_sphere_output_precision(-3);
set_sphere_output_precision
-----------------------------
SET 15
(1 row)

SELECT '( 1h 2m 30s , +1d 2m 30s)'::spoint;
spoint
------------------------------------------
(0.272707695624114 , 0.0181805130416076)
(1 row)

--
-- Check extra_float_digits after set_sphere_output_precision.
-- The change of extra_float_digits should not affect the precision of pgsphere
-- output because set_sphere_output_precision enables compatibility mode.
--
SELECT set_sphere_output_precision(10);
set_sphere_output_precision
-----------------------------
SET 10
(1 row)

SELECT '( 1h 2m 30s , +1d 2m 30s)'::spoint;
spoint
--------------------------------
(0.2727076956 , 0.01818051304)
(1 row)

SET extra_float_digits TO -6;
SELECT '( 1h 2m 30s , +1d 2m 30s)'::spoint;
spoint
--------------------------------
(0.2727076956 , 0.01818051304)
(1 row)

SET extra_float_digits TO -10;
SELECT '( 1h 2m 30s , +1d 2m 30s)'::spoint;
spoint
--------------------------------
(0.2727076956 , 0.01818051304)
(1 row)

--
-- Check reset_sphere_output_precision.
-- It should disable compatibility mode - extra_float_digits should work.
--
SELECT reset_sphere_output_precision();
reset_sphere_output_precision
-------------------------------
RESET
(1 row)

SELECT '( 1h 2m 30s , +1d 2m 30s)'::spoint;
spoint
----------------------
(0.27271 , 0.018181)
(1 row)

SET extra_float_digits TO -6;
SELECT '( 1h 2m 30s , +1d 2m 30s)'::spoint;
spoint
-----------------------------
(0.272707696 , 0.018180513)
(1 row)

Loading
0