8000 Add support for stamps with unlimited size for GalleySeries · postgrespro/libblobstamper@c26a528 · GitHub
[go: up one dir, main page]
Skip to content

Commit c26a528

Browse files
nataraj-hates-MS-for-stealing-githubnataraj-hates-MS-for-stealing-github
committed
Add support for stamps with unlimited size for GalleySeries
1 parent fc77793 commit c26a528

File tree

9 files changed

+158
-48
lines changed

9 files changed

+158
-48
lines changed

‎Makefile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ blobstamper/helpers.o \
1111
blobstamper/stamp.o \
1212
blobstamper/stamp_atomic.o \
1313
blobstamper/stamp_pg_type_geo.o \
14+
blobstamper/galley.o \
1415
blobstamper/dict.o \
1516

1617
WRAPPERS_OBJ = pg_op_wrappers.o

‎blobstamper/Makefile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
44
stamp.o \
55
stamp_atomic.o \
66
stamp_pg_type_geo.o \
7+
galley.o \
78
dict.o
89

910

‎blobstamper/blob.cpp

Lines changed: 1 addition & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,11 @@
1-
2-
31
#include <cstring>
42

53
#include "blob.h"
64
#include "helpers.h"
75

8-
96
#include "stamp.h"
107

8+
119
Blob::Blob (char * data_in, int size_in)
1210
{
1311
data = data_in;
@@ -75,31 +73,3 @@ Blob::DataDup(char *& data_out, size_t& size_out)
7573
//FIXME add out of memory check here!!!!
7674
memcpy(data_out, data + begin, size_out);
7775
}
78-
79-
80-
std::list<std::string>
81-
GalleySeries::Extract(Blob &blob)
82-
{
83-
std::list<std::string> res;
84-
85-
if (stamp.isFixedSize())
86-
{
87-
while (1)
88-
{
89-
std::string el = blob.ShiftSingleStampStr(stamp);
90-
if (el.empty())
91-
break;
92-
res.push_back(el);
93-
}
94-
}
95-
else
96-
{
97-
printf("Not implemented yet!");
98-
exit(1);
99-
}
100-
101-
return res;
102-
}
103-
104-
105-

‎blobstamper/blob.h

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -26,19 +26,5 @@ class Blob
2626
std::string ShiftSingleStampStr(StampGeneric &stmp);
2727
};
2828

29-
class GalleyBase
30-
{
31-
32-
};
33-
34-
class GalleySeries : public GalleyBase
35-
{
36-
protected:
37-
StampGeneric &stamp;
38-
public:
39-
GalleySeries(StampGeneric & stamp_arg) : stamp(stamp_arg) {};
40-
std::list<std::string> Extract(Blob &blob);
41-
};
42-
4329

4430
#endif /*BLOB_H*/

‎blobstamper/blobstamper.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,6 @@
44
#include "stamp_atomic.h"
55
#include "stamp_pg_type_geo.h"
< 9E88 /code>
66
#include "dict.h"
7+
#include "galley.h"
8+
79

‎blobstamper/galley.cpp

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
2+
3+
4+
#include "blob.h"
5+
#include "stamp.h"
6+
#include "stamp_atomic.h"
7+
#include "galley.h"
8+
#include <vector>
9+
10+
11+
std::list<std::string>
12+
GalleySeries::Extract(Blob &blob)
13+
{
14+
std::list<std::string> res;
15+
16+
if (stamp.isFixedSize())
17+
{
18+
while (1)
19+
{
20+
std::string el = blob.ShiftSingleStampStr(stamp);
21+
if (el.empty())
22+
break;
23+
res.push_back(el);
24+
}
25+
}
26+
else
27+
{
28+
if (stamp.maxSize() == -1) // if unlimited size
29+
{
30+
/*
31+
The idea of this part is following:
32+
We get two bytes from blob, and treat it as an oracle that says in how many parts blob shold be split.
33+
34+
We use stamp.minSize() and blob.Size() to determinate how many parts are possible at maximum (count_max)
35+
and normalize oracle so it would fit count_max in any case. Thus we get count_target the number of parts we
36+
wil have
37+
38+
Then we shift a size oracle for each part
39+
40+
Then we normalize size oracles to they in total will give us the size of remaining blob. Here we should
41+
keep in mind that part cant' be shorter than stamp.minSize()
42+
43+
Then we chop blob into selected sizes, and stap each chopped part with stamp.
44+
*/
45+
46+
/* Getting count oracle and normalze it to fit available size */
47+
size_t count_max = (blob.Size() - ORACLE_SIZE) / (stamp.minSize() + ORACLE_SIZE); //First oracle - for number of items, and second one is oracle for each item size
48+
ORACLE_STAMP stamp_oracle;
49+
ORACLE_TYPE *count_oracle;
50+
count_oracle = (ORACLE_TYPE *) blob.ShiftSingleStampBin(stamp_oracle);
51+
52+
ORACLE_TYPE count_target = count_max * (*count_oracle) / ORACLE_MAX + 1; /* +1 -- это грубая эмуляция округления вверх. oracle == ORACLE_MAX-1 == 65534 должен дать count_max*/
53+
if (count_target > count_max) count_target = count_max; // В случае если oracle оказался рваен ORACLE_MAX
54+
55+
/* Getting size oracles for each part */
56+
std::vector<ORACLE_TYPE> size_oracles;
57+
int size_oracle_total = 0;
58+
for(int i = 0; i<count_target; i++)
59+
{
60+
ORACLE_TYPE *o = (ORACLE_TYPE *) blob.ShiftSingleStampBin(stamp_oracle);
61+
size_oracles.push_back(*o);
62+
size_oracle_total += *o;
63+
free(o);
64+
}
65+
66+
/* Calculating available vairable size, that will be destributed between parts according to size oracles */
67+
int data_size = blob.Size();
68+
int fixed_data_size = stamp.minSize() * count_target;
69+
int var_data_size = data_size - fixed_data_size;
70+
71+
/* normalizing oracles so they fit total variable size, chop to parts and stamp parts */
72+
float remainder = 0; /* we do not want waste bytes because of rounding, so we keep the remainder, and reuse it. Thus we will use all bytes (alomost, may loose last one due to remainder=0.99999)*/
73+
for(ORACLE_TYPE o : size_oracles)
74+
{
75+
float el_size_f = stamp.minSize() + (float) o / size_oracle_total * var_data_size + remainder;
76+
int el_size = el_size_f;
77+
remainder = el_size_f - el_size;
78+
79+
Blob blob2 = blob.ShiftBytes(el_size);
80+
std::string str = blob2.ShiftSingleStampStr(stamp);
81+
res.push_back(str);
82+
}
83+
free(count_oracle);
84+
}
85+
else
86+
{
87+
printf("Not implemented yet!");
88+
exit(1);
89+
}
90+
}
91+
return res;
92+
}
93+
94+
95+
int
96+
GalleySeries::minSize()
97+
{
98+
if (stamp.isFixedSize())
99+
{
100+
return stamp.minSize(); // When size is fixed, series can have only one member with no extra data used
101+
}
102+
else
103+
{
104+
if (stamp.maxSize() == -1) // if unlimited size
105+
{
106+
return stamp.minSize() + ORACLE_SIZE * 2; // One -- count oracle, one -- size oracle
107+
}
108+
else
109+
{
110+
printf("Not implemented yet!");
111+
exit(1);
112+
}
113+
}
114+
}

‎blobstamper/galley.h

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
2+
#ifndef GALLEY_H
3+
#define GALLEY_H
4+
5+
6+
#include <limits.h>
7+
8+
#define ORACLE_STAMP StampBinInt16
9+
#define ORACLE_TYPE unsigned short int
10+
#define ORACLE_SIZE sizeof(ORACLE_TYPE)
11+
#define ORACLE_MAX USHRT_MAX
12+
13+
class GalleyBase
14+
{
15+
public:
16+
virtual int minSize() {return 1;};
17+
virtual int maxSize() {return 1;};
18+
virtual bool isFixedSize() {return 0;}
19+
20+
};
21+
22+
class GalleySeries : public GalleyBase
23+
{
24+
protected:
25+
StampGeneric &stamp;
26+
public:
27+
GalleySeries(StampGeneric & stamp_arg) : stamp(stamp_arg) {};
28+
std::list<std::string> Extract(Blob &blob);
29+
int minSize() override;
30+
int maxSize() override {return -1;}; /* Sereies always takes as much data as it can take */
31+
bool isFixedSize() override {return 0;} /* And not fixed size */
32+
33+
};
34+
35+
#endif /* GALLEY_H */

‎blobstamper/stamp.h

< CF7E div class="d-flex flex-row flex-justify-end flex-1 flex-order-1 flex-sm-order-2 flex-items-center">
Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,9 @@ class StampGeneric
1212
int min_size;
1313
int max_size;
1414
public:
15-
bool isFixedSize() {return is_fixed_size;}
16-
int minSize() {return min_size;}
17-
int maxSize() {return max_size;}
15+
virtual bool isFixedSize() {return is_fixed_size;}
16+
virtual int minSize() {return min_size;}
17+
virtual int maxSize() {return max_size;}
1818

1919
virtual void * Extract(Blob &blob);
2020
virtual std::string ExtractStr(Blob &blob) {printf ("22222\n"); return "";}

‎t/Makefile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ BLOB_STAMPER_OBJ = ../blobstamper/blob.o \
1818
../blobstamper/dict.o \
1919
../blobstamper/stamp.o \
2020
../blobstamper/stamp_atomic.o \
21+
../blobstamper/galley.o \
2122
../blobstamper/stamp_pg_type_geo.o \
2223
../pg_op_wrappers.o \
2324

0 commit comments

Comments
 (0)
0