8000 Use synchronous commit · osguydch/postgres_cluster@f73f993 · GitHub
[go: up one dir, main page]
Skip to content

Commit f73f993

Browse files
knizhnikknizhnik
committed
Use synchronous commit
2 parents 7dc0fe6 + 59befec commit f73f993

File tree

9 files changed

+110
-86
lines changed

9 files changed

+110
-86
lines changed

‎.dockerignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
*/*/.git
2+
.git
3+
.vscode

‎Dockerfile

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
# vim:set ft=dockerfile:
2+
FROM debian:jessie
3+
4+
# explicitly set user/group IDs
5+
RUN groupadd -r postgres --gid=999 && useradd -r -g postgres --uid=999 postgres
6+
7+
# make the "en_US.UTF-8" locale so postgres will be utf-8 enabled by default
8+
RUN apt-get update && apt-get install -y locales && rm -rf /var/lib/apt/lists/* \
9+
&& localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8
10+
ENV LANG en_US.utf8
11+
12+
# postgres build deps
13+
RUN apt-get update && apt-get install -y \
14+
git \
15+
make \
16+
gcc \
17+
gdb \
18+
libreadline-dev \
19+
bison \
20+
flex \
21+
zlib1g-dev \
22+
sudo \
23+
&& rm -rf /var/lib/apt/lists/*
24+
25+
RUN mkdir /pg && chown postgres:postgres /pg
26+
# We need that to allow editing of /proc/sys/kernel/core_pattern
27+
# from docker-entrypoint.sh
28+
RUN echo "postgres ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers
29+
30+
COPY ./ /pg/src
31+
RUN chown -R postgres:postgres /pg/src
32+
33+
USER postgres
34+
ENV CFLAGS -O0
35+
WORKDIR /pg
36+
37+
RUN cd /pg/src && \
38+
ls -la && \
39+
whoami && \
40+
./configure --enable-cassert --enable-debug --prefix=/pg/install && \
41+
make -j 4 install
42+
43+
ENV PATH /pg/install/bin:$PATH
44+
ENV PGDATA /pg/data
45+
46+
RUN cd /pg/src/contrib/raftable && make clean && make install
47+
RUN cd /pg/src/contrib/mmts && make clean && make install
48+
49+
ENTRYPOINT ["/pg/src/contrib/mmts/tests2/docker-entrypoint.sh"]
50+
51+
EXPOSE 5432
52+
CMD ["postgres"]

‎GNUmakefile.in

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,20 @@ include $(top_builddir)/src/Makefile.global
1010

1111
$(call recurse,all install,src config)
1212

13+
xcheck:
14+
#pip install -r tests2/requirements.txt
15+
# docker build -t pgmmts .
16+
cd contrib/mmts/tests2 && blockade destroy || true
17+
cd contrib/mmts/tests2 && docker rm node1 || true
18+
cd contrib/mmts/tests2 && docker rm node2 || true
19+
cd contrib/mmts/tests2 && docker rm node3 || true
20+
cd contrib/mmts/tests2 && docker network rm tests2_net || true
21+
cd contrib/mmts/tests2 && docker network rm tests2_net || true
22+
cd contrib/mmts/tests2 && blockade up
23+
sleep 20 # wait for mmts init
24+
cd contrib/mmts/tests2 && python test_recovery.py || true
25+
#cd contrib/mmts/tests2 && blockade destroy
26+
1327
all:
1428
+@echo "All of PostgreSQL successfully made. Ready to install."
1529

‎contrib/mmts/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ USER postgres
3131
ENV CFLAGS -O0
3232
WORKDIR /pg
3333

34-
ENV REBUILD 2
34+
ENV REBUILD 5
3535

3636
RUN cd /pg && \
3737
git clone https://github.com/postgrespro/postgres_cluster.git --depth 1 && \

‎contrib/mmts/tests2/docker-entrypoint.sh

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,13 +68,15 @@ if [ "$1" = 'postgres' ]; then
6868
listen_addresses='*'
6969
max_prepared_transactions = 100
7070
synchronous_commit = on
71+
fsync = off
7172
wal_level = logical
7273
max_worker_processes = 30
7374
max_replication_slots = 10
7475
max_wal_senders = 10
7576
shared_preload_libraries = 'raftable,multimaster'
76-
default_transaction_isolation = 'repeatable read'
77+
default_transaction_isolation = 'repeatable read'
7778
log_checkpoints = on
79+
checkpoint_timeout = 30
7880
log_autovacuum_min_duration = 0
7981
8082
raftable.id = $NODE_ID

‎contrib/mmts/tests2/lib/bank_client.py

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,11 +33,14 @@ def stop(self):
3333
for client in self._clients:
3434
client.stop()
3535

36-
def print_agg(self):
36+
def aggregate(self, echo=True):
3737
aggs = []
3838
for client in self._clients:
3939
aggs.append(client.history.aggregate())
4040

41+
if not echo:
42+
return aggs
43+
4144
columns = ['running', 'running_latency', 'max_latency', 'finish']
4245

4346
print("\t\t", end="")
@@ -61,10 +64,7 @@ def print_agg(self):
6164

6265
print("")
6366

64-
def set_acc_to_tx(self, max_acc):
65-
for client in self._clients:
66-
client.set_acc_to_tx(max_acc)
67-
67+
return aggs
6868

6969
class BankClient(object):
7070

@@ -141,7 +141,7 @@ def exec_tx(self, name, tx_block):
141141
tx_block(conn, cur)
142142
self.history.register_finish(event_id, 'Commit')
143143
except psycopg2.Error as e:
144-
print("=== node%d: %s" % (self.node_id, e.pgerror))
144+
# print("=== node%d: %s" % (self.node_id, e.pgerror))
145145
self.history.register_finish(event_id, e.pgerror)
146146
#time.sleep(0.2)
147147

‎contrib/mmts/tests2/lib/event_history.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ def aggregate(self):
8585
self.events = []
8686

8787
for value in self.running_events.itervalues():
88-
print(value)
88+
# print(value)
8989

9090
if value['name'] not in agg:
9191
agg[value['name']] = copy.deepcopy(self.agg_template)

‎contrib/mmts/tests2/test_recovery.py

Lines changed: 26 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -6,105 +6,56 @@
66
class RecoveryTest(unittest.TestCase):
77
@classmethod
88
def setUpClass(self):
9-
#subprocess.check_call(['blockade','up'])
109
self.clients = ClientCollection([
1110
"dbname=postgres host=127.0.0.1 user=postgres",
1211
"dbname=postgres host=127.0.0.1 user=postgres port=5433",
1312
"dbname=postgres host=127.0.0.1 user=postgres port=5434"
1413
])
14+
self.clients.start()
15+
time.sleep(5)
1516

1617
@classmethod
1718
def tearDownClass(self):
1819
print('tearDown')
19-
#subprocess.check_call(['blockade','join'])
20-
21-
# in case of error
2220
self.clients.stop()
23-
#self.clients[0].cleanup()
2421

25-
def test_0_0_0(self):
26-
print('### normalOpsTest ###')
22+
# def test_normal_operations(self):
23+
# print('### normalOpsTest ###')
24+
25+
# for i in range(3):
26+
# time.sleep(3)
27+
# aggs = self.clients.aggregate()
28+
# for agg in aggs:
29+
# # there were some commits
30+
# self.assertTrue( agg['transfer'] > 0 )
31+
32+
def test_node_prtition(self):
33+
print('### nodePartitionTest ###')
2734

28-
time.sleep(10)
2935
subprocess.check_call(['blockade','partition','node3'])
3036
print('### blockade node3 ###')
3137

32-
self.clients.set_acc_to_tx(10000)
33-
self.clients.start()
38+
# clear tx history
39+
self.clients.aggregate(echo=False)
3440

35-
for i in range(10):
41+
for i in range(3):
3642
time.sleep(3)
37-
self.clients.print_agg()
38-
print("\n")
43+
aggs = self.clients.aggregate()
44+
self.assertTrue( aggs[0]['transfer']['finish']['Commit'] > 0 )
45+
self.assertTrue( aggs[1]['transfer']['finish']['Commit'] > 0 )
46+
self.assertTrue( 'Commit' not in aggs[2]['transfer'] )
3947

4048
subprocess.check_call(['blockade','join'])
4149
print('### deblockade node3 ###')
4250

51+
# clear tx history
52+
self.clients.aggregate(echo=False)
53+
4354
for i in range(1000):
4455
time.sleep(3)
45-
t = datetime.datetime.now()
46-
self.clients.print_agg()
47-
print("took %f seconds" % ( (datetime.datetime.now()-t).total_seconds(),))
48-
print("\n")
49-
50-
self.clients.stop()
51-
52-
56+
aggs = self.clients.aggregate()
57+
print(i, aggs)
5358

54-
# def test_0_normal_operation(self):
55-
# print('### normalOpsTest ###')
56-
#
57-
# self.clients.set_acc_to_tx(10000)
58-
# self.clients.start()
59-
#
60-
# for i in range(5):
61-
# time.sleep(3)
62-
# for client in self.clients:
63-
# agg = client.history.aggregate()
64-
# print(agg)
65-
# self.assertTrue(agg['transfer']['finish']['Commit'] > 0)
66-
# print("\n")
67-
#
68-
# self.clients.stop()
69-
#
70-
# def test_1_distributed_deadlock(self):
71-
# print('### DDD test ###')
72-
#
73-
# self.clients.set_acc_to_tx(10)
74-
# self.clients.start()
75-
#
76-
# for i in range(5):
77-
# time.sleep(3)
78-
# for client in self.clients:
79-
# agg = client.history.aggregate()
80-
# print(agg)
81-
# self.assertTrue(agg['transfer']['finish']['Commit'] > 0)
82-
# print("\n")
83-
#
84-
# self.clients.stop()
85-
#
86-
# def test_2_node_disconnect(self):
87-
# print('### disconnectTest ###')
88-
#
89-
# self.clients.set_acc_to_tx(10000)
90-
# self.clients.start()
91-
#
92-
# subprocess.check_call(['blockade','partition','node3'])
93-
# print('Node3 disconnected')
94-
#
95-
# # give cluster some time to discover problem
96-
# time.sleep(3)
97-
#
98-
# for i in range(5):
99-
# time.sleep(3)
100-
# for client in self.clients:
101-
# agg = client.history.aggregate()
102-
# print(agg)
103-
# self.assertTrue(agg['transfer']['finish']['Commit'] > 0)
104-
# print("\n")
105-
#
106-
# subprocess.check_call(['blockade','join'])
107-
# self.clients.stop()
10859

10960
if __name__ == '__main__':
11061
unittest.main()

‎src/backend/access/transam/twophase.c

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1218,7 +1218,6 @@ ReadTwoPhaseFile(TransactionId xid, bool give_warnings)
12181218
if (crc_offset != MAXALIGN(crc_offset))
12191219
{
12201220
CloseTransientFile(fd);
1221-
fprintf(stderr, "wrong crc offset in two-phase file \"%s\"\n", path);
12221221
return NULL;
12231222
}
12241223

@@ -1245,7 +1244,6 @@ ReadTwoPhaseFile(TransactionId xid, bool give_warnings)
12451244
if (hdr->magic != TWOPHASE_MAGIC || hdr->total_len != stat.st_size)
12461245
{
12471246
pfree(buf);
1248-
fprintf(stderr, "muggle two-phase file \"%s\": no magic\n", path);
12491247
return NULL;
12501248
}
12511249

@@ -1622,6 +1620,10 @@ RecreateTwoPhaseFile(TransactionId xid, void *content, int len)
16221620
pg_crc32c statefile_crc;
16231621
int fd;
16241622

1623+
/* Crutch to fix crc and len check on 2pc file reading */
1624+
Assert( ((TwoPhaseFileHeader *) content)->total_len - sizeof(pg_crc32c) <= len);
1625+
len = ((TwoPhaseFileHeader *) content)->total_len - sizeof(pg_crc32c);
1626+
16251627
/* Recompute CRC */
16261628
INIT_CRC32C(statefile_crc);
16271629
COMP_CRC32C(statefile_crc, content, len);

0 commit comments

Comments
 (0)
0