10000 CQ: Get messages list from file when doing compaction · rabbitmq/rabbitmq-server@2955c4e · GitHub
[go: up one dir, main page]
Skip to content

Commit 2955c4e

Browse files
author
Loïc Hoguin
committed
CQ: Get messages list from file when doing compaction
Doing that from ets is far too expensive unfortunately.
1 parent e033d97 commit 2955c4e

File tree

2 files changed

+20
-21
lines changed

2 files changed

+20
-21
lines changed

‎deps/rabbit/src/rabbit_msg_store.erl

Lines changed: 19 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1913,21 +1913,25 @@ delete_file(File, State = #gc_state { file_summary_ets = FileSummaryEts,
19131913
ok
19141914
end.
19151915

1916-
load_and_vacuum_message_file(File, State) ->
1917-
Messages0 = index_select_all_from_file(File, State),
1918-
%% Cleanup messages that have 0 ref_count.
1919-
Messages = lists:foldl(fun
1920-
(Entry = #msg_location{ ref_count = 0 }, Acc) ->
1921-
ok = index_delete_object(Entry, State),
1922-
Acc;
1923-
(Entry, Acc) ->
1924-
[Entry|Acc]
1925-
end, [], Messages0),
1926-
lists:keysort(#msg_location.offset, Messages).
1927-
1928-
index_select_all_from_file(File, #gc_state { index_module = Index,
1929-
index_state = State }) ->
1930-
Index:select_all_from_file(File, State).
1916+
load_and_vacuum_message_file(File, State = #gc_state{ dir = Dir }) ->
1917+
%% Messages here will be end-of-file at start-of-list
1918+
{ok, Messages, _FileSize} =
1919+
scan_file_for_valid_messages(Dir, filenum_to_name(File)),
1920+
%% foldl will reverse so will end up with msgs in ascending offset order
1921+
lists:foldl(
1922+
fun ({MsgId, TotalSize, Offset}, Acc) ->
1923+
case index_lookup(MsgId, State) of
1924+
#msg_location { file = File, total_size = TotalSize,
1925+
offset = Offset, ref_count = 0 } = Entry ->
1926+
ok = index_delete_object(Entry, State),
1927+
Acc;
1928+
#msg_location { file = File, total_size = TotalSize,
1929+
offset = Offset } = Entry ->
1930+
[ Entry | Acc ];
1931+
_ ->
1932+
Acc
1933+
end
1934+
end, [], Messages).
19311935

19321936
scan_and_vacuum_message_file(File, State = #gc_state { dir = Dir }) ->
19331937
%% Messages here will be end-of-file at start-of-list

‎deps/rabbit/src/rabbit_msg_store_ets_index.erl

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
-behaviour(rabbit_msg_store_index).
1313

1414
-export([new/1, recover/1,
15-
lookup/2, select_from_file/3, select_all_from_file/2, insert/2, update/2, update_fields/3, delete/2,
15+
lookup/2, select_from_file/3, insert/2, update/2, update_fields/3, delete/2,
1616
delete_object/2, clean_up_temporary_reference_count_entries_without_file/1, terminate/1]).
1717

1818
-define(MSG_LOC_NAME, rabbit_msg_store_ets_index).
@@ -49,11 +49,6 @@ select_from_file(MsgIds, File, State) ->
4949
All = [lookup(Id, State) || Id <- MsgIds],
5050
[MsgLoc || MsgLoc=#msg_location{file=MsgFile} <- All, MsgFile =:= File].
5151

52-
%% Note that this function is not terribly efficient and should only be
53-
%% used for compaction or similar.
54-
select_all_from_file(File, State) ->
55-
ets:match_object(State #state.table, #msg_location { file = File, _ = '_' }).
56-
5752
insert(Obj, State) ->
5853
true = ets:insert_new(State #state.table, Obj),
5954
ok.

0 commit comments

Comments
 (0)
0