From 018674b82c6edfe81eb73764ef10dd363c20502d Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Thu, 5 Sep 2019 10:36:41 +0100 Subject: [PATCH 1/8] Add fetch request/response Initial attempt to define Fetch request/response protocol. Also, previous PR to add gset support had broken dialyzer, so specs corrected to reflect actual inputs/outputs of functions in riak_pb_dt_codec. --- src/riak_kv.proto | 18 +++++++++++++++++- src/riak_pb_dt_codec.erl | 6 +++--- src/riak_pb_messages.csv | 2 ++ 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/riak_kv.proto b/src/riak_kv.proto index 827c4c1a..30d0d5b4 100644 --- a/src/riak_kv.proto +++ b/src/riak_kv.proto @@ -346,4 +346,20 @@ message RpbRTEReq { optional bytes type = 13; // Bucket type, if not set we assume the 'default' type } -// RTE response - not defined, will return a RpbRTEResp on success or RpbErrorResp on failure \ No newline at end of file +// RTE response - not defined, will return a RpbRTEResp on success or RpbErrorResp on failure + +// Fetch request - retrieve an object by a queue name. +message RpbFetchReq{ + required bytes queuename = 1; +} + +// Fetch response - may be an empty queue, an object, a deleted tombstone +// If it is a deleted tombstone, it will return both an encoded_object and +// a deleted vclock. If the response is an object, just an encoded_object +// will be returned +message RpbFetchRsp{ + required bool queue_empty = 1; + optional bool deleted = 2; // Present only if queue_empty is false, true if object a tombstone + optional bytes replencoded_object = 3; // Special repl encoding using for object + optional bytes deleted_vclock = 4; +} \ No newline at end of file diff --git a/src/riak_pb_dt_codec.erl b/src/riak_pb_dt_codec.erl index 379a8164..628e8dd1 100644 --- a/src/riak_pb_dt_codec.erl +++ b/src/riak_pb_dt_codec.erl @@ -354,12 +354,12 @@ encode_set_update({remove_all, Members}, #setop{removes=R}=S) when is_list(Membe %% @doc Decodes a GSetOp message into a gset operation. --spec decode_gset_op(#setop{}) -> gset_op(). +-spec decode_gset_op(#gsetop{}) -> gset_op(). decode_gset_op(#gsetop{adds=A}) -> {add_all, A}. -%% @doc Encodes a set operation into a SetOp message. --spec encode_gset_op(gset_op()) -> #gsetop{}. +%% @doc Encodes a gset operation into a SetOp message. +-spec encode_gset_op(gset_op()|{update, [simple_gset_op()]}) -> #gsetop{}. encode_gset_op({update, Ops}) when is_list(Ops) -> lists:foldr(fun encode_gset_update/2, #gsetop{}, Ops); encode_gset_op({C, _}=Op) when add == C; add_all == C -> diff --git a/src/riak_pb_messages.csv b/src/riak_pb_messages.csv index e6ea186f..7312aa88 100644 --- a/src/riak_pb_messages.csv +++ b/src/riak_pb_messages.csv @@ -70,6 +70,8 @@ 104,TsTtbMsg,riak_ts 200,RpbRTEReq,riak_kv 201,RpbRTEResp,riak_kv +202,RpbFetchReq,riak_kv +203,RpbFetchResp,riak_kv 253,RpbAuthReq,riak 254,RpbAuthResp,riak 255,RpbStartTls,riak From 0c932253ae2241b1c401080739d61f3e24a242fd Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Thu, 5 Sep 2019 11:50:04 +0100 Subject: [PATCH 2/8] Add CRC check to fetch response --- src/riak_kv.proto | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/riak_kv.proto b/src/riak_kv.proto index 30d0d5b4..66ddeaba 100644 --- a/src/riak_kv.proto +++ b/src/riak_kv.proto @@ -360,6 +360,7 @@ message RpbFetchReq{ message RpbFetchRsp{ required bool queue_empty = 1; optional bool deleted = 2; // Present only if queue_empty is false, true if object a tombstone - optional bytes replencoded_object = 3; // Special repl encoding using for object - optional bytes deleted_vclock = 4; + optional uint32 crc_check = 3; // CRC check of encoded object + optional bytes replencoded_object = 4; // Special repl encoding using for object + optional bytes deleted_vclock = 5; } \ No newline at end of file From 68fa8c3aa41994bc85fd0233e11965adecb41c63 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Thu, 5 Sep 2019 21:55:40 +0100 Subject: [PATCH 3/8] Use resp not rsp For consistency --- src/riak_kv.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/riak_kv.proto b/src/riak_kv.proto index 66ddeaba..00c99a2d 100644 --- a/src/riak_kv.proto +++ b/src/riak_kv.proto @@ -357,7 +357,7 @@ message RpbFetchReq{ // If it is a deleted tombstone, it will return both an encoded_object and // a deleted vclock. If the response is an object, just an encoded_object // will be returned -message RpbFetchRsp{ +message RpbFetchResp{ required bool queue_empty = 1; optional bool deleted = 2; // Present only if queue_empty is false, true if object a tombstone optional uint32 crc_check = 3; // CRC check of encoded object From 9cabab93d9dc779a1fab4b3cc13e66ff4af8612b Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Tue, 26 Nov 2019 14:31:20 +0000 Subject: [PATCH 4/8] Add repl range request --- src/riak_kv.proto | 12 ++++++++++++ src/riak_pb_messages.csv | 1 + 2 files changed, 13 insertions(+) diff --git a/src/riak_kv.proto b/src/riak_kv.proto index a26eb661..49413be7 100644 --- a/src/riak_kv.proto +++ b/src/riak_kv.proto @@ -429,6 +429,18 @@ message RpbAaeFoldFetchClocksRangeReq{ optional uint32 last_mod_end = 42; } +message RpbAaeFoldReplKeysReq{ + optional bytes type = 1; + required bytes bucket = 2; + required bool key_range = 10; + optional bytes start_key = 11; + optional bytes end_key = 12; + required bool modified_range = 40; + optional uint32 last_mod_start = 41; + optional uint32 last_mod_end = 42; + required bytes queuename = 50; +} + message RpbAaeFoldFindKeysReq{ enum FindType { sibling_count = 0; diff --git a/src/riak_pb_messages.csv b/src/riak_pb_messages.csv index 8a91e1f9..bfee120c 100644 --- a/src/riak_pb_messages.csv +++ b/src/riak_pb_messages.csv @@ -79,6 +79,7 @@ 214,RpbAaeFoldFetchClocksRangeReq,riak_kv 215,RpbAaeFoldFindKeysReq,riak_kv 216,RpbAaeFoldObjectStatsReq,riak_kv +217,RpbAaeFoldReplKeysReq,riak_kv 220,RpbAaeFoldTreeResp,riak_kv 221,RpbAaeFoldKeyValueResp,riak_kv 222,RpbAaeFoldKeyCountResp,riak_kv From 2a92531beb72d08e14a584531bc9374a5411f590 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Mon, 9 Dec 2019 09:41:46 +0000 Subject: [PATCH 5/8] Extend AAE Fold --- src/riak_kv.proto | 83 ++++++++++++++++++++++++++++++++++++++++ src/riak_pb_messages.csv | 3 ++ 2 files changed, 86 insertions(+) diff --git a/src/riak_kv.proto b/src/riak_kv.proto index 49413be7..b84f1a8f 100644 --- a/src/riak_kv.proto +++ b/src/riak_kv.proto @@ -459,6 +459,89 @@ message RpbAaeFoldFindKeysReq{ required uint32 find_limit = 51; } +message RpbAaeFoldFindTombsReq{ + enum RpbTreeSize{ + xxsmall = 0; + xsmall = 1; + small = 2; + medium = 3; + large = 4; + xlarge = 5; + } + + optional bytes type = 1; + required bytes bucket = 2; + required bool key_range = 10; + optional bytes start_key = 11; + optional bytes end_key = 12; + required bool segment_filter = 30; + repeated uint32 id_filter = 31; + optional RpbTreeSize filter_tree_size = 32; + required bool modified_range = 40; + optional uint32 last_mod_start = 41; + optional uint32 last_mod_end = 42; +} + +message RpbAaeFoldReapTombsReq{ + enum RpbTreeSize{ + xxsmall = 0; + xsmall = 1; + small = 2; + medium = 3; + large = 4; + xlarge = 5; + } + enum ChangeMethod { + job = 0; + local = 1; + count = 2; + } + + optional bytes type = 1; + required bytes bucket = 2; + required bool key_range = 10; + optional bytes start_key = 11; + optional bytes end_key = 12; + required bool segment_filter = 30; + repeated uint32 id_filter = 31; + optional RpbTreeSize filter_tree_size = 32; + required bool modified_range = 40; + optional uint32 last_mod_start = 41; + optional uint32 last_mod_end = 42; + required ChangeMethod change_method = 50; + optional uint32 job_id = 51; +} + +message RpbAaeFoldEraseKeysReq{ + enum RpbTreeSize{ + xxsmall = 0; + xsmall = 1; + small = 2; + medium = 3; + large = 4; + xlarge = 5; + } + enum ChangeMethod { + job = 0; + local = 1; + count = 2; + } + + optional bytes type = 1; + required bytes bucket = 2; + required bool key_range = 10; + optional bytes start_key = 11; + optional bytes end_key = 12; + required bool segment_filter = 30; + repeated uint32 id_filter = 31; + optional RpbTreeSize filter_tree_size = 32; + required bool modified_range = 40; + optional uint32 last_mod_start = 41; + optional uint32 last_mod_end = 42; + required ChangeMethod change_method = 50; + optional uint32 job_id = 51; +} + message RpbAaeFoldObjectStatsReq{ optional bytes type = 1; required bytes bucket = 2; diff --git a/src/riak_pb_messages.csv b/src/riak_pb_messages.csv index bfee120c..dc765a63 100644 --- a/src/riak_pb_messages.csv +++ b/src/riak_pb_messages.csv @@ -80,9 +80,12 @@ 215,RpbAaeFoldFindKeysReq,riak_kv 216,RpbAaeFoldObjectStatsReq,riak_kv 217,RpbAaeFoldReplKeysReq,riak_kv +218,RpbAaeFoldFindTombsReq,riak_kv 220,RpbAaeFoldTreeResp,riak_kv 221,RpbAaeFoldKeyValueResp,riak_kv 222,RpbAaeFoldKeyCountResp,riak_kv +230,RpbAaeFoldReapTombsReq,riak_kv +231,RpbAaeFoldEraseKeysReq,riak_kv 253,RpbAuthReq,riak 254,RpbAuthResp,riak 255,RpbStartTls,riak From 5b5b47fbda4ecee3af86fc91eba5bbb2edc06bf0 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Tue, 10 Dec 2019 15:20:16 +0000 Subject: [PATCH 6/8] Add AAE Fold version lof list_buckets --- src/riak_kv.proto | 13 +++++++++++++ src/riak_pb_messages.csv | 2 ++ 2 files changed, 15 insertions(+) diff --git a/src/riak_kv.proto b/src/riak_kv.proto index b84f1a8f..6bd81e02 100644 --- a/src/riak_kv.proto +++ b/src/riak_kv.proto @@ -553,6 +553,19 @@ message RpbAaeFoldObjectStatsReq{ optional uint32 last_mod_end = 42; } +message RpbAaeFoldListBucketsReq{ + optional uint32 n_val = 1; +} + +message RpbAaeFoldListBucketsResp{ + repeated RpbAaeFoldBucket bucket_list = 1; +} + +message RpbAaeFoldBucket{ + optional bytes type = 1; + required bytes bucket = 2; +} + message RpbKeysValue { optional bytes type = 1; required bytes bucket = 2; diff --git a/src/riak_pb_messages.csv b/src/riak_pb_messages.csv index dc765a63..253f530e 100644 --- a/src/riak_pb_messages.csv +++ b/src/riak_pb_messages.csv @@ -84,6 +84,8 @@ 220,RpbAaeFoldTreeResp,riak_kv 221,RpbAaeFoldKeyValueResp,riak_kv 222,RpbAaeFoldKeyCountResp,riak_kv +223,RpbAaeFoldListBucketReq,riak_kv +224,RpbAaeFoldListBucketResp,riak_kv 230,RpbAaeFoldReapTombsReq,riak_kv 231,RpbAaeFoldEraseKeysReq,riak_kv 253,RpbAuthReq,riak From 6cdf6302b09d4e182ca9d1cb40126acf63d04e34 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Tue, 10 Dec 2019 16:28:05 +0000 Subject: [PATCH 7/8] Consistent naming --- src/riak_pb_messages.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/riak_pb_messages.csv b/src/riak_pb_messages.csv index 253f530e..acd1358e 100644 --- a/src/riak_pb_messages.csv +++ b/src/riak_pb_messages.csv @@ -84,8 +84,8 @@ 220,RpbAaeFoldTreeResp,riak_kv 221,RpbAaeFoldKeyValueResp,riak_kv 222,RpbAaeFoldKeyCountResp,riak_kv -223,RpbAaeFoldListBucketReq,riak_kv -224,RpbAaeFoldListBucketResp,riak_kv +223,RpbAaeFoldListBucketsReq,riak_kv +224,RpbAaeFoldListBucketsResp,riak_kv 230,RpbAaeFoldReapTombsReq,riak_kv 231,RpbAaeFoldEraseKeysReq,riak_kv 253,RpbAuthReq,riak From 3ad0e979fe6871d16d9dfbfe1fe07186572f31e3 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Tue, 11 Feb 2020 09:23:06 +0000 Subject: [PATCH 8/8] Remove hamcrest Does not appear to be referenced --- rebar.config | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rebar.config b/rebar.config index 7b123518..d6551a44 100644 --- a/rebar.config +++ b/rebar.config @@ -6,8 +6,7 @@ ]}. {deps, [ - {protobuffs, "0.9.*", {git, "https://github.com/basho/erlang_protobuffs.git", {tag, "0.9.1"}}}, - {hamcrest, "0.4.*", {git, "https://github.com/basho/hamcrest-erlang.git", {tag, "0.4.2"}}} + {protobuffs, "0.9.*", {git, "https://github.com/basho/erlang_protobuffs.git", {tag, "0.9.1"}}} ]}. {xref_checks, [