syntax = "proto3"; package waymaker.sketches; option go_package = "/apis/waymaker_sketches"; // In-memory probabilistic data structures: Bloom filter, // HyperLogLog, Count-Min Sketch, Top-K, t-digest. // // Storage model: // - State is opaque binary per filter, owned by the ring's // hash-owner for the filter's name. // - Periodic snapshot replication pushes the bytes to N-1 // secondaries via ReplicateProbState. Adoption sweep promotes // replicas to primary on ring shifts. // - Disk persistence: every ~60s, owned filters snapshot to // /probabilistic//.bin via tempfile + // rename. Hydrated on controller startup. // - Ephemeral by default — survives node failover + process // restart, full-cluster restart without prior snapshot loses // state. // // See specs/WIRE_SPEC.md (top-level) for the serialization byte // layout per type — cross-language clients implement against that // document for the replication state-transfer protocol. service WaymakerSketchesService { // ----- Bloom filter ----- rpc BloomReserve (BloomReserveRequest) returns (BloomReserveResponse); rpc BloomAdd (BloomAddRequest) returns (BloomAddResponse); rpc BloomMultiAdd (BloomMultiAddRequest) returns (BloomMultiAddResponse); rpc BloomExists (BloomExistsRequest) returns (BloomExistsResponse); rpc BloomMultiExists (BloomMultiExistsRequest) returns (BloomMultiExistsResponse); rpc BloomInfo (BloomInfoRequest) returns (BloomInfoResponse); rpc BloomDelete (BloomDeleteRequest) returns (BloomDeleteResponse); // ----- HyperLogLog ----- rpc HllReserve (HllReserveRequest) returns (HllReserveResponse); rpc HllAdd (HllAddRequest) returns (HllAddResponse); rpc HllCount (HllCountRequest) returns (HllCountResponse); rpc HllMerge (HllMergeRequest) returns (HllMergeResponse); rpc HllDelete (HllDeleteRequest) returns (HllDeleteResponse); // ----- Count-Min Sketch ----- rpc CmsReserve (CmsReserveRequest) returns (CmsReserveResponse); rpc CmsIncrBy (CmsIncrByRequest) returns (CmsIncrByResponse); rpc CmsQuery (CmsQueryRequest) returns (CmsQueryResponse); rpc CmsDelete (CmsDeleteRequest) returns (CmsDeleteResponse); // ----- Top-K ----- rpc TopKReserve (TopKReserveRequest) returns (TopKReserveResponse); rpc TopKAdd (TopKAddRequest) returns (TopKAddResponse); rpc TopKQuery (TopKQueryRequest) returns (TopKQueryResponse); rpc TopKList (TopKListRequest) returns (TopKListResponse); rpc TopKDelete (TopKDeleteRequest) returns (TopKDeleteResponse); // ----- t-digest ----- rpc TDigestCreate (TDigestCreateRequest) returns (TDigestCreateResponse); rpc TDigestAdd (TDigestAddRequest) returns (TDigestAddResponse); rpc TDigestQuantile (TDigestQuantileRequest) returns (TDigestQuantileResponse); rpc TDigestMinMax (TDigestMinMaxRequest) returns (TDigestMinMaxResponse); rpc TDigestDelete (TDigestDeleteRequest) returns (TDigestDeleteResponse); // Internal: snapshot replication. Primary pushes serialized // filter state to N-1 secondaries periodically. Version counter // dedupes out-of-order pushes. rpc ReplicateProbState (ReplicateProbStateRequest) returns (ReplicateProbStateResponse); } // ===== Bloom filter =========================================== message BloomReserveRequest { string name = 1; uint64 capacity = 2; double error_rate = 3; uint64 ttl_ms = 4; } message BloomReserveResponse { bool success = 1; string result_code = 2; string message = 3; } message BloomAddRequest { string name = 1; bytes item = 2; } message BloomAddResponse { bool success = 1; string result_code = 2; string message = 3; } message BloomMultiAddRequest { string name = 1; repeated bytes items = 2; } message BloomMultiAddResponse { bool success = 1; string result_code = 2; string message = 3; } message BloomExistsRequest { string name = 1; bytes item = 2; } message BloomExistsResponse { bool success = 1; string result_code = 2; string message = 3; bool exists = 4; } message BloomMultiExistsRequest { string name = 1; repeated bytes items = 2; } message BloomMultiExistsResponse { bool success = 1; string result_code = 2; string message = 3; repeated bool exists = 4; } message BloomInfoRequest { string name = 1; } message BloomInfoResponse { bool success = 1; string result_code = 2; string message = 3; uint64 capacity = 4; double error_rate = 5; uint64 bits_set = 6; uint64 bit_count = 7; uint32 hash_count = 8; uint64 items_added = 9; } message BloomDeleteRequest { string name = 1; } message BloomDeleteResponse { bool success = 1; string result_code = 2; string message = 3; } // ===== HyperLogLog ============================================ message HllReserveRequest { string name = 1; uint32 precision = 2; uint64 ttl_ms = 3; } message HllReserveResponse { bool success = 1; string result_code = 2; string message = 3; } message HllAddRequest { string name = 1; repeated bytes items = 2; } message HllAddResponse { bool success = 1; string result_code = 2; string message = 3; } message HllCountRequest { string name = 1; } message HllCountResponse { bool success = 1; string result_code = 2; string message = 3; uint64 estimate = 4; } message HllMergeRequest { string destination = 1; repeated string sources = 2; } message HllMergeResponse { bool success = 1; string result_code = 2; string message = 3; } message HllDeleteRequest { string name = 1; } message HllDeleteResponse { bool success = 1; string result_code = 2; string message = 3; } // ===== Count-Min Sketch ======================================= message CmsReserveRequest { string name = 1; uint64 width = 2; uint64 depth = 3; uint64 ttl_ms = 4; } message CmsReserveResponse { bool success = 1; string result_code = 2; string message = 3; } message CmsIncrByItem { bytes item = 1; uint64 count = 2; } message CmsIncrByRequest { string name = 1; repeated CmsIncrByItem items = 2; } message CmsIncrByResponse { bool success = 1; string result_code = 2; string message = 3; repeated uint64 counts = 4; } message CmsQueryRequest { string name = 1; repeated bytes items = 2; } message CmsQueryResponse { bool success = 1; string result_code = 2; string message = 3; repeated uint64 counts = 4; } message CmsDeleteRequest { string name = 1; } message CmsDeleteResponse { bool success = 1; string result_code = 2; string message = 3; } // ===== Top-K ================================================== message TopKReserveRequest { string name = 1; uint32 k = 2; uint64 width = 3; uint64 depth = 4; double decay = 5; uint64 ttl_ms = 6; } message TopKReserveResponse { bool success = 1; string result_code = 2; string message = 3; } message TopKAddRequest { string name = 1; repeated bytes items = 2; } message TopKAddResponse { bool success = 1; string result_code = 2; string message = 3; repeated bytes evicted = 4; } message TopKQueryRequest { string name = 1; repeated bytes items = 2; } message TopKQueryResponse { bool success = 1; string result_code = 2; string message = 3; repeated bool in_top_k = 4; } message TopKListRequest { string name = 1; } message TopKListResponse { bool success = 1; string result_code = 2; string message = 3; repeated TopKEntry entries = 4; } message TopKEntry { bytes item = 1; uint64 count = 2; } message TopKDeleteRequest { string name = 1; } message TopKDeleteResponse { bool success = 1; string result_code = 2; string message = 3; } // ===== t-digest =============================================== message TDigestCreateRequest { string name = 1; uint32 compression = 2; uint64 ttl_ms = 3; } message TDigestCreateResponse { bool success = 1; string result_code = 2; string message = 3; } message TDigestAddRequest { string name = 1; repeated double values = 2; } message TDigestAddResponse { bool success = 1; string result_code = 2; string message = 3; } message TDigestQuantileRequest { string name = 1; repeated double quantiles = 2; } message TDigestQuantileResponse { bool success = 1; string result_code = 2; string message = 3; repeated double values = 4; } message TDigestMinMaxRequest { string name = 1; } message TDigestMinMaxResponse { bool success = 1; string result_code = 2; string message = 3; double min = 4; double max = 5; } message TDigestDeleteRequest { string name = 1; } message TDigestDeleteResponse { bool success = 1; string result_code = 2; string message = 3; } // ===== Internal: snapshot replication ========================= enum ProbType { PROB_UNSPECIFIED = 0; PROB_BLOOM = 1; PROB_HLL = 2; PROB_CMS = 3; PROB_TOPK = 4; PROB_TDIGEST = 5; } message ReplicateProbStateRequest { ProbType type = 1; string name = 2; // Opaque binary snapshot — see specs/WIRE_SPEC.md "Probabilistic // subsystem" for the per-type byte layout. bytes snapshot = 3; uint64 version = 4; } message ReplicateProbStateResponse { bool success = 1; string result_code = 2; string message = 3; }