pb.proto

/*
 * Copyright (C) 2017 Dgraph Labs, Inc. and Contributors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


// Style guide for Protocol Buffer 3.
// Use CamelCase (with an initial capital) for message names – for example,
// SongServerRequest. Use underscore_separated_names for field names – for
// example, song_name.

syntax = "proto3";

package pb;

import "api.proto";
import "github.com/dgraph-io/badger/pb/pb.proto";

/* import "gogoproto/gogo.proto"; */

/* option (gogoproto.marshaler_all) = true; */
/* option (gogoproto.sizer_all) = true; */
/* option (gogoproto.unmarshaler_all) = true; */
/* option (gogoproto.goproto_getters_all) = true; */

message List {
	repeated fixed64 uids = 1;
}

message TaskValue {
	bytes val = 1;
	Posting.ValType val_type = 2;
}

message SrcFunction {
	string name = 1;
	repeated string args = 3;
	bool isCount = 4;
}

message Query {
	string attr = 1;
	repeated string langs = 2; // language list for attribute
	fixed64 after_uid = 3;  // Only return UIDs greater than this.
	bool do_count = 4;      // Are we just getting lengths?

	// Exactly one of uids and terms is populated.
	List uid_list = 5;

	// Function to generate or filter UIDs.
	SrcFunction src_func = 6;

	bool reverse = 7;  // Whether this is a reverse edge.

	FacetParams facet_param = 8; // which facets to fetch
	FilterTree facets_filter = 9; // filtering on facets : has Op (and/or/not) tree

	bool expand_all = 10; // expand all language variants.

	uint64 read_ts = 13;
	int32 cache = 14;
}

message ValueList {
	repeated TaskValue values = 1;
}

message LangList {
	repeated string lang = 1;
}

message Result {
	repeated List uid_matrix = 1;
	repeated ValueList value_matrix = 2;
	repeated uint32 counts = 3;
	bool intersect_dest = 4;
	repeated FacetsList facet_matrix = 5;
	repeated LangList lang_matrix = 6;
	bool list = 7;
}

message Order {
	string attr = 1;
	bool desc = 2;
	repeated string langs = 3;
}

message SortMessage {
	repeated Order order = 1;
	repeated List uid_matrix = 2;
	int32 count = 3;   // Return this many elements.
	int32 offset = 4;  // Skip this many elements.

	uint64 read_ts = 13;
}

message SortResult {
	repeated List uid_matrix = 1;
}

message RaftContext {
	fixed64 id = 1;
	uint32 group = 2;
	string addr = 3;
	uint64 snapshot_ts = 4;
}

// Member stores information about RAFT group member for a single RAFT node.
// Note that each server can be serving multiple RAFT groups. Each group would have
// one RAFT node per server serving that group.
message Member {
	fixed64 id = 1;
	uint32 group_id = 2;
	string addr = 3;
	bool leader = 4;
	bool am_dead = 5;
	uint64 last_update = 6;

	bool cluster_info_only = 13;
}

message Group {
	map<uint64, Member> members = 1; // Raft ID is the key.
	map<string, Tablet> tablets = 2; // Predicate + others are key.
	uint64 snapshot_ts          = 3; // Stores Snapshot transaction ts.
	uint64 checksum             = 4; // Stores a checksum.
}

message Enterprise {
	string entity = 1;
	uint64 maxNodes = 2;
	int64 expiryTs = 3;
	bool enabled = 4;
}

message ZeroProposal {
	map<uint32, uint64> snapshot_ts = 1; // Group ID -> Snapshot Ts.
	Member member = 2;
	Tablet tablet = 3;
	uint64 maxLeaseId = 4;
	uint64 maxTxnTs = 5;
	uint64 maxRaftId = 6;
	api.TxnContext txn = 7;
	string key = 8;  // Used as unique identifier for proposal id.
	string cid = 9; // Used as unique identifier for the cluster.
	Enterprise enterprise = 10;
}

// MembershipState is used to pack together the current membership state of all the nodes
// in the caller server; and the membership updates recorded by the callee server since
// the provided lastUpdate.
message MembershipState {
	uint64 counter = 1;  // used to find latest membershipState in case of race.
	map<uint32, Group> groups = 2;
	map<uint64, Member> zeros = 3;
	uint64 maxLeaseId = 4;
	uint64 maxTxnTs = 5;
	uint64 maxRaftId = 6;
	repeated Member removed = 7;
	string cid = 8; // Used to uniquely identify the Dgraph cluster.
	Enterprise enterprise = 9;
}

message ConnectionState {
	Member member = 1;
	MembershipState state = 2;
	uint64 max_pending = 3; // Used to determine the timstamp for reading after bulk load
}

message Tablet {
	uint32 group_id  = 1; // Served by which group.
	string predicate = 2;
	bool force       = 3; // Used while moving predicate.
	int64 space      = 7;
	bool remove      = 8;
  bool read_only   = 9; // If true, do not ask zero to serve any tablets.
}

message DirectedEdge {
	fixed64 entity             = 1;    // Subject or source node / UID.
	string attr                = 2;       // Attribute or predicate. Labels the edge.
	bytes value                = 3;       // Edge points to a value.
	Posting.ValType value_type = 4;  // The type of the value
	fixed64 value_id           = 5;   // Object or destination node / UID.
	string label               = 6;
	string lang                = 7;
	enum Op {
		SET = 0;
		DEL = 1;
	}
	Op op = 8;
	repeated api.Facet facets = 9;
}

message Mutations {
	uint32 group_id	= 1;
	uint64 start_ts	= 2;
	repeated DirectedEdge edges	= 3;
	repeated SchemaUpdate schema = 4;
	bool ignore_index_conflict = 5;
	repeated TypeUpdate types	= 6;
	enum DropOp {
		NONE = 0;
		ALL = 1;
		DATA = 2;
		TYPE = 3;
	}
	DropOp drop_op = 7;
	string drop_value = 8;
}

message Snapshot {
	RaftContext context = 1;
	uint64 index = 2;
	uint64 read_ts = 3;
	// done is used to indicate that snapshot stream was a success.
	bool done	= 4;
	// since_ts stores the ts of the last snapshot to support diff snap updates.
	uint64 since_ts = 5;
}

message Proposal {
	Mutations mutations    = 2;
	repeated KV kv         = 4;
	MembershipState state  = 5;
	string clean_predicate = 6;  // Delete the predicate which was moved to other group.
	string key             = 7;
	OracleDelta delta      = 8;
	Snapshot snapshot      = 9;  // Used to tell the group when to take snapshot.
	uint64 index           = 10; // Used to store Raft index, in raft.Ready.
}

message KVS {
 repeated pb.KV kv = 1;
 // done used to indicate if the stream of KVS is over.
 bool done      = 2;
}

// Posting messages.
message Posting {
	fixed64 uid = 1;
	bytes value = 2;
	enum ValType {
		DEFAULT = 0;
		BINARY = 1;
		INT = 2; // We treat it as int64.
		FLOAT = 3;
		BOOL = 4;
		DATETIME = 5;
		GEO = 6;
		UID = 7;
		PASSWORD = 8;
		STRING = 9;
    OBJECT = 10;
	}
	ValType val_type = 3;
	enum PostingType {
		REF=0;          // UID
		VALUE=1;        // simple, plain value
		VALUE_LANG=2;   // value with specified language
	}
	PostingType posting_type = 4;
	bytes lang_tag = 5; // Only set for VALUE_LANG
	string label = 6;
	repeated api.Facet facets = 9;

	// TODO: op is only used temporarily. See if we can remove it from here.
	uint32 op = 12;
	uint64 start_ts = 13;   // Meant to use only inmemory
	uint64 commit_ts = 14;  // Meant to use only inmemory
}

message UidBlock {
	uint64 base = 1;
	// deltas contains the deltas encoded with Varints. We don't store deltas as a list of integers,
	// because when the PB is brought to memory, Go would always use 8-bytes per integer. Instead,
	// storing it as a byte slice is a lot cheaper in memory.
	bytes deltas = 2;
	// num_uids is the number of UIDs in the block. We are including this because we want to 
	// swtich encoding to groupvarint encoding. Current avaialble open source version implements
	// encoding and decoding for uint32. We want to wrap it around our logic to use it here.
	// Default Blocksize is 256 so uint32 would be sufficient.
	uint32 num_uids = 3;
}

message UidPack {
	uint32 block_size        = 1;
	repeated UidBlock blocks = 2;
}

message PostingList {
	UidPack pack = 1; // Encoded list of uids in this posting list.
	repeated Posting postings = 2;
	uint64 commit_ts = 3; // More inclination towards smaller values.

  repeated uint64 splits = 4;
}

message FacetParam {
	string key = 1;
	string alias = 2;
}

message FacetParams {
	bool all_keys = 1; // keys should be in sorted order.
	repeated FacetParam param = 2;
}

message Facets {
	repeated api.Facet facets = 1;
}

message FacetsList {
	repeated Facets facets_list = 1;
}

message Function {
	string name = 1;          // Name of the function : eq, le
	string key = 2;           // Facet key over which to run the function.
	repeated string args = 3; // Arguments of the function.
}

// Op and Children are internal nodes and Func on leaves.
message FilterTree {
	string op = 1;
	repeated FilterTree children = 2;
	Function func = 3;
}

// Schema messages.
message SchemaRequest {
	uint32 group_id = 1;
	repeated string predicates = 2;
	// fields can be on of type, index, reverse or tokenizer
	repeated string fields = 3;

  repeated string types = 4;
}

message SchemaResult {
	repeated api.SchemaNode schema = 1 [deprecated=true];
}

message SchemaUpdate {
	string predicate = 1;
	Posting.ValType value_type = 2;
	enum Directive {
	   NONE = 0;
	   INDEX = 1;
	   REVERSE = 2;
	   DELETE = 3;
	}
	Directive directive = 3;
	repeated string tokenizer = 4;
	bool count = 5;
	bool list = 6;
	bool upsert = 8;
	bool lang = 9;

	// Fields required for type system.
	bool non_nullable = 10;
	bool non_nullable_list = 11;

	// If value_type is OBJECT, then this represents an object type with a
	// custom name. This field stores said name.
	string object_type_name = 12;

	// Deleted field:
	reserved 7;
	reserved "explicit";
}

message TypeUpdate {
	string type_name = 1;
	repeated SchemaUpdate fields = 2;
}

// Bulk loader proto.
message MapEntry {
	bytes key = 1;

	// Only one should be set.
	fixed64 uid = 2;
	Posting posting = 3;
}

message MovePredicatePayload {
	string predicate  = 1;
	uint32 source_gid = 2;
	uint32 dest_gid   = 3;
	uint64 txn_ts     = 4;
}

message TxnStatus {
	uint64 start_ts = 1;
	uint64 commit_ts = 2;
}

message OracleDelta {
	repeated TxnStatus txns             = 1;
	uint64 max_assigned                 = 2;
	map<uint32, uint64> group_checksums = 3;
	// implement tmax.
}

message TxnTimestamps {
	repeated uint64 ts = 1;
}

message PeerResponse {
	bool status = 1;
}

message RaftBatch {
	RaftContext context = 1;
	api.Payload payload = 2;
}

service Raft {
	rpc Heartbeat (api.Payload)        returns (stream api.Payload) {}
	rpc RaftMessage (stream RaftBatch) returns (api.Payload) {}
	rpc JoinCluster (RaftContext)      returns (api.Payload) {}
	rpc IsPeer (RaftContext)           returns (PeerResponse) {}
}

service Zero {
	// These 3 endpoints are for handling membership.
	rpc Connect (Member)               returns (ConnectionState) {}
	rpc UpdateMembership (Group)                 returns (api.Payload) {}
	rpc StreamMembership (api.Payload)       returns (stream MembershipState) {}

	rpc Oracle (api.Payload)           returns (stream OracleDelta) {}
	rpc ShouldServe (Tablet)           returns (Tablet) {}
	rpc AssignUids (Num)               returns (AssignedIds) {}
	rpc Timestamps (Num)               returns (AssignedIds) {}
	rpc CommitOrAbort (api.TxnContext) returns (api.TxnContext) {}
	rpc TryAbort (TxnTimestamps)       returns (OracleDelta) {}
}

service Worker {
	// Data serving RPCs.
	rpc Mutate (Mutations)                  returns (api.TxnContext) {}
	rpc ServeTask (Query)                   returns (Result) {}
	rpc StreamSnapshot (stream Snapshot)    returns (stream KVS) {}
	rpc Sort (SortMessage)                  returns (SortResult) {}
	rpc Schema (SchemaRequest)              returns (SchemaResult) {}
	rpc Backup (BackupRequest)              returns (Status) {}
	rpc Export (ExportRequest)              returns (Status) {}
	rpc ReceivePredicate(stream KVS)        returns (api.Payload) {}
	rpc MovePredicate(MovePredicatePayload) returns (api.Payload) {}
}

message Num {
	uint64 val = 1;
	bool read_only = 2;
	bool forwarded = 3; // True if this request was forwarded by a peer.
}

message AssignedIds {
	uint64 startId = 1;
	uint64 endId = 2;

	// The following is used for read only transactions.
	uint64 read_only = 5;
}

message SnapshotMeta {
	uint64 client_ts = 1;
	uint32 group_id = 2;
}

// Status describes a general status response.
// code: 0 = success, 0 != failure.
message Status {
	int32 code = 1;
	string msg = 2;
}

message BackupRequest {
	uint64 read_ts = 1;
  uint64 since_ts = 2;
	uint32 group_id = 3;
	string unix_ts = 4;
	string destination = 5;
	string access_key = 6;
	string secret_key = 7;
	string session_token = 8;

  // True if no credentials should be used to access the S3 or minio bucket.
  // For example, when using a bucket with a public policy.
  bool anonymous = 9;

  // The predicates to backup. All other predicates present in the group (e.g
  // stale data from a predicate move) will be ignored.
  repeated string predicates = 10;
}

message ExportRequest {
	uint32  group_id = 1;  // Group id to back up.
	uint64  read_ts  = 2;
	int64   unix_ts  = 3;
	string  format   = 4;
}

// A key stored in the format used for writing backups.
message BackupKey {
  enum KeyType {
    UNKNOWN = 0;
    DATA = 1;
    INDEX = 2;
    REVERSE = 3;
    COUNT = 4;
    COUNT_REV = 5;
    SCHEMA = 6;
    TYPE = 7;
  }

  KeyType type = 1;
  string attr = 2;
  uint64 uid = 3;
  uint64 start_uid = 4;
  string term = 5;
  uint32 count = 6;
}

// A posting list stored in the format used for writing backups.
message BackupPostingList {
	repeated uint64 uids = 1;
	repeated Posting postings = 2;
	uint64 commit_ts = 3;
  repeated uint64 splits = 4;
}

// vim: noexpandtab sw=2 ts=2