All Data Structures Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
src/include/citrusleaf/cf_proto.h
Go to the documentation of this file.
1 /******************************************************************************
2  * Copyright 2008-2013 by Aerospike.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a copy
5  * of this software and associated documentation files (the "Software"), to
6  * deal in the Software without restriction, including without limitation the
7  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
8  * sell copies of the Software, and to permit persons to whom the Software is
9  * furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
20  * IN THE SOFTWARE.
21  *****************************************************************************/
22 #pragma once
23 
24 #include <stddef.h>
25 #include <stdint.h>
26 
27 #ifdef __cplusplus
28 extern "C" {
29 #endif
30 
31 #define CL_PROTO_RESULT_OK 0
32 #define CL_PROTO_RESULT_FAIL_UNKNOWN 1 // unknown failure
33 #define CL_PROTO_RESULT_FAIL_NOTFOUND 2
34 #define CL_PROTO_RESULT_FAIL_GENERATION 3
35 #define CL_PROTO_RESULT_FAIL_PARAMETER 4
36 #define CL_PROTO_RESULT_FAIL_KEY_EXISTS 5 // if 'WRITE_ADD', could fail because already exists
37 #define CL_PROTO_RESULT_FAIL_BIN_EXISTS 6
38 #define CL_PROTO_RESULT_FAIL_CLUSTER_KEY_MISMATCH 7
39 #define CL_PROTO_RESULT_FAIL_PARTITION_OUT_OF_SPACE 8
40 #define CL_PROTO_RESULT_FAIL_TIMEOUT 9
41 #define CL_PROTO_RESULT_FAIL_NOXDS 10
42 #define CL_PROTO_RESULT_FAIL_UNAVAILABLE 11 // error returned during node down and partition isn't available
43 #define CL_PROTO_RESULT_FAIL_INCOMPATIBLE_TYPE 12 // op and bin type incompatibilty
44 #define CL_PROTO_RESULT_FAIL_RECORD_TOO_BIG 13
45 #define CL_PROTO_RESULT_FAIL_KEY_BUSY 14
46 // 15 and 16 used for 3.x functionality.
47 #define CL_PROTO_RESULT_FAIL_BIN_NOT_FOUND 17
48 
49 // Forward definitions
50 struct cl_bin_s;
51 
52 
53 
54 
55 // NOTE! This is the only place where datamodel.h is exported in the external
56 // proto.h. Maybe I should make a different (annotated) proto.h?
57 
58 /* cl_particle_type
59  * Particles are typed, which reflects their contents:
60  * NULL: no associated content (not sure I really need this internally?)
61  * INTEGER: a signed, 64-bit integer
62  * BIGNUM: a big number
63  * STRING: a null-terminated UTF-8 string
64  * BLOB: arbitrary-length binary data
65  * TIMESTAMP: milliseconds since 1 January 1970, 00:00:00 GMT
66  * DIGEST: an internal Aerospike key digest */
67 typedef enum {
81 
82 
83 /* SYNOPSIS
84  * Aerospike wire protocol
85  *
86  * Version 2
87  *
88  * Aerospike uses a message-oriented wire protocol to transfer information.
89  * Each message consists of a header, which determines the type and the length
90  * to follow. This is called the 'proto_msg'.
91  *
92  * these messages are vectored out to the correct handler. Over TCP, they can be
93  * pipelined (but not out of order). If we wish to support out of order responses,
94  * we should upgrade the protocol.
95  *
96  * the most common type of message is the cl_msg, a message which reads or writes
97  * a single row to the data store.
98  *
99  */
100 
101 
102 #define CL_PROTO_VERSION 2
103 #define CL_PROTO_TYPE_INFO 1 // ascii-format message for determining server info
104 #define CL_PROTO_TYPE_CL_MSG 3
105 #define CL_PROTO_TYPE_CL_MSG_COMPRESSED 4
106 
107 #define CL_RESULT_OK 0
108 #define CL_RESULT_FAIL 1
109 #define CL_RESULT_NOTFOUND 2
110 
111 #if defined(__APPLE__) || defined(CF_WINDOWS)
112 
113 #pragma pack(push, 1) // packing is now 1
114 typedef struct cl_proto_s {
115  uint64_t version :8;
116  uint64_t type :8;
117  uint64_t sz :48;
118 } cl_proto;
119 #pragma pack(pop) // packing is back to what it was
120 
121 #pragma pack(push, 1) // packing is now 1
122 /*
123  * zlib decompression API needs original size of the compressed data.
124  * So we need to transfer it to another end.
125  * This structure packs together -
126  * header + original size of data + compressed data
127  */
128 typedef struct cl_comp_proto_s {
129  cl_proto proto; // Protocol header
130  uint64_t org_sz; // Original size of compressed data hold in 'data'
131  uint8_t data[]; // Compressed data
132 } cl_comp_proto;
133 #pragma pack(pop) // packing is back to what it was
134 
135 
136 /* cl_msg_field
137  * Aerospike message field */
138 
139 #pragma pack(push, 1)
140 typedef struct cl_msg_field_s {
141  uint32_t field_sz; // get the data size through the accessor function, don't worry, it's a small macro
142  uint8_t type;
143  uint8_t data[];
144 } cl_msg_field;
145 #pragma pack(pop)
146 
147 #pragma pack(push, 1) // packing is now 1
148 typedef struct cl_msg_op_s {
149  uint32_t op_sz;
150  uint8_t op;
151  uint8_t particle_type;
152  uint8_t version;
153  uint8_t name_sz;
154  uint8_t name[]; // UTF-8
155  // there's also a value here but you can't have two variable size arrays
156 } cl_msg_op;
157 #pragma pack(pop) // packing is back to what it was
158 
159 /* cl_msg_key_s
160 */
161 // Not using it anywhere in libevent2 client
162 // Please be aware when using this for any other client
163 /*
164 #pragma pack(push, 1) // packing is now 1
165 typedef struct cl_msg_key_s {
166  cl_msg_field f;
167  uint8_t key[];
168 } cl_msg_key;
169 #pragma pack(pop) // packing is back to what it was
170 */
171 /* cl_msg_number_s
172 */
173 
174 #pragma pack(push, 1) // packing is now 1
175 typedef struct cl_msg_number_s {
176  uint32_t number;
177  cl_msg_field f;
178 } cl_msg_number;
179 #pragma pack(pop) // packing is back to what it was
180 
181 
182 /* cl_msg
183  * Aerospike message
184  * size: size of the payload, not including the header */
185 
186 #pragma pack(push, 1) // packing is now 1
187 typedef struct cl_msg_s {
188 /*00*/ uint8_t header_sz; // number of uint8_ts in this header
189 /*01*/ uint8_t info1; // bitfield about this request
190 /*02*/ uint8_t info2;
191 /*03*/ uint8_t info3;
192 /*04*/ uint8_t unused;
193 /*05*/ uint8_t result_code;
194 /*06*/ uint32_t generation;
195 /*10*/ uint32_t record_ttl;
196 /*14*/ uint32_t transaction_ttl;
197 /*18*/ uint16_t n_fields; // size in uint8_ts
198 /*20*/ uint16_t n_ops; // number of operations
199 /*22*/ uint8_t data[0]; // data contains first the fields, then the ops
200 } cl_msg;
201 #pragma pack(pop) // packing is back to what it was
202 
203 
204 /* cl_ms
205  * Aerospike message
206  * sz: size of the payload, not including the header */
207 
208 #pragma pack(push, 1) // packing is now 1
209 typedef struct as_msg_s {
210  cl_proto proto;
211  cl_msg m;
212 } as_msg;
213 #pragma pack(pop) // packing is back to what it was
214 
215 
216 #else
217 
218 typedef struct cl_proto_s {
219  uint8_t version;
220  uint8_t type;
221  uint64_t sz:48;
222  uint8_t data[];
223 } __attribute__ ((__packed__)) cl_proto;
224 
225 /*
226  * zlib decompression API needs original size of the compressed data.
227  * So we need to transfer it to another end.
228  * This structure packs together -
229  * header + original size of data + compressed data
230  */
231 typedef struct cl_comp_proto_s {
232  cl_proto proto; // Protocol header
233  uint64_t org_sz; // Original size of compressed data hold in 'data'
234  uint8_t data[]; // Compressed data
235 } cl_comp_proto;
236 
237  /* cl_msg_field
238  * Aerospike message field */
239 typedef struct cl_msg_field_s {
240  uint32_t field_sz; // get the data size through the accessor function, don't worry, it's a small macro
241  uint8_t type;
242  uint8_t data[];
243 } __attribute__((__packed__)) cl_msg_field;
244 
245 
246 typedef struct cl_msg_op_s {
247  uint32_t op_sz;
248  uint8_t op;
249  uint8_t particle_type;
250  uint8_t version;
251  uint8_t name_sz;
252  uint8_t name[]; // UTF-8
253  // there's also a value here but you can't have two variable size arrays
254 } __attribute__((__packed__)) cl_msg_op;
255 
256 
257 typedef struct cl_msg_key_s {
258  cl_msg_field f;
259  uint8_t key[];
260 } __attribute__ ((__packed__)) cl_msg_key;
261 
262 typedef struct cl_msg_number_s {
263  cl_msg_field f;
264  uint32_t number;
265 } __attribute__ ((__packed__)) cl_msg_number;
266 
267 
268 
269 /* cl_msg
270  * Aerospike message
271  * size: size of the payload, not including the header */
272 typedef struct cl_msg_s {
273 /*00*/ uint8_t header_sz; // number of uint8_ts in this header
274 /*01*/ uint8_t info1; // bitfield about this request
275 /*02*/ uint8_t info2;
276 /*03*/ uint8_t info3;
277 /*04*/ uint8_t unused;
278 /*05*/ uint8_t result_code;
279 /*06*/ uint32_t generation;
280 /*10*/ uint32_t record_ttl;
281 /*14*/ uint32_t transaction_ttl;
282 /*18*/ uint16_t n_fields; // size in uint8_ts
283 /*20*/ uint16_t n_ops; // number of operations
284 /*22*/ uint8_t data[]; // data contains first the fields, then the ops
285 } __attribute__((__packed__)) cl_msg;
286 
287 /* cl_ms
288  * Aerospike message
289  * sz: size of the payload, not including the header */
290 typedef struct as_msg_s {
291  cl_proto proto;
292  cl_msg m;
293 } __attribute__((__packed__)) as_msg;
294 
295 #endif
296 
297 // 0-19 STANDARD
298 #define CL_MSG_FIELD_TYPE_NAMESPACE 0 // UTF8 string
299 #define CL_MSG_FIELD_TYPE_SET 1
300 #define CL_MSG_FIELD_TYPE_KEY 2 // contains a key type
301 #define CL_MSG_FIELD_TYPE_BIN 3 // used for secondary key access - contains a bin, thus a name and value
302 #define CL_MSG_FIELD_TYPE_DIGEST_RIPE 4 // used to send the digest just computed to the server so it doesn't have to
303 #define CL_MSG_FIELD_TYPE_GU_TID 5
304 #define CL_MSG_FIELD_TYPE_DIGEST_RIPE_ARRAY 6
305 #define CL_MSG_FIELD_TYPE_TRID 7
306 // We are going to overload the OPTIONS field -- this will hold either SCAN
307 // options or QUERY options, depending on the type of call. THis is done with
308 // the expectation that the call will have only one or the other (Nov 20, 2013 tjl)
309 #define CL_MSG_FIELD_TYPE_SCAN_OPTIONS 8
310 #define CL_MSG_FIELD_TYPE_QUERY_OPTIONS 8
311 
312 // 20-29 RESERVED FOR SECONDARY INDEX
313 #define CL_MSG_FIELD_TYPE_INDEX_NAME 21
314 #define CL_MSG_FIELD_TYPE_INDEX_RANGE 22
315 #define CL_MSG_FIELD_TYPE_INDEX_FILTER 23
316 #define CL_MSG_FIELD_TYPE_INDEX_LIMIT 24
317 #define CL_MSG_FIELD_TYPE_INDEX_ORDER_BY 25
318 
319 // 30-39 RESEVED FOR UDF
320 #define CL_MSG_FIELD_TYPE_UDF_FILENAME 30
321 #define CL_MSG_FIELD_TYPE_UDF_FUNCTION 31
322 #define CL_MSG_FIELD_TYPE_UDF_ARGLIST 32
323 #define CL_MSG_FIELD_TYPE_UDF_OP 33
324 // NOTE: UDF_OP really holds "Stream" or "Record" UDF type. And, going forward
325 // from this point (Nov 21, 2013), we're going to have two fields that all will
326 // treat the same. Udf type will be (None, Record, Stream), and the Transaction
327 // Call type will be Query/Scan and the Transaction ResultType will be
328 // FOREGROUND or BACKGROUND (as specified in the query/scan options).
329 // Historical note: Somehow QUERY and SCAN took different paths and started
330 // using this field differently.
331 // QUERY CLIENT had: None, Record, Stream
332 // QUERY SERVER had: UDF, Aggregate, MR
333 // SCAN SERVER had: None, UDF, Background
334 // On the wire, we put: 0=Record and 1=Stream into field 33 (above)
335 #define CL_UDF_MSG_VAL_RECORD 0
336 #define CL_UDF_MSG_VAL_STREAM 1
337 
338 // 40-49 RESERVED FOR QUERY
339 #define CL_MSG_FIELD_TYPE_QUERY_BINLIST 40
340 
341 
342 #define CL_MSG_OP_READ 1 // read the value in question
343 #define CL_MSG_OP_WRITE 2 // write the value in question
344 #define CL_MSG_OP_WRITE_UNIQUE 3 // write a namespace-wide unique value
345 #define CL_MSG_OP_WRITE_NOW 4 // write the server-current time
346 #define CL_MSG_OP_INCR 5
347 #define CL_MSG_OP_APPEND_SEGMENT 6 // Append segment to a particle
348 #define CL_MSG_OP_APPEND_SEGMENT_EXT 7 // Extended append - with parameters
349 #define CL_MSG_OP_APPEND_SEGMENT_QUERY 8 // Query to return subset of segments
350 #define CL_MSG_OP_APPEND 9 // Add to an existing particle
351 #define CL_MSG_OP_PREPEND 10 // Add to the beginning of an existing particle
352 #define CL_MSG_OP_TOUCH 11 // Touch
353 
354 #define CL_MSG_OP_MC_INCR 129 // Memcache-compatible version of the increment command
355 #define CL_MSG_OP_MC_APPEND 130 // Memcache compatible append. Allow appending to ints.
356 #define CL_MSG_OP_MC_PREPEND 131 // Memcache compatile prepend. Allow prepending to ints.
357 #define CL_MSG_OP_MC_TOUCH 132 // Memcache compatible touch - does not change generation count
358 
359 #define CL_MSG_INFO1_READ (1 << 0) // contains a read operation
360 #define CL_MSG_INFO1_GET_ALL (1 << 1) // get all bins, period
361 #define CL_MSG_INFO1_GET_ALL_NODATA (1 << 2) // get all bins WITHOUT data (currently unimplemented)
362 #define CL_MSG_INFO1_VERIFY (1 << 3) // verify is a GET transaction that includes data, and assert if the data aint right
363 #define CL_MSG_INFO1_XDS (1 << 4) // operation is being performed by XDS
364 #define CL_MSG_INFO1_NOBINDATA (1 << 5) // do not get information about bins and its data
365 
366 #define CL_MSG_INFO2_WRITE (1 << 0) // contains a write semantic
367 #define CL_MSG_INFO2_DELETE (1 << 1) // fling a record into the belly of Moloch
368 #define CL_MSG_INFO2_GENERATION (1 << 2) // pay attention to the generation
369 #define CL_MSG_INFO2_GENERATION_GT (1 << 3) // apply write if new generation >= old, good for restore
370 #define CL_MSG_INFO2_GENERATION_DUP (1 << 4) // if a generation collision, create a duplicate
371 #define CL_MSG_INFO2_CREATE_ONLY (1 << 5) // write record only if it doesn't exist
372 #define CL_MSG_INFO2_BIN_CREATE_ONLY (1 << 6) // write bin only if it doesn't exist
373 #define CL_MSG_INFO2_WRITE_MERGE (1 << 7) // merge this with current
374 
375 #define CL_MSG_INFO3_LAST (1 << 0) // this is the last of a multi-part message
376 #define CL_MSG_INFO3_TRACE (1 << 1) // apply server trace logging for this transaction
377 #define CL_MSG_INFO3_TOMBSTONE (1 << 2) // if set on response, a version was a delete tombstone
378 #define CL_MSG_INFO3_UPDATE_ONLY (1 << 3) // update existing record only, do not create new record
379 #define CL_MSG_INFO3_CREATE_OR_REPLACE (1 << 4) // completely replace existing record, or create new record
380 #define CL_MSG_INFO3_REPLACE_ONLY (1 << 5) // completely replace existing record, do not create new record
381 #define CL_MSG_INFO3_BIN_REPLACE_ONLY (1 << 6) // replace existing bin, do not create new bin
382 
383 
384 static inline uint8_t * cl_msg_op_get_value_p(cl_msg_op *op)
385 {
386  return ( ((uint8_t *)op) + sizeof(cl_msg_op) + op->name_sz);
387 }
388 
389 static inline uint32_t cl_msg_op_get_value_sz(cl_msg_op *op)
390 {
391  return( op->op_sz - (4 + op->name_sz) );
392 }
393 
394 static inline uint32_t cl_msg_field_get_value_sz(cl_msg_field *f)
395 {
396  return( f->field_sz - 1 );
397 }
398 
399 static inline cl_msg_field *
400 cl_msg_field_get_next(cl_msg_field *mf)
401 {
402  return ( (cl_msg_field *) (((uint8_t *)mf) + sizeof(mf->field_sz) + mf->field_sz) );
403 }
404 
405 
406 /* cl_msg_field_get
407  * Retrieve a specific field from a message */
408 static inline cl_msg_field *
409 cl_msg_field_get(cl_msg *msg, uint8_t type)
410 {
411  uint16_t n;
412  cl_msg_field *fp = NULL;
413 
414  fp = (cl_msg_field *)msg->data;
415  for (n = 0; n < msg->n_fields; n++) {
416 
417  if (fp->type == type)
418  break;
419 
420  fp = cl_msg_field_get_next(fp);
421  }
422  if (n == msg->n_fields)
423  return(NULL);
424  else
425  return(fp);
426 }
427 
428 /* cl_msg_field_getnext
429  * iterator for all fields of a particular type
430  * First time through: pass 0 as current, you'll get a field
431  * next time: pass the current as current
432  * you'll get null when there are no more
433  */
434 static inline cl_msg_op *
436 {
437  return ( (cl_msg_op *) (((uint8_t *) op) + sizeof(op->op_sz) + op->op_sz ) );
438 }
439 
440 
441 static inline cl_msg_op *
442 cl_msg_op_iterate(cl_msg *msg, cl_msg_op *current, int *n)
443 {
444  // skip over the fields the first time
445  if (!current) {
446  if (msg->n_ops == 0) return(0); // short cut
447  cl_msg_field *mf = (cl_msg_field *) msg->data;
448  for (uint32_t i = 0; i < msg->n_fields; i++)
449  mf = cl_msg_field_get_next(mf);
450  current = (cl_msg_op *) mf;
451  *n = 0;
452  return(current);
453  }
454  (*n)++;
455  if (*n >= msg->n_ops) return(0);
456  return ( cl_msg_op_get_next( current ) );
457 
458 }
459 
460 
461 /* cl_msg_size_get
462  * Get the size of a message */
463 static inline size_t
465 {
466  return( sizeof(cl_proto) + proto->sz);
467 }
468 
469 
470 /* Function declarations */
471 extern void cl_proto_swap_to_be(cl_proto *m);
472 extern void cl_proto_swap_from_be(cl_proto *m);
473 extern void cl_msg_swap_header_to_be(cl_msg *m);
474 extern void cl_msg_swap_header_from_be(cl_msg *m);
475 extern void cl_msg_swap_field_to_be(cl_msg_field *mf);
476 extern void cl_msg_swap_field_from_be(cl_msg_field *mf);
477 extern void cl_msg_swap_op_to_be(cl_msg_op *op);
478 extern void cl_msg_swap_op_from_be(cl_msg_op *op);
479 
480 #ifdef __cplusplus
481 } // end extern "C"
482 #endif
static uint32_t cl_msg_op_get_value_sz(cl_msg_op *op)
static uint8_t * cl_msg_op_get_value_p(cl_msg_op *op)
void cl_msg_swap_header_from_be(cl_msg *m)
static cl_msg_op * cl_msg_op_iterate(cl_msg *msg, cl_msg_op *current, int *n)
void cl_proto_swap_from_be(cl_proto *m)
void cl_msg_swap_header_to_be(cl_msg *m)
cl_msg_field f
static cl_msg_field * cl_msg_field_get_next(cl_msg_field *mf)
void cl_msg_swap_field_to_be(cl_msg_field *mf)
void cl_msg_swap_field_from_be(cl_msg_field *mf)
struct cl_proto_s __attribute__((__packed__)) cl_proto
void cl_proto_swap_to_be(cl_proto *m)
static cl_msg_field * cl_msg_field_get(cl_msg *msg, uint8_t type)
void cl_msg_swap_op_from_be(cl_msg_op *op)
static cl_msg_op * cl_msg_op_get_next(cl_msg_op *op)
static uint32_t cl_msg_field_get_value_sz(cl_msg_field *f)
void cl_msg_swap_op_to_be(cl_msg_op *op)
static size_t cl_proto_size_get(cl_proto *proto)