Return to gcs.h CVS log

Up to [Development] / failsafe / FailSafe / cluster_services / include

Initial checkin

/*
 * Copyright (c) 2000 Silicon Graphics, Inc.  All Rights Reserved.
 * 
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of version 2.1 of the GNU Lesser General Public
 * License as published by the Free Software Foundation.
 * 
 * This program is distributed in the hope that it would be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 * 
 * Further, this software is distributed without any warranty that it is
 * free of the rightful claim of any third person regarding infringement
 * or the like.  Any license provided herein, whether implied or
 * otherwise, applies only to this software file.  Patent licenses, if
 * any, provided herein do not apply to combinations of this program with
 * other software, or any other product whatsoever.
 * 
 * You should have received a copy of the GNU Lesser General Public
 * License along with this program; if not, write the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307,
 * USA.
 * 
 * Contact information: Silicon Graphics, Inc., 1600 Amphitheatre Pkwy,
 * Mountain View, CA 94043, or:
 * 
 * http://www.sgi.com 
 * 
 * For further information regarding this notice, see: 
 * 
 * http://oss.sgi.com/projects/GenInfo/NoticeExplan
 */
/* 
 * gcs.h
 *
 * This file contains GCS client interface constructs
 */
#ifndef GCS_H
#define GCS_H
#ident "$Id: gcs.h,v 1.1 2000/08/31 19:16:32 vasa Exp $"

#include <ci_log.h>
#include <ci_misc.h>
#include <ci_ipc.h>

#define GCS_DAEMON "ha_gcd"

typedef  struct cl_register* gcs_hdl_t;	

/* Message types */
typedef enum msg_type_s {
	GCSMSG_CLIENT,
	GCSMSG_MEMBERSHIP,
	GCSMSG_ADMIN,
	GCSMSG_ADMIN_REPLY,
	GCSMSG_REGISTER_REPLY,
	GCSMSG_UNREGISTER_REPLY,
	GCSMSG_NEWGCD,
	GCSMSG_NEWGCD_REPLY,
	GCSMSG_DOWN_MEMBERSHIP,
	GCSMSG_GCS_QUERY,
	GCSMSG_GCS_QUERY_REPLY
} msg_type_t;

typedef struct gcs_msg {
	ci_instid_t        sender_instid;
	msg_type_t	msg_type;
	uint32_t	msg_size;
	char*		msg_ptr; 	/* Buffer of size msg_size */
} gcs_msg_t;

/* change this value later */
#define MAX_INSTANCES 16

#define GCS_MAX_INSTID		100
#define GCS_MAX_GROUPID		100	

#define NULL_INSTID		(-1)	/* Null local instance id */
#define NULL_GCSHANDLE		0	/* Null GCS handle */

#define GCD_MAX_NUM_GROUPS	32

/* defining another macro for clients indicate that GCD can allocate any
   inst-id for them. */

#define GCS_ANY_INSTID		(short)0xfffd /* -3 */

/* invalid global inst_id - changed to -2 so as not to clash
   with the value of GCS_BROADCAST (-1). */
#define INVALID_INSTID		0xfffffffe

/* Value of dest_instid used to do broadcast gcs_send */
#define GCS_BROADCAST		0xffffffff

/* invalid group id. Group id's are int32_t, so this can be -1. */
#define INVALID_GROUPID		(-1)
#define GCS_SRM_GROUPID		10	/* default value if not read from CDB */
#define GCS_FAILSAFE_GROUPID	11	/* default value if not read from CDB */

/* Bitmasks used for obtaining instance id and node id */
#define NODEID(num)		(ci_nodeid_t)((num & 0xffff0000) >> 16)
#define INSTID(num)		(short)(num & 0x0000ffff)
#define NODEINSTID(node, inst)	((node<<16) | inst)

typedef int32_t gcs_groupid_t;


typedef enum {
	GCSCHG_INSTANCE_FAILURE,
	GCSCHG_INSTANCE_UNREGISTER,
	GCSCHG_INSTANCE_JOIN,
	GCSCHG_NODE_FAILURE,
	GCSCHG_NODE_UNKNOWN,
	GCSCHG_UNKNOWN
} membr_chng_t;

/* Client side membership data structs. 
 * These structures are defined so that client side structs are independant of
 * server side structs. The library is responsible for translating info 
 * provided by server to clients via these data structs.
 */

/* currently identical to server membership_t */

typedef struct gcs_membership_s {
	int32_t	group_id; /* == -1, entry not used */
	int32_t num_instances;
	ci_instid_t inst_ids[MAX_INSTANCES]; 
} gcs_membership_t;

/* identical to  membr_msg_t */
typedef struct gcs_membr_msg_s {
	int32_t		num_instid;
    
    	/* INSTANCE_FAILURE|INSTANCE_UNREGISTER |INSTANCE_JOIN|
	   INSTANCE_UNKNOWN 
	 */
	membr_chng_t	cause;	   
    	/* there can be MAX_INSTANCES per group */
    	ci_instid_t 	ids[MAX_INSTANCES]; 
} gcs_membr_msg_t;

typedef struct gcs_membr_msg_s gcs_membr_diff_t;

typedef struct gcs_mbrship_msg_s {
    gcs_membership_t membr;
    gcs_membr_diff_t  diff;
} gcs_mbrship_msg_t;

/* end of client side membership data structs */

#define	GCS_NOSIG			CI_IPC_NOSIG
#define GCS_INITIAL_HB_INTERVAL      	CI_IPC_INITIAL_HB_INTERVAL   /* initial hb default */

typedef struct  gcs_cdb_params_s{
    	uint32_t	valid;
    	size_t		mem_buf_size;
    	ci_clock_t	resend_q_t;
    	ci_clock_t	proposal_q_t;
	ci_clock_t    	commit_q_t;
    	ci_clock_t	cms_pulse_t;
    	ci_clock_t	cms_mon_t;
    	ci_clock_t	gcd_loop_t;
} gcs_params_t;


/* Initial sequence number for a group. */
#define NULLSEQNO	0

ci_err_t	gcs_set_nodename(char *name);	/* DEBUG only */
ci_err_t	gcs_send(gcs_hdl_t gcs_handle, ci_instid_t dest_instid,
			 gcs_msg_t *clientmsg );
ci_err_t	gcs_recv(gcs_hdl_t gcs_handle, gcs_msg_t *clientmsg, 
			 boolean_t check);
msg_type_t	gcs_get_msg_type(gcs_msg_t *msg);
ci_instid_t	gcs_get_sender_instid(gcs_msg_t *msg);
ci_instid_t	gcs_get_globalid(gcs_hdl_t);
ci_instid_t	gcs_get_globalid_of_peer(gcs_hdl_t, short );

ci_err_t	gcs_pulse(gcs_hdl_t gcs_handle, ci_clock_t heartbeat);
ci_err_t	gcs_unregister(gcs_hdl_t gcs_handle);
ci_err_t	gcs_register(gcs_hdl_t *gcsh, int32_t group_id, short *inst_id,
			     size_t msgpool_size, gcs_membership_t* members, 
			     uint32_t sig, ci_clock_t);

int32_t         gcs_get_msg_seqno(gcs_hdl_t gcsh);
int32_t         gcs_get_group_msg_seqno(gcs_hdl_t gcsh);
size_t		gcs_get_max_msg_size(gcs_hdl_t);
size_t		gcs_get_gcd_mem_size(void);

ci_err_t	gcs_query_params(gcs_params_t *params);


/* Functions for getting membership info */

uint32_t   	gcs_mem_get_no_of_members(gcs_membership_t *memp);
boolean_t 	gcs_mem_is_member(gcs_membership_t *memp, ci_instid_t inst);
ci_err_t	gcs_mem_get_instids(gcs_membership_t *memp, uint32_t *count, 
				    ci_instid_t **inst_ids);
ci_err_t	gcs_mem_get_instids_on_node(gcs_membership_t *memp, 
					    ci_nodeid_t nodeid, 
					    uint32_t *count, 
					    ci_instid_t ** inst_ids);
ci_err_t	gcs_mem_get_member_nodes(gcs_membership_t *memp, 
					 uint32_t *count, 
					 ci_nodeid_t** nodeids);
boolean_t 	gcs_mem_has_member_on_node(gcs_membership_t *memp, 
					   ci_nodeid_t nodeid );
ci_instid_t	gcs_mem_get_first_instance(gcs_membership_t *memp);
void		gcs_mem_print_membership(gcs_membership_t *memp, int loglvl);
void		gcs_mem_remove_instid(gcs_membership_t *memp, 
				       ci_instid_t inst);
ci_err_t	gcs_mem_add_instid(gcs_membership_t *memp, ci_instid_t inst);



uint32_t		gcs_get_no_of_members(gcs_hdl_t gcsh);
ci_err_t	gcs_get_instids(gcs_hdl_t gcsh, uint32_t *count, 
				ci_instid_t **inst_ids);
ci_err_t	gcs_get_instids_on_node(gcs_hdl_t gcsh, ci_nodeid_t nodeid, 
					uint32_t *count, 
					ci_instid_t** inst_ids);
ci_err_t	gcs_get_member_nodes(gcs_hdl_t gcsh, uint32_t *count,
				     ci_nodeid_t** nodeid);

boolean_t	gcs_is_member(gcs_hdl_t gcsh, ci_instid_t inst);
boolean_t 	gcs_has_member_on_node(gcs_hdl_t gcsh, ci_nodeid_t nodeid );
ci_instid_t	gcs_get_first_instance(gcs_hdl_t gcsh);
void		gcs_print_membership(gcs_hdl_t gcsh, int loglvl);
gcs_membership_t*   gcs_get_membership_copy(gcs_hdl_t gcsh);
gcs_membr_diff_t*   gcs_get_membership_diff(gcs_hdl_t gcsh);
void 		gcs_destroy_membership(gcs_membership_t *memp);


/* Admin Features */

#define GCS_ADMIN_GROUP		0

typedef enum gcs_admin_action {
	GCS_GROUP_LOCAL_INFO = 0,
	GCS_GROUP_GLOBAL_INFO,
	GCS_INFO,
	GCS_SET_GROUP_PARAMS = 0x20,
	GCS_STIMULUS = 0x40,
	GCS_RESPONSE
} gcs_admin_action_t;

/* Setting group parameters */

typedef struct gcs_group_params {
      /* all tunable parameters for group(s) go here */
	gcs_groupid_t group_id;
	uint32_t    init_q_timeout;
	uint32_t    proposal_q_timeout;
	uint32_t    commit_q_timeout;
} gcs_group_params_t;

/* contains per-group per node information */

typedef struct gcs_group_local_info_s {
	int32_t		group_id;  
	uint32_t	num_instances;
	
	/* msgs i sent = m'ship due to client join + 
	 *               m'ship due to failed clients + 
	 *               msgs on clients' behalf + 
	 *               reply to client unreg 
	 */
	uint32_t	msgs_i_sent; 

	/* msgs recd = msgs read from socket */
	uint32_t	msgs_recd_from_peer;
	
	/* registration requests are not included 
	 * msgs_from_client = msgs on clients' behalf + reply to client unreg
	 */
	uint32_t	msgs_from_client;
  
	/* msgs resent */
	uint32_t	msgs_resent;
  
	/* Clients registered with this group */
	uint32_t	clients_reg;
	/* Clients unregistered gracefully */
	uint32_t	clients_unreg;
	/* clients forced out of membership */
	uint32_t	clients_failed;

	/* queue lengths */
	uint32_t	initiating_queue_len;
	uint32_t	pending_queue_len;
        uint32_t        down_node_mbrship_queue_len;
        uint32_t	early_down_queue_len;
  
} gcs_group_local_info_t;

/* contains cluster wide per group information */

typedef struct gcs_group_global_info_s {
	int32_t		group_id;  
	gcs_membership_t mbrship;
	uint32_t	num_instances;

	/* msgs sent by all GCDs for this group */
	uint32_t	tot_msgs_in_grp;

	/* Clients registered with this group */
	uint32_t	clients_reg;
	/* Clients unregistered gracefully */
	uint32_t	clients_unreg;
	/* clients forced out of membership */
	uint32_t	clients_failed;
} gcs_group_global_info_t;

/* for obtaining information about  gcd */

typedef struct gcs_info_s {

	boolean_t	is_coordinator;
	ci_nodeid_t	coord_node;
	ci_nodeid_t	next_gcd;

	/* GCD max msg size */
	size_t		gcd_msg_size;

	/* CMS related values */
	ci_clock_t	cms_hb_timeout;

	/* all clients & local membership changes */

	/* number of groups which have clients registered */
	int32_t		active_groups[GCD_MAX_NUM_GROUPS];
	
	/* total number of clients currently registered, across all nodes 
	 * across all groups
	 */
	uint32_t	tot_clients;
	
	/* Current clients served */
	uint32_t	clients_reg;
	/* Clients unregistered gracefully */
	uint32_t	clients_unreg;
	/* clients forced out of membership */
	uint32_t	clients_failed;

	/* any more values we can think of - tunable params, timeouts, etc.. */
	ci_clock_t	init_timeout;
	ci_clock_t	proposal_timeout;
	ci_clock_t	commit_timeout;

	/* Co-ordinator q lengths */
  	uint32_t	coord_init_q_len;
        uint32_t	coord_pending_q_len;
} gcs_info_t;

ci_err_t gcs_admin(gcs_hdl_t gcsh, gcs_admin_action_t act, void *datap);

/*
 * Use memory-mapped clock rather than times() system call.
 */
#define	times	ci_times
extern time_t	ci_times(struct tms *);

/* Memory SPECS for GCD */
/* Currently used a max of 32 blocks of 64K each */
#define GCD_MAX_MEM_BLKS	32
#define GCD_MEM_BUF_SIZE	(32 * 1024)

/* GCD CLI */

/* copied from gcd_defs.h - required for cli - validation of input values */

#define GCD_DEFAULT_PROPOSAL_TIMEOUT	10000  /* 10 s */
#define GCD_DEFAULT_COMMIT_TIMEOUT	10000  /* 10 s */
#define GCD_DEFAULT_RESEND_TIMEOUT	10000  /* 10 s */

#define GCD_CMS_PULSE_TIME		6000	/* 6 s */
#define GCD_CMS_MON_TIME    		30000   /* 30 s */

#define GCD_LOOP_TIME 			120000  /*  120s   */

#define GCD_MAX_NUM_GROUPS		32
#define GCD_DEFAULT_MAX_NUM_CLIENTS	255

/* User must specify a size atleast equal to this. 
   GCD needs about 20K for its initial data structures. */

#define GCD_MIN_MEM_SIZE		( 32 * 1024)

/*
 * Some other minimums - for user configurable vals.
 */
#define GCD_MIN_RESEND_TIMEOUT		10000	/* 10s */
#define GCD_MIN_PROPOSAL_TIMEOUT	10000	/* 10s */
#define GCD_MIN_COMMIT_TIMEOUT		10000	/* 10s */
#define GCD_MIN_CMS_PULSE_TIME		5000   /* 5s */
#define GCD_MIN_CMS_MON_TIME		10000    /* 10s */
#define GCD_MIN_LOOP_TIME 		100000  /* 100s */

/*
 * For GCD mem buf size, we need a maximum since this is
 * directly related to the largest message that can be sent
 * via UDP. The max msg size that can be sent by UDP is 64K,
 * and that includes the IP and UDP headers (which add up to
 * 28 bytes today), so effectively UDP can send 
 * (64Kbytes - 28bytes) for GCD. Out of this, we need space
 * for the GCD header and the secure library header. 
 * 
 * Let's just allow clients msgs to be of size <= 63K.
 */
#define GCD_MAX_MEM_SIZE		(63 * 1024)

/*  upto here */

#define GCS_PARAM_MEM_BUF 	0x00001
#define GCS_PARAM_RESEND_T 	0x00002
#define GCS_PARAM_PROPOSAL_T	0x00004
#define GCS_PARAM_COMMIT_T	0x00008
#define GCS_PARAM_CMS_PULSE_T	0x00020
#define GCS_PARAM_CMS_MON_T	0x00040
#define GCS_PARAM_GCD_LOOP_T	0x00080


ci_err_t gcs_cdb_set_value(cdb_handle_t cdbh, char* clustername,
			   gcs_params_t *keyvals, boolean_t local);
ci_err_t gcs_cdb_get_value(cdb_handle_t cdbh, char *clustername, 
			   gcs_params_t *keyvals, boolean_t local, 
			   char *errorstr);
ci_err_t gcs_cdb_check_value(gcs_params_t *keyvals, char *errorstr);

#endif /* GCS_H */