/*

	MPI wrappers which measure time taken.
	Version 10

	

	Used for analyzing performance of an MPI code.

	Contents
		MPI_Init		C and Fortran
		MPI_Finalize		C and Fortran

		MPI_Comm_rank		C
		MPI_Get_count		C
		MPI_Ibsend           	C		(with detail stats)
		MPI_Probe		C
		MPI_Recv		C and Fortran   (with detail stats)
		MPI_Irecv		C and Fortran
		MPI_Send           	C and Fortran	(with detail stats)
		MPI_Isend           	C and Fortran	(with detail stats)
		MPI_Ssend           	C and Fortran	(with detail stats)
		MPI_Wait		C and Fortran   (with detail stats)
		MPI_Waitall		C and Fortran   (with detail stats)

		MPI_Barrier		C and Fortran
		MPI_Alltoall		C and Fortran
		MPI_Alltoallv		C and Fortran
		MPI_Reduce		C and Fortran
		MPI_Allreduce		C and Fortran
		MPI_Bcast		C and Fortran
		MPI_Gather		C and Fortran
		MPI_Gatherv		C and Fortran
		MPI_Scatter		C and Fortran
		MPI_Scatterv		C and Fortran

        Version History:
                o version 8 from Frank Kampe added MPI_Isend,
                  MPI_Irecv, and MPI_Waitall.
                o version 9 fixes the MPI_Scatter wrapper
                o version 9b fixes the mpi_gatherv_ wrapper
                o version 10 adds message length statistics
                  gathering to MPI_Ibsend, MPI_Isend, MPI_Ssend,
                  and MPI_Isend



 */


#include <mpi.h>
#include <stdio.h>

/*
 * CONSTANTS
 */

#define MAXRANKS 1024		/* must be set >= total number of MPI	*/
				/* processes.				*/

/*
 * UTILITIES
 */

#define TIMEIT_NORETURN(func,  var)    {  \
	double t0, t1; \
	t0 = MPI_Wtime(); \
 	rv = func; \
	t1=MPI_Wtime(); \
	var += t1-t0; \
}


#define TIMEIT(func,  var)    {  \
	int rv; \
	TIMEIT_NORETURN(func,  var) \
	return rv; \
}

/*
 * DATA
 */

static double tstart, tend;

static int datatype_numbytes[40] = {0,1,2,4,sizeof(long),1,2,4,sizeof(long),
                                    4,8,16,8,4,4,8,8,16,1,1,1,2,4,8,4,8,16,
                                    1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1};

/*************************************************************************
 * Wrappers
 */

/* MPI_Comm_rank */
static long comm_rank_cnt;
static double comm_rank_time;
int
MPI_Comm_rank(MPI_Comm comm, int *rank)
{
	comm_rank_cnt++;
	TIMEIT( PMPI_Comm_rank(comm, rank), comm_rank_time );
}

/* MPI_Get_count */
static long get_count_cnt;
static double get_count_time;
int
MPI_Get_count(MPI_Status *status, MPI_Datatype type, int *count)
{
	get_count_cnt++;
	TIMEIT( PMPI_Get_count(status, type, count), get_count_time);
}


/* MPI_Ibsend */
static long ibsend_cnt, ibsend_cnts[MAXRANKS];
static double ibsend_time, ibsend_times[MAXRANKS];
static long ibsend_datacnt;
int
MPI_Ibsend(void *buf, int count, MPI_Datatype type, int des, int tag,
	MPI_Comm comm, MPI_Request *request)
{
	int rv;
	double t0, t1;

	ibsend_cnt++;
	if (comm == MPI_COMM_WORLD) {
		ibsend_cnts[des]++;
	}

	t0 = MPI_Wtime(); 
	rv = PMPI_Ibsend(buf, count, type, des, tag, comm, request);
	t1 = MPI_Wtime(); 
	ibsend_time += t1-t0;
	if (comm == MPI_COMM_WORLD) {
		ibsend_times[des] += t1-t0;
	}
        ibsend_datacnt += datatype_numbytes[type]*count;

	return rv;
}

/* MPI_Probe */
static long probe_cnt, probe_cnts_as, probe_cnts[MAXRANKS];
static double probe_time, probe_times_as, probe_times[MAXRANKS];
int
MPI_Probe(int source, int tag, MPI_Comm comm, MPI_Status *status)
{
	int rv;
	double t0, t1;

	probe_cnt++;

	if (source == MPI_ANY_SOURCE) {
		probe_cnts_as++;
	} else if (comm == MPI_COMM_WORLD) {
		probe_cnts[source]++;
	}

	t0 = MPI_Wtime(); 
	rv = PMPI_Probe(source, tag, comm, status);
	t1 = MPI_Wtime(); 
	probe_time += t1-t0;
	if (source == MPI_ANY_SOURCE) {
		probe_times_as += t1-t0;
	} else if (comm == MPI_COMM_WORLD) {
		probe_times[source] += t1-t0;
	}
	return rv;
}

/* MPI_Recv */
static long recv_cnt, recv_cnts_as, recv_cnts[MAXRANKS];
static double recv_time, recv_times_as, recv_times[MAXRANKS];
static long recv_datacnt;

static long recv_nomsg_cnt, recv_nomsg_cnts[MAXRANKS];
static double recv_nomsg_time;
static double recv_nomsg_times[MAXRANKS];
int
MPI_Recv(void *buf, int count, MPI_Datatype type, int src, int tag, 
	MPI_Comm comm, MPI_Status *status)

{
	int rv, flag;
	double t0, twait, t1;

	rv = PMPI_Iprobe(src, tag, comm, &flag, status);

	t0 = MPI_Wtime(); 
	twait = t0;
	if (!flag) {
		while (!flag) {
			PMPI_Iprobe(src, tag, comm, &flag, status);
		}
		twait = MPI_Wtime(); 
	}
	
	rv = PMPI_Recv(buf, count, type, src, tag, comm, status);
	t1 = MPI_Wtime(); 

	recv_cnt++;
	recv_time += t1-t0;
	if (comm == MPI_COMM_WORLD) {
		if (src == MPI_ANY_SOURCE) 
			src = status->MPI_SOURCE;
		recv_cnts[src]++;
		recv_times[src] += t1-t0;

		if (twait > t0) {
			recv_nomsg_cnt ++;
			recv_nomsg_cnts[src] ++;
			recv_nomsg_time += twait-t0;
			recv_nomsg_times[src] += twait-t0;
		}
	}

	recv_datacnt += status->size;

	return rv;
}

void
mpi_recv_(void *buf, int *count, MPI_Datatype *type, int *src, int *tag,
        MPI_Comm *comm, MPI_Status *status, int *ierr)
{
	*ierr = MPI_Recv(buf, *count, *type, *src, *tag, *comm, status);
}

/* MPI_Irecv */
static long irecv_cnt, irecv_cnts_as, irecv_cnts[MAXRANKS];
static double irecv_time, irecv_times_as, irecv_times[MAXRANKS];
static long irecv_datacnt;

static long irecv_nomsg_cnt, irecv_nomsg_cnts[MAXRANKS];
static double irecv_nomsg_time;
static double irecv_nomsg_times[MAXRANKS];
int
MPI_Irecv(void *buf, int count, MPI_Datatype type, int src, int tag, 
	MPI_Comm comm, MPI_Request *req)

{
	int rv, flag;
	double t0, twait, t1;
        MPI_Status status;

	rv = PMPI_Iprobe(src, tag, comm, &flag, &status);

	t0 = MPI_Wtime(); 
	
	rv = PMPI_Irecv(buf, count, type, src, tag, comm, req);
	t1 = MPI_Wtime(); 

	irecv_cnt++;
	irecv_time += t1-t0;
	if ((comm == MPI_COMM_WORLD) && flag) {
		if (src == MPI_ANY_SOURCE) 
			src = status.MPI_SOURCE;
		irecv_cnts[src]++;
		irecv_times[src] += t1-t0;

	}

	if( flag ) irecv_datacnt += status.size;

	return rv;
}

void
mpi_irecv_(void *buf, int *count, MPI_Datatype *type, int *src, int *tag,
        MPI_Comm *comm, MPI_Request *req, int *ierr)
{
	*ierr = MPI_Irecv(buf, *count, *type, *src, *tag, *comm, req);
}


/* MPI_Send */
static long send_cnt, send_cnts[MAXRANKS];
static double send_time, send_times[MAXRANKS];
static long send_datacnt;
int
MPI_Send(void *buf, int count, MPI_Datatype type, int des, int tag,
	MPI_Comm comm)
{
	int rv;
	double t0, t1;

	send_cnt++;
	if (comm == MPI_COMM_WORLD) {
		send_cnts[des]++;
	}

	t0 = MPI_Wtime(); 
	rv = PMPI_Send(buf, count, type, des, tag, comm);
	t1 = MPI_Wtime(); 
	send_time += t1-t0;
	if (comm == MPI_COMM_WORLD) {
		send_times[des] += t1-t0;
	}
        send_datacnt += datatype_numbytes[type]*count;

	return rv;
}

void
mpi_send_(void *buf, int *count, MPI_Datatype *type, int *des, int *tag,
        MPI_Comm *comm, int *ierr)
{
	*ierr = MPI_Send(buf, *count, *type, *des, *tag, *comm);
}

/* MPI_Isend */
static long isend_cnt, isend_cnts[MAXRANKS];
static double isend_time, isend_times[MAXRANKS];
static long isend_datacnt;
int
MPI_Isend(void *buf, int count, MPI_Datatype type, int des, int tag,
	MPI_Comm comm, MPI_Request *req)
{
	int rv;
	double t0, t1;

	isend_cnt++;
	if (comm == MPI_COMM_WORLD) {
		isend_cnts[des]++;
	}

	t0 = MPI_Wtime(); 
	rv = PMPI_Isend(buf, count, type, des, tag, comm, req);
	t1 = MPI_Wtime(); 
	isend_time += t1-t0;
	if (comm == MPI_COMM_WORLD) {
		isend_times[des] += t1-t0;
	}
        isend_datacnt += datatype_numbytes[type]*count;

	return rv;
}

void
mpi_isend_(void *buf, int *count, MPI_Datatype *type, int *des, int *tag,
        MPI_Comm *comm, MPI_Request *req, int *ierr)
{
	*ierr = MPI_Isend(buf, *count, *type, *des, *tag, *comm, req);
}

/* MPI_Ssend */
static long ssend_cnt, ssend_cnts[MAXRANKS];
static double ssend_time, ssend_times[MAXRANKS];
static long ssend_datacnt;
int
MPI_Ssend(void *buf, int count, MPI_Datatype type, int des, int tag,
        MPI_Comm comm)
{
        int rv;
        double t0, t1;

        ssend_cnt++;
        if (comm == MPI_COMM_WORLD) {
                ssend_cnts[des]++;
        }

        t0 = MPI_Wtime();
        rv = PMPI_Ssend(buf, count, type, des, tag, comm);
        t1 = MPI_Wtime();
        ssend_time += t1-t0;
        if (comm == MPI_COMM_WORLD) {
                ssend_times[des] += t1-t0;
        }
        ssend_datacnt += datatype_numbytes[type]*count;

        return rv;
}

void
mpi_ssend_(void *buf, int *count, MPI_Datatype *type, int *des, int *tag,
        MPI_Comm *comm, int *ierr)
{
        *ierr = MPI_Ssend(buf, *count, *type, *des, *tag, *comm);
}

/* MPI_Wait */
static long wait_datacnt;

static long wait_cnt, wait_cnts[MAXRANKS];
static double wait_time, wait_times[MAXRANKS];
int
MPI_Wait(MPI_Request *request, MPI_Status *status)
{
	int rv;
	wait_cnt++;
	TIMEIT_NORETURN( PMPI_Wait(request, status), wait_time);

	wait_datacnt += status->size;
	return rv;
}

void
mpi_wait_(MPI_Request *request, MPI_Status *status, int *ierr)
{
	*ierr = MPI_Wait(request, status);
}

/* MPI_Waitall */
static long waitall_datacnt;

static long waitall_cnt, waitall_cnts[MAXRANKS];
static long waitall_numreq;
static double waitall_time, waitall_times[MAXRANKS];
int
MPI_Waitall(int numreq, MPI_Request *request, MPI_Status *status)
{
	int rv,i;
	waitall_cnt++;
	TIMEIT_NORETURN( PMPI_Waitall(numreq, request, status), waitall_time);

        for (i = 0; i < numreq; i++,status++)
	waitall_datacnt += status->size;
        waitall_numreq += numreq;
	return rv;
}

void
mpi_waitall_(int *numreq, MPI_Request *request, MPI_Status *status, int *ierr)
{
	*ierr = MPI_Waitall(*numreq, request, status);
}
/* MPI_Barrier */
static long barrier_cnt;
static double barrier_time;
int
MPI_Barrier(MPI_Comm comm)
{
	barrier_cnt++;
	TIMEIT( PMPI_Barrier(comm), barrier_time );
}

void
mpi_barrier_(MPI_Comm *comm, int *ierr)
{
	*ierr = MPI_Barrier(*comm);
}


/* MPI_Alltoall */
static long alltoall_cnt;
static double alltoall_time;
static long alltoall_datacnt;
int
MPI_Alltoall(void *sendbuf, int sendcount, MPI_Datatype sendtype,
        void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm)
{
	alltoall_cnt++;
	alltoall_datacnt += sendcount;
	TIMEIT( PMPI_Alltoall(sendbuf, sendcount, sendtype, recvbuf,
		recvcount, recvtype, comm), alltoall_time );
}

void
mpi_alltoall_( void *sendbuf, int *sendcount, MPI_Datatype *sendtype,
	void *recvbuf, int *recvcnt, MPI_Datatype *recvtype,
	MPI_Comm *comm, int *ierr)
{
        *ierr = MPI_Alltoall(sendbuf, *sendcount, *sendtype, recvbuf,
		*recvcnt, *recvtype, *comm);
}


/* MPI_Alltoallv */
static long alltoallv_cnt;
static double alltoallv_time;
int
MPI_Alltoallv(void *sendbuf, int *sendcnts, int *sdispls,
	MPI_Datatype sendtype, void *recvbuf, int *recvcnts, int *rdispls,
	MPI_Datatype recvtype, MPI_Comm comm)
{
        alltoallv_cnt++;
        TIMEIT( PMPI_Alltoallv(sendbuf, sendcnts, sdispls, sendtype, recvbuf,
		recvcnts, rdispls, recvtype, comm), alltoallv_time );
}

void
mpi_alltoallv_( void *sendbuf, int *sendcnts, int *sdispls,
	MPI_Datatype *sendtype, void *recvbuf, int *recvcnts, int *rdispls,
	MPI_Datatype *recvtype, MPI_Comm *comm, int *ierr)
{
        *ierr = MPI_Alltoallv(sendbuf, sendcnts, sdispls, *sendtype, recvbuf,
		recvcnts, rdispls, *recvtype, *comm);
}


/* MPI_Reduce */
static long reduce_cnt;
static double reduce_time;
int
MPI_Reduce(void *sendbuf, void *recvbuf, int count, MPI_Datatype type,
	MPI_Op op, int root, MPI_Comm comm)
{
	reduce_cnt++;
	TIMEIT( PMPI_Reduce(sendbuf, recvbuf, count, type, op, root, comm),
		reduce_time );
}
 
void
mpi_reduce_( void *sendbuf, void *recvbuf, int *count, MPI_Datatype *type,
	MPI_Op *op, int *root, MPI_Comm *comm, int *ierr)
{
        *ierr = MPI_Reduce(sendbuf, recvbuf, *count, *type, *op, *root, *comm);
}


/* MPI_Allreduce */
static long allreduce_cnt;
static double allreduce_time;
int
MPI_Allreduce(void *sendbuf, void *recvbuf, int count, MPI_Datatype type,
	MPI_Op op, MPI_Comm comm)
{
	allreduce_cnt++;
	TIMEIT( PMPI_Allreduce(sendbuf, recvbuf, count, type, op, comm),
		allreduce_time );
}

void
mpi_allreduce_( void *sendbuf, void *recvbuf, int *count,
	MPI_Datatype *type, MPI_Op *op, MPI_Comm *comm,
	int *ierr)
{
        *ierr = MPI_Allreduce(sendbuf, recvbuf, *count, *type, *op,
		*comm);
}

/* MPI_Bcast */
static long bcast_cnt;
static double bcast_time;
int
MPI_Bcast(void *buffer, int count, MPI_Datatype type, int root, MPI_Comm comm)
{
	bcast_cnt++;
	TIMEIT( PMPI_Bcast(buffer, count, type, root, comm),
		bcast_time );

}

void
mpi_bcast_( void *buffer, int *count, MPI_Datatype *type, int *root,
	MPI_Comm *comm, int *ierr)
{
	*ierr = MPI_Bcast(buffer, *count, *type, *root, *comm);
}


/* MPI_Gather */
static long gather_cnt;
static double gather_time;
int
MPI_Gather(void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf,
	int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm)
{
	gather_cnt++;
	TIMEIT( PMPI_Gather(sendbuf, sendcount, sendtype, recvbuf,
		recvcount, recvtype, root, comm),
		gather_time );
}

void
mpi_gather_( void *sendbuf, int *sendcnt, MPI_Datatype *sendtype, void *recvbuf,
	int *recvcount, MPI_Datatype *recvtype, int *root, MPI_Comm *comm,
	int *ierr)
{
        *ierr = MPI_Gather(sendbuf, *sendcnt, *sendtype, recvbuf, *recvcount,
		*recvtype, *root, *comm);
}



/* MPI_Gatherv */
static long gatherv_cnt;
static double gatherv_time;
int
MPI_Gatherv(void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf,
	int *recvcounts, int *displs, MPI_Datatype recvtype, int root,
	MPI_Comm comm)
{
	gatherv_cnt++;
	TIMEIT( PMPI_Gatherv(sendbuf, sendcount, sendtype, recvbuf,
        	recvcounts, displs, recvtype, root, comm),
		gatherv_time );
}

void
mpi_gatherv_(void *sendbuf, int *sendcnt, MPI_Datatype *sendtype,
	void *recvbuf, int *recvcnts, int *displs, MPI_Datatype *recvtype,
	int *root, int *ierr, MPI_Comm *comm)
{
	*ierr = MPI_Gatherv(sendbuf, *sendcnt, *sendtype, recvbuf, recvcnts,
		displs, *recvtype, *root, *comm);
}


/* MPI_Scatter */
static long scatter_cnt;
static double scatter_time;
int
MPI_Scatter(void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf,
        int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm)
{
        scatter_cnt++;
        TIMEIT( PMPI_Scatter(sendbuf, sendcount, sendtype, recvbuf,
                recvcount, recvtype, root, comm),
                scatter_time );
}

void
mpi_scatter_( void *sendbuf, int *sendcnt, MPI_Datatype *sendtype,
	void *recvbuf, int *recvcount, MPI_Datatype *recvtype, int *root,
	MPI_Comm *comm, int *ierr)
{
        *ierr = MPI_Scatter(sendbuf, *sendcnt, *sendtype, recvbuf, *recvcount,
                *recvtype, *root, *comm);
}


/* MPI_Scatterv */
static long scatterv_cnt;
static double scatterv_time;
int
MPI_Scatterv(void *sendbuf, int *sendcounts, int *displs,
	MPI_Datatype sendtype, void *recvbuf, int recvcount,
	MPI_Datatype recvtype, int root, MPI_Comm comm)
{
	scatterv_cnt++;
        TIMEIT( PMPI_Scatterv(sendbuf, sendcounts, displs, sendtype,
		recvbuf, recvcount, recvtype, root, comm), 
		scatterv_time );
}

void
mpi_scatterv_( void *sendbuf, int *sendcnts, int *displs,
	MPI_Datatype *sendtype, void *recvbuf, int *recvcnt,
	MPI_Datatype *recvtype, int *root, MPI_Comm *comm, int *ierr)
{
	*ierr = MPI_Scatterv(sendbuf, sendcnts, displs, *sendtype, recvbuf,
		*recvcnt, *recvtype, *root, *comm);
}










/*************************************************************************/

/*
 * MPI_Init Wrapper
 */
int
MPI_Init( int  *argc, char ***argv)
{
	int rv, nranks, me;
	rv = PMPI_Init( argc, argv);
	tstart = MPI_Wtime();
	PMPI_Comm_rank(MPI_COMM_WORLD, &me);
	PMPI_Comm_size(MPI_COMM_WORLD, &nranks);
	if (nranks > MAXRANKS) {
		if (me == 0) {
			fprintf(stderr,"\
profiling lib max MPI proc limit of %d exceeded\n", MAXRANKS);
			exit(-1);
		}
	}
	return rv;
}

void
mpi_init_(int *ier)
{
	MPI_Init(NULL, NULL);
	*ier = 0;
}


/*
 * MPI_Finalize Wrapper
 * 
 * This wrapper prints out the statistics gathered.
 */
int
MPI_Finalize()
{
	FILE *f;
	double tk = MPI_Wtick();
	int i,j,rv;
	int rank, nranks, ncols;
	MPI_Comm_rank(MPI_COMM_WORLD, &rank);
	MPI_Comm_size(MPI_COMM_WORLD, &nranks);

	PMPI_Barrier(MPI_COMM_WORLD);
	tend = MPI_Wtime();
	f = fopen("MPI_PROFILING_STATS", "a");	
	/* rank 0 prints the title and column headings */
	if (rank == 0) {
		/* section title */
		fprintf(f,"Summary counts and timings\n\n");
		fprintf(f,"Total job time %e sec\n",tend-tstart);
		fprintf(f,"Total MPI processes %d\n",nranks);
		fprintf(f,"Wtime resolution is %e sec\n",tk);
		fflush(f);
	}
        PMPI_Barrier(MPI_COMM_WORLD);

	/* 
	 * Print summary counts and timings.
	 */
	for (i=0; i<nranks; i++) {
	    if (rank == i) {
		long ave;
		char *fmt = "%10s calls %ld\ttime %e\n";
		char *fmtdat, *fmtdat2;

		fprintf(f,"\nactivity on process rank %d\n",rank);
		fprintf(f,fmt, "comm_rank",comm_rank_cnt, comm_rank_time);

		fprintf(f,fmt, "get_count",get_count_cnt, get_count_time);
		if (ibsend_cnt==0) ave = 0; else ave = ibsend_datacnt/ibsend_cnt;
		fmtdat2 =
"%10s calls %ld\ttime %.5e   avg datacnt %ld\n";
		fprintf(f,fmtdat2, "ibsend",ibsend_cnt, ibsend_time, ave);
		fprintf(f,fmt, "probe",probe_cnt, probe_time);

		if (recv_cnt==0) ave = 0; else ave = recv_datacnt/recv_cnt;

		fmtdat =
"%10s calls %ld\ttime %.5e   avg datacnt %ld   waits %ld  wait time %.5e\n";
		fprintf(f,fmtdat, "recv",recv_cnt, recv_time,ave,
			recv_nomsg_cnt, recv_nomsg_time);
		fmtdat =
"%10s calls %ld\ttime %.5e   datacnt %ld\n";

		fprintf(f,fmtdat, "irecv",irecv_cnt, irecv_time,irecv_datacnt);
		if (send_cnt==0) ave = 0; else ave = send_datacnt/send_cnt;
		fprintf(f,fmtdat2, "send",send_cnt, send_time, ave);
		if (ssend_cnt==0) ave = 0; else ave = ssend_datacnt/ssend_cnt;
		fprintf(f,fmtdat2, "ssend",ssend_cnt, ssend_time, ave);
		if (isend_cnt==0) ave = 0; else ave = isend_datacnt/isend_cnt;
		fprintf(f,fmtdat2, "isend",isend_cnt, isend_time, ave);

		if (wait_cnt==0) ave = 0; else ave = wait_datacnt/wait_cnt;
		fmtdat =
"%10s calls %ld\ttime %.5e   avg datacnt %ld\n";
		fprintf(f,fmtdat, "wait",wait_cnt, wait_time, ave);
		if (waitall_numreq==0) ave = 0; else ave = waitall_datacnt/waitall_numreq;
		fmtdat =
"%10s calls %ld\ttime %.5e   # of Reqs %ld  avg datacnt %ld\n";
		fprintf(f,fmtdat, "waitall",waitall_cnt, waitall_time, waitall_numreq, ave);

		fprintf(f,fmt, "barrier",  barrier_cnt,   barrier_time);

		fmtdat =
"%10s calls %ld\ttime %.5e   avg datacnt %ld\n";
		if (alltoall_cnt==0)
			ave = 0;
		else
			ave = alltoall_datacnt/alltoall_cnt;
		fprintf(f,fmtdat, "alltoall", alltoall_cnt,  alltoall_time,
			ave);
		fprintf(f,fmt, "alltoallv",alltoallv_cnt, alltoallv_time);
		fprintf(f,fmt, "reduce",   reduce_cnt,    reduce_time);
		fprintf(f,fmt, "allreduce",allreduce_cnt, allreduce_time);
		fprintf(f,fmt, "bcast",bcast_cnt, bcast_time);
		fprintf(f,fmt, "gather",gather_cnt, gather_time);
		fprintf(f,fmt, "gatherv",gatherv_cnt, gatherv_time);
		fprintf(f,fmt, "scatter",scatter_cnt, scatter_time);
		fprintf(f,fmt, "scatterv",scatterv_cnt, scatterv_time);
		fflush(f);
	    }
	    PMPI_Barrier(MPI_COMM_WORLD);
	}

/* 
  * Print point-to-point counts and timings.
  */
	ncols = nranks;

	if (ncols > 10) ncols = 10;

	/******************
	 * recv
	 */
	/* rank 0 prints the title and column headings */
	PMPI_Barrier(MPI_COMM_WORLD);
	if (rank == 0) {
		/* section title */
		fprintf(f,"\nrecv profile\n\n");
		
		/* column headings */
		fprintf(f,"             cnt/sec for all remote ranks\n");

		fprintf(f,"%5s","local");
		fprintf(f,"%13s","ANY_SOURCE");
		for (i=0; i<ncols; i++)
			fprintf(f,"%9d    ",i);
		fprintf(f,"\n");
		fprintf(f,"%5s","rank");

		fprintf(f,"\n");
		fflush(f);
	}
	PMPI_Barrier(MPI_COMM_WORLD);

	for (i=0; i<nranks; i++) {
	    if (rank == i) {
		if (recv_cnt != 0) {
		  fprintf(f,"%5d ",i);
		  fprintf(f,"%4ld/%.1e ",
				recv_cnts_as, recv_times_as);
		  for (j=0; j<ncols; j++)
			fprintf(f,"%4ld/%.1e ",
				recv_cnts[j],recv_times[j]);
		  fprintf(f,"\n");
		  fflush(f);
	         }
	    }
	    PMPI_Barrier(MPI_COMM_WORLD);
	}

        /******************
         * recv waits
         */
        /* rank 0 prints the title and column headings */
        PMPI_Barrier(MPI_COMM_WORLD);
        if (rank == 0) {
                /* section title */
                fprintf(f,"\nrecv wait for data profile\n\n");

                /* column headings */
                fprintf(f,"             cnt/sec for all remote ranks\n");

                fprintf(f,"%5s","local");
                for (i=0; i<ncols; i++)
                        fprintf(f,"%9d    ",i);
                fprintf(f,"\n");
                fprintf(f,"%5s","rank");

                fprintf(f,"\n");
                fflush(f);
        }
        PMPI_Barrier(MPI_COMM_WORLD);

        for (i=0; i<nranks; i++) {
            if (rank == i) {
                if (recv_nomsg_cnt != 0) {
                  fprintf(f,"%5d ",i);
                  for (j=0; j<ncols; j++)
                        fprintf(f,"%4ld/%.1e ",
                                recv_nomsg_cnts[j],recv_nomsg_times[j]);
                  fprintf(f,"\n");
                  fflush(f);
                 }
            }
            PMPI_Barrier(MPI_COMM_WORLD);
        }


	/******************
	 * send
	 */
	/* rank 0 prints the title and column headings */
	PMPI_Barrier(MPI_COMM_WORLD);
	if (rank == 0) {
		/* section title */
		fprintf(f,"\nsend profile\n\n");
		
		/* column headings */
		fprintf(f,"             cnt/sec for all destination ranks\n");

		fprintf(f,"%5s","src");
		for (i=0; i<ncols; i++)
			fprintf(f,"%9d    ",i);
		fprintf(f,"\n");
		fprintf(f,"%5s","rank");

		fprintf(f,"\n");
		fflush(f);
	}
	PMPI_Barrier(MPI_COMM_WORLD);

	for (i=0; i<nranks; i++) {
	    if (rank == i) {
		if (send_cnt != 0) {
		  fprintf(f,"%5d ",i);
		  for (j=0; j<ncols; j++)
			fprintf(f,"%4ld/%.1e ",
				send_cnts[j],send_times[j]);
		  fprintf(f,"\n");
		  fflush(f);
	         }
	    }
	    PMPI_Barrier(MPI_COMM_WORLD);
	}

        /******************
         * ssend
         */
        /* rank 0 prints the title and column headings */
        PMPI_Barrier(MPI_COMM_WORLD);
        if (rank == 0) {
                /* section title */
                fprintf(f,"\nssend profile\n\n");

                /* column headings */
                fprintf(f,"             cnt/sec for all destination ranks\n");

                fprintf(f,"%5s","src");
                for (i=0; i<ncols; i++)
                        fprintf(f,"%9d    ",i);
                fprintf(f,"\n");
                fprintf(f,"%5s","rank");

                fprintf(f,"\n");
                fflush(f);
        }
        PMPI_Barrier(MPI_COMM_WORLD);

        for (i=0; i<nranks; i++) {
            if (rank == i) {
                if (ssend_cnt != 0) {
                  fprintf(f,"%5d ",i);
                  for (j=0; j<ncols; j++)
                        fprintf(f,"%4ld/%.1e ",
                                ssend_cnts[j],ssend_times[j]);
                  fprintf(f,"\n");
                  fflush(f);
                 }
            }
            PMPI_Barrier(MPI_COMM_WORLD);
        }



	/******************
	 * ibsend
	 */
	/* rank 0 prints the title and column headings */
	PMPI_Barrier(MPI_COMM_WORLD);
	if (rank == 0) {
		/* section title */
		fprintf(f,"\nibsend profile\n\n");
		
		/* column headings */
		fprintf(f,"             cnt/sec for all destination ranks\n");

		fprintf(f,"%5s","src");
		for (i=0; i<ncols; i++)
			fprintf(f,"%9d    ",i);
		fprintf(f,"\n");
		fprintf(f,"%5s","rank");

		fprintf(f,"\n");
		fflush(f);
	}
	PMPI_Barrier(MPI_COMM_WORLD);

	for (i=0; i<nranks; i++) {
	    if (rank == i) {
		if (ibsend_cnt != 0) {
		  fprintf(f,"%5d ",i);
		  for (j=0; j<ncols; j++)
			fprintf(f,"%4ld/%.1e ",
				ibsend_cnts[j],ibsend_times[j]);
		  fprintf(f,"\n");
		  fflush(f);
	         }
	    }
	    PMPI_Barrier(MPI_COMM_WORLD);
	}
	
/*
static long ibsend_cnt, ibsend_cnts[MAXRANKS];
static double ibsend_time, ibsend_times[MAXRANKS];
static long probe_cnt, probe_cnts_as, probe_cnts[MAXRANKS];
static double probe_time, probe_times_as, probe_times[MAXRANKS];
static long recv_cnt, recv_cnts_as, recv_cnts[MAXRANKS];
static double recv_time, recv_times_as, recv_times[MAXRANKS];
static long send_cnt, send_cnts[MAXRANKS];
static double send_time, send_times[MAXRANKS];
static long wait_cnt, wait_cnts[MAXRANKS];
static double wait_time, wait_times[MAXRANKS];
*/

	rv = PMPI_Finalize();
	return rv;
}

void
mpi_finalize_(int *ier)
{
	MPI_Finalize();
	*ier = 0;
}