Skip to content

Commit

Permalink
Added more output file options, new routine to distribute jobs.
Browse files Browse the repository at this point in the history
  • Loading branch information
tiagopereira committed Jan 3, 2011
1 parent b84dfbd commit f4f7521
Show file tree
Hide file tree
Showing 12 changed files with 1,210 additions and 191 deletions.
23 changes: 17 additions & 6 deletions rh15d_mpi/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ ONE_D_OBJS = anglequad.o feautrier.o formal.o hydrostat.o multiatmos.o \
rh15d.o piecestokes.o piecewise.o project.o riiplane.o \
writeflux_xdr.o writegeom_xdr.o readatmos_ncdf.o \
background_p.o brs_p.o readj_p.o scatter_p.o writespect_p.o \
initial_p.o
initial_p.o distribute_jobs.o writeinputdata_p.o

ONEDOBJ = solve15d.o iter_1d.o feautrier.o

Expand Down Expand Up @@ -79,13 +79,18 @@ backgrcontr.o: ../rh.h ../atom.h ../atmos.h \
background_p.o: ../rh.h ../atom.h ../atmos.h ../spectrum.h \
../constant.h ../background.h ../error.h \
../statistics.h ../inputs.h ../xdr.h parallel.h \
background_p.c
io.h background_p.c
$(CC) $(CFLAGS) -Wall -c -o $@ background_p.c

brs_p.o: ../rh.h ../atom.h ../atmos.h ../spectrum.h \
../error.h ../xdr.h parallel.h brs_p.c
../error.h ../xdr.h parallel.h io.h brs_p.c
$(CC) $(CFLAGS) -Wall -c -o $@ brs_p.c

distribute_jobs.o: ../rh.h ../atom.h ../atmos.h geometry.h \
../inputs.h parallel.h io.h distribute_jobs.c
$(CC) $(CFLAGS) -Wall -c -o $@ distribute_jobs.c


feautrier.o: ../rh.h ../atom.h ../atmos.h \
geometry.h ../spectrum.h ../error.h

Expand Down Expand Up @@ -120,11 +125,12 @@ piecewise.o: ../rh.h ../atom.h ../atmos.h \
project.o: ../rh.h ../atom.h ../atmos.h \
geometry.h

readatmos_ncdf.o: ../rh.h ../atmos.h ../inputs.h geometry.h parallel.h readatmos_ncdf.c
readatmos_ncdf.o: ../rh.h ../atmos.h ../inputs.h geometry.h parallel.h \
io.h readatmos_ncdf.c
$(CC) $(CFLAGS) -Wall -c -o $@ readatmos_ncdf.c

readj_p.o: ../rh.h ../atom.h ../atmos.h ../spectrum.h parallel.h \
../background.h ../inputs.h ../error.h readj_p.c
../background.h ../inputs.h ../error.h io.h readj_p.c
$(CC) $(CFLAGS) -Wall -c -o $@ readj_p.c

rh15d.o: ../rh.h ../atom.h ../atmos.h \
Expand All @@ -146,6 +152,11 @@ solveray.o: ../rh.h ../atom.h ../atmos.h \
geometry.h ../spectrum.h ../background.h \
../statistics.h ../inputs.h ../error.h ../xdr.h

writeinputdata_p.o: ../rh.h ../atom.h ../atmos.h \
geometry.h ../error.h ../inputs.h \
parallel.h io.h writeinputdata_p.c
$(CC) $(CFLAGS) -Wall -c -o $@ writeinputdata_p.c

writeflux_xdr.o: ../rh.h ../atom.h ../atmos.h \
geometry.h ../spectrum.h ../constant.h \
../error.h ../xdr.h
Expand All @@ -155,7 +166,7 @@ writegeom_xdr.o: ../rh.h ../atom.h ../atmos.h \

writespect_p.o: ../rh.h ../atom.h ../atmos.h \
../spectrum.h ../constant.h ../error.h \
../inputs.h ../xdr.h parallel.h writespect_p.c
../inputs.h ../xdr.h parallel.h io.h writespect_p.c
$(CC) $(CFLAGS) -Wall -c -o $@ writespect_p.c


Expand Down
4 changes: 1 addition & 3 deletions rh15d_mpi/background_p.c
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,6 @@
#include <math.h>
#include <stdlib.h>
#include <string.h>
#include <netcdf.h>

#include "rh.h"
#include "atom.h"
Expand All @@ -122,10 +121,9 @@
#include "xdr.h"
#include "geometry.h"
#include "parallel.h"
#include "io.h"

#define COMMENT_CHAR "#"
#define FILE_EXT ".dat"


/* --- Function prototypes -- -------------- */

Expand Down
32 changes: 12 additions & 20 deletions rh15d_mpi/brs_p.c
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <netcdf.h>

#include "rh.h"
#include "atom.h"
Expand All @@ -24,20 +23,14 @@
#include "xdr.h"
#include "geometry.h"
#include "parallel.h"

#define BRS_DOT_OUT "brs.out"
#define BRS_FILE "scratch/brs_out_p"
#define BRS_EXT ".ncdf"
#define HASLINE_VAR "hasline"
#define ISPOL_VAR "ispolarised"
#define BGREC_VAR "backgrrecno"
#include "io.h"

/* --- Global variables -- -------------- */

extern Atmosphere atmos;
extern Spectrum spectrum;
extern char messageStr[];
extern BackgroundData bgdat;
extern IO_data io;
extern MPI_data mpi;

/* ------- begin -------------------------- init_ncdf_BRS.c --------- */
Expand Down Expand Up @@ -106,12 +99,11 @@ void init_ncdf_BRS(void)
/* End define mode */
if ((ierror = nc_enddef(ncid))) ERR(ierror,routineName);

/* Copy stuff to Background data struct */
bgdat.brs_ncid = ncid;
bgdat.brs_hl_var = hl_var;
bgdat.brs_ip_var = ip_var;
bgdat.brs_nrec_var = nrec_var;
bgdat.brs_fname = file_brs;
/* Copy stuff to the IO data struct */
io.brs_ncid = ncid;
io.brs_hl_var = hl_var;
io.brs_ip_var = ip_var;
io.brs_nrec_var = nrec_var;

return;
}
Expand All @@ -125,7 +117,7 @@ void close_ncdf_BRS(void)
const char routineName[] = "close_ncdf_BRS";
int ierror;

if ((ierror = nc_close(bgdat.brs_ncid))) ERR(ierror,routineName);
if ((ierror = nc_close(io.brs_ncid))) ERR(ierror,routineName);
return;
}
/* ------- end ---------------------------- close_ncdf_BRS.c --------- */
Expand All @@ -142,7 +134,7 @@ void writeBRS_ncdf(void)
size_t start[] = {0, 0};
size_t count[] = {1, 1};

ncid = bgdat.brs_ncid;
ncid = io.brs_ncid;

/* For unlimited dimension */
start[0] = (size_t) mpi.task;
Expand All @@ -166,12 +158,12 @@ void writeBRS_ncdf(void)

/* write data to file */
count[1] = (size_t) spectrum.Nspect;
if ((ierror = nc_put_vara_ubyte(bgdat.brs_ncid, bgdat.brs_hl_var, start, count,
if ((ierror = nc_put_vara_ubyte(io.brs_ncid, io.brs_hl_var, start, count,
hasline ))) ERR(ierror,routineName);
if ((ierror = nc_put_vara_ubyte(bgdat.brs_ncid, bgdat.brs_ip_var, start, count,
if ((ierror = nc_put_vara_ubyte(io.brs_ncid, io.brs_ip_var, start, count,
ispolarised ))) ERR(ierror,routineName);
count[1] = (size_t) Nrecno;
if ((ierror = nc_put_vara_long( bgdat.brs_ncid, bgdat.brs_nrec_var, start, count,
if ((ierror = nc_put_vara_long( io.brs_ncid, io.brs_nrec_var, start, count,
atmos.backgrrecno ))) ERR(ierror,routineName);


Expand Down
187 changes: 187 additions & 0 deletions rh15d_mpi/distribute_jobs.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,187 @@
/* ------- file: -------------------------- distribute_jobs.c ------------
Version: rh2.0, 1.5-D plane-parallel
Author: Tiago Pereira ([email protected])
Last modified: Wed Dec 29 20:58:00 2010 --
-------------------------- ----------RH-- */

/* --- Divides all the 1D columns between the different MPI processes --- */

#include <ctype.h>
#include <stdlib.h>
#include <math.h>
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>


#include "rh.h"
#include "atom.h"
#include "atmos.h"
#include "geometry.h"
#include "inputs.h"
#include "parallel.h"
#include "io.h"


/* --- Function prototypes -- -------------- */

int *intrange(int start, int end, int step, int *N);
int *get_tasks(long ntotal, int size);
int **get_taskmap(int *ntasks, int *my_start);
void finish_jobs(void);

/* --- Global variables -- -------------- */

extern MPI_data mpi;
extern InputData input;
extern char messageStr[];
extern NCDF_Atmos_file infile;


/* ------- begin -------------------------- distribute_jobs.c --- */
void distribute_jobs(void)
{
/* The end product of this routine must be:
For each process:
* mpi.Ntasks --DONE
* An array of rank (mpi.Ntasks,2) that gives (mpi.ix, mpi.iy)
for each task index --DONE (mpi.my_tasks)
* A map of (mpi.ix, mpi.iy) -> (xi, yi) --DONE (two 1D arrays: mpi.xnum, mpi.ynum)
*/

int i,*tasks, my_start;


mpi.backgrrecno = 0;

/* Sanitise input */
if ((input.p15d_x0 < 0)) input.p15d_x0 = 0;
if ((input.p15d_y0 < 0)) input.p15d_y0 = 0;

if ((input.p15d_x0 > input.p15d_x1)) input.p15d_x0 = input.p15d_x1;
if ((input.p15d_y0 > input.p15d_y1)) input.p15d_y0 = input.p15d_y1;

if ((input.p15d_x1 <= 0) || (input.p15d_x1 > infile.nx))
input.p15d_x1 = infile.nx;
if ((input.p15d_y1 <= 0) || (input.p15d_y1 > infile.ny))
input.p15d_y1 = infile.ny;

if ((input.p15d_xst < 1)) input.p15d_xst = 1;
if ((input.p15d_yst < 1)) input.p15d_yst = 1;

/* Calculate array maps of (mpi.ix/iy) > xi/yi */
mpi.xnum = intrange(input.p15d_x0, input.p15d_x1, input.p15d_xst, &mpi.nx);
mpi.ynum = intrange(input.p15d_y0, input.p15d_y1, input.p15d_yst, &mpi.ny);

/* Calculate tasks and distribute */
tasks = get_tasks(mpi.nx*mpi.ny, mpi.size);
mpi.Ntasks = tasks[mpi.rank];
mpi.taskmap = get_taskmap(tasks, &my_start);

//printf("MMM = %d\n",mpi.my_tasks[10][1]);

/*
printf("Process %d: %d tasks, mystart = %d\n",mpi.rank,mpi.Ntasks, my_start);
for (i=0; i < mpi.Ntasks; i++){
printf("Process %d: task %d : (x,y) = (%d, %d)\n",mpi.rank,i, mpi.taskmap[i+my_start][0],
mpi.taskmap[i+my_start][1]);
printf("Process %d: task %d : (x,y) = (%d, %d)\n",mpi.rank,i, mpi.my_tasks[i][0],
mpi.my_tasks[i][1]);
}
*/

free(tasks);

return;
}
/* ------- end -------------------------- distribute_jobs.c --- */

/* ------- begin -------------------------- get_taskmap.c ----- --- */
int **get_taskmap(int *ntasks, int *my_start)
{
int i, j, k, *start, **taskmap;

taskmap = matrix_int(mpi.nx*mpi.ny, 2);
start = (int *) malloc(mpi.size * sizeof(int));

/* Create map of tasks */
k = 0;
for (i=0; i < mpi.nx; i++) {
for (j=0; j < mpi.ny; j++) {
taskmap[k][0] = j;
taskmap[k][1] = i;
++k;
}
}

/* distribute tasks */
k = 0;
for (i=0; i < mpi.size; i++) {
start[i] = k;
k += ntasks[i];
}

/* pointer to each process's starting task */
*my_start = start[mpi.rank];

free(start);

return taskmap;
}
/* ------- end -------------------------- get_taskmap.c ----- --- */

/* ------- begin -------------------------- get_tasks.c ------- --- */
int *get_tasks(long ntotal, int size)
/* Divides the ntotal tasks by 'size' processes */
{
int i, *tasks;

tasks = (int *) malloc(size * sizeof(int));

for (i=0; i < size; i++) tasks[i] = ntotal/size;

/* Distribute remaining */
if ((ntotal % size) > 0) {
for (i=0; i < ntotal % size; i++) ++tasks[i];
}

return tasks;
}
/* ------- end -------------------------- get_tasks.c ------- --- */

/* ------- begin -------------------------- intrange.c -------- --- */
int *intrange(int start, int end, int step, int *N)
/* Mimics Python's range function. Also gives a pointer
to the number of elements. */
{
int i, *arange;

*N = (end - start) / step;
if ((end - start) % step > 0) ++*N;

arange = (int *) malloc(*N * sizeof(int));

arange[0] = start;
for (i=1; i < *N; i++) {
arange[i] = arange[i-1] + step;
}

return arange;
}
/* ------- end -------------------------- intrange.c -------- --- */

/* ------- begin -------------------------- finish_jobs.c ------- --- */
void finish_jobs(void)
/* Frees from memory stuff used for job control */
{

free(mpi.xnum);
free(mpi.ynum);
free(mpi.taskmap);

}
/* ------- end -------------------------- finish_jobs.c ------- --- */
5 changes: 3 additions & 2 deletions rh15d_mpi/geometry.h
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ typedef struct {
int ncid, nx_id, ny_id, nz_id, nhyd_id;
int T_varid, ne_varid, vz_varid, nh_varid;
size_t nx, ny, nz, NHydr;
double *x, *y;
} NCDF_Atmos_file;


Expand All @@ -55,10 +56,10 @@ void getBoundary(Geometry *geometry);
void MULTIatmos(Atmosphere *atmos, Geometry *geometry);
void writeGeometry(Geometry *geometry);

void init_ncdf(Atmosphere *atmos, Geometry *geometry, NCDF_Atmos_file *infile);
void init_ncdf_atmos(Atmosphere *atmos, Geometry *geometry, NCDF_Atmos_file *infile);
void readAtmos_ncdf(int xi, int yi, Atmosphere *atmos, Geometry *geometry,
NCDF_Atmos_file *infile);
void close_atmos_ncdf(Atmosphere *atmos, Geometry *geometry, NCDF_Atmos_file *infile);
void close_ncdf_atmos(Atmosphere *atmos, Geometry *geometry, NCDF_Atmos_file *infile);


/* --- Formal solution related -- -------------- */
Expand Down
Loading

0 comments on commit f4f7521

Please sign in to comment.