Commit a955563e authored by gsaxena's avatar gsaxena

Parallelized the ./examples/performance_test_substrates.cpp example.

parent 155f54cf
......@@ -116,6 +116,8 @@ main_experiment: ./examples/main_experiment.cpp $(BioFVM_OBJECTS) $(pugixml_OBJE
$(COMPILE_COMMAND) -o ./examples/3DtumorX ./examples/3D_tumor_example_X.cpp $(BioFVM_OBJECTS) $(pugixml_OBJECTS)
perfVoxels: ./examples/performance_test_voxels.cpp $(BioFVM_OBJECTS) $(pugixml_OBJECTS)
$(COMPILE_COMMAND) -o ./examples/perfVoxels ./examples/performance_test_voxels.cpp $(BioFVM_OBJECTS) $(pugixml_OBJECTS)
perfSubstrates: ./examples/performance_test_substrates.cpp $(BioFVM_OBJECTS) $(pugixml_OBJECTS)
$(COMPILE_COMMAND) -o ./examples/perfSubstrates ./examples/performance_test_substrates.cpp $(BioFVM_OBJECTS) $(pugixml_OBJECTS)
clean:
rm -f *.o
......
......@@ -57,10 +57,13 @@
#include "../BioFVM.h"
/*----------------Include MPI Header-------------*/
#include <mpi.h>
/*----------------------------------------------*/
using namespace BioFVM;
int omp_num_threads = 8; // set number of threads for parallel computing
// set this to # of CPU cores x 2 (for hyperthreading)
//int omp_num_threads = 8; // set number of threads for parallel computing, set this to # of CPU cores x 2 (for hyperthreading)
//setup Mersenne Twister random generator
std::random_device rd;
......@@ -71,16 +74,15 @@ int num_substrates;
void supply_function( Microenvironment* microenvironment, int voxel_index , std::vector<double>* write_here )
{
double max_xyz=500;
double min_xyz=-500;
double strip_width=40;
double max_xyz=500; //Related to maximum values of the domain
double min_xyz=-500; //Related to minimum values of the domain
double strip_width=40; //Defines an inner border around all 6 boundaries
(*write_here)[0] = 0;
if( abs(max_xyz-microenvironment->voxels(voxel_index).center[0]) < strip_width || abs(microenvironment->voxels(voxel_index).center[0]- min_xyz)< strip_width
|| abs(max_xyz-microenvironment->voxels(voxel_index).center[1]) < strip_width || abs(microenvironment->voxels(voxel_index).center[1]- min_xyz)< strip_width
|| abs(max_xyz-microenvironment->voxels(voxel_index).center[2]) < strip_width || abs(microenvironment->voxels(voxel_index).center[2]- min_xyz)< strip_width )
{
// std::cout<<"test"<<std::endl;
for(int i=0;i< num_substrates;i++)
(*write_here)[i] = substrates_sources[i];
}
......@@ -129,6 +131,55 @@ void process_output(double t, double dt, double mesh_resolution)
int main( int argc, char* argv[] )
{
unsigned int mpi_Error;
int mpi_Requested_level = MPI_THREAD_FUNNELED;
int mpi_Provided_level;
MPI_Comm mpi_Comm = MPI_COMM_WORLD;
MPI_Comm mpi_Cart_comm;
int mpi_Size, mpi_Rank;
int mpi_Dims[3], mpi_Is_periodic[3], mpi_Coords[3];
int mpi_Reorder;
mpi_Error = MPI_Init_thread(NULL, NULL, mpi_Requested_level, &mpi_Provided_level);
MPI_Barrier(mpi_Comm);
if(mpi_Error != MPI_SUCCESS)
MPI_Abort(mpi_Comm,-1);
MPI_Comm_size(mpi_Comm, &mpi_Size);
MPI_Comm_rank(mpi_Comm, &mpi_Rank);
if(mpi_Provided_level != mpi_Requested_level && mpi_Rank == 0)
{
std::cout<<"The MPI Requested Level is not available, please lower the level and try again"<<std::endl;
MPI_Abort(mpi_Comm, -1);
}
/*------------------------------------------------------------*/
/* MPI Comm Size and Rank */
/* Right now it should be 1-D X decomposition */
/*------------------------------------------------------------*/
mpi_Dims[0] = 1;
mpi_Dims[1] = mpi_Size; //Number of Y processes, since Y-processes divide X-axis, we have X-decomposition
mpi_Dims[2] = 1;
mpi_Is_periodic[0]=0;
mpi_Is_periodic[1]=0;
mpi_Is_periodic[2]=0;
mpi_Reorder = 0;
/*------------------------------------------------------------*/
/* Create MPI topology, find coords */
/*------------------------------------------------------------*/
MPI_Cart_create(mpi_Comm, 3, mpi_Dims, mpi_Is_periodic, mpi_Reorder, &mpi_Cart_comm);
MPI_Cart_coords(mpi_Cart_comm, mpi_Rank, 3, mpi_Coords);
double t = 0.0;
double t_output_interval = 1.0;
double t_next_output_time = 0;
......@@ -141,8 +192,9 @@ int main( int argc, char* argv[] )
num_substrates=atoi(argv[1]);
substrates_sources.resize(num_substrates);
// openmp setup
omp_set_num_threads(omp_num_threads);
// openmp setup - This is being controlled from within the SLURM submission script
//omp_set_num_threads(omp_num_threads);
// set random seed
srand(2);
......@@ -164,8 +216,9 @@ int main( int argc, char* argv[] )
microenvironment.decay_rates[i] = get_rand(0.05, 0.15);
substrates_sources.push_back(get_rand(30.0,60.0));
}
double minX=-500, minY=-500, minZ=-500, maxX=500, maxY=500, maxZ=500;//, mesh_resolution=10;
microenvironment.resize_space_uniform( minX,maxX,minY,maxY,minZ,maxZ, mesh_resolution );
microenvironment.resize_space_uniform( minX,maxX,minY,maxY,minZ,maxZ, mesh_resolution, mpi_Dims, mpi_Coords );
// register the diffusion solver
microenvironment.diffusion_decay_solver = diffusion_decay_solver__constant_coefficients_LOD_3D;
......@@ -180,31 +233,41 @@ int main( int argc, char* argv[] )
}
// display information
microenvironment.display_information( std::cout );
if(mpi_Rank == 0)
microenvironment.display_information( std::cout );
//Need to call barrier as timing calls are beginning
MPI_Barrier(MPI_COMM_WORLD);
BioFVM::RUNTIME_TIC();
BioFVM::TIC();
int output_index=0;
std::cout<<"number of agents: "<< all_basic_agents.size() <<std::endl;
while( t < t_max )
{
if( fabs( t - t_next_output_time ) < dt/10.0 )
{
process_output(t, dt, mesh_resolution);
if(mpi_Rank == 0)
process_output(t, dt, mesh_resolution);
t_next_output_time += t_output_interval;
next_output_index++;
}
// simulate microenvironment
microenvironment.simulate_bulk_sources_and_sinks( dt );
microenvironment.simulate_diffusion_decay( dt );
microenvironment.simulate_diffusion_decay( dt, mpi_Size, mpi_Rank, mpi_Coords, mpi_Dims, mpi_Cart_comm );
t += dt;
output_index++;
}
process_output(t_max, dt, mesh_resolution);
if(mpi_Rank == 0)
process_output(t_max, dt, mesh_resolution);
BioFVM::RUNTIME_TOC();
write_report(BioFVM::runtime_stopwatch_value());
std::cout << "done!" << std::endl;
MPI_Barrier(MPI_COMM_WORLD);
if(mpi_Rank == 0)
write_report(BioFVM::runtime_stopwatch_value());
MPI_Finalize();
return 0;
}
\ No newline at end of file
#!/bin/bash
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=2
#SBATCH --cpus-per-task=24
#SBATCH -t 00:60:00
#SBATCH -o output-%j
#SBATCH -e error-%j
#SBATCH --exclusive
export OMP_DISPLAY_ENV=true
export OMP_NUM_THREADS=24
export OMP_PROC_BIND=spread
export OMP_PLACES=threads
mpiexec --map-by ppr:1:socket:pe=24 --report-bindings ./examples/perfSubstrates 100
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment