Commit c8a5bf8b authored by Gaurav Saxena's avatar Gaurav Saxena
Browse files

BioFVM with X decomposition - obtaining much better timings than the Z...

BioFVM with X decomposition - obtaining much better timings than the Z decomposition. Also remember to use the ppr notation in the script file. Best results being obtained from 2 processes per node i.e. one per NUMA region and then nodes can be increased.
parent 975859a6
...@@ -778,7 +778,7 @@ void Microenvironment::simulate_bulk_sources_and_sinks( double dt ) ...@@ -778,7 +778,7 @@ void Microenvironment::simulate_bulk_sources_and_sinks( double dt )
void Microenvironment::simulate_cell_sources_and_sinks( std::vector<Basic_Agent*>& basic_agent_list , double dt ) void Microenvironment::simulate_cell_sources_and_sinks( std::vector<Basic_Agent*>& basic_agent_list , double dt )
{ {
//#pragma omp parallel for #pragma omp parallel for
for( int i=0 ; i < basic_agent_list.size() ; i++ ) for( int i=0 ; i < basic_agent_list.size() ; i++ )
{ {
basic_agent_list[i]->simulate_secretion_and_uptake( this , dt ); basic_agent_list[i]->simulate_secretion_and_uptake( this , dt );
......
This diff is collapsed.
...@@ -2,8 +2,15 @@ VERSION := 1.1.6 ...@@ -2,8 +2,15 @@ VERSION := 1.1.6
PROGRAM_NAME := make_lib PROGRAM_NAME := make_lib
#For GCC
CC := mpic++ CC := mpic++
#For Intel
#CC := mpiicpc
#For TAU
#CC := tau_cxx.sh
# change this to your own CPU archicture. # change this to your own CPU archicture.
# Here is a list for gcc 4.9.0 # Here is a list for gcc 4.9.0
# https://gcc.gnu.org/onlinedocs/gcc-4.9.0/gcc/i386-and-x86-64-Options.html#i386-and-x86-64-Options # https://gcc.gnu.org/onlinedocs/gcc-4.9.0/gcc/i386-and-x86-64-Options.html#i386-and-x86-64-Options
...@@ -24,7 +31,10 @@ ARCH := native # best auto-tuning ...@@ -24,7 +31,10 @@ ARCH := native # best auto-tuning
# ARCH := silvermont # ARCH := silvermont
# ARCH := nocona #64-bit pentium 4 or later # ARCH := nocona #64-bit pentium 4 or later
CFLAGS := -march=$(ARCH) -O3 -fomit-frame-pointer -mfpmath=both -fopenmp -m64 -std=c++11 -g #First for GCC and second for ICPC
CFLAGS := -march=$(ARCH) -O2 -fomit-frame-pointer -mfpmath=both -fopenmp -m64 -std=c++11 -g
#CFLAGS := -march=$(ARCH) -O2 -fomit-frame-pointer -qopenmp -m64 -std=c++11 -g
BioFVM_OBJECTS := BioFVM_vector.o BioFVM_matlab.o BioFVM_utilities.o BioFVM_mesh.o \ BioFVM_OBJECTS := BioFVM_vector.o BioFVM_matlab.o BioFVM_utilities.o BioFVM_mesh.o \
BioFVM_microenvironment.o BioFVM_solvers.o BioFVM_basic_agent.o \ BioFVM_microenvironment.o BioFVM_solvers.o BioFVM_basic_agent.o \
......
[s06r1b59:192760] MCW rank 0 bound to socket 0[core 0[hwt 0]], socket 0[core 1[hwt 0]], socket 0[core 2[hwt 0]], socket 0[core 3[hwt 0]], socket 0[core 4[hwt 0]], socket 0[core 5[hwt 0]], socket 0[core 6[hwt 0]], socket 0[core 7[hwt 0]], socket 0[core 8[hwt 0]], socket 0[core 9[hwt 0]], socket 0[core 10[hwt 0]], socket 0[core 11[hwt 0]], socket 0[core 12[hwt 0]], socket 0[core 13[hwt 0]], socket 0[core 14[hwt 0]], socket 0[core 15[hwt 0]], socket 0[core 16[hwt 0]], socket 0[core 17[hwt 0]], socket 0[core 18[hwt 0]], socket 0[core 19[hwt 0]], socket 0[core 20[hwt 0]], socket 0[core 21[hwt 0]], socket 0[core 22[hwt 0]], socket 0[core 23[hwt 0]]: [B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B][./././././././././././././././././././././././.]
[s06r1b59:192760] MCW rank 1 bound to socket 1[core 24[hwt 0]], socket 1[core 25[hwt 0]], socket 1[core 26[hwt 0]], socket 1[core 27[hwt 0]], socket 1[core 28[hwt 0]], socket 1[core 29[hwt 0]], socket 1[core 30[hwt 0]], socket 1[core 31[hwt 0]], socket 1[core 32[hwt 0]], socket 1[core 33[hwt 0]], socket 1[core 34[hwt 0]], socket 1[core 35[hwt 0]], socket 1[core 36[hwt 0]], socket 1[core 37[hwt 0]], socket 1[core 38[hwt 0]], socket 1[core 39[hwt 0]], socket 1[core 40[hwt 0]], socket 1[core 41[hwt 0]], socket 1[core 42[hwt 0]], socket 1[core 43[hwt 0]], socket 1[core 44[hwt 0]], socket 1[core 45[hwt 0]], socket 1[core 46[hwt 0]], socket 1[core 47[hwt 0]]: [./././././././././././././././././././././././.][B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B]
[s06r1b61:272153] MCW rank 2 bound to socket 0[core 0[hwt 0]], socket 0[core 1[hwt 0]], socket 0[core 2[hwt 0]], socket 0[core 3[hwt 0]], socket 0[core 4[hwt 0]], socket 0[core 5[hwt 0]], socket 0[core 6[hwt 0]], socket 0[core 7[hwt 0]], socket 0[core 8[hwt 0]], socket 0[core 9[hwt 0]], socket 0[core 10[hwt 0]], socket 0[core 11[hwt 0]], socket 0[core 12[hwt 0]], socket 0[core 13[hwt 0]], socket 0[core 14[hwt 0]], socket 0[core 15[hwt 0]], socket 0[core 16[hwt 0]], socket 0[core 17[hwt 0]], socket 0[core 18[hwt 0]], socket 0[core 19[hwt 0]], socket 0[core 20[hwt 0]], socket 0[core 21[hwt 0]], socket 0[core 22[hwt 0]], socket 0[core 23[hwt 0]]: [B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B][./././././././././././././././././././././././.]
[s06r1b65:389755] MCW rank 6 bound to socket 0[core 0[hwt 0]], socket 0[core 1[hwt 0]], socket 0[core 2[hwt 0]], socket 0[core 3[hwt 0]], socket 0[core 4[hwt 0]], socket 0[core 5[hwt 0]], socket 0[core 6[hwt 0]], socket 0[core 7[hwt 0]], socket 0[core 8[hwt 0]], socket 0[core 9[hwt 0]], socket 0[core 10[hwt 0]], socket 0[core 11[hwt 0]], socket 0[core 12[hwt 0]], socket 0[core 13[hwt 0]], socket 0[core 14[hwt 0]], socket 0[core 15[hwt 0]], socket 0[core 16[hwt 0]], socket 0[core 17[hwt 0]], socket 0[core 18[hwt 0]], socket 0[core 19[hwt 0]], socket 0[core 20[hwt 0]], socket 0[core 21[hwt 0]], socket 0[core 22[hwt 0]], socket 0[core 23[hwt 0]]: [B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B][./././././././././././././././././././././././.]
[s06r1b61:272153] MCW rank 3 bound to socket 1[core 24[hwt 0]], socket 1[core 25[hwt 0]], socket 1[core 26[hwt 0]], socket 1[core 27[hwt 0]], socket 1[core 28[hwt 0]], socket 1[core 29[hwt 0]], socket 1[core 30[hwt 0]], socket 1[core 31[hwt 0]], socket 1[core 32[hwt 0]], socket 1[core 33[hwt 0]], socket 1[core 34[hwt 0]], socket 1[core 35[hwt 0]], socket 1[core 36[hwt 0]], socket 1[core 37[hwt 0]], socket 1[core 38[hwt 0]], socket 1[core 39[hwt 0]], socket 1[core 40[hwt 0]], socket 1[core 41[hwt 0]], socket 1[core 42[hwt 0]], socket 1[core 43[hwt 0]], socket 1[core 44[hwt 0]], socket 1[core 45[hwt 0]], socket 1[core 46[hwt 0]], socket 1[core 47[hwt 0]]: [./././././././././././././././././././././././.][B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B]
[s06r1b65:389755] MCW rank 7 bound to socket 1[core 24[hwt 0]], socket 1[core 25[hwt 0]], socket 1[core 26[hwt 0]], socket 1[core 27[hwt 0]], socket 1[core 28[hwt 0]], socket 1[core 29[hwt 0]], socket 1[core 30[hwt 0]], socket 1[core 31[hwt 0]], socket 1[core 32[hwt 0]], socket 1[core 33[hwt 0]], socket 1[core 34[hwt 0]], socket 1[core 35[hwt 0]], socket 1[core 36[hwt 0]], socket 1[core 37[hwt 0]], socket 1[core 38[hwt 0]], socket 1[core 39[hwt 0]], socket 1[core 40[hwt 0]], socket 1[core 41[hwt 0]], socket 1[core 42[hwt 0]], socket 1[core 43[hwt 0]], socket 1[core 44[hwt 0]], socket 1[core 45[hwt 0]], socket 1[core 46[hwt 0]], socket 1[core 47[hwt 0]]: [./././././././././././././././././././././././.][B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B]
[s06r1b64:351888] MCW rank 4 bound to socket 0[core 0[hwt 0]], socket 0[core 1[hwt 0]], socket 0[core 2[hwt 0]], socket 0[core 3[hwt 0]], socket 0[core 4[hwt 0]], socket 0[core 5[hwt 0]], socket 0[core 6[hwt 0]], socket 0[core 7[hwt 0]], socket 0[core 8[hwt 0]], socket 0[core 9[hwt 0]], socket 0[core 10[hwt 0]], socket 0[core 11[hwt 0]], socket 0[core 12[hwt 0]], socket 0[core 13[hwt 0]], socket 0[core 14[hwt 0]], socket 0[core 15[hwt 0]], socket 0[core 16[hwt 0]], socket 0[core 17[hwt 0]], socket 0[core 18[hwt 0]], socket 0[core 19[hwt 0]], socket 0[core 20[hwt 0]], socket 0[core 21[hwt 0]], socket 0[core 22[hwt 0]], socket 0[core 23[hwt 0]]: [B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B][./././././././././././././././././././././././.]
[s06r1b64:351888] MCW rank 5 bound to socket 1[core 24[hwt 0]], socket 1[core 25[hwt 0]], socket 1[core 26[hwt 0]], socket 1[core 27[hwt 0]], socket 1[core 28[hwt 0]], socket 1[core 29[hwt 0]], socket 1[core 30[hwt 0]], socket 1[core 31[hwt 0]], socket 1[core 32[hwt 0]], socket 1[core 33[hwt 0]], socket 1[core 34[hwt 0]], socket 1[core 35[hwt 0]], socket 1[core 36[hwt 0]], socket 1[core 37[hwt 0]], socket 1[core 38[hwt 0]], socket 1[core 39[hwt 0]], socket 1[core 40[hwt 0]], socket 1[core 41[hwt 0]], socket 1[core 42[hwt 0]], socket 1[core 43[hwt 0]], socket 1[core 44[hwt 0]], socket 1[core 45[hwt 0]], socket 1[core 46[hwt 0]], socket 1[core 47[hwt 0]]: [./././././././././././././././././././././././.][B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B]
OPENMP DISPLAY ENVIRONMENT BEGIN
_OPENMP = '201511'
OPENMP DISPLAY ENVIRONMENT BEGIN
_OPENMP = '201511'
OMP_DYNAMIC = 'FALSE'
OMP_NESTED = 'FALSE'
OMP_NUM_THREADS = '24'
OMP_SCHEDULE = 'DYNAMIC'
OMP_PROC_BIND = 'SPREAD'
OMP_PLACES = '{ OMP_DYNAMIC = 'FALSE'
OMP_NESTED = 'FALSE'
OMP_NUM_THREADS = '24'
OMP_SCHEDULE = 'DYNAMIC'
OMP_PROC_BIND = 'SPREAD'
OMP_PLACES = '{024},{25},{26},{27},{28},{29},{30},{},{1},{2},{3},{4},{5},{6},{31},{32},{33},{34},{35},{36},{37},{38},{7},{8},{9},{10},{11},{12},{13},39},{40},{41},{42},{43},{44},{45},{46},{47}{14},{15},{16},{17},{18},{19},{20},{21},{22},'
OMP_STACKSIZE = '0'
OMP_WAIT_POLICY = 'PASSIVE'
OMP_THREAD_LIMIT = '4294967295'
OMP_MAX_ACTIVE_LEVELS = '2147483647'
OMP_CANCELLATION = 'FALSE'
OMP_DEFAULT_DEVICE = '0'
OMP_MAX_TASK_PRIORITY = '0'
OPENMP DISPLAY ENVIRONMENT END
{23}'
OMP_STACKSIZE = '0'
OMP_WAIT_POLICY = 'PASSIVE'
OMP_THREAD_LIMIT = '4294967295'
OMP_MAX_ACTIVE_LEVELS = '2147483647'
OMP_CANCELLATION = 'FALSE'
OMP_DEFAULT_DEVICE = '0'
OMP_MAX_TASK_PRIORITY = '0'
OPENMP DISPLAY ENVIRONMENT END
OPENMP DISPLAY ENVIRONMENT BEGIN
_OPENMP = '201511'
OMP_DYNAMIC = 'FALSE'
OMP_NESTED = 'FALSE'
OMP_NUM_THREADS = '24'
OMP_SCHEDULE = 'DYNAMIC'
OMP_PROC_BIND = 'SPREAD'
OMP_PLACES = '{0},{1},{2},{3},{4},{5},{6},{
OPENMP DISPLAY ENVIRONMENT BEGIN
_OPENMP = '201511'
OMP_DYNAMIC = 'FALSE'
7},{8},{9},{10},{11},{
OPENMP DISPLAY ENVIRONMENT BEGIN
_OPENMP = '201511'
OMP_DYNAMIC = 'FALSE'
OMP_NESTED = 'FALSE'
OMP_NUM_THREADS = '24'
OMP_SCHEDULE = 'DYNAMIC'
OMP_PROC_BIND = 'SPREAD'
OMP_PLACES = '{12},{13},{14},{15},{16},{ OMP_NESTED = 'FALSE'
OMP_NUM_THREADS = '24'
OMP_SCHEDULE = 'DYNAMIC'
OMP_PROC_BIND = 'SPREAD'
OMP_PLACES = '{17},{18},{19},{20},{21},{0},{1},{222},{23}'
OMP_STACKSIZE = '0'
OMP_WAIT_POLICY = 'PASSIVE'
OMP_THREAD_LIMIT = '4294967295'
OMP_MAX_ACTIVE_LEVELS = '2147483647'
OMP_CANCELLATION = 'FALSE'
},{3},{4},{5},{6 OMP_DEFAULT_DEVICE = '0'
OMP_MAX_TASK_PRIORITY = '0'
OPENMP DISPLAY ENVIRONMENT END
24},{25},{26},{27},{28
OPENMP DISPLAY ENVIRONMENT BEGIN
_OPENMP = '201511'
OMP_DYNAMIC = 'FALSE'
OMP_NESTED = 'FALSE'
OMP_NUM_THREADS = '24'
OMP_SCHEDULE = 'DYNAMIC'
OMP_PROC_BIND = 'SPREAD'
OMP_PLACES = '{24},{25},{26},{27},{28},{29},{30},{31},{32},{33},{34},{35},{36},{37},{38},{39},{40},{41},{42},{43},{44},{45},{},{7},{8},{9},{10},{11},{12},{1346},{47}'
OMP_STACKSIZE = '0'
OMP_WAIT_POLICY = 'PASSIVE'
OMP_THREAD_LIMIT = '4294967295'
OMP_MAX_ACTIVE_LEVELS = '2147483647'
OMP_CANCELLATION = 'FALSE'
OMP_DEFAULT_DEVICE = '0'
OMP_MAX_TASK_PRIORITY = '0'
OPENMP DISPLAY ENVIRONMENT END
},{29},{30},{31},{32},{33},{34},{35},{36},},{14},{15},{16},{17},{18},{19},{20},{21},{22},{23}'
OMP_STACKSIZE = '0'
OMP_WAIT_POLICY = 'PASSIVE'
OMP_THREAD_LIMIT = '4294967295'
OMP_MAX_ACTIVE_LEVELS = '2147483647'
OMP_CANCELLATION = 'FALSE'
OMP_DEFAULT_DEVICE = '0'
{37},{38},{39},{40},{41},{42},{43},{44},{45},{46},{47}'
OMP_STACKSIZE = '0'
OMP_WAIT_POLICY = 'PASSIVE'
OMP_THREAD_LIMIT = '4294967295'
OMP_MAX_ACTIVE_LEVELS = '2147483647'
OMP_CANCELLATION = 'FALSE'
OMP_DEFAULT_DEVICE = '0'
OMP_MAX_TASK_PRIORITY = '0'
OPENMP DISPLAY ENVIRONMENT END
OMP_MAX_TASK_PRIORITY = '0'
OPENMP DISPLAY ENVIRONMENT END
OPENMP DISPLAY ENVIRONMENT BEGIN
_OPENMP = '201511'
OMP_DYNAMIC = 'FALSE'
OMP_NESTED = 'FALSE'
OMP_NUM_THREADS = '24'
OMP_SCHEDULE = 'DYNAMIC'
OMP_PROC_BIND = 'SPREAD'
OMP_PLACES = '{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11},{12},{13},{14},{15},{16},
OPENMP DISPLAY ENVIRONMENT BEGIN
_OPENMP = '201511'
OMP_DYNAMIC = 'FALSE'
OMP_NESTED = 'FALSE'
OMP_NUM_THREADS = '24'
OMP_SCHEDULE = 'DYNAMIC'
OMP_PROC_BIND = 'SPREAD'
OMP_PLACES = '{24},{25},{26},{27{17},{18},{19},{20},{21},{22},{23}'
OMP_STACKSIZE = '0'
OMP_WAIT_POLICY = 'PASSIVE'
OMP_THREAD_LIMIT = '4294967295'
OMP_MAX_ACTIVE_LEVELS = '2147483647'
OMP_CANCELLATION = 'FALSE'
OMP_DEFAULT_DEVICE = '0'
OMP_MAX_TASK_PRIORITY = '0'
OPENMP DISPLAY ENVIRONMENT END
},{28},{29},{30},{31},{32},{33},{34},{35},{36},{37},{38},{39},{40},{41},{42},{43},{44},{45},{46},{47}'
OMP_STACKSIZE = '0'
OMP_WAIT_POLICY = 'PASSIVE'
OMP_THREAD_LIMIT = '4294967295'
OMP_MAX_ACTIVE_LEVELS = '2147483647'
OMP_CANCELLATION = 'FALSE'
OMP_DEFAULT_DEVICE = '0'
OMP_MAX_TASK_PRIORITY = '0'
OPENMP DISPLAY ENVIRONMENT END
No preview for this file type
...@@ -55,6 +55,7 @@ ...@@ -55,6 +55,7 @@
#include <fstream> #include <fstream>
#include <random> #include <random>
/*----------------Include MPI Header-------------*/ /*----------------Include MPI Header-------------*/
#include <mpi.h> #include <mpi.h>
/*----------------------------------------------*/ /*----------------------------------------------*/
...@@ -64,7 +65,7 @@ ...@@ -64,7 +65,7 @@
using namespace BioFVM; using namespace BioFVM;
int omp_num_threads = 1; // set number of threads for parallel computing, set this to # of CPU cores x 2 (for hyperthreading) //int omp_num_threads = 48; // set number of threads for parallel computing, set this to # of CPU cores x 2 (for hyperthreading)
...@@ -222,10 +223,10 @@ void create_point_sources(double cell_radius, double dt, int num_sources, Microe ...@@ -222,10 +223,10 @@ void create_point_sources(double cell_radius, double dt, int num_sources, Microe
/* this class is also generating an ID, and we do not want the constructor ID */ /* this class is also generating an ID, and we do not want the constructor ID */
/*------------------------------------------------------------------------------------------------*/ /*------------------------------------------------------------------------------------------------*/
for(int r=0; r<mpi_Size; r++) //for(int r=0; r<mpi_Size; r++)
{ //{
if(mpi_Rank == r) // if(mpi_Rank == r)
{ // {
for(int k=0 ; k<sources_per_proc; k++) for(int k=0 ; k<sources_per_proc; k++)
{ {
Basic_Agent *temp_point_source = create_basic_agent(); Basic_Agent *temp_point_source = create_basic_agent();
...@@ -236,15 +237,15 @@ void create_point_sources(double cell_radius, double dt, int num_sources, Microe ...@@ -236,15 +237,15 @@ void create_point_sources(double cell_radius, double dt, int num_sources, Microe
tempPoint[i] = list_of_coords[3*k + i]; tempPoint[i] = list_of_coords[3*k + i];
temp_point_source->assign_position(tempPoint, mpi_Rank, mpi_Dims); temp_point_source->assign_position(tempPoint, mpi_Rank, mpi_Dims);
temp_point_source->ID = list_of_IDs[k]; //Added this statement to overwrite Constructor generated ID. temp_point_source->ID = list_of_IDs[k]; //Added this statement to overwrite Constructor generated ID.
std::cout <<"SOURCE:" << "Rank = "<< mpi_Rank << "(" << tempPoint[0] << "," << tempPoint[1] << "," << tempPoint[2] << ")" << "ID = " << list_of_IDs[k]<<std::endl; //std::cout <<"SOURCE:" << "Rank = "<< mpi_Rank << "(" << tempPoint[0] << "," << tempPoint[1] << "," << tempPoint[2] << ")" << "ID = " << list_of_IDs[k]<<std::endl;
temp_point_source->set_total_volume( (4.0/3.0)*pi*pow(cell_radius,3.0) ); temp_point_source->set_total_volume( (4.0/3.0)*pi*pow(cell_radius,3.0) );
(*temp_point_source->secretion_rates)[0]=10; (*temp_point_source->secretion_rates)[0]=10;
(*temp_point_source->saturation_densities)[0]=1; (*temp_point_source->saturation_densities)[0]=1;
temp_point_source->set_internal_uptake_constants(dt); temp_point_source->set_internal_uptake_constants(dt);
} }
} // }
MPI_Barrier(mpi_Cart_comm); // MPI_Barrier(mpi_Cart_comm);
} //}
} }
...@@ -388,39 +389,41 @@ void create_point_sinks(double cell_radius, double dt, int num_sinks, Microenvir ...@@ -388,39 +389,41 @@ void create_point_sinks(double cell_radius, double dt, int num_sinks, Microenvir
/* this class is also generating an ID, and we do not want the constructor ID */ /* this class is also generating an ID, and we do not want the constructor ID */
/*------------------------------------------------------------------------------------------------*/ /*------------------------------------------------------------------------------------------------*/
for(int r = 0; r< mpi_Size; r++) //for(int r = 0; r< mpi_Size; r++)
{ //{
if(mpi_Rank == r) // if(mpi_Rank == r)
{ // {
for(int k=0 ; k<sinks_per_proc; k++) for(int k=0 ; k<sinks_per_proc; k++)
{ {
Basic_Agent *temp_point_sink = create_basic_agent(); Basic_Agent *temp_point_sink = create_basic_agent();
temp_point_sink->register_microenvironment(&microenvironment); temp_point_sink->register_microenvironment(&microenvironment);
for(int i=0 ; i<3 ; i++) for(int i=0 ; i<3 ; i++)
tempPoint[i] = list_of_coords[3*k + i]; tempPoint[i] = list_of_coords[3*k + i];
temp_point_sink->assign_position(tempPoint, mpi_Rank, mpi_Dims); temp_point_sink->assign_position(tempPoint, mpi_Rank, mpi_Dims);
std::cout << "SINK:" << "Rank = "<< mpi_Rank << "(" << tempPoint[0] << "," << tempPoint[1] << "," << tempPoint[2] << ")" <<"ID = " << list_of_IDs[k] << std::endl; //std::cout << "SINK:" << "Rank = "<< mpi_Rank << "(" << tempPoint[0] << "," << tempPoint[1] << "," << tempPoint[2] << ")" <<"ID = " << list_of_IDs[k] << std::endl;
temp_point_sink->ID = list_of_IDs[k]; //Added this statement to overwrite Constructor generated ID. temp_point_sink->ID = list_of_IDs[k]; //Added this statement to overwrite Constructor generated ID.
temp_point_sink->set_total_volume( (4.0/3.0)*pi*pow(cell_radius,3.0) ); temp_point_sink->set_total_volume( (4.0/3.0)*pi*pow(cell_radius,3.0) );
(*temp_point_sink->uptake_rates)[0]=0.8; (*temp_point_sink->uptake_rates)[0]=0.8;
temp_point_sink->set_internal_uptake_constants(dt); temp_point_sink->set_internal_uptake_constants(dt);
} }
} // }
MPI_Barrier(mpi_Cart_comm); // MPI_Barrier(mpi_Cart_comm);
} //}
} }
int main( int argc, char* argv[] ) int main( int argc, char* argv[] )
{ {
double minX=0, minY=0, minZ=0, maxX=1000, maxY=1000, maxZ=1000, mesh_resolution=10; double minX=0, minY=0, minZ=0, maxX=1440, maxY=1440, maxZ=1440, mesh_resolution=10;
/*--------------------------------------------*/ /*--------------------------------------------*/
/* Initialize MPI */ /* Initialize MPI */
/*--------------------------------------------*/ /*--------------------------------------------*/
double t_start, t_end, t_total_start, t_total_end;
unsigned int mpi_Error; unsigned int mpi_Error;
int mpi_Requested_level = MPI_THREAD_FUNNELED; int mpi_Requested_level = MPI_THREAD_FUNNELED;
int mpi_Provided_level; int mpi_Provided_level;
...@@ -430,8 +433,11 @@ int main( int argc, char* argv[] ) ...@@ -430,8 +433,11 @@ int main( int argc, char* argv[] )
int mpi_Dims[3], mpi_Is_periodic[3], mpi_Coords[3]; int mpi_Dims[3], mpi_Is_periodic[3], mpi_Coords[3];
int mpi_Reorder; int mpi_Reorder;
mpi_Error = MPI_Init_thread(NULL, NULL, mpi_Requested_level, &mpi_Provided_level);
mpi_Error = MPI_Init_thread(NULL, NULL, mpi_Requested_level, &mpi_Provided_level);
MPI_Barrier(mpi_Comm);
t_total_start = MPI_Wtime();
if(mpi_Error != MPI_SUCCESS) if(mpi_Error != MPI_SUCCESS)
MPI_Abort(mpi_Comm,-1); MPI_Abort(mpi_Comm,-1);
...@@ -451,23 +457,23 @@ int main( int argc, char* argv[] ) ...@@ -451,23 +457,23 @@ int main( int argc, char* argv[] )
mpi_Dims[0] = 1; mpi_Dims[0] = 1;
mpi_Dims[1] = 1; mpi_Dims[1] = mpi_Size; //Number of Y processes, since Y-processes divide X-axis, we have X-decomposition
mpi_Dims[2] = mpi_Size; mpi_Dims[2] = 1;
//MPI_Dims_create(mpi_Size,3,mpi_Dims); //MPI_Dims_create(mpi_Size,3,mpi_Dims); //-->no need as we already know the decomposition i.e. 1-D Z-decomposition
mpi_Is_periodic[0]=0; mpi_Is_periodic[0]=0;
mpi_Is_periodic[1]=0; mpi_Is_periodic[1]=0;
mpi_Is_periodic[2]=0; mpi_Is_periodic[2]=0;
mpi_Reorder = 0; mpi_Reorder = 0;
if(mpi_Rank == 0) // if(mpi_Rank == 0)
{ // {
std::cout<<"Total MPI processes = "<<mpi_Size<<std::endl; // std::cout<<"Total MPI processes = "<<mpi_Size<<std::endl;
std::cout<<"Thread level provided = "<<mpi_Provided_level<<std::endl; // std::cout<<"Thread level provided = "<<mpi_Provided_level<<std::endl;
} // }
/*------------------------------------------------------------*/ /*------------------------------------------------------------*/
/* Create MPI topology, find coords */ /* Create MPI topology, find coords */
/*------------------------------------------------------------*/ /*------------------------------------------------------------*/
MPI_Cart_create(mpi_Comm, 3, mpi_Dims, mpi_Is_periodic, mpi_Reorder, &mpi_Cart_comm); MPI_Cart_create(mpi_Comm, 3, mpi_Dims, mpi_Is_periodic, mpi_Reorder, &mpi_Cart_comm);
...@@ -477,7 +483,7 @@ int main( int argc, char* argv[] ) ...@@ -477,7 +483,7 @@ int main( int argc, char* argv[] )
/* OpenMP threads set */ /* OpenMP threads set */
/*--------------------------------------------*/ /*--------------------------------------------*/
omp_set_num_threads(omp_num_threads); //omp_set_num_threads(omp_num_threads);
/*--------------------------------------------*/ /*--------------------------------------------*/
/* Create Microenvironment */ /* Create Microenvironment */
...@@ -492,15 +498,19 @@ int main( int argc, char* argv[] ) ...@@ -492,15 +498,19 @@ int main( int argc, char* argv[] )
microenvironment.mesh.units = "microns"; microenvironment.mesh.units = "microns";
microenvironment.time_units = "minutes"; microenvironment.time_units = "minutes";
if(mpi_Rank == 0) /*if(mpi_Rank == 0)
std::cout << "Total Voxels = " << microenvironment.mesh.voxels.size() << std::endl; std::cout << "Total Voxels = " << microenvironment.mesh.voxels.size() << std::endl;*/
/*------------------------------------------------------------*/ /*------------------------------------------------------------*/
/* Resize local Microenvironment on each process */ /* Resize local Microenvironment on each process */
/*------------------------------------------------------------*/ /*------------------------------------------------------------*/
t_start = MPI_Wtime();
microenvironment.resize_space_uniform( minX,maxX,minY,maxY,minZ,maxZ, mesh_resolution, mpi_Dims, mpi_Coords); microenvironment.resize_space_uniform( minX,maxX,minY,maxY,minZ,maxZ, mesh_resolution, mpi_Dims, mpi_Coords);
microenvironment.display_information( std::cout ); t_end = MPI_Wtime();
if(mpi_Rank==0)
std::cout<<"TIME FOR RESIZING MICROENVIRONMENT = "<< (t_end-t_start)<< std::endl;
//microenvironment.display_information( std::cout );
std::vector<double> center(3); std::vector<double> center(3);
...@@ -508,20 +518,30 @@ int main( int argc, char* argv[] ) ...@@ -508,20 +518,30 @@ int main( int argc, char* argv[] )
center[1] = (microenvironment.mesh.bounding_box[1]+microenvironment.mesh.bounding_box[4])/2; center[1] = (microenvironment.mesh.bounding_box[1]+microenvironment.mesh.bounding_box[4])/2;
center[2] = (microenvironment.mesh.bounding_box[2]+microenvironment.mesh.bounding_box[5])/2; center[2] = (microenvironment.mesh.bounding_box[2]+microenvironment.mesh.bounding_box[5])/2;
double stddev_squared = -100.0 * 100.0; double stddev_squared = -100.0 * 100.0;
std::vector<double> one( microenvironment.density_vector(0).size() , 1.0 ); std::vector<double> one( microenvironment.density_vector(0).size() , 1.0 );
int num_voxels = microenvironment.number_of_voxels();
std::cout << "Total voxels are = "<<num_voxels; // int num_voxels = microenvironment.number_of_voxels();
//#pragma omp parallel for // std::cout << "Total voxels are = "<<num_voxels;
t_start = MPI_Wtime();
#pragma omp parallel for
for( int i=0; i < microenvironment.number_of_voxels() ; i++ ) for( int i=0; i < microenvironment.number_of_voxels() ; i++ )
{ {
std::vector<double> displacement = microenvironment.voxels(i).center - center; std::vector<double> displacement = microenvironment.voxels(i).center - center;
double distance_squared = norm_squared( displacement ); double distance_squared = norm_squared( displacement );
double coeff = distance_squared; double coeff = distance_squared;
coeff /= stddev_squared; coeff /= stddev_squared;
microenvironment.density_vector(i)[0]= exp( coeff ); microenvironment.density_vector(i)[0]= exp( coeff );
} }
microenvironment.write_to_matlab( "initial_concentration.mat", mpi_Rank, mpi_Size, mpi_Cart_comm ); t_end = MPI_Wtime();
if(mpi_Rank==0)
std::cout<<"TIME FOR GENERATING GAUSSIAN PROFILE = "<< (t_end-t_start)<< std::endl;
t_start = MPI_Wtime();
microenvironment.write_to_matlab( "initial_concentration.mat", mpi_Rank, mpi_Size, mpi_Cart_comm );
t_end = MPI_Wtime();
if(mpi_Rank==0)
std::cout<<"TIME FOR WRITING INITIAL CONCENTRATION FILE = "<< (t_end-t_start)<< std::endl;
// register the diffusion solver // register the diffusion solver
microenvironment.diffusion_decay_solver = diffusion_decay_solver__constant_coefficients_LOD_3D; microenvironment.diffusion_decay_solver = diffusion_decay_solver__constant_coefficients_LOD_3D;
...@@ -531,53 +551,48 @@ int main( int argc, char* argv[] ) ...@@ -531,53 +551,48 @@ int main( int argc, char* argv[] )
microenvironment.decay_rates[0] = 0.01; microenvironment.decay_rates[0] = 0.01;
// display information // display information
microenvironment.display_information( std::cout ); //microenvironment.display_information( std::cout );
double dt = 0.01; double dt = 0.01;
double cell_radius = 5; double cell_radius = 5;
int num_sources = 500; int num_sources = 500;
int num_sinks = 500; int num_sinks = 500;
t_start = MPI_Wtime();
create_point_sources(cell_radius, dt, num_sources, microenvironment, mpi_Dims, mpi_Rank, mpi_Size, mpi_Cart_comm); create_point_sources(cell_radius, dt, num_sources, microenvironment, mpi_Dims, mpi_Rank, mpi_Size, mpi_Cart_comm);
create_point_sinks(cell_radius, dt, num_sinks, microenvironment, mpi_Dims, mpi_Rank, mpi_Size, mpi_Cart_comm); create_point_sinks(cell_radius, dt, num_sinks, microenvironment, mpi_Dims, mpi_Rank, mpi_Size, mpi_Cart_comm);
t_end = MPI_Wtime();
/*for(int i=0; i< 500;i++) if(mpi_Rank==0)
{ std::cout<<"TIME FOR CREATING ALL BASIC AGENTS = "<< (t_end-t_start)<< std::endl;
std::vector<double> tempPoint(3,0.0);
for( int j=0; j < 3 ; j++ )
{ tempPoint[j] = UniformRandom()*1000; }
Basic_Agent * temp_point_source = create_basic_agent();
temp_point_source->register_microenvironment(&microenvironment);
temp_point_source->assign_position(tempPoint);
temp_point_source->set_total_volume( (4.0/3.0)*pi*pow(cell_radius,3.0) );
(*temp_point_source->secretion_rates)[0]=10;
(*temp_point_source->saturation_densities)[0]=1;
temp_point_source->set_internal_uptake_constants(dt);
for( int j=0; j < 3 ; j++ )
{ tempPoint[j] = UniformRandom()*1000; }
Basic_Agent * temp_point_sink = create_basic_agent();
temp_point_sink->register_microenvironment(&microenvironment);
temp_point_sink->assign_position(tempPoint);
temp_point_sink->set_total_volume( (4.0/3.0)*pi*pow(cell_radius,3.0) );
(*temp_point_sink->uptake_rates)[0]=0.8;
temp_point_sink->set_internal_uptake_constants(dt);
}
*/
double t = 0.0; double t = 0.0;
double t_max=5; double t_max=5;
while( t < t_max ) t_start = MPI_Wtime();
while( t < t_max )
{ {
microenvironment.simulate_cell_sources_and_sinks( dt ); microenvironment.simulate_cell_sources_and_sinks( dt );
microenvironment.simulate_diffusion_decay( dt, mpi_Size, mpi_Rank, mpi_Coords, mpi_Dims, mpi_Cart_comm ); microenvironment.simulate_diffusion_decay( dt, mpi_Size, mpi_Rank, mpi_Coords, mpi_Dims, mpi_Cart_comm );
t += dt; t += dt;
} }
microenvironment.write_to_matlab( "final.mat", mpi_Rank, mpi_Size, mpi_Cart_comm ); //Remember to use Parallel Version ! t_end = MPI_Wtime();
std::cout<<"done!"<<std::endl; if(mpi_Rank==0)
std::cout<<"TIME FOR SIMULATING (SOURCES+SINKS+DIFFUSION) = "<< (t_end-t_start)<< std::endl;
t_start = MPI_Wtime();
microenvironment.write_to_matlab( "final.mat", mpi_Rank, mpi_Size, mpi_Cart_comm ); //Remember to use Parallel Version !
t_end = MPI_Wtime();
if(mpi_Rank==0)
std::cout<<"TIME FOR WRITING FINAL FILE = "<< (t_end-t_start)<< std::endl;
MPI_Barrier(mpi_Comm);
t_total_end = MPI_Wtime();
if(mpi_Rank == 0)
std::cout<<"TOTAL PROGRAM EXECUTION TIME = " <<(t_total_end-t_total_start)<<std::endl;
//std::cout<<"done!"<<std::endl;
/*-------------------------------------------------------------------------*/ /*-------------------------------------------------------------------------*/
......
No preview for this file type
TIME FOR RESIZING MICROENVIRONMENT = 0.758827
TIME FOR GENERATING GAUSSIAN PROFILE = 0.00794553
TIME FOR WRITING INITIAL CONCENTRATION FILE = 0.34451
TIME FOR CREATING ALL BASIC AGENTS = 0.000949931
TIME FOR SIMULATING (SOURCES+SINKS+DIFFUSION) = 8.48919
TIME FOR WRITING FINAL FILE = 0.280464
TOTAL PROGRAM EXECUTION TIME = 9.88661