diff options
Diffstat (limited to 'src/Panda/Array.C')
| -rw-r--r-- | src/Panda/Array.C | 649 |
1 files changed, 649 insertions, 0 deletions
diff --git a/src/Panda/Array.C b/src/Panda/Array.C new file mode 100644 index 0000000..e2fd7eb --- /dev/null +++ b/src/Panda/Array.C @@ -0,0 +1,649 @@ +#include "definitions.h" +#include "MPIFS.h" +#include "Array.h" + +#include "external/IEEEIO/src/Arch.h" + +extern "C" { + int IOsizeOf(int); + int IOreadAttributeInfo(IOFile, char *,int *, int *); + int IOreadAttribute(IOFile,int,void*); +} + +extern int global_system_type_; +extern MPIFS* MPIFS_global_obj; +extern int SUBCHUNK_SIZE; + +/*************************************************************************** + * Class: Array + * Description: This is a user-visible class. This is used to describe the + * global array. It also stores pointers to local chunks of + * data. + * + * Instance-variables: + * name_ - name of the array + * rank_ - rank of the array (inherited variable) + * size_ - size of the array (elements) in the various dimensions + * element_size_ - size of each array element (in bytes) + * compute_node_layout_ - layout of the compute nodes + * io_node_layout_ - layout of the io nodes + * subchunk_layout_ - layout of the subchunks + * natural_chunked - whether the array is natural chunked + * compute_node_alloc_policy - chunk dist policy on compute nodes + * io_node_alloc_policy - chunk dist policy on the io nodes + **************************************************************************** + */ + +Array::Array() : Template() +{ + subchunk_layout_ = NULL; + element_size_ = 0; + natural_chunked_ = NO; + sub_chunked_ = NO; + overlap_ = NO; + io_strategy_ = SIMPLE_IO; +} + +/* This function is used on the compute nodes to create the array * + * object. In this case there is no user-specified subchunking and * + * the chunk distribution on the compute nodes is 1 chunk per * + * compute node and round-robin on the io nodes. */ +Array::Array(char *name, int rank, int *sizearray, int elt_size, + ArrayLayout *mem_layout, Distribution *mem_dist, + ArrayLayout *io_layout, Distribution *io_dist):Template(rank, sizearray) +{ + do_init(name, rank, sizearray, elt_size, mem_layout, mem_dist, + io_layout, io_dist, NULL, NULL, REGULAR, ROUND_ROBIN, HPF); + /* call function to allocate chunk_list */ + if (sizearray) allocate_chunks(COMPUTE_NODE); + overlap_ = NO; +} + +/* This function is used on the compute nodes to create the array * + * object. In this case there is no user-specified subchunking and * + * the chunk distribution on the compute nodes is 1 chunk per * + * compute node and round-robin on the io nodes. Also in this case* + * the user specifies the data ptr to be used. */ +Array::Array(char *name, int rank, int *sizearray, int elt_size, + ArrayLayout *mem_layout, Distribution *mem_dist, + ArrayLayout *io_layout, Distribution *io_dist, + char *data_ptr) : Template(rank, sizearray) +{ + char *ptr = data_ptr; + do_init(name, rank, sizearray, elt_size, mem_layout, mem_dist, + io_layout, io_dist, NULL, NULL, REGULAR, ROUND_ROBIN, HPF); + /* call function to allocate chunk_list */ + if (sizearray) allocate_chunks(COMPUTE_NODE, 1 , &ptr, 0); + overlap_ = NO; +} + +/* This function is used on the compute nodes to create the array * + * object. In this case there is no user-specified subchunking and * + * the chunk distribution on the compute nodes is 1 chunk per * + * compute node and round-robin on the io nodes. Also in this case* + * the user specifies the data ptr to be used and stencil width. */ +Array::Array(char *name, int rank, int *sizearray, int elt_size, + ArrayLayout *mem_layout, Distribution *mem_dist, + ArrayLayout *io_layout, Distribution *io_dist, + char *data_ptr, int stencil_width) : Template(rank, sizearray) +{ + char *ptr = data_ptr; + do_init(name, rank, sizearray, elt_size, mem_layout, mem_dist, + io_layout, io_dist, NULL, NULL, REGULAR, ROUND_ROBIN, HPF); + /* call function to allocate chunk_list */ + if (sizearray) allocate_chunks(COMPUTE_NODE, 1 , &ptr, stencil_width); + if (stencil_width > 0) overlap_ = YES; + else overlap_ = NO; +} + +/* This function is used on the compute nodes to create the array * + * object. In this case there is user-specified subchunking and * + * the chunk distribution on the compute nodes is 1 chunk per * + * compute node and round-robin on the io nodes. */ +Array::Array(char *name, int rank, int *sizearray, int elt_size, + ArrayLayout *mem_layout, Distribution *mem_dist, + ArrayLayout *io_layout, Distribution *io_dist, + ArrayLayout *sub_layout, Distribution* sub_dist) + : Template(rank, sizearray) +{ + do_init(name, rank, sizearray, elt_size, mem_layout, mem_dist, + io_layout, io_dist, sub_layout, sub_dist, + REGULAR, ROUND_ROBIN, HPF); + /* call function to allocate chunk_list */ + if (sizearray) allocate_chunks(COMPUTE_NODE); + overlap_ = NO; +} + +/* This function is used on the compute nodes to create the array * + * object. In this case there is user-specified subchunking and * + * the chunk distribution on the compute nodes is 1 chunk per * + * compute node and round-robin on the io nodes. This function is * + * used to when the user provides the data_ptr. */ +Array::Array(char *name, int rank, int *sizearray, int elt_size, + ArrayLayout *mem_layout, Distribution *mem_dist, + ArrayLayout *io_layout, Distribution *io_dist, + ArrayLayout *sub_layout, Distribution* sub_dist, + char *data_ptr) : Template(rank, sizearray) +{ + char *ptr = data_ptr; + do_init(name, rank, sizearray, elt_size, mem_layout, mem_dist, + io_layout, io_dist, sub_layout, sub_dist, + REGULAR, ROUND_ROBIN, HPF); + /* call function to allocate chunk_list */ + if (sizearray) allocate_chunks(COMPUTE_NODE, 1, &ptr, 0); + overlap_ = NO; +} + +/* Initializes the state of the array object. the chunks are allocated * + * via another function */ +void Array::do_init(char *name, int rank, int *sizearray, int elt_size, + ArrayLayout *mem_layout, Distribution *mem_dist, + ArrayLayout *io_layout, Distribution *io_dist, + ArrayLayout *subchunk_layout, Distribution *subchunk_dist, + ChunkAllocPolicy comp_node_policy, ChunkAllocPolicy io_node_policy, + Block_Distribution block_dist) +{ + io_strategy_ = SIMPLE_IO; + + name_ = (char *) malloc(sizeof(char)*(strlen(name)+5)); + strcpy(name_, name); + ieee_size_ = elt_size; + element_size_ = IOsizeOf(ieee_size_); + + compute_node_layout_ = new RegularDistribution(rank, mem_layout, mem_dist, + comp_node_policy, block_dist); + if (io_layout) + io_node_layout_ = new RegularDistribution(rank, io_layout, io_dist, + io_node_policy, block_dist); + else io_node_layout_ = NULL; + if (subchunk_layout) + subchunk_layout_ = new RegularDistribution(rank, subchunk_layout, + subchunk_dist, ROUND_ROBIN, + block_dist); + else subchunk_layout_ = NULL; + + /* Check if there is any sub-chunking */ + if (subchunk_layout_) sub_chunked_ = YES; + else sub_chunked_ = NO; + + /* Check if there is any natural chuunking */ + if (compute_node_layout_->equal(io_node_layout_)) natural_chunked_ = YES; + else natural_chunked_ = NO; +} + +/* This function is used to initialize the array objects on the io * + * node side. */ +Array::Array(int **schema_buf) +{ + int* ptr = *schema_buf; + + io_strategy_ = *ptr++; + op_type_ = *ptr++; + int len = *ptr++; + name_ = (char *) malloc(len+1); + for (int i=0; i< len; i++) name_[i] = (char) *ptr++; + name_[len] = '\0'; + rank_ = *ptr++; + + if (*ptr++ > 0) { + size_ = (int *) malloc(sizeof(int) * rank_); + for(int i=0; i < rank_; i++) size_[i] = *ptr++; + } else size_ = NULL; + + element_size_ = *ptr++; + ieee_size_ = *ptr++; + natural_chunked_ = (Boolean) *ptr++; + sub_chunked_ = (Boolean) *ptr++; + overlap_ = (Boolean) *ptr++; + + compute_node_layout_ = unpack_layout(&ptr); + io_node_layout_ = unpack_layout(&ptr); + + if (sub_chunked_) subchunk_layout_ = unpack_layout(&ptr); + else subchunk_layout_ = NULL; + + *schema_buf = ptr; +} + +ArrayDistribution *Array::unpack_layout(int **schema_buf) +{ + int *ptr = *schema_buf; + int type = *ptr++; + ArrayDistribution *tmp; + + if (type == UNSET) tmp = NULL; + else if (type == Regular) tmp = new RegularDistribution(&ptr); + else if (type == Irregular) {printf("Irregular is not supported\n"); exit(0);} + else tmp = NULL; + + *schema_buf = ptr; + return tmp; +} + +/* Allocate chunks - Currently only used on the compute node side */ +void Array::allocate_chunks(int node_type) +{ + int my_rank; + Chunk *new_chunk; + + if (node_type == COMPUTE_NODE) { + /* First find out what kind of system we have (MPI or sequential) */ + if (global_system_type_ == MPI_SYSTEM) { + /* Allocate a single chunk with index=compute_node_rank */ + my_rank = MPIFS_global_obj->my_rank(COMPUTE_NODE); + new_chunk = new Chunk(this, my_rank, COMPUTE_NODE, ALLOC); + compute_node_layout_->add_last(new_chunk); + } else if (global_system_type_ == UNIX_SYSTEM) { + /* There is only one kind of Allocation policy */ + int num = compute_node_layout_->total_elements(); + for (my_rank=0; my_rank<num; my_rank++) { + new_chunk = new Chunk(this, my_rank, COMPUTE_NODE, ALLOC); + compute_node_layout_->add_last(new_chunk); + } + } else printf("Unsupported filesystem\n"); + } else if (node_type == IO_NODE) { + printf("Will have to do this later\n"); + } else { + printf("Error: Don't know the node type\n"); + } +} + +/* Allocate chunks with user-specified data pointer. This function + * currently supports only the REGULAR distribution of chunks in + * the MPI-based file system and + * should be called only on the compute node side + */ +void Array::allocate_chunks(int node_type, int num_ptrs, + char **data_ptr, int stencil_width) +{ + int my_rank; + Chunk *new_chunk; + + if (node_type == COMPUTE_NODE) { + /* First find out what kind of system we have (MPI or sequential) */ + if (global_system_type_ == MPI_SYSTEM) { + /* Allocate a single chunk with index=compute_node_rank */ + my_rank = MPIFS_global_obj->my_rank(COMPUTE_NODE); + new_chunk = new Chunk(this, my_rank, COMPUTE_NODE, NO_ALLOC); + new_chunk->set_data_ptr(data_ptr[0]); + new_chunk->set_stencil_width(stencil_width); + compute_node_layout_->add_last(new_chunk); + } else if (global_system_type_ == UNIX_SYSTEM) { + /* There is only one kind of Allocation policy */ + int num = compute_node_layout_->total_elements(); + for (my_rank=0; my_rank<num; my_rank++) { + new_chunk = new Chunk(this, my_rank, COMPUTE_NODE, NO_ALLOC); + new_chunk->set_data_ptr(data_ptr[my_rank]); + new_chunk->set_stencil_width(stencil_width); + compute_node_layout_->add_last(new_chunk); + } + } else printf("Unsupported filesystem\n"); + } else if (node_type == IO_NODE) { + printf("Will have to do this later\n"); + } else { + printf("Error: Don't know the node type\n"); + } +} + +Array::~Array() +{ + if (name_) free(name_); + name_ = NULL; + if (compute_node_layout_) delete(compute_node_layout_); + if (io_node_layout_) delete(io_node_layout_); + if (subchunk_layout_) delete(subchunk_layout_); + compute_node_layout_ = io_node_layout_ = subchunk_layout_ = NULL; +} + +/* We are not packing the chunk information here */ +void Array::pack(int** schema_buf, int *schema_size) +{ + int *ptr, *head; + int i, len; + + ptr = (int *) malloc(sizeof(int)*100); + head = ptr; + + *ptr++ = io_strategy_; + *ptr++ = op_type_; + len = strlen(name_); + *ptr++ = len; + for(i=0; i<len;i++) *ptr++ = (int) name_[i]; + *ptr++ = rank_; + if (size_) { *ptr++ = 1; for(int i=0; i < rank_;i++) *ptr++ = size_[i]; } + else *ptr++ = 0; + *ptr++ = element_size_; + *ptr++ = ieee_size_; + *ptr++ = (int)natural_chunked_; + *ptr++ = (int)sub_chunked_; + *ptr++ = (int)overlap_; + + if (compute_node_layout_) compute_node_layout_->pack(&ptr); + else *ptr++ = (int)UNSET; + if (io_node_layout_) io_node_layout_->pack(&ptr); + else *ptr++ = (int)UNSET; + if (sub_chunked_) subchunk_layout_->pack(&ptr); + + *schema_size = (int)(ptr - head); + *schema_buf = head; +} + +ArrayDistribution* Array::layout(int layout_type) +{ + switch(layout_type) { + case COMPUTE_NODE: + return compute_node_layout_; + case IO_NODE: + return io_node_layout_; + case SUB_CHUNK: + return subchunk_layout_; + default: + printf("Invalid type\n"); + return NULL; + } +} + +/* The following two functions are used for regular layouts (HPF-style) only */ +/* Given a chunk index and node type, this function returns the * + * the relative node number on which the chunk resides */ +int Array::which_node(int chunk_id, int node_type) +{ + if (node_type == COMPUTE_NODE) + if (compute_node_layout_->alloc_policy() == REGULAR) return chunk_id; + else { + printf("Unsupported chunk alloc type\n"); + exit(1); + } + else if (node_type == IO_NODE) { + printf("Currently this is unsupported\n"); + exit(1); + } else { + printf("Unsupported node type\n"); + exit(1); + } + return -1; +} + +int Array::which_node(int chunk_id, int node_type, int num_io_nodes) +{ + if (node_type == IO_NODE){ + switch(io_node_layout_->alloc_policy()){ + case ROUND_ROBIN: + return(chunk_id % num_io_nodes); + + default: + printf("Error in which_node(int,int,int).. Invalid distribution type\n"); + exit(1); + } + } else if (node_type == COMPUTE_NODE) { + switch(compute_node_layout_->alloc_policy()){ + case REGULAR: + return chunk_id; + + default: + printf("Error in which_node(int,int,int)... Invalid distribution type\n"); + exit(1); + } + } else { + printf("Error in which_node(int,int,int)... Invalid node type\n"); + exit(1); + } + return -1; +} + +Chunk* Array::get_next_chunk() +{ + return compute_node_layout_->get_next_chunk(); +} + +/* The following seven functions are called by compute nodes only */ +/* Given a chunk index, find the chunk */ +Chunk* Array::find_chunk(int id) +{ + List *list = compute_node_layout_->chunk_list(); + Cell *list_ptr = list != NULL ? list->head_: NULL; + Chunk *chunk_ptr; + + while (list_ptr) { + chunk_ptr = (Chunk *)list_ptr->item(); + if (chunk_ptr->chunk_id() == id) return chunk_ptr; + list_ptr = list_ptr->next(); + } + return NULL; +} + +int Array::element_size(){return element_size_;} +int Array::ieee_size(){return ieee_size_;} + +Boolean Array::nat_chunked(){return natural_chunked_;} + +Boolean Array::sub_chunked(){return sub_chunked_;} + +/* This function needs to be checked and refined */ +void Array::make_sub_chunks(Chunk *chunk) +{ + Distribution *subchunk_dist; + int *subchunk_layout_sizes; + int i, tmp_size, dim, val_dim; + int *chunk_size = chunk->size(); + + if (sub_chunked_) { + printf("Error: Array already subchunked\n"); + exit(1); + } else { + subchunk_dist = (Distribution *) malloc(sizeof(Distribution)*rank_); + subchunk_layout_sizes = (int*) malloc(sizeof(int)*rank_); + tmp_size = chunk->total_size_in_bytes(); + if (tmp_size < SUBCHUNK_SIZE){ + for(i=0;i<rank_;i++){ + subchunk_dist[i] = BLOCK; + subchunk_layout_sizes[i] = 1; + } + } else { + tmp_size = element_size_; + i = rank_; + while(tmp_size < SUBCHUNK_SIZE){ + i--; + tmp_size *= chunk_size[i]; + } + dim =i; + tmp_size /=chunk_size[i]; + val_dim = SUBCHUNK_SIZE / tmp_size; + for(i=0;i<dim;i++){ + subchunk_dist[i] = BLOCK; + subchunk_layout_sizes[i] = chunk_size[i]; + } + subchunk_dist[dim] = BLOCK; + subchunk_layout_sizes[dim] = (chunk_size[i] + val_dim -1)/val_dim; + for(i=dim+1;i<rank_; i++){ + subchunk_dist[i] = BLOCK; + subchunk_layout_sizes[i] = 1; + } + } + ArrayLayout *tmp_layout = new ArrayLayout(rank_, subchunk_layout_sizes); + subchunk_layout_ = new RegularDistribution(rank_, tmp_layout, + subchunk_dist, ROUND_ROBIN, HPF); + sub_chunked_ = YES; + free(subchunk_layout_sizes); + free(subchunk_dist); + } +} + +int Array::array_info() +{ + List *list = compute_node_layout_->chunk_list(); + Cell *list_ptr = list->head_; + Chunk *chunk_ptr; + int ret =0; + + while(list_ptr) { + chunk_ptr = (Chunk *)list_ptr->item(); + ret += chunk_ptr->total_size_in_bytes(); + list_ptr = list_ptr->next(); + } + return ret; +} + +/* Called only on the I/O node side */ +int Array::get_next_index(Chunk *&chunk, int old_val, int io_node_num, + int num_io_nodes, int max) +{ + int ret = io_node_layout_->get_next_index(chunk, old_val, io_node_num, + num_io_nodes, max); + if (io_node_layout_->distribution_type() == Regular) + if (ret < max) chunk->init(this, ret, IO_NODE, NO_ALLOC); + return ret; +} + +/* This function should be called only on the compute node side and * + * make sense only for the regular distribution of chunks, */ +void Array::set_data_ptr(char *data_ptr) +{ + List *list = compute_node_layout_->chunk_list(); + Chunk *chunk_ptr; + + if (list && list->head_){ + chunk_ptr = (Chunk *) list->head_->item(); + chunk_ptr->set_data_ptr(data_ptr); + } else { + printf("Error: No chunks present - cannot set data ptr\n"); + } +} + + +/* This function should be called only on the compute node side and * + * make sense only for the regular distribution of chunks, */ +char* Array::get_data_ptr() +{ + List *list = compute_node_layout_->chunk_list(); + Chunk *chunk_ptr; + + if (list && list->head_){ + chunk_ptr = (Chunk *) list->head_->item(); + return ((char *)chunk_ptr->data_ptr()); + } else { + printf("Error: No chunks present - cannot set data ptr\n"); + return NULL; + } +} + +Boolean Array::overlaped() +{ + return overlap_; +} + +void Array::read_schema_file(IOFile file_ptr) +{ + int *base = (int *)malloc(sizeof(int) * rank_); + int *size = (int *)malloc(sizeof(int) * rank_); + int index, length, datatype; + Chunk *new_chunk; + + index = IOreadAttributeInfo(file_ptr, "chunk_origin", &datatype, &length); + if (index >=0 ) { // the attribute exists + IOreadAttribute(file_ptr, index, base); + index = IOreadAttributeInfo(file_ptr, "chunk_size",&datatype,&length); + if (index < 0) { printf("Error in reading attributes\n"); exit(0); } + IOreadAttribute(file_ptr, index, size); + new_chunk = new Chunk(this, base, size); + } else { + for (int j=0; j<rank_; j++) base[j] = 0; + new_chunk = new Chunk(this, base, size_); + } + io_node_layout_ = new IrregularDistribution(1, &new_chunk); + free(base); + free(size); +} + +/* The collective io operation to write out the arrays. */ +void Array::timestep() +{ + int *schema, schema_size; + int node_type = MPIFS_global_obj->node_type(); + + op_type_ = TIMESTEP; + if (node_type == COMPUTE_NODE){ + MPIFS_global_obj->send_array_schema(this); + MPIFS_global_obj->compute_node_io_loop(this); + } + else if (node_type == PART_TIME_COMPUTE) + MPIFS_global_obj->compute_node_io_loop(this); + else { + pack(&schema, &schema_size); + MPIFS_global_obj->part_time_io_node_loop(schema, schema_size, this); + } +} + +/* The collective io operation to write out the arrays. */ +void Array::checkpoint() +{ + int *schema, schema_size; + int node_type = MPIFS_global_obj->node_type(); + + op_type_ = CHECKPOINT; + if (node_type == COMPUTE_NODE){ + MPIFS_global_obj->send_array_schema(this); + MPIFS_global_obj->compute_node_io_loop(this); + } + else if (node_type == PART_TIME_COMPUTE) + MPIFS_global_obj->compute_node_io_loop(this); + else { + pack(&schema, &schema_size); + MPIFS_global_obj->part_time_io_node_loop(schema, schema_size, this); + } +} + +/* The collective io operation to read in the arrays from a * + * checkpoint file. Currently (for testing purposes) this * + * does not happen. */ +void Array::restart() +{ + int *schema, schema_size; + int node_type = MPIFS_global_obj->node_type(); + + op_type_ = RESTART; + if (node_type == COMPUTE_NODE){ + MPIFS_global_obj->send_array_schema(this); + MPIFS_global_obj->compute_node_io_loop(this); + } + else if (node_type == PART_TIME_COMPUTE) + MPIFS_global_obj->compute_node_io_loop(this); + else { + pack(&schema, &schema_size); + MPIFS_global_obj->part_time_io_node_loop(schema, schema_size , this); + } +} + +void Array::read_timestep() +{ + int *schema, schema_size; + int node_type = MPIFS_global_obj->node_type(); + + op_type_ = READ_TIMESTEP; + if (node_type == COMPUTE_NODE){ + MPIFS_global_obj->send_array_schema(this); + MPIFS_global_obj->compute_node_io_loop(this); + } + else if (node_type == PART_TIME_COMPUTE) + MPIFS_global_obj->compute_node_io_loop(this); + else { + pack(&schema, &schema_size); + MPIFS_global_obj->part_time_io_node_loop(schema, schema_size , this); + } +} + +int Array::op_type() { return op_type_; } +int Array::io_strategy() { return io_strategy_; } + +void Array::init(int rank, int ieee_size, int *size, int node_type) +{ + rank_ = rank; + ieee_size_ = ieee_size; + element_size_ = IOsizeOf(ieee_size_); + size_ = size; + if (node_type == COMPUTE_NODE) { + int my_rank = MPIFS_global_obj->my_rank(COMPUTE_NODE); + Chunk *new_chunk = new Chunk(this, my_rank, COMPUTE_NODE, ALLOC); + compute_node_layout_->add_last(new_chunk); + } +} |