1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
|
/* This is a simple interface for c applications to use Panda */
#include <stdio.h>
#include <stdlib.h>
#include "definitions.h"
#include "Panda.h"
#include "MPIFS.h"
#include "ArrayLayout.h"
#include "Array.h"
extern "C" { int Panda_Create(int, int); }
extern "C" { void Panda_Finalize(); }
extern "C" { void PandaTimestep(struct ArrayInfo *); }
extern "C" { void *PandaReadTimestep(struct ArrayInfo *); }
extern "C" { void Panda_WriteAttribute(char *, char *, int, int, void *); }
extern "C" { void *Panda_ReadAttribute(char *, char *, int *, int *); }
extern "C" { Boolean PandaIsNewFile(char *); }
typedef struct ArrayInfo {
char* name_; /* array name */
int rank_; /* rank */
int* size_; /* glbal size of the array */
int esize_; /* size of each element */
int mem_rank_; /* compute processor topology - rank */
int* mem_layout_; /* compute processor topology - mesh */
Distribution* mem_dist_; /* compute processor topology - dist */
int disk_rank_; /* io processor topology - rank */
int* disk_layout_; /* io processor topology - mesh */
Distribution* disk_dist_; /* io processor topology - dist */
char* data_; /* data pointer belonging to me */
int stencil_width_; /* stencil width */
struct ArrayInfo *next_; /* next element */
} ArrayInfo;
Panda *global_bear = NULL;
extern MPIFS *MPIFS_global_obj;
int Panda_Create(int ioproc_every, int is_part_time_mode)
{
int i, my_app_size, my_rank, *world_ranks;
int io_nodes;
/* if (io_nodes > 1) {
printf("Warning: Write Chunks instead of Write arrays.\n");
printf("There might be errors in Attributes write\n");
}*/
MPI_Comm_size(MPI_COMM_WORLD, &my_app_size);
MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
world_ranks = (int *) malloc(sizeof(int)*my_app_size);
for (i=0; i<my_app_size; i++) world_ranks[i] = i;
io_nodes = (my_app_size - 1) / ioproc_every + 1;
if (is_part_time_mode) {
if (my_rank < io_nodes) { /* part-time io nodes */
global_bear = new Panda(PART_TIME_IO, my_rank, my_app_size, world_ranks,
my_rank, io_nodes, world_ranks);
//printf("##### Panda proc %d/%d PART_TIME_IO\n", my_rank, my_app_size);
} else { /* part-time compute nodes */
global_bear = new Panda(PART_TIME_COMPUTE, my_rank, my_app_size,
world_ranks, -1, io_nodes, world_ranks);
//printf("##### Panda proc %d/%d PART_TIME_COMPUTE\n", my_rank, my_app_size);
}
} else {
// printf("Warning: Full-time I/O nodes is not integrated with Cactus yet, ");
// printf("due to the communicator problem. Panda's part is done, though\n");
if (my_rank < io_nodes) { /* full-time io nodes */
global_bear = new Panda(IO_NODE, 0, my_rank, io_nodes, world_ranks);
delete global_bear;
free(world_ranks);
return 1;
} else { /* compute nodes */
for (i=0; i<(my_app_size-io_nodes); i++) world_ranks[i] += io_nodes;
global_bear = new Panda(COMPUTE_NODE, 1, my_rank-io_nodes,
my_app_size-io_nodes, world_ranks);
}
}
free(world_ranks);
return 0;
}
void Panda_Finalize()
{
if (global_bear) delete global_bear;
}
void PandaTimestep(ArrayInfo *ptr)
{
/* Test if Panda_Create() has been called */
if (global_bear == NULL) {
printf("Panda object is not created yet - Use Panda_Create(...)\n");
return;
}
/* Create array information */
ArrayLayout *mem_layout, *disk_layout;
Array *array;
mem_layout = new ArrayLayout(ptr->mem_rank_, ptr->mem_layout_);
disk_layout = new ArrayLayout(ptr->disk_rank_, ptr->disk_layout_);
array = new Array(ptr->name_, ptr->rank_, ptr->size_, ptr->esize_,
mem_layout, ptr->mem_dist_,
disk_layout, ptr->disk_dist_,
ptr->data_, ptr->stencil_width_);
global_bear->app_barrier();
// printf("---------------- Panda Timestep -------------------\n");
//printf("name %s rank %d size %d %d %d esize %d mem_layout %d %d %d disk_layout %d stencil_width_ %d\n", ptr->name_, ptr->rank_, ptr->size_[0], ptr->size_[1], ptr->size_[2], ptr->esize_, ptr->mem_layout_[0], ptr->mem_layout_[1], ptr->mem_layout_[2], ptr->disk_layout_[0], ptr->stencil_width_);
array->timestep();
delete mem_layout;
delete disk_layout;
delete array;
}
void *PandaReadTimestep(ArrayInfo *ptr)
{
/* Test if Panda_Create() has been called */
if (global_bear == NULL) {
printf("Panda object is not created yet - Use Panda_Create(...)\n");
return NULL;
}
/* Create array information */
ArrayLayout *mem_layout, *disk_layout;
Array *array;
mem_layout = new ArrayLayout(ptr->mem_rank_, ptr->mem_layout_);
disk_layout = NULL;
array = new Array(ptr->name_, ptr->rank_, ptr->size_, ptr->esize_,
mem_layout, ptr->mem_dist_,
disk_layout, ptr->disk_dist_,
ptr->data_, ptr->stencil_width_);
printf("---------------- Panda ReadTimestep -------------------\n");
global_bear->app_barrier();
array->read_timestep();
void *data = (void *)array->get_data_ptr();
array->set_data_ptr(NULL);
delete mem_layout;
delete array;
return data;
}
void Panda_WriteAttribute(char *fname, char *name, int esize,
int count, void *data)
{
Attribute *attr = new Attribute();
attr->write(fname, name, esize, count, data);
delete attr;
}
void *Panda_ReadAttribute(char *fname, char *name, int *type, int *count)
{
Attribute *attr = new Attribute();
attr->read(fname, name);
void *data = attr->get_data_ptr();
attr->set_data_ptr(NULL);
*type = attr->esize();
*count = attr->count();
delete attr;
return data;
}
Boolean PandaIsNewFile(char *fname)
{
return MPIFS_global_obj->is_new_file(fname);
}
|
