Skip to content

Commit

Permalink
Debug case for non-working test.
Browse files Browse the repository at this point in the history 
The script tests/tst_memory_policy.py fails with a cuda illegal access.
The intention is to get help from NESAP to get a functional test.
  • Loading branch information
@nksauter
nksauter committed Jun 13, 2024
1 parent 1df01e6 commit e94da3a
Show file tree
Hide file tree
Showing 6 changed files with 664 additions and 171 deletions.
135 changes: 135 additions & 0 deletions simtbx/kokkos/detector.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,5 +92,140 @@ namespace simtbx { namespace Kokkos {
}
}

template<>
void kokkos_detector<small_whitelist_policy>::hello(){
SCITBX_EXAMINE("small small small");
}
template<>
void kokkos_detector<large_array_policy>::hello(){
SCITBX_EXAMINE("large large large");
}

template<> void
kokkos_detector<large_array_policy>::each_image_allocate() {
resize(m_rangemap, m_total_pixel_count);
resize(m_omega_reduction, m_total_pixel_count);
resize(m_max_I_x_reduction, m_total_pixel_count);
resize(m_max_I_y_reduction, m_total_pixel_count);
resize(m_floatimage, m_total_pixel_count);

resize(m_maskimage, m_total_pixel_count);
kokkostbx::transfer_shared2kokkos(m_sdet_vector, metrology.sdet);
kokkostbx::transfer_shared2kokkos(m_fdet_vector, metrology.fdet);
kokkostbx::transfer_shared2kokkos(m_odet_vector, metrology.odet);
kokkostbx::transfer_shared2kokkos(m_pix0_vector, metrology.pix0);
kokkostbx::transfer_shared2kokkos(m_distance, metrology.dists);
kokkostbx::transfer_shared2kokkos(m_Xbeam, metrology.Xbeam);
kokkostbx::transfer_shared2kokkos(m_Ybeam, metrology.Ybeam);
fence();

// metrology.show();

// printf(" rangemap size:%d\n", m_rangemap.span());
// printf(" omega_reduction size:%d\n", m_omega_reduction.span());
// printf(" max_I_x_reduction size:%d\n", m_max_I_x_reduction.span());
// printf(" max_I_y_reduction size:%d\n", m_max_I_y_reduction.span());
// printf(" maskimage size:%d\n", m_maskimage.span());
// printf(" floatimage size:%d\n", m_floatimage.span());
// printf(" sdet_vector size:%d\n", m_sdet_vector.span());
// printf(" fdet_vector size:%d\n", m_fdet_vector.span());
// printf(" odet_vector size:%d\n", m_odet_vector.span());
// printf(" pix0_vector size:%d\n", m_pix0_vector.span());
// printf(" distance size:%d\n", m_distance.span());
// printf(" Xbeam size:%d\n", m_Xbeam.span());
// printf(" Ybeam size:%d\n", m_Ybeam.span());

// print_view(m_fdet_vector);
// print_view(m_odet_vector, 1, 3);

// printf("DONE.\n");
}
template<> void
kokkos_detector<small_whitelist_policy>::each_image_allocate() {
resize(m_maskimage, m_total_pixel_count);
kokkostbx::transfer_shared2kokkos(m_sdet_vector, metrology.sdet);
kokkostbx::transfer_shared2kokkos(m_fdet_vector, metrology.fdet);
kokkostbx::transfer_shared2kokkos(m_odet_vector, metrology.odet);
kokkostbx::transfer_shared2kokkos(m_pix0_vector, metrology.pix0);
kokkostbx::transfer_shared2kokkos(m_distance, metrology.dists);
kokkostbx::transfer_shared2kokkos(m_Xbeam, metrology.Xbeam);
kokkostbx::transfer_shared2kokkos(m_Ybeam, metrology.Ybeam);
fence();
}

template<>
void
kokkos_detector<large_array_policy>::set_active_pixels_on_GPU(af::shared<std::size_t> active_pixel_list_value) {
m_active_pixel_size = active_pixel_list_value.size();
kokkostbx::transfer_shared2kokkos(m_active_pixel_list, active_pixel_list_value);
active_pixel_list = active_pixel_list_value;
}

template<>
void
kokkos_detector<small_whitelist_policy>::set_active_pixels_on_GPU(af::shared<std::size_t> active_pixel_list_value) {
m_active_pixel_size = active_pixel_list_value.size();
kokkostbx::transfer_shared2kokkos(m_active_pixel_list, active_pixel_list_value);
active_pixel_list = active_pixel_list_value;
resize(m_rangemap, m_active_pixel_size);
resize(m_omega_reduction, m_active_pixel_size);
resize(m_max_I_x_reduction, m_active_pixel_size);
resize(m_max_I_y_reduction, m_active_pixel_size);
resize(m_floatimage, m_active_pixel_size);
resize(m_accumulate_floatimage, m_active_pixel_size);
fence();
}

template<> af::shared<double>
kokkos_detector<large_array_policy>::get_whitelist_raw_pixels(af::shared<std::size_t> selection) {
hello();
//return the data array for the multipanel detector case, but only for whitelist pixels
vector_size_t active_pixel_selection = vector_size_t("active_pixel_selection", selection.size());
kokkostbx::transfer_shared2kokkos(active_pixel_selection, selection);

size_t output_pixel_size = selection.size();
vector_cudareal_t active_pixel_results = vector_cudareal_t("active_pixel_results", output_pixel_size);

auto temp = m_accumulate_floatimage;

parallel_for("get_active_pixel_selection",
range_policy(0, output_pixel_size),
KOKKOS_LAMBDA (const int i) {
size_t index = active_pixel_selection( i );
active_pixel_results( i ) = temp( index );
});

af::shared<double> output_array(output_pixel_size, af::init_functor_null<double>());
kokkostbx::transfer_kokkos2shared(output_array, active_pixel_results);

SCITBX_ASSERT(output_array.size() == output_pixel_size);
return output_array;
}
template<> af::shared<double>
kokkos_detector<small_whitelist_policy>::get_whitelist_raw_pixels(af::shared<std::size_t> selection) {
SCITBX_CHECK_POINT;
hello();
//return the data array for the multipanel detector case, but only for whitelist pixels

std::size_t output_pixel_size = selection.size();
//vector_cudareal_t active_pixel_results = vector_cudareal_t("active_pixel_results", output_pixel_size);

//auto temp = m_accumulate_floatimage;

//parallel_for("get_active_pixel_selection2",
// range_policy(0, output_pixel_size),
// KOKKOS_LAMBDA (const int i) {
// active_pixel_results( i ) = temp( i );
//});

af::shared<double> output_array(output_pixel_size, af::init_functor_null<double>());
SCITBX_CHECK_POINT;
kokkostbx::transfer_kokkos2shared(output_array, m_accumulate_floatimage);//active_pixel_results);
SCITBX_CHECK_POINT;

SCITBX_ASSERT(output_array.size() == output_pixel_size);
return output_array;
}

} // Kokkos
} // simtbx
82 changes: 7 additions & 75 deletions simtbx/kokkos/detector.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,10 +41,8 @@ struct packed_metrology{
af::shared<double>Ybeam;
};

struct large_array_policy {
};
struct small_whitelist_policy {
};
struct large_array_policy {};
struct small_whitelist_policy {};

template <typename MemoryPolicy>
struct kokkos_detector
Expand All @@ -63,7 +61,7 @@ struct kokkos_detector
//void write_raw_pixels(simtbx::nanoBragg::nanoBragg&);
//af::flex_double get_raw_pixels();
//void set_active_pixels_on_GPU(af::shared<std::size_t>);
//af::shared<double> get_whitelist_raw_pixels(af::shared<std::size_t>);
af::shared<double> get_whitelist_raw_pixels(af::shared<std::size_t>);
inline void each_image_free(){} //no op in Kokkos
int h_deviceID;

Expand Down Expand Up @@ -157,46 +155,7 @@ struct kokkos_detector
return view_floatimage;
};

inline void
each_image_allocate() {
resize(m_rangemap, m_total_pixel_count);
resize(m_omega_reduction, m_total_pixel_count);
resize(m_max_I_x_reduction, m_total_pixel_count);
resize(m_max_I_y_reduction, m_total_pixel_count);

resize(m_maskimage, m_total_pixel_count);
resize(m_floatimage, m_total_pixel_count);

kokkostbx::transfer_shared2kokkos(m_sdet_vector, metrology.sdet);
kokkostbx::transfer_shared2kokkos(m_fdet_vector, metrology.fdet);
kokkostbx::transfer_shared2kokkos(m_odet_vector, metrology.odet);
kokkostbx::transfer_shared2kokkos(m_pix0_vector, metrology.pix0);
kokkostbx::transfer_shared2kokkos(m_distance, metrology.dists);
kokkostbx::transfer_shared2kokkos(m_Xbeam, metrology.Xbeam);
kokkostbx::transfer_shared2kokkos(m_Ybeam, metrology.Ybeam);
fence();

// metrology.show();

// printf(" rangemap size:%d\n", m_rangemap.span());
// printf(" omega_reduction size:%d\n", m_omega_reduction.span());
// printf(" max_I_x_reduction size:%d\n", m_max_I_x_reduction.span());
// printf(" max_I_y_reduction size:%d\n", m_max_I_y_reduction.span());
// printf(" maskimage size:%d\n", m_maskimage.span());
// printf(" floatimage size:%d\n", m_floatimage.span());
// printf(" sdet_vector size:%d\n", m_sdet_vector.span());
// printf(" fdet_vector size:%d\n", m_fdet_vector.span());
// printf(" odet_vector size:%d\n", m_odet_vector.span());
// printf(" pix0_vector size:%d\n", m_pix0_vector.span());
// printf(" distance size:%d\n", m_distance.span());
// printf(" Xbeam size:%d\n", m_Xbeam.span());
// printf(" Ybeam size:%d\n", m_Ybeam.span());

// print_view(m_fdet_vector);
// print_view(m_odet_vector, 1, 3);

// printf("DONE.\n");
}
void each_image_allocate();

inline void
scale_in_place(const double& factor){
Expand All @@ -206,6 +165,8 @@ struct kokkos_detector
});
}

void set_active_pixels_on_GPU(af::shared<std::size_t> active_pixel_list_value);

inline void
write_raw_pixels(simtbx::nanoBragg::nanoBragg& nB) {
//only implement the monolithic detector case, one panel
Expand Down Expand Up @@ -242,37 +203,8 @@ struct kokkos_detector
return output_array;
}

inline void
set_active_pixels_on_GPU(af::shared<std::size_t> active_pixel_list_value) {
m_active_pixel_size = active_pixel_list_value.size();
kokkostbx::transfer_shared2kokkos(m_active_pixel_list, active_pixel_list_value);
active_pixel_list = active_pixel_list_value;
}

inline af::shared<double>
get_whitelist_raw_pixels(af::shared<std::size_t> selection) {
//return the data array for the multipanel detector case, but only for whitelist pixels
vector_size_t active_pixel_selection = vector_size_t("active_pixel_selection", selection.size());
kokkostbx::transfer_shared2kokkos(active_pixel_selection, selection);
void hello();

size_t output_pixel_size = selection.size();
vector_cudareal_t active_pixel_results = vector_cudareal_t("active_pixel_results", output_pixel_size);

auto temp = m_accumulate_floatimage;

parallel_for("get_active_pixel_selection",
range_policy(0, output_pixel_size),
KOKKOS_LAMBDA (const int i) {
size_t index = active_pixel_selection( i );
active_pixel_results( i ) = temp( index );
});

af::shared<double> output_array(output_pixel_size, af::init_functor_null<double>());
kokkostbx::transfer_kokkos2shared(output_array, active_pixel_results);

SCITBX_ASSERT(output_array.size() == output_pixel_size);
return output_array;
}
};


Expand Down
Loading

0 comments on commit e94da3a

Please sign in to comment.