SRTM example (wip)

examples/CMakeLists.txt

@@ -1,3 +1,5 @@
+find_package(ZLIB REQUIRED)
+
 file(GLOB PSEMEK_EXAMPLES RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/*")
 list(REMOVE_ITEM PSEMEK_EXAMPLES "CMakeLists.txt")
 
@@ -6,6 +8,6 @@ foreach(example ${PSEMEK_EXAMPLES})
 	set(TARGET_NAME psemek-example-${TARGET_NAME})
 	psemek_add_executable(${TARGET_NAME} ${example})
 	if(TARGET ${TARGET_NAME})
-		target_link_libraries(${TARGET_NAME} PUBLIC psemek)
+		target_link_libraries(${TARGET_NAME} PUBLIC psemek ZLIB::ZLIB)
 	endif()
 endforeach()

examples/srtm.cpp

@@ -35,6 +35,9 @@
 #include <atomic>
 #include <unordered_map>
 #include <unordered_set>
+#include <filesystem>
+
+#include <zlib.h>
 
 // TODO: use LRU cache for tile generation requests, combine with threadpool
 // TODO: fix frustum culling
@@ -97,14 +100,10 @@ struct height_provider
 
 float height_provider::height_at(geom::vector<float, 3> const & v)
 {
-//	static std::string const data_path = "/home/lisyarus/data/srtm/dem/";
+	static std::filesystem::path const data_path = "/home/lisyarus/data/srtm/dem";
-	static std::string const data_path = "/home/lisyarus/data/srtm/test/";
 
 	float const lat = geom::deg(std::asin(v[2]));
 
-//	if (std::abs(lat) > 60.f) return 0.f;
-//	return -10.f;
-
 	float const lon = geom::deg(std::atan2(v[0], -v[1]));
 
 	int ilat = std::floor(lat);
@@ -130,27 +129,34 @@ float height_provider::height_at(geom::vector<float, 3> const & v)
 	{
 		{
 			std::lock_guard lock{datums_mutex};
-			if (datums.size() > 40)
+			if (datums.size() > 100)
 				datums.clear();
 		}
 
 		std::ostringstream os;
-//		os << "ASTGTMV003_";
 		if (ilat >= 0)
-			os << 'N' << std::setw(2) << std::setfill('0') << ilat;
+			os << "N/" << std::setw(2) << std::setfill('0') << ilat << '/';
 		else
-			os << 'S' << std::setw(2) << std::setfill('0') << (-ilat);
+			os << "S/" << std::setw(2) << std::setfill('0') << (-ilat) << '/';
 		if (ilon >= 0)
-			os << 'E' << std::setw(3) << std::setfill('0') << ilon;
+			os << "E/" << std::setw(3) << std::setfill('0') << ilon << '/';
 		else
-			os << 'W' << std::setw(3) << std::setfill('0') << (-ilon);
+			os << "W/" << std::setw(3) << std::setfill('0') << (-ilon) << '/';
 
-		os << ".gray";
+		os << "data.zip";
-//		os << ".zip";
+
+		std::filesystem::path const filename = data_path / os.str();
+
+		if (!std::filesystem::exists(filename) || !std::filesystem::is_regular_file(filename))
+		{
+			std::lock_guard lock{no_datums_mutex};
+			no_datums.insert(id);
+			return 0.f;
+		}
+
+		std::size_t zsize = std::filesystem::file_size(filename);
 
-		std::string const filename = data_path + os.str();
 		std::ifstream ifs(filename, std::ios::binary);
-
 		if (!ifs)
 		{
 			std::lock_guard lock{no_datums_mutex};
@@ -158,10 +164,26 @@ float height_provider::height_at(geom::vector<float, 3> const & v)
 			return 0.f;
 		}
 
-//		return 10.f;
+		std::vector<char> zdata(zsize);
+		ifs.read(zdata.data(), zdata.size());
+		ifs.close();
 
 		std::unique_ptr<std::uint16_t[]> data{new std::uint16_t[3601 * 3601]};
-		ifs.read(reinterpret_cast<char *>(data.get()), 3601 * 3601 * 2);
+
+		z_stream infstream;
+		infstream.zalloc = Z_NULL;
+		infstream.zfree = Z_NULL;
+		infstream.opaque = Z_NULL;
+		// setup "b" as the input and "c" as the compressed output
+		infstream.avail_in = zdata.size(); // size of input
+		infstream.next_in = (Bytef *)zdata.data(); // input char array
+		infstream.avail_out = 3601 * 3601 * 2; // size of output
+		infstream.next_out = (Bytef *)data.get(); // output char array
+
+		// the actual DE-compression work.
+		inflateInit(&infstream);
+		inflate(&infstream, Z_NO_FLUSH);
+		inflateEnd(&infstream);
 
 		{
 			std::lock_guard lock{datums_mutex};
@@ -196,6 +218,40 @@ static constexpr int node_child_size = 1 << node_child_depth;
 static constexpr int node_child_count = node_child_size * node_child_size;
 static constexpr int max_child_level = 20 - node_size_log2 - node_child_depth;
 
+struct node_cache
+{
+	static std::filesystem::path data_path() { return "/home/lisyarus/data/srtm/cache"; }
+
+	std::optional<std::vector<std::int16_t>> load(std::size_t uid);
+	void store(std::size_t uid, std::vector<std::int16_t> const & data);
+};
+
+std::optional<std::vector<std::int16_t>> node_cache::load(std::size_t uid)
+{
+	std::ostringstream oss;
+	oss << std::hex << uid;
+
+	std::ifstream file(data_path() / oss.str(), std::ios::binary);
+	if (!file)
+		return std::nullopt;
+
+	std::vector<std::int16_t> data(((node_size + 2) * (node_size + 1)) / 2);
+	file.read(reinterpret_cast<char *>(data.data()), data.size() * sizeof(data[0]));
+	return data;
+}
+
+void node_cache::store(std::size_t uid, std::vector<std::int16_t> const & data)
+{
+	std::ostringstream oss;
+	oss << std::hex << uid;
+
+	std::ofstream file(data_path() / oss.str(), std::ios::binary);
+	if (!file)
+		return;
+
+	file.write(reinterpret_cast<char const *>(data.data()), data.size() * sizeof(data[0]));
+}
+
 struct node
 {
 	geom::vector<float, 3> v[3];
@@ -208,6 +264,7 @@ struct node
 struct node_controller
 {
 	node_controller();
+	~node_controller();
 
 	node * root(int f);
 
@@ -215,6 +272,8 @@ struct node_controller
 
 	std::size_t loader_queue_size() const { return loader_.queue_size(); }
 
+	void preload(int max_level);
+
 private:
 	struct node_impl
 		: node
@@ -223,6 +282,7 @@ private:
 		gfx::array array;
 		gfx::buffer height_buffer;
 		std::unique_ptr<node> children[node_child_count];
+		std::size_t uid;
 
 		std::vector<std::int16_t> height_data;
 		std::atomic<bool> height_data_ready = false;
@@ -242,8 +302,10 @@ private:
 	std::unique_ptr<node_impl> roots_[20];
 
 	height_provider height_provider_;
+	node_cache node_cache_;
 
-	util::threadpool loader_{"load", 4};
+	util::threadpool loader_{"load", 1};
+	std::atomic<bool> cancel_ = false;
 
 	std::size_t node_count_ = 0;
 
@@ -282,11 +344,17 @@ node_controller::node_controller()
 	index_buffer_.load(indices, gl::STATIC_DRAW);
 }
 
+node_controller::~node_controller()
+{
+	cancel_ = true;
+}
+
 node * node_controller::root(int f)
 {
 	if (!roots_[f])
 	{
 		auto n = make_node();
+		n->uid = (1 << 5) | f;
 
 		auto face = cg::faces(icosahedron_)[f];
 		n->v[0] = icosahedron_.vertices[face[0]] - geom::point<float, 3>::zero();
@@ -299,6 +367,40 @@ node * node_controller::root(int f)
 	return roots_[f].get();
 }
 
+void node_controller::preload(int)
+{
+	std::string strid;
+	auto visit = util::recursive([this, &strid](auto & self, node * n, int level) -> void
+	{
+		auto nn = static_cast<node_impl *>(n);
+		log::info() << "Loading " << strid << "...";
+		util::clock<> clock;
+		nn->load_heights();
+		while (!nn->height_data_ready)
+			std::this_thread::yield();
+		auto const time = clock.count();
+		log::info() << "Loading " << strid << " done (" << time << "s)";
+
+		if (level + 1 < max_child_level && (time >= 0.05 || level < 5))
+		{
+			std::string old_strid = strid;
+			for (int i = 0; i < node_child_count; ++i)
+			{
+				strid = old_strid + "/" + std::to_string(i);
+				self(n->child(i), level + 1);
+			}
+
+			strid = old_strid;
+		}
+	});
+
+	for (int f = 0; f < 20; ++f)
+	{
+		strid = std::to_string(f);
+		visit(root(f), 0);
+	}
+}
+
 std::unique_ptr<node_controller::node_impl> node_controller::make_node()
 {
 	auto n = std::make_unique<node_controller::node_impl>();
@@ -321,7 +423,6 @@ bool node_controller::node_impl::draw(int level)
 	{
 		if (!height_data_loader_dispatched)
 		{
-			height_data_loader_dispatched = true;
 			load_heights();
 		}
 
@@ -348,6 +449,7 @@ node * node_controller::node_impl::child(int id)
 	if (!children[id])
 	{
 		auto n = controller->make_node();
+		n->uid = (uid << (2 * node_child_depth)) | id;
 
 		int i0, j0, i1, j1, i2, j2;
 
@@ -399,7 +501,18 @@ node * node_controller::node_impl::child(int id)
 
 void node_controller::node_impl::load_heights()
 {
+	height_data_loader_dispatched = true;
 	controller->loader_.dispatch([this]{
+		if (controller->cancel_) return;
+
+		auto cached = controller->node_cache_.load(uid);
+		if (cached)
+		{
+			height_data = std::move(*cached);
+			height_data_ready = true;
+			return;
+		}
+
 		height_data.assign(((node_size + 2) * (node_size + 1)) / 2, 0);
 
 		auto * out = height_data.data();
@@ -417,10 +530,14 @@ void node_controller::node_impl::load_heights()
 		{
 			for (int j = 0; j <= i; ++j)
 			{
+				if (controller->cancel_) return;
+
 				*out++ = static_cast<std::int16_t>(controller->height_provider_.height_at(at(i, j)));
 			}
 		}
 
+		controller->node_cache_.store(uid, height_data);
+
 		height_data_ready = true;
 	});
 }
@@ -498,8 +615,8 @@ out vec4 out_color;
 
 void main()
 {
-	float l = (0.5 + dot(normalize(g_normal), u_light) * 0.5);
+	float l = max(0.2, dot(normalize(g_normal), u_light));
-	out_color = vec4(g_color * l, 1.0);
+	out_color = vec4(pow(g_color * l, vec3(1.0 / 2.2)), 1.0);
 })";
 
 static char const tile_far_fs[] =
@@ -514,8 +631,8 @@ out vec4 out_color;
 void main()
 {
 	vec3 normal = cross(dFdx(pos), dFdy(pos));
-	float l = (0.5 + dot(normalize(normal), u_light) * 0.5);
+	float l = max(0.2, dot(normalize(normal), u_light));
-	out_color = vec4(color * l, 1.0);
+	out_color = vec4(pow(color * l, vec3(1.0 / 2.2)), 1.0);
 })";
 
 struct srtm_app
@@ -567,13 +684,26 @@ srtm_app::srtm_app()
 	selected_mesh.setup<geom::point<float, 3>>();
 
 	{
-		util::array<gfx::color_rgb, 1> colors({5});
+		util::array<gfx::color_rgb, 1> colors({16});
 
-		colors(0) = {0, 127, 0};
+		auto * c = colors.data();
-		colors(1) = {127, 127, 0};
+
-		colors(2) = {127, 63, 0};
+		*c++ = {0, 63, 0};
-		colors(3) = {127, 0, 0};
+		*c++ = {0, 127, 0};
-		colors(4) = {191, 191, 191};
+		*c++ = {63, 127, 0};
+		*c++ = {127, 127, 0};
+		*c++ = {95, 95, 0};
+		*c++ = {63, 63, 0};
+		*c++ = {95, 63, 0};
+		*c++ = {127, 95, 0};
+		*c++ = {127, 63, 0};
+		*c++ = {127, 31, 0};
+		*c++ = {127, 0, 0};
+		*c++ = {95, 0, 0};
+		*c++ = {63, 0, 0};
+		*c++ = {191, 191, 191};
+		*c++ = {159, 159, 191};
+		*c++ = {127, 127, 191};
 		color_map.load(colors);
 		color_map.clamp();
 		color_map.linear_filter();
@@ -591,6 +721,8 @@ srtm_app::srtm_app()
 		color_map_neg.clamp();
 		color_map_neg.linear_filter();
 	}
+
+//	nodes.preload(6);
 }
 
 void srtm_app::on_resize(int width, int height)
@@ -615,12 +747,12 @@ void srtm_app::update()
 
 	if (is_key_down(SDLK_q))
 	{
-		camera.rotateXY(-0.04f);
+		camera.rotateXY(- 4.f * dt);
 	}
 
 	if (is_key_down(SDLK_e))
 	{
-		camera.rotateXY(0.04f);
+		camera.rotateXY(4.f * dt);
 	}
 
 	float const camera_speed = std::min(5.f, geom::distance(camera.pos, geom::point<float, 3>::zero()) - 1.f);
@@ -693,7 +825,8 @@ void srtm_app::present()
 
 	std::vector<std::string> info;
 
-	gl::ClearColor(0.9f, 0.9f, 0.9f, 0.f);
+//	gl::ClearColor(0.9f, 0.9f, 0.9f, 0.f);
+	gl::ClearColor(0.0f, 0.0f, 0.0f, 0.f);
 	gl::Clear(gl::COLOR_BUFFER_BIT | gl::DEPTH_BUFFER_BIT);
 
 	gl::LineWidth(2.f);
@@ -721,17 +854,19 @@ void srtm_app::present()
 	auto const frustum = cg::frustum(camera_transform);
 	(void)frustum;
 
+	auto light = geom::normalized(geom::vector{camera_pos[0]-camera_pos[1], camera_pos[1]+camera_pos[0], 0.f});
+
 	tile_close_program.bind();
 	tile_close_program["u_transform"] = camera_transform;
 	tile_close_program["u_N"] = static_cast<int>(node_size);
-	tile_close_program["u_light"] = geom::vector<float, 3>{0.f, 0.f, 1.f};
+	tile_close_program["u_light"] = light;
 	tile_close_program["u_colormap"] = 0;
 	tile_close_program["u_colormap_neg"] = 1;
 	tile_close_program["u_far"] = camera.far_clip;
 	tile_far_program.bind();
 	tile_far_program["u_transform"] = camera_transform;
 	tile_far_program["u_N"] = static_cast<int>(node_size);
-	tile_far_program["u_light"] = geom::vector<float, 3>{0.f, 0.f, 1.f};
+	tile_far_program["u_light"] = light;
 	tile_far_program["u_colormap"] = 0;
 	tile_far_program["u_colormap_neg"] = 1;
 	tile_far_program["u_far"] = camera.far_clip;
@@ -798,12 +933,15 @@ void srtm_app::present()
 			auto c = camera_pos - o;
 			if (geom::det(v[0], v[1], c) >= 0.f && geom::det(v[1], v[2], c) >= 0.f && geom::det(v[2], v[0], c) >= 0.f)
 				distance = std::min(distance, geom::length(c) - 1.f);
-			if (auto d = edge(v[0], v[1], c); d)
+			else
-				distance = std::min(distance, *d);
+			{
-			if (auto d = edge(v[1], v[2], c); d)
+				if (auto d = edge(v[0], v[1], c); d)
-				distance = std::min(distance, *d);
+					distance = std::min(distance, *d);
-			if (auto d = edge(v[2], v[0], c); d)
+				if (auto d = edge(v[1], v[2], c); d)
-				distance = std::min(distance, *d);
+					distance = std::min(distance, *d);
+				if (auto d = edge(v[2], v[0], c); d)
+					distance = std::min(distance, *d);
+			}
 			distance = std::min(distance, geom::length(c - v[0]));
 			distance = std::min(distance, geom::length(c - v[1]));
 			distance = std::min(distance, geom::length(c - v[2]));
@@ -897,7 +1035,7 @@ void srtm_app::present()
 		opts.x = gfx::painter::x_align::left;
 		opts.y = gfx::painter::y_align::top;
 		opts.f = gfx::painter::font::font_9x12;
-		opts.c = gfx::cyan;
+		opts.c = gfx::gray;
 		opts.scale = 2.f;
 
 		for (int l = 0; l < info.size(); ++l)
@@ -905,7 +1043,7 @@ void srtm_app::present()
 			painter.text({10.f + 1.f, 10.f + 24.f * l + 1.f}, info[l], opts);
 		}
 
-		opts.c = gfx::black;
+		opts.c = gfx::white;
 
 		for (int l = 0; l < info.size(); ++l)
 		{