From 036e653f01062ea2f76badd1fc511afe952bf64b Mon Sep 17 00:00:00 2001 From: Aang23 Date: Fri, 16 Feb 2024 22:47:18 +0100 Subject: [PATCH] Allow loading products/datasets over http(s) --- pipelines/IM-1.json | 3 +- .../noaa_apt/module_noaa_apt_decoder.cpp | 1 + .../eos/module_eos_instruments.cpp | 1 + .../fengyun3/module_fy3_instruments.cpp | 1 + .../hinode/module_hinode_instruments.cpp | 1 + .../jpss/module_jpss_instruments.cpp | 1 + .../msumr/module_meteor_msumr_lrpt.cpp | 1 + .../meteor/module_meteor_dump_instruments.cpp | 1 + .../meteor/module_meteor_instruments.cpp | 1 + .../metop/module_metop_instruments.cpp | 1 + src-core/products/dataset.cpp | 37 +++++++++++++++++++ src-core/products/dataset.h | 20 +--------- src-core/products/image_products.cpp | 37 +++++++++++++++++-- src-core/products/products.cpp | 29 +++++++++++---- src-interface/viewer/viewer.cpp | 12 ++++-- tools/proj_test/main.cpp | 2 +- 16 files changed, 114 insertions(+), 35 deletions(-) create mode 100644 src-core/products/dataset.cpp diff --git a/pipelines/IM-1.json b/pipelines/IM-1.json index 9788641f2..ddd34fb54 100644 --- a/pipelines/IM-1.json +++ b/pipelines/IM-1.json @@ -34,7 +34,8 @@ "rs_type": "rs239", "rs_usecheck": true }*/ - /* "ccsds_conv_concat_decoder": { + /*"ccsds_conv_concat_decoder": { + "conv_rate": "7/8", "constellation": "bpsk_90", "cadu_size": 2048, "viterbi_ber_thresold": 0.3, diff --git a/plugins/analog_support/noaa_apt/module_noaa_apt_decoder.cpp b/plugins/analog_support/noaa_apt/module_noaa_apt_decoder.cpp index 90a6a977f..44a29c24a 100644 --- a/plugins/analog_support/noaa_apt/module_noaa_apt_decoder.cpp +++ b/plugins/analog_support/noaa_apt/module_noaa_apt_decoder.cpp @@ -6,6 +6,7 @@ #include "common/utils.h" #include "products/image_products.h" #include "products/dataset.h" +#include "nlohmann/json_utils.h" #include "common/dsp/filter/firdes.h" #include "resources.h" diff --git a/plugins/eos_support/eos/module_eos_instruments.cpp b/plugins/eos_support/eos/module_eos_instruments.cpp index d6f2c3f70..9fcaf84a6 100644 --- a/plugins/eos_support/eos/module_eos_instruments.cpp +++ b/plugins/eos_support/eos/module_eos_instruments.cpp @@ -11,6 +11,7 @@ #include "products/dataset.h" #include "resources.h" #include "instruments/modis/modis_histmatch.h" +#include "nlohmann/json_utils.h" #include "common/calibration.h" #include "instruments/modis/calibrator/modis_calibrator.h" diff --git a/plugins/fengyun3_support/fengyun3/module_fy3_instruments.cpp b/plugins/fengyun3_support/fengyun3/module_fy3_instruments.cpp index 73c2e3570..898dc8c3e 100644 --- a/plugins/fengyun3_support/fengyun3/module_fy3_instruments.cpp +++ b/plugins/fengyun3_support/fengyun3/module_fy3_instruments.cpp @@ -14,6 +14,7 @@ #include "resources.h" #include "instruments/mersi_histmatch.h" #include "instruments/mersi_offset_interleaved.h" +#include "nlohmann/json_utils.h" namespace fengyun3 { diff --git a/plugins/hinode_support/hinode/module_hinode_instruments.cpp b/plugins/hinode_support/hinode/module_hinode_instruments.cpp index d1a41b0c6..4cbf6734b 100644 --- a/plugins/hinode_support/hinode/module_hinode_instruments.cpp +++ b/plugins/hinode_support/hinode/module_hinode_instruments.cpp @@ -8,6 +8,7 @@ #include "common/ccsds/ccsds_weather/demuxer.h" #include "products/products.h" #include "products/dataset.h" +#include "nlohmann/json_utils.h" #include "common/codings/reedsolomon/reedsolomon.h" diff --git a/plugins/jpss_support/jpss/module_jpss_instruments.cpp b/plugins/jpss_support/jpss/module_jpss_instruments.cpp index b229ae40b..f8702ed4c 100644 --- a/plugins/jpss_support/jpss/module_jpss_instruments.cpp +++ b/plugins/jpss_support/jpss/module_jpss_instruments.cpp @@ -13,6 +13,7 @@ #include "products/dataset.h" #include "resources.h" #include "common/calibration.h" +#include "nlohmann/json_utils.h" namespace jpss { diff --git a/plugins/meteor_support/meteor/instruments/msumr/module_meteor_msumr_lrpt.cpp b/plugins/meteor_support/meteor/instruments/msumr/module_meteor_msumr_lrpt.cpp index d2afa3fab..d27f434d8 100644 --- a/plugins/meteor_support/meteor/instruments/msumr/module_meteor_msumr_lrpt.cpp +++ b/plugins/meteor_support/meteor/instruments/msumr/module_meteor_msumr_lrpt.cpp @@ -14,6 +14,7 @@ #include "products/dataset.h" #include "resources.h" #include "common/utils.h" +#include "nlohmann/json_utils.h" #define BUFFER_SIZE 8192 diff --git a/plugins/meteor_support/meteor/module_meteor_dump_instruments.cpp b/plugins/meteor_support/meteor/module_meteor_dump_instruments.cpp index f21d1176b..c3e626fc1 100644 --- a/plugins/meteor_support/meteor/module_meteor_dump_instruments.cpp +++ b/plugins/meteor_support/meteor/module_meteor_dump_instruments.cpp @@ -9,6 +9,7 @@ #include "common/tracking/tle.h" #include "products/dataset.h" #include "resources.h" +#include "nlohmann/json_utils.h" namespace meteor { diff --git a/plugins/meteor_support/meteor/module_meteor_instruments.cpp b/plugins/meteor_support/meteor/module_meteor_instruments.cpp index 1fb804b14..923360996 100644 --- a/plugins/meteor_support/meteor/module_meteor_instruments.cpp +++ b/plugins/meteor_support/meteor/module_meteor_instruments.cpp @@ -10,6 +10,7 @@ #include "common/tracking/tle.h" #include "products/dataset.h" #include "resources.h" +#include "nlohmann/json_utils.h" namespace meteor { diff --git a/plugins/noaa_metop_support/metop/module_metop_instruments.cpp b/plugins/noaa_metop_support/metop/module_metop_instruments.cpp index 5447f9cf5..7cfbf89ad 100644 --- a/plugins/noaa_metop_support/metop/module_metop_instruments.cpp +++ b/plugins/noaa_metop_support/metop/module_metop_instruments.cpp @@ -14,6 +14,7 @@ #include "products/dataset.h" #include "common/tracking/tle.h" #include "resources.h" +#include "nlohmann/json_utils.h" namespace metop { diff --git a/src-core/products/dataset.cpp b/src-core/products/dataset.cpp new file mode 100644 index 000000000..6c61730f8 --- /dev/null +++ b/src-core/products/dataset.cpp @@ -0,0 +1,37 @@ +#include "dataset.h" +#include "nlohmann/json_utils.h" +#include "common/utils.h" + +namespace satdump +{ + void ProductDataSet::save(std::string path) + { + nlohmann::json json_ojb; + json_ojb["satellite"] = satellite_name; + json_ojb["timestamp"] = timestamp; + json_ojb["products"] = products_list; + + saveJsonFile(path + "/dataset.json", json_ojb); + } + + void ProductDataSet::load(std::string path) + { + nlohmann::json json_ojb; + + if (path.find("http") == 0) + { + std::string res; + if (perform_http_request(path, res)) + throw std::runtime_error("Could not download from : " + path); + json_ojb = nlohmann::json::parse(res); + } + else + { + json_ojb = loadJsonFile(path); + } + + satellite_name = json_ojb["satellite"].get(); + timestamp = json_ojb["timestamp"].get(); + products_list = json_ojb["products"].get>(); + } +} diff --git a/src-core/products/dataset.h b/src-core/products/dataset.h index c6960facc..86fb382e5 100644 --- a/src-core/products/dataset.h +++ b/src-core/products/dataset.h @@ -1,7 +1,6 @@ #pragma once #include "nlohmann/json.hpp" -#include "nlohmann/json_utils.h" namespace satdump { @@ -11,22 +10,7 @@ namespace satdump double timestamp; // Rough UTC Timestamp of the dataset std::vector products_list; // List of products to load, relative to this file - inline void save(std::string path) - { - nlohmann::json json_ojb; - json_ojb["satellite"] = satellite_name; - json_ojb["timestamp"] = timestamp; - json_ojb["products"] = products_list; - - saveJsonFile(path + "/dataset.json", json_ojb); - } - - inline void load(std::string path) - { - nlohmann::json json_ojb = loadJsonFile(path); - satellite_name = json_ojb["satellite"].get(); - timestamp = json_ojb["timestamp"].get(); - products_list = json_ojb["products"].get>(); - } + void save(std::string path); + void load(std::string path); }; } diff --git a/src-core/products/image_products.cpp b/src-core/products/image_products.cpp index 2040e1dc4..675fbe506 100644 --- a/src-core/products/image_products.cpp +++ b/src-core/products/image_products.cpp @@ -9,6 +9,7 @@ #include "common/lua/lua_utils.h" #include "common/calibration.h" #include "core/plugin.h" +#include "common/utils.h" namespace satdump { @@ -98,11 +99,26 @@ namespace satdump if (contents.contains("save_as_matrix")) save_as_matrix = contents["save_as_matrix"].get(); + std::string tmp_path = std::filesystem::temp_directory_path().string(); + image::Image img_matrix; if (save_as_matrix) { - if (std::filesystem::exists(directory + "/" + contents["images"][0]["file"].get())) - img_matrix.load_img(directory + "/" + contents["images"][0]["file"].get()); + if (file.find("http") == 0) + { + std::string res; + if (perform_http_request(directory + "/" + contents["images"][0]["file"].get(), res)) + throw std::runtime_error("Could not download from : " + directory + "/" + contents["images"][0]["file"].get()); + std::ofstream(tmp_path + "/satdumpdltmp.tmp", std::ios::binary).write((char *)res.data(), res.size()); + img_matrix.load_img(tmp_path + "/satdumpdltmp.tmp"); + if (std::filesystem::exists(tmp_path + "/satdumpdltmp.tmp")) + std::filesystem::remove(tmp_path + "/satdumpdltmp.tmp"); + } + else + { + if (std::filesystem::exists(directory + "/" + contents["images"][0]["file"].get())) + img_matrix.load_img(directory + "/" + contents["images"][0]["file"].get()); + } } for (size_t c = 0; c < contents["images"].size(); c++) @@ -116,8 +132,21 @@ namespace satdump if (!save_as_matrix) { - if (std::filesystem::exists(directory + "/" + contents["images"][c]["file"].get())) - img_holder.image.load_img(directory + "/" + contents["images"][c]["file"].get()); + if (file.find("http") == 0) + { + std::string res; + if (perform_http_request(directory + "/" + contents["images"][c]["file"].get(), res)) + throw std::runtime_error("Could not download from : " + directory + "/" + contents["images"][c]["file"].get()); + std::ofstream(tmp_path + "/satdumpdltmp.tmp", std::ios::binary).write((char *)res.data(), res.size()); + img_holder.image.load_img(tmp_path + "/satdumpdltmp.tmp"); + if (std::filesystem::exists(tmp_path + "/satdumpdltmp.tmp")) + std::filesystem::remove(tmp_path + "/satdumpdltmp.tmp"); + } + else + { + if (std::filesystem::exists(directory + "/" + contents["images"][c]["file"].get())) + img_holder.image.load_img(directory + "/" + contents["images"][c]["file"].get()); + } } else { diff --git a/src-core/products/products.cpp b/src-core/products/products.cpp index 2396d6dac..99d93966e 100644 --- a/src-core/products/products.cpp +++ b/src-core/products/products.cpp @@ -9,6 +9,7 @@ #include "processor/image_processor.h" #include "processor/radiation_processor.h" #include "processor/scatterometer_processor.h" +#include "common/utils.h" namespace satdump { @@ -32,15 +33,26 @@ namespace satdump logger->info(file); // Read file std::vector cbor_data; - std::ifstream in_file(file, std::ios::binary); - while (!in_file.eof()) + if (file.find("http") == 0) { - uint8_t b; - in_file.read((char *)&b, 1); - cbor_data.push_back(b); + std::string res; + if (perform_http_request(file, res)) + throw std::runtime_error("Could not download from : " + file); + for (auto &v : res) + cbor_data.push_back(*((uint8_t *)&v)); + } + else + { + std::ifstream in_file(file, std::ios::binary); + while (!in_file.eof()) + { + uint8_t b; + in_file.read((char *)&b, 1); + cbor_data.push_back(b); + } + in_file.close(); + cbor_data.pop_back(); } - in_file.close(); - cbor_data.pop_back(); contents = nlohmann::json::from_cbor(cbor_data); instrument_name = contents["instrument"].get(); @@ -56,7 +68,8 @@ namespace satdump std::shared_ptr final_products; Products raw_products; - if (std::filesystem::is_directory(path)) + if (std::filesystem::is_directory(path) || + (path.find("http") == 0 && path.find_last_of("cbor") != path.size() - 5)) path = path + "/product.cbor"; raw_products.load(path); diff --git a/src-interface/viewer/viewer.cpp b/src-interface/viewer/viewer.cpp index d13c6ed22..8d69eea2e 100644 --- a/src-interface/viewer/viewer.cpp +++ b/src-interface/viewer/viewer.cpp @@ -110,7 +110,7 @@ namespace satdump void ViewerApplication::loadProductsInViewer(std::string path, std::string dataset_name) { - if (std::filesystem::exists(path)) + if (std::filesystem::exists(path) || path.find("http") == 0) { std::shared_ptr products = loadProducts(path); @@ -301,7 +301,10 @@ namespace satdump ImGui::Separator(); ImGui::Text("Load Dataset :"); - if (select_dataset_dialog.draw()) + bool vd = select_dataset_dialog.draw(); + ImGui::SameLine(); + vd |= ImGui::Button("Load##datasetdialog"); + if (vd) { ui_thread_pool.push([this](int) { @@ -315,7 +318,10 @@ namespace satdump } }); } ImGui::Text("Load Products :"); - if (select_products_dialog.draw()) + bool vp = select_products_dialog.draw(); + ImGui::SameLine(); + vp |= ImGui::Button("Load##productdialog"); + if (vp) { ui_thread_pool.push([this](int) { diff --git a/tools/proj_test/main.cpp b/tools/proj_test/main.cpp index 0d94a5e2f..c2b658459 100644 --- a/tools/proj_test/main.cpp +++ b/tools/proj_test/main.cpp @@ -57,7 +57,7 @@ int main(int /*argc*/, char *argv[]) printf("\n%s\n", img_pro.contents.dump(4).c_str()); satdump::ImageCompositeCfg rgb_cfg; - rgb_cfg.equation = "ch2,ch2,ch1"; //"(ch7421+ch7422+ch7423+ch7242)/4"; + rgb_cfg.equation = "1-ch4"; //"(ch7421+ch7422+ch7423+ch7242)/4"; // rgb_cfg.equation = "1-ch37"; // rgb_cfg.equation = "1-ch33,1-ch34,1-ch35"; //"(ch3 * 0.4 + ch2 * 0.6) * 2.2 - 0.15, ch2 * 2.2 - 0.15, ch1 * 2.2 - 0.15"; rgb_cfg.individual_equalize = true;