ham-ref/satdump
1
0
Fork 0
mirror of https://github.com/SatDump/SatDump synced 2026-08-13 17:47:30 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Allow loading products/datasets over http(s)

Browse source
This commit is contained in:
Aang23 2024-02-16 22:47:18 +01:00
parent 01178ec5f3
commit 036e653f01
16 changed files with 114 additions and 35 deletions
Download patch file Download diff file Expand all files Collapse all files

3
pipelines/IM-1.json
View file

@ -34,7 +34,8 @@
"rs_type": "rs239",
"rs_usecheck": true
}*/
/* "ccsds_conv_concat_decoder": {
/*"ccsds_conv_concat_decoder": {
"conv_rate": "7/8",
"constellation": "bpsk_90",
"cadu_size": 2048,
"viterbi_ber_thresold": 0.3,

1
plugins/analog_support/noaa_apt/module_noaa_apt_decoder.cpp
View file

@ -6,6 +6,7 @@
#include "common/utils.h"
#include "products/image_products.h"
#include "products/dataset.h"
#include "nlohmann/json_utils.h"
#include "common/dsp/filter/firdes.h"
#include "resources.h"

1
plugins/eos_support/eos/module_eos_instruments.cpp
View file

@ -11,6 +11,7 @@
#include "products/dataset.h"
#include "resources.h"
#include "instruments/modis/modis_histmatch.h"
#include "nlohmann/json_utils.h"
#include "common/calibration.h"
#include "instruments/modis/calibrator/modis_calibrator.h"

1
plugins/fengyun3_support/fengyun3/module_fy3_instruments.cpp
View file

@ -14,6 +14,7 @@
#include "resources.h"
#include "instruments/mersi_histmatch.h"
#include "instruments/mersi_offset_interleaved.h"
#include "nlohmann/json_utils.h"
namespace fengyun3
{

1
plugins/hinode_support/hinode/module_hinode_instruments.cpp
View file

@ -8,6 +8,7 @@
#include "common/ccsds/ccsds_weather/demuxer.h"
#include "products/products.h"
#include "products/dataset.h"
#include "nlohmann/json_utils.h"
#include "common/codings/reedsolomon/reedsolomon.h"

1
plugins/jpss_support/jpss/module_jpss_instruments.cpp
View file

@ -13,6 +13,7 @@
#include "products/dataset.h"
#include "resources.h"
#include "common/calibration.h"
#include "nlohmann/json_utils.h"
namespace jpss
{

1
plugins/meteor_support/meteor/instruments/msumr/module_meteor_msumr_lrpt.cpp
View file

@ -14,6 +14,7 @@
#include "products/dataset.h"
#include "resources.h"
#include "common/utils.h"
#include "nlohmann/json_utils.h"
#define BUFFER_SIZE 8192

1
plugins/meteor_support/meteor/module_meteor_dump_instruments.cpp
View file

@ -9,6 +9,7 @@
#include "common/tracking/tle.h"
#include "products/dataset.h"
#include "resources.h"
#include "nlohmann/json_utils.h"
namespace meteor
{

1
plugins/meteor_support/meteor/module_meteor_instruments.cpp
View file

@ -10,6 +10,7 @@
#include "common/tracking/tle.h"
#include "products/dataset.h"
#include "resources.h"
#include "nlohmann/json_utils.h"
namespace meteor
{

1
plugins/noaa_metop_support/metop/module_metop_instruments.cpp
View file

@ -14,6 +14,7 @@
#include "products/dataset.h"
#include "common/tracking/tle.h"
#include "resources.h"
#include "nlohmann/json_utils.h"
namespace metop
{

37
src-core/products/dataset.cpp Normal file
View file

@ -0,0 +1,37 @@
#include "dataset.h"
#include "nlohmann/json_utils.h"
#include "common/utils.h"
namespace satdump
{
void ProductDataSet::save(std::string path)
{
nlohmann::json json_ojb;
json_ojb["satellite"] = satellite_name;
json_ojb["timestamp"] = timestamp;
json_ojb["products"] = products_list;
saveJsonFile(path + "/dataset.json", json_ojb);
}
void ProductDataSet::load(std::string path)
{
nlohmann::json json_ojb;
if (path.find("http") == 0)
{
std::string res;
if (perform_http_request(path, res))
throw std::runtime_error("Could not download from : " + path);
json_ojb = nlohmann::json::parse(res);
}
else
{
json_ojb = loadJsonFile(path);
}
satellite_name = json_ojb["satellite"].get<std::string>();
timestamp = json_ojb["timestamp"].get<double>();
products_list = json_ojb["products"].get<std::vector<std::string>>();
}
}

20
src-core/products/dataset.h
View file

@ -1,7 +1,6 @@
#pragma once
#include "nlohmann/json.hpp"
#include "nlohmann/json_utils.h"
namespace satdump
{
@ -11,22 +10,7 @@ namespace satdump
double timestamp; // Rough UTC Timestamp of the dataset
std::vector<std::string> products_list; // List of products to load, relative to this file
inline void save(std::string path)
{
nlohmann::json json_ojb;
json_ojb["satellite"] = satellite_name;
json_ojb["timestamp"] = timestamp;
json_ojb["products"] = products_list;
saveJsonFile(path + "/dataset.json", json_ojb);
}
inline void load(std::string path)
{
nlohmann::json json_ojb = loadJsonFile(path);
satellite_name = json_ojb["satellite"].get<std::string>();
timestamp = json_ojb["timestamp"].get<double>();
products_list = json_ojb["products"].get<std::vector<std::string>>();
}
void save(std::string path);
void load(std::string path);
};
}

37
src-core/products/image_products.cpp
View file

@ -9,6 +9,7 @@
#include "common/lua/lua_utils.h"
#include "common/calibration.h"
#include "core/plugin.h"
#include "common/utils.h"
namespace satdump
{
@ -98,11 +99,26 @@ namespace satdump
if (contents.contains("save_as_matrix"))
save_as_matrix = contents["save_as_matrix"].get<bool>();
std::string tmp_path = std::filesystem::temp_directory_path().string();
image::Image<uint16_t> img_matrix;
if (save_as_matrix)
{
if (std::filesystem::exists(directory + "/" + contents["images"][0]["file"].get<std::string>()))
img_matrix.load_img(directory + "/" + contents["images"][0]["file"].get<std::string>());
if (file.find("http") == 0)
{
std::string res;
if (perform_http_request(directory + "/" + contents["images"][0]["file"].get<std::string>(), res))
throw std::runtime_error("Could not download from : " + directory + "/" + contents["images"][0]["file"].get<std::string>());
std::ofstream(tmp_path + "/satdumpdltmp.tmp", std::ios::binary).write((char *)res.data(), res.size());
img_matrix.load_img(tmp_path + "/satdumpdltmp.tmp");
if (std::filesystem::exists(tmp_path + "/satdumpdltmp.tmp"))
std::filesystem::remove(tmp_path + "/satdumpdltmp.tmp");
}
else
{
if (std::filesystem::exists(directory + "/" + contents["images"][0]["file"].get<std::string>()))
img_matrix.load_img(directory + "/" + contents["images"][0]["file"].get<std::string>());
}
}
for (size_t c = 0; c < contents["images"].size(); c++)
@ -116,8 +132,21 @@ namespace satdump
if (!save_as_matrix)
{
if (std::filesystem::exists(directory + "/" + contents["images"][c]["file"].get<std::string>()))
img_holder.image.load_img(directory + "/" + contents["images"][c]["file"].get<std::string>());
if (file.find("http") == 0)
{
std::string res;
if (perform_http_request(directory + "/" + contents["images"][c]["file"].get<std::string>(), res))
throw std::runtime_error("Could not download from : " + directory + "/" + contents["images"][c]["file"].get<std::string>());
std::ofstream(tmp_path + "/satdumpdltmp.tmp", std::ios::binary).write((char *)res.data(), res.size());
img_holder.image.load_img(tmp_path + "/satdumpdltmp.tmp");
if (std::filesystem::exists(tmp_path + "/satdumpdltmp.tmp"))
std::filesystem::remove(tmp_path + "/satdumpdltmp.tmp");
}
else
{
if (std::filesystem::exists(directory + "/" + contents["images"][c]["file"].get<std::string>()))
img_holder.image.load_img(directory + "/" + contents["images"][c]["file"].get<std::string>());
}
}
else
{

29
src-core/products/products.cpp
View file

@ -9,6 +9,7 @@
#include "processor/image_processor.h"
#include "processor/radiation_processor.h"
#include "processor/scatterometer_processor.h"
#include "common/utils.h"
namespace satdump
{
@ -32,15 +33,26 @@ namespace satdump
logger->info(file);
// Read file
std::vector<uint8_t> cbor_data;
std::ifstream in_file(file, std::ios::binary);
while (!in_file.eof())
if (file.find("http") == 0)
{
uint8_t b;
in_file.read((char *)&b, 1);
cbor_data.push_back(b);
std::string res;
if (perform_http_request(file, res))
throw std::runtime_error("Could not download from : " + file);
for (auto &v : res)
cbor_data.push_back(*((uint8_t *)&v));
}
else
{
std::ifstream in_file(file, std::ios::binary);
while (!in_file.eof())
{
uint8_t b;
in_file.read((char *)&b, 1);
cbor_data.push_back(b);
}
in_file.close();
cbor_data.pop_back();
}
in_file.close();
cbor_data.pop_back();
contents = nlohmann::json::from_cbor(cbor_data);
instrument_name = contents["instrument"].get<std::string>();
@ -56,7 +68,8 @@ namespace satdump
std::shared_ptr<Products> final_products;
Products raw_products;
if (std::filesystem::is_directory(path))
if (std::filesystem::is_directory(path) ||
(path.find("http") == 0 && path.find_last_of("cbor") != path.size() - 5))
path = path + "/product.cbor";
raw_products.load(path);

12
src-interface/viewer/viewer.cpp
View file

@ -110,7 +110,7 @@ namespace satdump
void ViewerApplication::loadProductsInViewer(std::string path, std::string dataset_name)
{
if (std::filesystem::exists(path))
if (std::filesystem::exists(path) || path.find("http") == 0)
{
std::shared_ptr<Products> products = loadProducts(path);
@ -301,7 +301,10 @@ namespace satdump
ImGui::Separator();
ImGui::Text("Load Dataset :");
if (select_dataset_dialog.draw())
bool vd = select_dataset_dialog.draw();
ImGui::SameLine();
vd |= ImGui::Button("Load##datasetdialog");
if (vd)
{
ui_thread_pool.push([this](int)
{
@ -315,7 +318,10 @@ namespace satdump
} });
}
ImGui::Text("Load Products :");
if (select_products_dialog.draw())
bool vp = select_products_dialog.draw();
ImGui::SameLine();
vp |= ImGui::Button("Load##productdialog");
if (vp)
{
ui_thread_pool.push([this](int)
{

2
tools/proj_test/main.cpp
View file

@ -57,7 +57,7 @@ int main(int /*argc*/, char *argv[])
printf("\n%s\n", img_pro.contents.dump(4).c_str());
satdump::ImageCompositeCfg rgb_cfg;
rgb_cfg.equation = "ch2,ch2,ch1"; //"(ch7421+ch7422+ch7423+ch7242)/4";
rgb_cfg.equation = "1-ch4"; //"(ch7421+ch7422+ch7423+ch7242)/4";
// rgb_cfg.equation = "1-ch37";
// rgb_cfg.equation = "1-ch33,1-ch34,1-ch35"; //"(ch3 * 0.4 + ch2 * 0.6) * 2.2 - 0.15, ch2 * 2.2 - 0.15, ch1 * 2.2 - 0.15";
rgb_cfg.individual_equalize = true;