From f9a679ffc2d8400820ab4ab74acdfd4768e66b17 Mon Sep 17 00:00:00 2001 From: attermann Date: Tue, 5 Dec 2023 11:06:49 -0700 Subject: [PATCH] WIP: Initial implementation of persistence Persistence implemented with the help of the ArduinoJson library using MessagePack format. --- library.json | 34 +++ platformio.ini | 2 + src/Bytes.h | 14 + src/Identity.cpp | 72 +++-- src/Identity.h | 2 + src/Interface.cpp | 12 + src/Interface.h | 13 + src/Packet.cpp | 12 + src/Packet.h | 13 + src/Reticulum.cpp | 17 +- src/Reticulum.h | 14 + src/Test/Test.cpp | 3 +- src/Test/Test.h | 1 + src/Test/TestPersistence.cpp | 502 +++++++++++++++++++++++++++++++++++ src/Transport.cpp | 18 +- src/Transport.h | 86 ++++-- src/Utilities/OS.cpp | 144 ++++++++++ src/Utilities/OS.h | 15 ++ src/Utilities/Persistence.h | 89 +++++++ src/main.cpp | 22 +- 20 files changed, 1031 insertions(+), 54 deletions(-) create mode 100644 library.json create mode 100644 src/Test/TestPersistence.cpp create mode 100644 src/Utilities/OS.cpp create mode 100644 src/Utilities/Persistence.h diff --git a/library.json b/library.json new file mode 100644 index 0000000..0f7509b --- /dev/null +++ b/library.json @@ -0,0 +1,34 @@ +{ + "name": "reticulum-cpp", + "version": "0.1.1", + "description": "Port of Reticulum Network Stack to C++ specifically but not exclusively targeting 32-bit and better MCUs", + "keywords": "reticulum, mcu, esp32", + "repository": + { + "type": "git", + "url": "https://github.com/attermann/reticulum-cpp.git" + }, + "authors": + [ + { + "name": "Chad Attermann", + "email": "attermann@gmail.com", + "maintainer": true + } + ], + "license": "Apache-2.0", + "homepage": "https://reticulum.network/", + "dependencies": { + "ArduinoJson": "~6.21.3", + "rweather/Crypto": "~0.4.0", + "sandeepmistry/LoRa": "~0.8.0", + "WiFi": "~2.0.0" + }, + "frameworks": "*", + "platforms": "*", + "export": {}, + "exclude": [ + ".github", + "src/main.cpp" + ] +} \ No newline at end of file diff --git a/platformio.ini b/platformio.ini index 792e58a..bfa6dc2 100644 --- a/platformio.ini +++ b/platformio.ini @@ -23,6 +23,7 @@ build_flags = -Isrc -DNATIVE lib_deps = + ArduinoJson@^6.21.3 rweather/Crypto@^0.4.0 lib_compat_mode = off @@ -54,6 +55,7 @@ build_flags = -Wno-format -Isrc lib_deps = + ArduinoJson@^6.21.3 rweather/Crypto@^0.4.0 sandeepmistry/LoRa@^0.8.0 WiFi@^2.0.0 diff --git a/src/Bytes.h b/src/Bytes.h index aa6fb0c..8408c05 100644 --- a/src/Bytes.h +++ b/src/Bytes.h @@ -2,6 +2,8 @@ #include "Log.h" +#include + #include #include #include @@ -257,3 +259,15 @@ inline RNS::Bytes& operator << (RNS::Bytes& lhbytes, uint8_t rhbyte) { lhbytes.append(rhbyte); return lhbytes; } + +namespace ArduinoJson { + inline bool convertToJson(const RNS::Bytes& src, JsonVariant dst) { + return dst.set(src.toHex()); + } + inline void convertFromJson(JsonVariantConst src, RNS::Bytes& dst) { + dst.assignHex(src.as()); + } + inline bool canConvertFromJson(JsonVariantConst src, const RNS::Bytes&) { + return src.is(); + } +} diff --git a/src/Identity.cpp b/src/Identity.cpp index c55aae0..21c5935 100644 --- a/src/Identity.cpp +++ b/src/Identity.cpp @@ -36,16 +36,16 @@ void Identity::createKeys() { _object->_prv_bytes = _object->_prv->private_bytes(); debug("Identity::createKeys: prv bytes: " + _object->_prv_bytes.toHex()); - // CRYPTO: create encryption public keys - _object->_pub = _object->_prv->public_key(); - _object->_pub_bytes = _object->_pub->public_bytes(); - debug("Identity::createKeys: pub bytes: " + _object->_pub_bytes.toHex()); - // CRYPTO: create signature private keys _object->_sig_prv = Cryptography::Ed25519PrivateKey::generate(); _object->_sig_prv_bytes = _object->_sig_prv->private_bytes(); debug("Identity::createKeys: sig prv bytes: " + _object->_sig_prv_bytes.toHex()); + // CRYPTO: create encryption public keys + _object->_pub = _object->_prv->public_key(); + _object->_pub_bytes = _object->_pub->public_bytes(); + debug("Identity::createKeys: pub bytes: " + _object->_pub_bytes.toHex()); + // CRYPTO: create signature public keys _object->_sig_pub = _object->_sig_prv->public_key(); _object->_sig_pub_bytes = _object->_sig_pub->public_bytes(); @@ -127,20 +127,58 @@ void Identity::load_public_key(const Bytes& pub_bytes) { } bool Identity::load(const char* path) { -/* - try: - with open(path, "rb") as key_file: - prv_bytes = key_file.read() - return self.load_private_key(prv_bytes) - return False - except Exception as e: - RNS.log("Error while loading identity from "+str(path), RNS.LOG_ERROR) - RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR) -*/ - // MOCK - return true; + extreme("Reading identity key from storage..."); + try { + Bytes prv_bytes = OS::read_file(path); + if (prv_bytes) { + return load_private_key(prv_bytes); + } + else { + return false; + } + } + catch (std::exception& e) { + error("Error while loading identity from " + std::string(path)); + error("The contained exception was: " + std::string(e.what())); + } + return false; } +/* +Saves the identity to a file. This will write the private key to disk, +and anyone with access to this file will be able to decrypt all +communication for the identity. Be very careful with this method. + +:param path: The full path specifying where to save the identity. +:returns: True if the file was saved, otherwise False. +*/ +bool Identity::to_file(const char* path) { + extreme("Writing identity key to storage..."); + try { + return OS::write_file(get_private_key(), path); + } + catch (std::exception& e) { + error("Error while saving identity to " + std::string(path)); + error("The contained exception was: " + std::string(e.what())); + } + return false; +} + + +/* +Create a new :ref:`RNS.Identity` instance from a file. +Can be used to load previously created and saved identities into Reticulum. + +:param path: The full path to the saved :ref:`RNS.Identity` data +:returns: A :ref:`RNS.Identity` instance, or *None* if the loaded data was invalid. +*/ +/*static*/ const Identity Identity::from_file(const char* path) { + Identity identity(false); + if (identity.load(path)) { + return identity; + } + return {Type::NONE}; +} /*static*/ void Identity::remember(const Bytes& packet_hash, const Bytes& destination_hash, const Bytes& public_key, const Bytes& app_data /*= {Bytes::NONE}*/) { if (public_key.size() != Type::Identity::KEYSIZE/8) { diff --git a/src/Identity.h b/src/Identity.h index 97ec6eb..075c032 100644 --- a/src/Identity.h +++ b/src/Identity.h @@ -90,6 +90,7 @@ namespace RNS { _object->_hexhash = _object->_hash.toHex(); }; bool load(const char* path); + bool to_file(const char* path); inline const Bytes get_salt() const { assert(_object); return _object->_hash; } inline const Bytes get_context() const { return {Bytes::NONE}; } @@ -103,6 +104,7 @@ namespace RNS { void prove(const Packet& packet, const Destination& destination) const; void prove(const Packet& packet) const; + static const Identity from_file(const char* path); static void remember(const Bytes& packet_hash, const Bytes& destination_hash, const Bytes& public_key, const Bytes& app_data = {Bytes::NONE}); static Identity recall(const Bytes& destination_hash); static Bytes recall_app_data(const Bytes& destination_hash); diff --git a/src/Interface.cpp b/src/Interface.cpp index f47fbf8..0e6c21d 100644 --- a/src/Interface.cpp +++ b/src/Interface.cpp @@ -73,3 +73,15 @@ void Interface::process_announce_queue() { RNS.log("The announce queue for this interface has been cleared.", RNS.LOG_ERROR) */ } + +void ArduinoJson::convertFromJson(JsonVariantConst src, RNS::Interface& dst) { + if (!src.isNull()) { + RNS::Bytes hash; + hash.assignHex(src.as()); + // Query transport for matching interface + dst = Transport::find_interface_from_hash(hash); + } + else { + dst = {RNS::Type::NONE}; + } +} diff --git a/src/Interface.h b/src/Interface.h index e94d489..c183db7 100644 --- a/src/Interface.h +++ b/src/Interface.h @@ -176,3 +176,16 @@ namespace RNS { }; } + +namespace ArduinoJson { + inline bool convertToJson(const RNS::Interface& src, JsonVariant dst) { + if (!src) { + return dst.set(nullptr); + } + return dst.set(src.get_hash().toHex()); + } + void convertFromJson(JsonVariantConst src, RNS::Interface& dst); + inline bool canConvertFromJson(JsonVariantConst src, const RNS::Interface&) { + return src.is() && strlen(src.as()) == 64; + } +} diff --git a/src/Packet.cpp b/src/Packet.cpp index d0706f9..2fff788 100644 --- a/src/Packet.cpp +++ b/src/Packet.cpp @@ -554,3 +554,15 @@ void PacketReceipt::check_timeout() { } } } + +void ArduinoJson::convertFromJson(JsonVariantConst src, RNS::Packet& dst) { + if (!src.isNull()) { + RNS::Bytes hash; + hash.assignHex(src.as()); + // Query transport for matching interface + dst = Transport::get_cached_packet(hash); + } + else { + dst = {RNS::Type::NONE}; + } +} diff --git a/src/Packet.h b/src/Packet.h index c42c504..5b6387a 100644 --- a/src/Packet.h +++ b/src/Packet.h @@ -274,3 +274,16 @@ namespace RNS { }; } + +namespace ArduinoJson { + inline bool convertToJson(const RNS::Packet& src, JsonVariant dst) { + if (!src) { + return dst.set(nullptr); + } + return dst.set(src.get_hash().toHex()); + } + void convertFromJson(JsonVariantConst src, RNS::Packet& dst); + inline bool canConvertFromJson(JsonVariantConst src, const RNS::Packet&) { + return src.is() && strlen(src.as()) == 64; + } +} diff --git a/src/Reticulum.cpp b/src/Reticulum.cpp index 71ef477..cdea374 100644 --- a/src/Reticulum.cpp +++ b/src/Reticulum.cpp @@ -2,6 +2,7 @@ #include "Transport.h" #include "Log.h" +#include "Utilities/OS.h" #include @@ -13,6 +14,8 @@ using namespace RNS; using namespace RNS::Type::Reticulum; +/*static*/ std::string Reticulum::storagepath; + /*static*/ bool Reticulum::__transport_enabled = false; /*static*/ bool Reticulum::__use_implicit_proof = true; /*static*/ bool Reticulum::__allow_probes = false; @@ -40,6 +43,8 @@ Reticulum::Reticulum() : _object(new Object()) { // CBA TEST Transport __transport_enabled = true; + Utilities::OS::setup(); + #ifdef ARDUINO // Stir in the Ethernet MAC address. //byte mac[6]; @@ -51,9 +56,9 @@ Reticulum::Reticulum() : _object(new Object()) { //RNG.addNoiseSource(noise); #endif -/* - RNS.vendor.platformutils.platform_checks() + //z RNS.vendor.platformutils.platform_checks() +/* TODO if configdir != None: Reticulum.configdir = configdir else: @@ -73,7 +78,15 @@ Reticulum::Reticulum() : _object(new Object()) { Reticulum.cachepath = Reticulum.configdir+"/storage/cache" Reticulum.resourcepath = Reticulum.configdir+"/storage/resources" Reticulum.identitypath = Reticulum.configdir+"/storage/identities" +*/ +// CBA MOCK +#ifdef ARDUINO + storagepath = ""; +#else + storagepath = "."; +#endif +/* TODO Reticulum.__transport_enabled = False Reticulum.__use_implicit_proof = True Reticulum.__allow_probes = False diff --git a/src/Reticulum.h b/src/Reticulum.h index 0ff95df..5538830 100644 --- a/src/Reticulum.h +++ b/src/Reticulum.h @@ -4,6 +4,7 @@ #include "Type.h" #include +#include #include #include @@ -12,6 +13,19 @@ namespace RNS { class Reticulum { public: + + //z router = None + //z config = None + + // The default configuration path will be expanded to a directory + // named ".reticulum" inside the current users home directory + //z userdir = os.path.expanduser("~") + //z configdir = None + //z configpath = "" + //p storagepath = "" + static std::string storagepath; + //z cachepath = "" + static bool __transport_enabled; static bool __use_implicit_proof; static bool __allow_probes; diff --git a/src/Test/Test.cpp b/src/Test/Test.cpp index 3b21b84..de6ecba 100644 --- a/src/Test/Test.cpp +++ b/src/Test/Test.cpp @@ -1,7 +1,5 @@ #include "Test.h" -#include "Log.h" - void test() { testOS(); @@ -9,5 +7,6 @@ void test() { testCollections(); testReference(); testCrypto(); + testPersistence(); } diff --git a/src/Test/Test.h b/src/Test/Test.h index f7095b6..c7b861f 100644 --- a/src/Test/Test.h +++ b/src/Test/Test.h @@ -6,3 +6,4 @@ void testBytes(); void testCollections(); void testReference(); void testCrypto(); +void testPersistence(); diff --git a/src/Test/TestPersistence.cpp b/src/Test/TestPersistence.cpp new file mode 100644 index 0000000..4a821c1 --- /dev/null +++ b/src/Test/TestPersistence.cpp @@ -0,0 +1,502 @@ +//#include + +#include "Transport.h" +#include "Interfaces/LoRaInterface.h" +#include "Utilities/Persistence.h" +#include "Utilities/OS.h" +#include "Log.h" +#include "Bytes.h" + +#include + +#include +#include +#include +#include +#include + + +class Test { +public: + Test() {} + Test(const char* foo, const char* fee) : _foo(foo), _fee(fee) {} + const std::string toString() { return std::string("Test(") + _foo + "," + _fee + ")"; } + std::string _foo; + std::string _fee; +}; + +namespace ArduinoJson { + template <> + struct Converter { + static bool toJson(const Test& src, JsonVariant dst) { + dst["foo"] = src._foo; + dst["fee"] = src._fee; + return true; + } + + static Test fromJson(JsonVariantConst src) { + return Test(src["foo"], src["fee"]); + } + + static bool checkJson(JsonVariantConst src) { + return src["foo"].is() && src["fee"].is(); + } + }; +} + + +#ifdef ARDUINO +const char test_file_path[] = "/test_file"; +#else +const char test_file_path[] = "test_file"; +#endif +const char test_file_data[] = "test data"; + +void testWrite() { + RNS::Bytes data(test_file_data); + if (RNS::Utilities::OS::write_file(data, test_file_path)) { + RNS::extreme("wrote: " + std::to_string(data.size()) + " bytes"); + } + else { + RNS::extreme("write failed"); + assert(false); + } +} + +void testRead() { + RNS::Bytes data = RNS::Utilities::OS::read_file(test_file_path); + if (data) { + RNS::extreme("read: " + std::to_string(data.size()) + " bytes"); + RNS::extreme("data: " + data.toString()); + assert(data.size() == strlen(test_file_data)); + assert(memcmp(data.data(), test_file_data, strlen(test_file_data)) == 0); + //assert(RNS::Utilities::OS::remove_file(test_file_path)); + } + else { + RNS::extreme("read failed"); + assert(false); + } +} + + +#ifdef ARDUINO +const char test_object_path[] = "/test_object"; +#else +const char test_object_path[] = "test_object"; +#endif + +void testSerializeObject() { + + Test test("bar", "fum"); + + StaticJsonDocument<1024> doc; + doc["foo"] = test._foo; + doc["fee"] = test._fee; + + RNS::Bytes data; + size_t size = 1024; + //size_t length = serializeJson(doc, data.writable(size), size); + size_t length = serializeMsgPack(doc, data.writable(size), size); + if (length < size) { + data.resize(length); + } + RNS::extreme("serialized " + std::to_string(length) + " bytes"); + if (length > 0) { + if (RNS::Utilities::OS::write_file(data, test_object_path)) { + RNS::extreme("wrote: " + std::to_string(data.size()) + " bytes"); + } + else { + RNS::extreme("write failed"); + assert(false); + } + } + else { + RNS::extreme("failed to serialize"); + assert(false); + } + +} + +void testDeserializeObject() { + + //StaticJsonDocument<8192> doc; + DynamicJsonDocument doc(8192); + + RNS::Bytes data = RNS::Utilities::OS::read_file(test_object_path); + if (data) { + RNS::extreme("read: " + std::to_string(data.size()) + " bytes"); + //RNS::extreme("data: " + data.toString()); + //DeserializationError error = deserializeJson(doc, data.data()); + DeserializationError error = deserializeMsgPack(doc, data.data()); + if (!error) { + JsonObject root = doc.as(); + for (JsonPair kv : root) { + RNS::extreme("key: " + std::string(kv.key().c_str()) + " value: " + kv.value().as()); + } + } + else { + RNS::extreme("failed to deserialize"); + assert(false); + } + //assert(RNS::Utilities::OS::remove_file(test_object_path)); + } + else { + RNS::extreme("read failed"); + assert(false); + } + +} + + +#ifdef ARDUINO +const char test_vector_path[] = "/test_vector"; +#else +const char test_vector_path[] = "test_vector"; +#endif + +void testSerializeVector() { + + //std::vector vector; + //vector.push_back("foo"); + //vector.push_back("bar"); + std::vector vector; + vector.push_back({"bar1", "fum1"}); + vector.push_back({"bar2", "fum2"}); + + StaticJsonDocument<4096> doc; + //copyArray(vector, doc.createNestedArray("vector")); + //doc["vector"] = vector; + //copyArray(vector, doc); + doc = vector; + +/* + DynamicJsonBuffer jsonBuffer; + JsonObject& root = jsonBuffer.createObject(); + + //JsonObject& weather = root.createNestedObject("weather"); + //weather["temperature"] = 12; + //weather["condition"] = "cloudy"; + + //JsonArray& coords = root.createNestedArray("coords"); + //coords.add(48.7507371, 7); + //coords.add(2.2625587, 7); + JsonArray& arr = root.createNestedArray("vec"); + for (auto entry : vec) { + arr.add( + } +*/ + + RNS::Bytes data; + size_t size = 4096; + size_t length = serializeJson(doc, data.writable(size), size); + //size_t length = serializeMsgPack(doc, data.writable(size), size); + if (length < size) { + data.resize(length); + } + RNS::extreme("testSerializeVector: serialized " + std::to_string(length) + " bytes"); + if (length > 0) { + if (RNS::Utilities::OS::write_file(data, test_vector_path)) { + RNS::extreme("testSerializeVector: wrote: " + std::to_string(data.size()) + " bytes"); + } + else { + RNS::extreme("testSerializeVector: write failed"); + assert(false); + } + } + else { + RNS::extreme("testSerializeVector: failed to serialize"); + assert(false); + } + +} + +void testDeserializeVector() { + + // Compute the required size + const int capacity = + JSON_ARRAY_SIZE(2) + + 2*JSON_OBJECT_SIZE(3) + + 4*JSON_OBJECT_SIZE(1); + + //StaticJsonDocument<8192> doc; + DynamicJsonDocument doc(8192); + + RNS::Bytes data = RNS::Utilities::OS::read_file(test_vector_path); + if (data) { + RNS::extreme("testDeserializeVector: read: " + std::to_string(data.size()) + " bytes"); + //RNS::extreme("testDeserializeVector: data: " + data.toString()); + DeserializationError error = deserializeJson(doc, data.data()); + //DeserializationError error = deserializeMsgPack(doc, data.data()); + if (!error) { + RNS::extreme("testDeserializeVector: successfully deserialized"); + + std::vector vector = doc.as>(); + for (auto& test : vector) { + RNS::extreme("testDeserializeVector: entry: " + test.toString()); + } + + //JsonArray arr = doc.as(); + //for (JsonVariant elem : arr) { + // RNS::extreme("testDeserializeVector: entry: " + elem.as().toString()); + //} + } + else { + RNS::extreme("testDeserializeVector: failed to deserialize"); + assert(false); + } + //assert(RNS::Utilities::OS::remove_file(test_vector_path)); + } + else { + RNS::extreme("testDeserializeVector: read failed"); + assert(false); + } + +} + + +#ifdef ARDUINO +const char test_map_path[] = "/test_map"; +#else +const char test_map_path[] = "test_map"; +#endif + +void testSerializeMap() { + + //std::map map; + //map.insert({"foo", "bar"}); + //map.insert({"fee", "fum"}); + std::map map; + map.insert({"one", {"bar1", "fum1"}}); + map.insert({"two", {"bar2", "fum2"}}); + + StaticJsonDocument<4096> doc; + //copyArray(map, doc.createNestedArray("map")); + //doc["map"] = map; + //copyArray(map, doc); + doc = map; + + RNS::Bytes data; + size_t size = 4096; + size_t length = serializeJson(doc, data.writable(size), size); + //size_t length = serializeMsgPack(doc, data.writable(size), size); + if (length < size) { + data.resize(length); + } + RNS::extreme("testSerializeMap: serialized " + std::to_string(length) + " bytes"); + if (length > 0) { + if (RNS::Utilities::OS::write_file(data, test_map_path)) { + RNS::extreme("testSerializeMap: wrote: " + std::to_string(data.size()) + " bytes"); + } + else { + RNS::extreme("testSerializeMap: write failed"); + assert(false); + } + } + else { + RNS::extreme("testSerializeMap: failed to serialize"); + assert(false); + } + +} + +void testDeserializeMap() { + + // Compute the required size + const int capacity = + JSON_ARRAY_SIZE(2) + + 2*JSON_OBJECT_SIZE(3) + + 4*JSON_OBJECT_SIZE(1); + + //StaticJsonDocument<8192> doc; + DynamicJsonDocument doc(8192); + + RNS::Bytes data = RNS::Utilities::OS::read_file(test_map_path); + if (data) { + RNS::extreme("testDeserializeMap: read: " + std::to_string(data.size()) + " bytes"); + //RNS::extreme("testDeserializeMap: data: " + data.toString()); + DeserializationError error = deserializeJson(doc, data.data()); + //DeserializationError error = deserializeMsgPack(doc, data.data()); + if (!error) { + RNS::extreme("testDeserializeMap: successfully deserialized"); + + std::map map(doc.as>()); + for (auto& [str, test] : map) { + RNS::extreme("testDeserializeMap: entry: " + str + " = " + test.toString()); + } + + //JsonObject root = doc.as(); + //for (JsonPair kv : root) { + // RNS::extreme("testDeserializeMap: entry: " + std::string(kv.key().c_str()) + " = " + kv.value().as().toString()); + //} + } + else { + RNS::extreme("testDeserializeMap: failed to deserialize"); + assert(false); + } + //assert(RNS::Utilities::OS::remove_file(test_map_path)); + } + else { + RNS::extreme("testDeserializeMap: testDeserializeVector: read failed"); + assert(false); + } + +} + +#ifdef ARDUINO +const char test_destination_table_path[] = "/test_destination_table"; +#else +const char test_destination_table_path[] = "test_destination_table"; +#endif + +void testSerializeDestinationTable() { + + //RNS::Bytes empty; + //std::set blobs; + //RNS::Interface interface({RNS::Type::NONE}); + RNS::Interfaces::LoRaInterface lora_interface; + //RNS::Packet packet({RNS::Type::NONE}); + static std::map map; + //DestinationEntry(double time, const Bytes& received_from, uint8_t announce_hops, double expires, const std::set& random_blobs, Interface& receiving_interface, const Packet& packet) : + //RNS::Transport::DestinationEntry entry_one(1.0, empty, 1, 0.0, blobs, interface, packet); + RNS::Bytes received; + received.assignHex("deadbeef"); + RNS::Transport::DestinationEntry entry_one; + entry_one._timestamp = 1.0; + entry_one._received_from = received; + entry_one._receiving_interface = lora_interface; + RNS::Bytes one; + one.assignHex("1111111111111111"); + map.insert({one, entry_one}); + //RNS::Transport::DestinationEntry entry_two(2.0, empty, 1, 0.0, blobs, interface, packet); + RNS::Transport::DestinationEntry entry_two; + entry_two._timestamp = 2.0; + entry_two._received_from = received; + entry_two._receiving_interface = lora_interface; + RNS::Bytes two; + two.assignHex("2222222222222222"); + map.insert({two, entry_two}); + + for (auto& [hash, test] : map) { + RNS::extreme("testSerializeDestinationTable: entry: " + hash.toHex() + " = (" + std::to_string(test._timestamp) + "," + test._received_from.toHex() + ")"); + } + + StaticJsonDocument<4096> doc; + doc = map; + + RNS::Bytes data; + size_t size = 4096; + size_t length = serializeJson(doc, data.writable(size), size); + //size_t length = serializeMsgPack(doc, data.writable(size), size); + if (length < size) { + data.resize(length); + } + RNS::extreme("testSerializeDestinationTable: serialized " + std::to_string(length) + " bytes"); + if (length > 0) { + if (RNS::Utilities::OS::write_file(data, test_destination_table_path)) { + RNS::extreme("testSerializeDestinationTable: wrote: " + std::to_string(data.size()) + " bytes"); + } + else { + RNS::extreme("testSerializeDestinationTable: write failed"); + assert(false); + } + } + else { + RNS::extreme("testSerializeDestinationTable: failed to serialize"); + assert(false); + } + +} + +void testDeserializeDestinationTable() { + + // Compute the required size + const int capacity = + JSON_ARRAY_SIZE(2) + + 2*JSON_OBJECT_SIZE(3) + + 4*JSON_OBJECT_SIZE(1); + + //StaticJsonDocument<8192> doc; + DynamicJsonDocument doc(8192); + + RNS::Bytes data = RNS::Utilities::OS::read_file(test_destination_table_path); + if (data) { + RNS::extreme("testDeserializeDestinationTable: read: " + std::to_string(data.size()) + " bytes"); + //RNS::extreme("testDeserializeVector: data: " + data.toString()); + DeserializationError error = deserializeJson(doc, data.data()); + //DeserializationError error = deserializeMsgPack(doc, data.data()); + if (!error) { + RNS::extreme("testDeserializeDestinationTable: successfully deserialized"); + + static std::map map(doc.as>()); + for (auto& [hash, test] : map) { + RNS::extreme("testDeserializeDestinationTable: entry: " + hash.toHex() + " = (" + std::to_string(test._timestamp) + "," + test._received_from.toHex() + ")"); + } + + //JsonObject root = doc.as(); + //for (JsonPair kv : root) { + // RNS::extreme("testDeserializeDestinationTable: entry: " + std::string(kv.key().c_str()) + " = " + kv.value().as().toString()); + //} + } + else { + RNS::extreme("testDeserializeDestinationTable: failed to deserialize"); + assert(false); + } + //assert(RNS::Utilities::OS::remove_file(test_destination_table_path)); + } + else { + RNS::extreme("testDeserializeDestinationTable: read failed"); + assert(false); + } + +} + + +void testPersistence() { + RNS::head("Running testPersistence...", RNS::LOG_EXTREME); + testWrite(); + testRead(); + testSerializeObject(); + testDeserializeObject(); + testSerializeVector(); + testDeserializeVector(); + testSerializeMap(); + testDeserializeMap(); + testSerializeDestinationTable(); + testDeserializeDestinationTable(); +} + +/* +void setUp(void) { + // set stuff up here +} + +void tearDown(void) { + // clean stuff up here +} + +int runUnityTests(void) { + UNITY_BEGIN(); + RUN_TEST(testCollections); + return UNITY_END(); +} + +// For native dev-platform or for some embedded frameworks +int main(void) { + return runUnityTests(); +} + +// For Arduino framework +void setup() { + // Wait ~2 seconds before the Unity test runner + // establishes connection with a board Serial interface + delay(2000); + + runUnityTests(); +} +void loop() {} + +// For ESP-IDF framework +void app_main() { + runUnityTests(); +} +*/ diff --git a/src/Transport.cpp b/src/Transport.cpp index f563bd7..bd2ce85 100644 --- a/src/Transport.cpp +++ b/src/Transport.cpp @@ -81,15 +81,17 @@ using namespace RNS::Utilities; _owner = reticulum_instance; if (!_identity) { - //z transport_identity_path = Reticulum::storagepath+"/transport_identity" - //z if (os.path.isfile(transport_identity_path)) { - //z identity = Identity.from_file(transport_identity_path); - //z } + std::string transport_identity_path = Reticulum::storagepath + "/transport_identity"; +// CBA TEST +//OS::remove_file(transport_identity_path.c_str()); + if (OS::file_exists(transport_identity_path.c_str())) { + _identity = Identity::from_file(transport_identity_path.c_str()); + } if (!_identity) { verbose("No valid Transport Identity in storage, creating..."); - _identity = new Identity(); - //z _identity.to_file(transport_identity_path); + _identity = Identity(); + _identity.to_file(transport_identity_path.c_str()); } else { verbose("Loaded Transport Identity from storage"); @@ -135,9 +137,8 @@ using namespace RNS::Utilities; // TODO /* if RNS.Reticulum.transport_enabled(): - destination_table_path = RNS.Reticulum.storagepath+"/destination_table" - tunnel_table_path = RNS.Reticulum.storagepath+"/tunnels" + destination_table_path = RNS.Reticulum.storagepath+"/destination_table" if os.path.isfile(destination_table_path) and not Transport.owner.is_connected_to_shared_instance: serialised_destinations = [] try: @@ -183,6 +184,7 @@ using namespace RNS::Utilities; except Exception as e: RNS.log("Could not load destination table from storage, the contained exception was: "+str(e), RNS.LOG_ERROR) + tunnel_table_path = RNS.Reticulum.storagepath+"/tunnels" if os.path.isfile(tunnel_table_path) and not Transport.owner.is_connected_to_shared_instance: serialised_tunnels = [] try: diff --git a/src/Transport.h b/src/Transport.h index 0a2c98f..3858084 100644 --- a/src/Transport.h +++ b/src/Transport.h @@ -4,6 +4,8 @@ #include "Bytes.h" #include "Type.h" +#include + #include #include #include @@ -54,11 +56,12 @@ namespace RNS { */ class Transport { - private: + public: // CBA TODO Analyze safety of using Inrerface references here // CBA TODO Analyze safety of using Packet references here class DestinationEntry { public: + DestinationEntry() {} DestinationEntry(double time, const Bytes& received_from, uint8_t announce_hops, double expires, const std::set& random_blobs, Interface& receiving_interface, const Packet& packet) : _timestamp(time), _received_from(received_from), @@ -74,11 +77,11 @@ namespace RNS { Bytes _received_from; uint8_t _hops = 0; double _expires = 0; - std::set _random_blobs; + const std::set _random_blobs; // CBA TODO does this need to be a reference in order for virtual method callbacks to work? - Interface& _receiving_interface; + Interface _receiving_interface = {Type::NONE}; //const Packet& _announce_packet; - Packet _announce_packet; + const Packet _announce_packet = {Type::NONE}; }; // CBA TODO Analyze safety of using Inrerface references here @@ -101,17 +104,17 @@ namespace RNS { double _timestamp = 0; double _retransmit_timeout = 0; uint8_t _retries = 0; - Bytes _received_from; + const Bytes _received_from; uint8_t _hops = 0; // CBA Storing packet reference causes memory issues, presumably because orignal packet is being destroyed // MUST use instance instad of reference!!! //const Packet& _packet; - Packet _packet; + const Packet _packet = {Type::NONE}; uint8_t _local_rebroadcasts = 0; bool _block_rebroadcasts = false; // CBA TODO does this need to be a reference in order for virtual method callbacks to work? //const Interface& _attached_interface; - Interface _attached_interface; + const Interface _attached_interface = {Type::NONE}; }; // CBA TODO Analyze safety of using Inrerface references here @@ -131,16 +134,16 @@ namespace RNS { } public: double _timestamp = 0; - Bytes _next_hop; + const Bytes _next_hop; // CBA TODO does this need to be a reference in order for virtual method callbacks to work? //const Interface& _outbound_interface; - Interface _outbound_interface; + const Interface _outbound_interface = {Type::NONE}; uint8_t _remaining_hops = 0; // CBA TODO does this need to be a reference in order for virtual method callbacks to work? //const Interface& _receiving_interface; - Interface _receiving_interface; + const Interface _receiving_interface = {Type::NONE}; uint8_t _hops = 0; - Bytes _destination_hash; + const Bytes _destination_hash; bool _validated = false; double _proof_timeout = 0; }; @@ -157,10 +160,10 @@ namespace RNS { public: // CBA TODO does this need to be a reference in order for virtual method callbacks to work? //const Interface& _receiving_interface; - Interface _receiving_interface; + const Interface _receiving_interface = {Type::NONE}; // CBA TODO does this need to be a reference in order for virtual method callbacks to work? //const Interface& _outbound_interface; - Interface _outbound_interface; + const Interface _outbound_interface = {Type::NONE}; double _timestamp = 0; }; @@ -174,11 +177,11 @@ namespace RNS { { } public: - Bytes _destination_hash; + const Bytes _destination_hash; double _timeout = 0; // CBA TODO does this need to be a reference in order for virtual method callbacks to work? //const Interface& _requesting_interface; - Interface _requesting_interface; + const Interface _requesting_interface = {Type::NONE}; }; public: @@ -312,4 +315,55 @@ namespace RNS { static Identity _identity; }; -} \ No newline at end of file +} + +namespace ArduinoJson { + + // ArduinoJSON serialization support for RNS::Transport::DestinationEntry + template <> + struct Converter { + static bool toJson(const RNS::Transport::DestinationEntry& src, JsonVariant dst) { + dst["timestamp"] = src._timestamp; + dst["received_from"] = src._received_from; + dst["announce_hops"] = src._hops; + dst["expires"] = src._expires; + //dst["random_blobs"] = src._random_blobs; + dst["receiving_interface"] = src._receiving_interface; + dst["packet"] = src._announce_packet; + return true; + } + static RNS::Transport::DestinationEntry fromJson(JsonVariantConst src) { +/**/ + RNS::Transport::DestinationEntry dst; + dst._timestamp = src["timestamp"]; + dst._received_from = src["received_from"]; + dst._hops = src["announce_hops"]; + dst._expires = src["expires"]; + //dst._random_blobs = src["random_blobs"]; + dst._receiving_interface = src["receiving_interface"]; + //dst._announce_packet = src["packet"]; +/**/ +/* + //RNS::Transport::DestinationEntry dst(src["timestamp"], src["received_from"], src["announce_hops"], src["expires"], src["random_blobs"], src["receiving_interface"], src["packet"]); + RNS::Transport::DestinationEntry dst( + src["timestamp"].as(), + src["received_from"].as(), + src["announce_hops"].as(), + src["expires"].as(), + src["random_blobs"].as>(), + src["receiving_interface"].as(), + src["packet"].as() + ); +*/ + return dst; + } + static bool checkJson(JsonVariantConst src) { + return + src["timestamp"].is() && + src["received_from"].is() && + src["announce_hops"].is() && + src["expires"].is(); + } + }; + +} diff --git a/src/Utilities/OS.cpp b/src/Utilities/OS.cpp new file mode 100644 index 0000000..565575d --- /dev/null +++ b/src/Utilities/OS.cpp @@ -0,0 +1,144 @@ +#include "OS.h" + +#include "../Log.h" + +#ifdef ARDUINO +#include +#include +#else +#include +#include +#include +#include +#endif + +using namespace RNS; +using namespace RNS::Utilities; + +/*static*/ void OS::setup() { + +#ifdef ARDUINO + // Setup filesystem + if (!LittleFS.begin(true)){ + Serial.println("LittleFS Mount Failed"); + return; + } +#endif + +} + +/*static*/ bool OS::file_exists(const char* file_path) { +#ifdef ARDUINO + File file = LittleFS.open(file_path, FILE_READ); + if (file) { +#else + FILE* file = fopen(file_path, "r"); + if (file != nullptr) { +#endif + //RNS::extreme("file_exists: file exists, closing file"); +#ifdef ARDUINO + file.close(); +#else + fclose(file); +#endif + return true; + } + else { + RNS::extreme("file_exists: failed to open file " + std::string(file_path)); + return false; + } +} + +/*static*/ const Bytes OS::read_file(const char* file_path) { + Bytes data; +#ifdef ARDUINO + File file = LittleFS.open(file_path, FILE_READ); + if (file) { + size_t size = file.size(); + size_t read = file.readBytes((char*)data.writable(size), size); +#else + FILE* file = fopen(file_path, "r"); + if (file != nullptr) { + fseek(file, 0, SEEK_END); + size_t size = ftell(file); + rewind(file); + //size_t read = fread(data.writable(size), size, 1, file); + size_t read = fread(data.writable(size), 1, size, file); +#endif + RNS::extreme("read_file: read " + std::to_string(read) + " bytes from file " + std::string(file_path)); + if (read != size) { + RNS::extreme("read_file: failed to read file " + std::string(file_path)); + data.clear(); + } + //RNS::extreme("read_file: closing input file"); +#ifdef ARDUINO + file.close(); +#else + fclose(file); +#endif + } + else { + RNS::extreme("read_file: failed to open input file " + std::string(file_path)); + } + return data; +} + +/*static*/ bool OS::write_file(const Bytes& data, const char* file_path) { + bool success = false; +#ifdef ARDUINO + File file = LittleFS.open(file_path, FILE_WRITE); + if (file) { + size_t wrote = file.write(data.data(), data.size()); +#else + FILE* file = fopen(file_path, "w"); + if (file != nullptr) { + //size_t wrote = fwrite(data.data(), data.size(), 1, file); + size_t wrote = fwrite(data.data(), 1, data.size(), file); +#endif + RNS::extreme("write_file: wrote " + std::to_string(wrote) + " bytes to file " + std::string(file_path)); + if (wrote == data.size()) { + success = true; + } + else { + RNS::extreme("write_file: failed to write file " + std::string(file_path)); + } + //RNS::extreme("write_file: closing output file"); +#ifdef ARDUINO + file.close(); +#else + fclose(file); +#endif + } + else { + RNS::extreme("write_file: failed to open output file " + std::string(file_path)); + } + return success; +} + +/*static*/ bool OS::remove_file(const char* file_path) { +#ifdef ARDUINO + return LittleFS.remove(file_path); +#else + return (remove(file_path) == 0); +#endif +} + +/*static*/ bool OS::rename_file(const char* from_file_path, const char* to_file_path) { +#ifdef ARDUINO + return LittleFS.rename(from_file_path, to_file_path); +#else + return (rename(from_file_path, to_file_path) == 0); +#endif +} + +/*static*/ bool OS::create_directory(const char* directory_path) { +#ifdef ARDUINO + return LittleFS.mkdir(directory_path); +#else + struct stat st = {0}; + if (stat(directory_path, &st) == 0) { + return true; + } + return (mkdir(directory_path, 0700) == 0); +#endif +} diff --git a/src/Utilities/OS.h b/src/Utilities/OS.h index f2405e0..20324a8 100644 --- a/src/Utilities/OS.h +++ b/src/Utilities/OS.h @@ -1,16 +1,24 @@ #pragma once +#include "../Bytes.h" + #include #include #include #include #include +#ifdef ARDUINO +#include "Arduino.h" +#endif + namespace RNS { namespace Utilities { class OS { public: + static void setup(); + // sleep for specified milliseconds //static inline void sleep(float seconds) { ::sleep(seconds); } #ifdef ARDUINO @@ -30,6 +38,13 @@ namespace RNS { namespace Utilities { //static inline float round(float value, uint8_t precision) { return std::round(value / precision) * precision; } static inline double round(double value, uint8_t precision) { return std::round(value / precision) * precision; } + static bool file_exists(const char* file_path); + static const RNS::Bytes read_file(const char* file_path); + static bool write_file(const RNS::Bytes& data, const char* file_path); + static bool remove_file(const char* file_path); + static bool rename_file(const char* from_file_path, const char* to_file_path); + static bool create_directory(const char* directory_path); + }; } } diff --git a/src/Utilities/Persistence.h b/src/Utilities/Persistence.h new file mode 100644 index 0000000..eefb2d5 --- /dev/null +++ b/src/Utilities/Persistence.h @@ -0,0 +1,89 @@ + +#include "Bytes.h" + +#include +#include +#include + +#include + +namespace ArduinoJson { + + // ArduinoJSON serialization support for std::vector + template + struct Converter > { + static void toJson(const std::vector& src, JsonVariant dst) { + JsonArray array = dst.to(); + for (T item : src) + array.add(item); + } + static std::vector fromJson(JsonVariantConst src) { + std::vector dst; + for (T item : src.as()) + dst.push_back(item); + return dst; + } + static bool checkJson(JsonVariantConst src) { + JsonArrayConst array = src; + bool result = array; + for (JsonVariantConst item : array) + result &= item.is(); + return result; + } + }; + + // ArduinoJSON serialization support for std::map + template + struct Converter > { + static void toJson(const std::map& src, JsonVariant dst) { + JsonObject obj = dst.to(); + for (const auto& item : src) + obj[item.first] = item.second; + } + static std::map fromJson(JsonVariantConst src) { + std::map dst; + for (JsonPairConst item : src.as()) + dst[item.key().c_str()] = item.value().as(); + return dst; + } + static bool checkJson(JsonVariantConst src) { + JsonObjectConst obj = src; + bool result = obj; + for (JsonPairConst item : obj) + result &= item.value().is(); + return result; + } + }; + + // ArduinoJSON serialization support for std::map + template + struct Converter > { + static void toJson(const std::map& src, JsonVariant dst) { + JsonObject obj = dst.to(); + for (const auto& item : src) { + //obj[item.first] = item.second; + obj[item.first.toHex()] = item.second; + } + } + static std::map fromJson(JsonVariantConst src) { + std::map dst; + for (JsonPairConst item : src.as()) { + //dst[item.key().c_str()] = item.value().as(); + RNS::Bytes key; + key.assignHex(item.key().c_str()); + //dst[key] = item.value().as(); + dst.insert({key, item.value().as()}); + } + return dst; + } + static bool checkJson(JsonVariantConst src) { + JsonObjectConst obj = src; + bool result = obj; + for (JsonPairConst item : obj) { + result &= item.value().is(); + } + return result; + } + }; + +} diff --git a/src/main.cpp b/src/main.cpp index babdc28..d408d2e 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -120,9 +120,10 @@ void run_tests() { RNS::LogLevel loglevel = RNS::loglevel(); //RNS::loglevel(RNS::LOG_WARNING); RNS::loglevel(RNS::LOG_EXTREME); - test(); + //test(); //testReference(); //testCrypto(); + testPersistence(); RNS::loglevel(loglevel); RNS::extreme("Finished running tests"); //return; @@ -134,6 +135,7 @@ void run_tests() { } #endif +#if defined(RUN_RETICULUM) void reticulum_announce() { if (destination) { RNS::head("Announcing destination...", RNS::LOG_EXTREME); @@ -145,8 +147,7 @@ void reticulum_announce() { } } -#if defined(RUN_RETICULUM) -void setup_reticulum() { +void reticulum_setup() { RNS::info("Setting up Reticulum..."); try { @@ -177,7 +178,7 @@ void setup_reticulum() { #ifdef UDP_INTERFACE RNS::head("Starting UDPInterface...", RNS::LOG_EXTREME); - udp_interface.start("some_ssid", "some_password", 2424); + udp_interface.start("some_ssid", "some_password"); #endif #ifdef LORA_INTERFACE @@ -251,13 +252,14 @@ void setup_reticulum() { destination.receive(recv_packet); #endif + RNS::head("Ready!", RNS::LOG_EXTREME); } catch (std::exception& e) { - RNS::error(std::string("!!! Exception in setup_reticulum: ") + e.what() + " !!!"); + RNS::error(std::string("!!! Exception in reticulum_setup: ") + e.what() + " !!!"); } } -void teardown_reticulum() { +void reticulum_teardown() { RNS::info("Tearing down Reticulum..."); try { @@ -275,7 +277,7 @@ void teardown_reticulum() { } catch (std::exception& e) { - RNS::error(std::string("!!! Exception in teardown_reticulum: ") + e.what() + " !!!"); + RNS::error(std::string("!!! Exception in reticulum_teardown: ") + e.what() + " !!!"); } } #endif @@ -317,11 +319,11 @@ void setup() { #endif #if defined(RUN_RETICULUM) - setup_reticulum(); + reticulum_setup(); #endif #ifdef ARDUINO - Serial.print("Goodbye from T-Beam on PlatformIO!\n"); + //Serial.print("Goodbye from T-Beam on PlatformIO!\n"); #endif } @@ -380,7 +382,9 @@ int main(void) { if (ch > 0) { switch (ch) { case 'a': +#if defined(RUN_RETICULUM) reticulum_announce(); +#endif break; case 'q': run = false;