From 46a9cc3ee98c8f4e989dd277590d192bfbe0c718 Mon Sep 17 00:00:00 2001 From: ag Date: Tue, 26 Nov 2024 10:43:02 -0700 Subject: [PATCH] Prototype cleanup 1 --- src/tui.py => docs/Actions/Actions.md | 0 docs/Components/hopper.md | 7 +- docs/Config/Args.md | 0 docs/Config/Context.md | 16 ++ docs/Cryptography/WhaleSong.md | 116 +++++----- docs/Daisy/Cache.md | 13 +- docs/Daisy/Catch.md | 4 +- docs/Daisy/Credential.md | 7 + docs/Daisy/CryptographyUtil.md | 28 +++ docs/Daisy/Daisy.md | 11 +- docs/Daisy/Index.md | 25 +++ docs/Daisy/Ref.md | 13 ++ docs/Daisy/Soil.md | 16 -- docs/Daisy/Store.md | 33 ++- docs/Packets/HeaderPacket.md | 8 +- .../Messages/Protocols/bubble/Bubble.md | 7 + .../Messages/Protocols/catch/IndexSync.md | 7 + .../Messages/Protocols/catch/Request.md | 7 + .../Messages/Protocols/catch/Response.md | 7 + .../Protocols/cryptography/Handshake.md | 7 + .../Messages/Protocols/hopper/Request.md | 7 + .../Messages/Protocols/hopper/Response.md | 7 + .../Messages/Protocols/map/Announce.md | 7 + docs/Packets/Packet.md | 2 +- docs/Packets/SubMessage.md | 4 +- docs/Packets/SubPacket.md | 4 +- docs/Services/Action.md | 59 +++++ docs/Siph/map.md | 10 +- docs/Splash/serve.md | 18 +- docs/Sponge/Protocols/Yellow.md | 86 +++++++ docs/Sponge/Protocols/bubble.md | 11 + docs/Sponge/Protocols/catch.md | 11 + docs/Sponge/Protocols/cryptography.md | 11 + docs/Sponge/Protocols/hopper.md | 11 + docs/Sponge/Protocols/map.md | 13 ++ docs/Sponge/base.md | 92 ++++---- docs/Transceiver/Transceiver.md | 9 +- docs/readme.md | 182 ++++++++++----- docs/run.md | 68 +++--- docs/stale/Soil.md | 3 + docs/stale/ui.md | 3 + docs/tlog.md | 35 +++ docs/ui.md | 74 ------ src/.piermesh | 5 +- src/Actions/Actions.rst | 0 src/Components/hopper.py | 16 +- src/Config/Args.rst | 0 src/Config/Context.py | 21 +- src/Config/Context.rst | 5 + src/Cryptography/WhaleSong.py | 210 +++++++++--------- src/Cryptography/WhaleSong.rst | 6 +- src/Daisy/Cache.py | 52 +++-- src/Daisy/Catch.py | 33 ++- src/Daisy/Credential.py | 3 + src/Daisy/Credential.rst | 6 + src/Daisy/CryptographyUtil.py | 62 ++++-- src/Daisy/CryptographyUtil.rst | 6 + src/Daisy/Daisy.py | 29 +-- src/Daisy/Index.py | 51 ++++- src/Daisy/Index.rst | 6 + src/Daisy/Ref.py | 14 +- src/Daisy/Ref.rst | 6 + src/Daisy/Soil.py | 49 ---- src/Daisy/Soil.rst | 6 - src/Daisy/Store.py | 77 ++++++- src/Packets/HeaderPacket.py | 18 +- src/Packets/Message.py | 73 ++++-- .../Messages/Protocols/bubble/Bubble.py | 28 +++ .../Messages/Protocols/bubble/Bubble.rst | 6 + .../Messages/Protocols/catch/IndexSync.py | 30 +++ .../Messages/Protocols/catch/IndexSync.rst | 6 + .../Messages/Protocols/catch/Request.py | 37 +++ .../Messages/Protocols/catch/Request.rst | 6 + .../Messages/Protocols/catch/Response.py | 31 +++ .../Messages/Protocols/catch/Response.rst | 6 + .../Protocols/cryptography/Handshake.py | 39 +++- .../Protocols/cryptography/Handshake.rst | 6 + .../Messages/Protocols/hopper/Request.py | 41 +++- .../Messages/Protocols/hopper/Request.rst | 6 + .../Messages/Protocols/hopper/Response.py | 30 ++- .../Messages/Protocols/hopper/Response.rst | 6 + .../Messages/Protocols/map/Announce.py | 27 ++- .../Messages/Protocols/map/Announce.rst | 6 + src/Packets/Packet.py | 5 +- src/Packets/SinglePacket.py | 3 - src/Packets/SubMessage.py | 54 ++++- src/Packets/SubPacket.rst | 2 +- src/Services/Action.py | 41 +++- src/Services/Action.rst | 6 + src/Splash/serve.py | 82 ++++--- src/Splash/serve.rst | 2 +- src/Sponge/Protocols/Yellow.py | 82 ++++++- src/Sponge/Protocols/Yellow.rst | 10 + src/Sponge/Protocols/bubble.py | 1 + src/Sponge/Protocols/bubble.rst | 6 + src/Sponge/Protocols/catch.py | 2 +- src/Sponge/Protocols/catch.rst | 6 + src/Sponge/Protocols/cryptography.rst | 6 + src/Sponge/Protocols/daisy.py | 1 + src/Sponge/Protocols/hopper.py | 1 - src/Sponge/Protocols/hopper.rst | 6 + src/Sponge/Protocols/map.rst | 6 + src/Sponge/base.py | 96 ++++---- src/Sponge/base.rst | 12 - src/Transceiver/Transceiver.py | 3 +- src/index.rst | 11 +- src/mlookup.json | 8 + src/run.py | 162 +++++++------- src/run.rst | 1 + src/tlog.py | 49 ++-- src/tlog.rst | 6 + src/ui.py | 150 ------------- src/ui.rst | 5 - src/ui.tcss | 23 -- 114 files changed, 1949 insertions(+), 1023 deletions(-) rename src/tui.py => docs/Actions/Actions.md (100%) create mode 100644 docs/Config/Args.md create mode 100644 docs/Config/Context.md create mode 100644 docs/Daisy/Credential.md create mode 100644 docs/Daisy/CryptographyUtil.md create mode 100644 docs/Daisy/Index.md create mode 100644 docs/Daisy/Ref.md delete mode 100644 docs/Daisy/Soil.md create mode 100644 docs/Packets/Messages/Protocols/bubble/Bubble.md create mode 100644 docs/Packets/Messages/Protocols/catch/IndexSync.md create mode 100644 docs/Packets/Messages/Protocols/catch/Request.md create mode 100644 docs/Packets/Messages/Protocols/catch/Response.md create mode 100644 docs/Packets/Messages/Protocols/cryptography/Handshake.md create mode 100644 docs/Packets/Messages/Protocols/hopper/Request.md create mode 100644 docs/Packets/Messages/Protocols/hopper/Response.md create mode 100644 docs/Packets/Messages/Protocols/map/Announce.md create mode 100644 docs/Services/Action.md create mode 100644 docs/Sponge/Protocols/Yellow.md create mode 100644 docs/Sponge/Protocols/bubble.md create mode 100644 docs/Sponge/Protocols/catch.md create mode 100644 docs/Sponge/Protocols/cryptography.md create mode 100644 docs/Sponge/Protocols/hopper.md create mode 100644 docs/Sponge/Protocols/map.md create mode 100644 docs/stale/Soil.md create mode 100644 docs/stale/ui.md create mode 100644 docs/tlog.md delete mode 100644 docs/ui.md create mode 100644 src/Actions/Actions.rst create mode 100644 src/Config/Args.rst create mode 100644 src/Config/Context.rst create mode 100644 src/Daisy/Credential.rst create mode 100644 src/Daisy/CryptographyUtil.rst create mode 100644 src/Daisy/Index.rst create mode 100644 src/Daisy/Ref.rst delete mode 100644 src/Daisy/Soil.py delete mode 100644 src/Daisy/Soil.rst create mode 100644 src/Packets/Messages/Protocols/bubble/Bubble.rst create mode 100644 src/Packets/Messages/Protocols/catch/IndexSync.rst create mode 100644 src/Packets/Messages/Protocols/catch/Request.rst create mode 100644 src/Packets/Messages/Protocols/catch/Response.rst create mode 100644 src/Packets/Messages/Protocols/cryptography/Handshake.rst create mode 100644 src/Packets/Messages/Protocols/hopper/Request.rst create mode 100644 src/Packets/Messages/Protocols/hopper/Response.rst create mode 100644 src/Packets/Messages/Protocols/map/Announce.rst create mode 100644 src/Services/Action.rst create mode 100644 src/Sponge/Protocols/Yellow.rst create mode 100644 src/Sponge/Protocols/bubble.rst create mode 100644 src/Sponge/Protocols/catch.rst create mode 100644 src/Sponge/Protocols/cryptography.rst create mode 100644 src/Sponge/Protocols/hopper.rst create mode 100644 src/Sponge/Protocols/map.rst create mode 100644 src/mlookup.json create mode 100644 src/tlog.rst delete mode 100644 src/ui.py delete mode 100644 src/ui.rst delete mode 100644 src/ui.tcss diff --git a/src/tui.py b/docs/Actions/Actions.md similarity index 100% rename from src/tui.py rename to docs/Actions/Actions.md diff --git a/docs/Components/hopper.md b/docs/Components/hopper.md index 1d535922..f445fa0e 100644 --- a/docs/Components/hopper.md +++ b/docs/Components/hopper.md @@ -6,13 +6,18 @@ -### Components.hopper.get(url: str, params=None) +### Components.hopper.downloadFile(url, text=True, mimeType=None) + +Download resource from url and convert it to text or a data url + +### Components.hopper.get(url: str, params=None, followTags=None) http/s get request * **Parameters:** * **url** (*str*) * **params** – Requests (library) parameters + * **followTags** – None or list of tags to download the src/href from ### Components.hopper.post(url: str, params=None) diff --git a/docs/Config/Args.md b/docs/Config/Args.md new file mode 100644 index 00000000..e69de29b diff --git a/docs/Config/Context.md b/docs/Config/Context.md new file mode 100644 index 00000000..d9eb0816 --- /dev/null +++ b/docs/Config/Context.md @@ -0,0 +1,16 @@ + + +# Context + +### *class* Config.Context.Context(subsets: dict = {}, \*\*kwargs) + +Generic context data structure, current subclassed for use in filters, see Sponge/Protocols/Yellow.py + +[πŸ”— Source](https://git.utopic.work/PierMesh/piermesh/src/branch/main/Config/Context.py) + +#### ctx + +Dictionary of context values + +* **Type:** + dict diff --git a/docs/Cryptography/WhaleSong.md b/docs/Cryptography/WhaleSong.md index 76d59ed2..001d3fa4 100644 --- a/docs/Cryptography/WhaleSong.md +++ b/docs/Cryptography/WhaleSong.md @@ -1,22 +1,11 @@ - + -# WhaleSong: Diffie hellman ephemeral Fernet based encryption +# WhaleSong -### *class* Cryptography.WhaleSong.DHEFern(cache, nodeNickname, cLog) +### *class* Cryptography.WhaleSong.Transport(cache, nodeNickname, daisyCryptography, psk) [πŸ”— Source](https://git.utopic.work/PierMesh/piermesh/src/branch/main/Cryptography/WhaleSong.py) -#### cLog - -Method reference to run.Node.cLog so we can log to the ui from here - -#### loadedParams - -In memory representations of cryptography parameters - -* **Type:** - dict - #### loadedKeys In memory representations of cryptography keys @@ -36,7 +25,7 @@ Name of node for isolating configs when running multiple nodes Daisy cache for use in storing cryptography information * **Type:** - Components.daisy.Cache + [Daisy.Cache.Cache](/PierMesh/piermesh/src/branch/main/docs/Daisy/Cache.md#Daisy.Cache.Cache) #### publicKey @@ -46,20 +35,52 @@ Public key for node Private key for node -#### checkInMem(store: str, nodeID: str) +#### daisyCryptography -Check if parameters or keys are loaded for node of nodeID +Record cryptography reference -* **Parameters:** - **store** (*str*) – Whether to check loaded keys or parameters +* **Type:** + [Daisy.CryptographyUtil.SteelPetal](/PierMesh/piermesh/src/branch/main/docs/Daisy/CryptographyUtil.md#Daisy.CryptographyUtil.SteelPetal) -#### decrypt(data, nodeID: str) +#### addPeerEphemeralKey(onodeID, peerEphemeralKey: bytes) -Decrypt bytes and return either str or dict (TODO: Check whether to msgpack load) +Add a peer node’s epehemeral key for session encryption -#### encrypt(data, nodeID: str, isDict: bool = True) +onodeID +: Node identifier -Do Fernet encryption +peerEphemeralKey: bytes +: Serialized ephemeral key + +#### addPublickey(onodeID, publicKey, forSelf: bool = False) + +Add a public key for a given node including this one + +onodeID +: Node identifier + +publicKey +: Public key to add + +forSelf: bool +: Whether to add key for this node + +#### decrypt(data, onodeID: str, nonce, tag) + +Decrypt bytes and return either str or dict depending on result + +onodeID: str +: Node identifier + +nonce +: Encryption nonce + +tag +: Encryption tag + +#### encrypt(data, nodeID: str, isDict: bool = True, pskEncrypt=False) + +Encrypt given data with AES GCM data : Either bytes or dict to encrypt @@ -67,53 +88,42 @@ data isDict: bool : Whether data is a dictionary -#### genKeyPair(paramsOverride=False, setSelf: bool = True) +pskEncrypt: bool +: Whether to encrypt with pre-shared key -Generate public and private keys from self.params (TODO: Gen from passed params) +#### genOurEphemeralKey(onodeID) -paramsOverride -: False or parameters to use (TODO) +Generate epehemeral key for session encryption with given node -setSelf: bool -: Whether to set self.privateKey and self.publicKey +#### genStaticKey(onodeID) -#### genParams() +Generate static key for session encryption with given node -Generate Diffie Hellman parameters +#### generateSessionKey(onodeID) -#### getParamsBytes() +Generate session key for transport encryption -Get bytes encoded from self.parameters (TODO: Encode from store) +onodeID +: Node identifier -#### getRecord(store: str, key: str) +#### getRecord(store: str, key: str, ephemeral=False) Get record from store: store with key: key -#### getSalt() - -Get random salt - #### initStore(store: str) Initialize store: store -#### keyDerive(pubKey: bytes, salt: bytes, nodeID: str, params: bytes) +#### kdf(bytesX) -Derive shared key using Diffie Hellman +Key derivation function -pubKey: bytes -: Public key +#### sessionSetup(onodeID, peerEphemeralKey: bytes) -nodeID: str -: PierMesh node ID +Set up transport encryption session -params: bytes -: Encryption parameters +onodeID +: Node identifier -#### loadParamBytes(pemBytes: bytes) - -Load parameters to self.params from given bytes (TODO: Load from store) - -#### loadRecordToMem(store: str, nodeID: str) - -Load record of nodeID from store to either keys or pameters +peerEphemeralKey: bytes +: Serialized ephemeral key diff --git a/docs/Daisy/Cache.md b/docs/Daisy/Cache.md index da5d608a..99d57fa8 100644 --- a/docs/Daisy/Cache.md +++ b/docs/Daisy/Cache.md @@ -2,19 +2,20 @@ # Daisy based cache -### *class* Daisy.Cache.Cache(filepaths=None, cacheFile=None, path: str = 'daisy', walk: bool = False, isCatch: bool = False) +### *class* Daisy.Cache.Cache(daisyCryptography, filepaths=None, cacheFile=None, path: str = 'daisy', walk: bool = False) -In memory collection of Daisy records +In memory collection of Daisy records, provides a search functionality currently utilized by Daisy.Catch.Catch [πŸ”— Source](https://git.utopic.work/PierMesh/piermesh/src/branch/main/Daisy/Cache.py) -#### create(path: str, data: dict) +#### create(path: str, data: dict, remote=False) Create new record * **Parameters:** * **path** (*str*) – Path to create record at * **data** (*dict*) – Data to populate record with + * **remote** (*bool*) – Whether this is a reference to a distributed file (not implemented yet) #### get(path: str) @@ -29,10 +30,10 @@ Reload from disk to memory #### search(keydict: dict, strict: bool = True) -Search cache for record for records with values +Search cache for record for records with keys and values matching those +in the keydict keydict: dict -: Values to search for strict: bool -: Whether to require values match +: Whether to require all keys/values match diff --git a/docs/Daisy/Catch.md b/docs/Daisy/Catch.md index 60c30511..dedd5429 100644 --- a/docs/Daisy/Catch.md +++ b/docs/Daisy/Catch.md @@ -2,7 +2,7 @@ # Daisy Catch cache -### *class* Daisy.Catch.Catch(path: str = 'catch', filepaths=None, catchFile=None, walk: bool = False) +### *class* Daisy.Catch.Catch(daisyCryptography, path: str = 'catch', filepaths=None, catchFile=None, walk: bool = False) Sub class of Cache for handling catchs @@ -10,7 +10,7 @@ Sub class of Cache for handling catchs [πŸ”— Source](https://git.utopic.work/PierMesh/piermesh/src/branch/main/Daisy/Catch.py) -#### get(head: str, tail: str, fins=None) +#### get(head: str, body: str, fins=None) Get catch by pieces diff --git a/docs/Daisy/Credential.md b/docs/Daisy/Credential.md new file mode 100644 index 00000000..a86d3c7a --- /dev/null +++ b/docs/Daisy/Credential.md @@ -0,0 +1,7 @@ + + +# Credential + +### *class* Daisy.Credential.Credential(nodeNickname, credentialName, extension, daisyCryptography) + +Currently unused credential class, will be fleshed out for credentialed access to the web ui diff --git a/docs/Daisy/CryptographyUtil.md b/docs/Daisy/CryptographyUtil.md new file mode 100644 index 00000000..6895cd34 --- /dev/null +++ b/docs/Daisy/CryptographyUtil.md @@ -0,0 +1,28 @@ + + +# CryptographyUtil + +### *class* Daisy.CryptographyUtil.SteelPetal(key: str, nonce=None, testData=None) + +Cryptography utility for encrypting files + +#### decrypt(data: bytes) + +Decrypt encrypted binary data + +data: bytes +: Data to decrypt + +#### encrypt(data: bytes) + +Encrypt binary data + +data: bytes +: Data to encrypt + +#### pad(key: str) + +Pad key to make it usable + +key: str +: User’s plain text key diff --git a/docs/Daisy/Daisy.md b/docs/Daisy/Daisy.md index 8fa072f4..85de96d8 100644 --- a/docs/Daisy/Daisy.md +++ b/docs/Daisy/Daisy.md @@ -4,7 +4,7 @@ # Daisy -### *class* Daisy.Daisy.Daisy(filepath: str, templates: dict = {}, template: bool = False, prefillDict: bool = False) +### *class* Daisy.Daisy.Daisy(filepath: str, daisyCryptography, templates: dict = {}, template: bool = False, prefillDict: bool = False, remote=False) Base class for Daisy data representation @@ -33,6 +33,13 @@ Get record dictionary from memory * **Return type:** dict +#### json_to_msg(path: str) + +Convert json at the path plus .json to a msgpack binary + +* **Parameters:** + **path** (*str*) – Path to json minus the extension + #### read(decrypt: bool = False, decryptKey=False) Read record from disk to memory @@ -47,7 +54,7 @@ Lists contents of directory if object is a directory, otherwise return None #### write(override=False, encrypt: bool = False, encryptKey=None, recur: bool = False) -Write record to disk +Write record to disk, note: use override with updated record to update record * **Parameters:** * **override** – Either false or a dictionary of values to set on the record diff --git a/docs/Daisy/Index.md b/docs/Daisy/Index.md new file mode 100644 index 00000000..3f5e8b51 --- /dev/null +++ b/docs/Daisy/Index.md @@ -0,0 +1,25 @@ + + +# Index + +### *class* Daisy.Index.Index(nodeNickname: str, daisyCryptography, prefill: list = [], indexedFields: list = [], autoIndex: bool = True) + +A searchable index of records, this is currently only half implemented +but works enough to hold our remote catch index + +#### addEntry(entry: dict) + +Add a record to the index + +entry: dict +: Record to add to the index + +#### search(keydict: dict, strict: bool = True) + +Search index for record for records with values + +keydict: dict +: Keys/Values to search for + +strict: bool +: Whether to require all keys/values match diff --git a/docs/Daisy/Ref.md b/docs/Daisy/Ref.md new file mode 100644 index 00000000..8d4d0978 --- /dev/null +++ b/docs/Daisy/Ref.md @@ -0,0 +1,13 @@ + + +# Ref + +### *class* Daisy.Ref.Ref(metadata: dict, path: str, remoteNodeID: str) + +Reference to a remote record + +metadata: dict +: Data to fill record with, should only be metadata + +path: str +: Where to store data locally diff --git a/docs/Daisy/Soil.md b/docs/Daisy/Soil.md deleted file mode 100644 index 86ec6868..00000000 --- a/docs/Daisy/Soil.md +++ /dev/null @@ -1,16 +0,0 @@ - - -# Soil: Daisy signal management - -### *class* Daisy.Soil.Compound(cache, isCatch: bool = False) - -File system watcher to propagate disk changes - -[πŸ”— Source](https://git.utopic.work/PierMesh/piermesh/src/branch/main/Daisy/Soil.py) - -#### on_any_event(event) - -Called when a CRUD operation is performed on a record file - -* **Parameters:** - **event** – Event object provided by watchdog diff --git a/docs/Daisy/Store.md b/docs/Daisy/Store.md index 53610431..ec770d9a 100644 --- a/docs/Daisy/Store.md +++ b/docs/Daisy/Store.md @@ -2,12 +2,39 @@ # Store: Daisy key value store -### *class* Daisy.Store.Store(store: str, path: str, nodeNickname: str) +### *class* Daisy.Store.Store(store: str, path: str, nodeNickname: str, daisyCryptography) Key value store [πŸ”— Source](https://git.utopic.work/PierMesh/piermesh/src/branch/main/Daisy/Store.py) -#### getRecord(key: str) +#### epehemeral -#### update(entry: str, data, recur: bool = True) +Memory only records + +* **Type:** + dict + +#### createEmpty(key: str) + +* **Parameters:** + * **key** (*str*) + * **key** – Key to create empty record at + +#### getRecord(key: str, ephemeral=False) + +Get record at key + +* **Parameters:** + * **key** (*str*) + * **ephemeral** (*bool*) – Whether key is only in memory, used for session cryptography credentials currently + +#### update(entry: str, data, recur: bool = True, write=True) + +Update given record + +* **Parameters:** + * **entry** (*str*) – Key to update record of + * **data** – Data to update record with + * **recur** (*bool*) – Whether to iterate over data + * **write** (*bool*) – Whether record is ephemeral diff --git a/docs/Packets/HeaderPacket.md b/docs/Packets/HeaderPacket.md index 237877db..6a2ab8f1 100644 --- a/docs/Packets/HeaderPacket.md +++ b/docs/Packets/HeaderPacket.md @@ -2,7 +2,7 @@ # Header packet: Metadata packet -### *class* Packets.HeaderPacket.Header(packetsID: int, packetCount: int, sender: int, senderDisplayName: int, recipient: int, recipientNode: int, subpacket: bool = False, wantFullResponse: bool = False, packetsClass: int = 0, pAction: int = -1) +### *class* Packets.HeaderPacket.Header(packetsID: int, packetCount: int, sender: int, senderDisplayName: int, sourceNode: int, recipient: int, recipientNode: int, wantFullResponse: bool = False, packetsClass: int = 0, pAction: int = -1, target=True) Metadata packet for messages @@ -52,7 +52,7 @@ Whether a response should be sent when the message completes reception (TODO) #### pAction -3 digit (maximum) pAction ID for mapping precise actions within a protocol (TODO) +3 digit (maximum) pAction ID for mapping precise actions within a protocol * **Type:** int @@ -61,6 +61,6 @@ Whether a response should be sent when the message completes reception (TODO) Dump packet to msgpack encoded binary for transmission -#### usePreset(path: str) +#### usePreset(path: str, daisyCryptography) -Add preset fields to the packet +Add preset fields to the packet, currently unused diff --git a/docs/Packets/Messages/Protocols/bubble/Bubble.md b/docs/Packets/Messages/Protocols/bubble/Bubble.md new file mode 100644 index 00000000..d45e810c --- /dev/null +++ b/docs/Packets/Messages/Protocols/bubble/Bubble.md @@ -0,0 +1,7 @@ + + +# bubble.Bubble + +### *class* Packets.Messages.Protocols.bubble.Bubble.Bubble(sender, senderID, sourceNode, recipient, recipientNode, cryptographyInfo, data) + +Send data from peer to peer diff --git a/docs/Packets/Messages/Protocols/catch/IndexSync.md b/docs/Packets/Messages/Protocols/catch/IndexSync.md new file mode 100644 index 00000000..4e1fc844 --- /dev/null +++ b/docs/Packets/Messages/Protocols/catch/IndexSync.md @@ -0,0 +1,7 @@ + + +# catch.IndexSync + +### *class* Packets.Messages.Protocols.catch.IndexSync.IndexSync(sender, senderID, sourceNode, recipient, recipientNode, cryptographyInfo, index, target=False) + +Sync indices of Catchs across nodes diff --git a/docs/Packets/Messages/Protocols/catch/Request.md b/docs/Packets/Messages/Protocols/catch/Request.md new file mode 100644 index 00000000..c0afb22d --- /dev/null +++ b/docs/Packets/Messages/Protocols/catch/Request.md @@ -0,0 +1,7 @@ + + +# catch.Request + +### *class* Packets.Messages.Protocols.catch.Request.CatchRequest(sender, senderID, sourceNode, recipient, recipientNode, cryptographyInfo, head, body, fins, pskEncrypt=False) + +Request Catch (website) from another node diff --git a/docs/Packets/Messages/Protocols/catch/Response.md b/docs/Packets/Messages/Protocols/catch/Response.md new file mode 100644 index 00000000..42e5436e --- /dev/null +++ b/docs/Packets/Messages/Protocols/catch/Response.md @@ -0,0 +1,7 @@ + + +# catch.Response + +### *class* Packets.Messages.Protocols.catch.Response.CatchResponse(sender, senderID, sourceNode, recipient, recipientNode, cryptographyInfo, html, pskEncrypt=False) + +Send local Catch (website) to user who requested it diff --git a/docs/Packets/Messages/Protocols/cryptography/Handshake.md b/docs/Packets/Messages/Protocols/cryptography/Handshake.md new file mode 100644 index 00000000..cf19fadd --- /dev/null +++ b/docs/Packets/Messages/Protocols/cryptography/Handshake.md @@ -0,0 +1,7 @@ + + +# cryptography.Handshake + +### *class* Packets.Messages.Protocols.cryptography.Handshake.Handshake(sender, senderID, recipient, recipientNode, cryptographyInfo, onodeID, sourceNode) + +Provides the ephemeral key for session encryption diff --git a/docs/Packets/Messages/Protocols/hopper/Request.md b/docs/Packets/Messages/Protocols/hopper/Request.md new file mode 100644 index 00000000..434a4970 --- /dev/null +++ b/docs/Packets/Messages/Protocols/hopper/Request.md @@ -0,0 +1,7 @@ + + +# hopper.Request + +### *class* Packets.Messages.Protocols.hopper.Request.HopperRequest(sender, senderID, sourceNode, recipient, recipientNode, url: str, params: dict, method: str, cryptographyInfo) + +Proxy request to main internet from remote node diff --git a/docs/Packets/Messages/Protocols/hopper/Response.md b/docs/Packets/Messages/Protocols/hopper/Response.md new file mode 100644 index 00000000..2955db42 --- /dev/null +++ b/docs/Packets/Messages/Protocols/hopper/Response.md @@ -0,0 +1,7 @@ + + +# hopper.Response + +### *class* Packets.Messages.Protocols.hopper.Response.HopperResponse(sender, senderID, sourceNode, recipient, recipientNode, response, cryptographyInfo) + +Send proxied request back to requester diff --git a/docs/Packets/Messages/Protocols/map/Announce.md b/docs/Packets/Messages/Protocols/map/Announce.md new file mode 100644 index 00000000..401b52c9 --- /dev/null +++ b/docs/Packets/Messages/Protocols/map/Announce.md @@ -0,0 +1,7 @@ + + +# map.Announce + +### *class* Packets.Messages.Protocols.map.Announce.AnnounceMessage(sender, senderID, sourceNode, cryptographyInfo, mapping) + +Announce the network map details and public key of the node for discovery diff --git a/docs/Packets/Packet.md b/docs/Packets/Packet.md index c8f37aa9..ab2eedf9 100644 --- a/docs/Packets/Packet.md +++ b/docs/Packets/Packet.md @@ -2,7 +2,7 @@ # Packet: Base packet -### *class* Packets.Packet.Packet(data: bytes, packetsID: int = -1, packetNumber=False, packetCount: int = 1, packetsClass: int = -1) +### *class* Packets.Packet.Packet(data: bytes, packetsID: int = -1, packetNumber=False, packetCount: int = 1, packetsClass: int = -1, primaryMessage=None) Base class for Packets diff --git a/docs/Packets/SubMessage.md b/docs/Packets/SubMessage.md index 10176c07..faeba941 100644 --- a/docs/Packets/SubMessage.md +++ b/docs/Packets/SubMessage.md @@ -2,6 +2,6 @@ # SubMessage: Additional data for larger messages -### *class* Packets.SubMessage.SubMessage +### *class* Packets.SubMessage.SubMessage(sender, senderID, sourceNode, recipient, recipientNode, cryptographyInfo, protocolID, pAction, data, target=True, primaryMessage=None) -TODO +SubMessage to a primary message, enables us to send more/dynamic data diff --git a/docs/Packets/SubPacket.md b/docs/Packets/SubPacket.md index 958298a5..2b6a0cdb 100644 --- a/docs/Packets/SubPacket.md +++ b/docs/Packets/SubPacket.md @@ -2,6 +2,6 @@ # SubPacket: Packets for submessages -### *class* Packets.SubPacket.SubPacket +### *class* Packets.SubMessage.SubMessage(sender, senderID, sourceNode, recipient, recipientNode, cryptographyInfo, protocolID, pAction, data, target=True, primaryMessage=None) -TODO +SubMessage to a primary message, enables us to send more/dynamic data diff --git a/docs/Services/Action.md b/docs/Services/Action.md new file mode 100644 index 00000000..587f3411 --- /dev/null +++ b/docs/Services/Action.md @@ -0,0 +1,59 @@ + + +# Services.Action + +### *class* Services.Action.Action(action, data, sender=None, senderID=None, sourceNode=None, recipient=None, recipientNode=None) + +Generic action class for triggering actions from sub processes on the main thread + +[πŸ”— Source](https://git.utopic.work/PierMesh/piermesh/src/branch/main/Services/Action.py) + +#### action + +Action to run + +* **Type:** + str + +#### data + +Data to pass to action + +* **Type:** + dict + +#### sender + +Sender identifier + +* **Type:** + str + +#### senderID + +Sender second level identifier + +* **Type:** + str + +#### sourceNode + +Sending node + +* **Type:** + str + +#### recipient + +Peer identifier to route to + +* **Type:** + str + +#### recipientNode + +Intended destination node identifier + +#### getAction() + +#### getData() diff --git a/docs/Siph/map.md b/docs/Siph/map.md index 0cadab23..a40b04cf 100644 --- a/docs/Siph/map.md +++ b/docs/Siph/map.md @@ -36,7 +36,7 @@ Map of PierMesh node IDs to MeshTastic node IDs * **Type:** dict -#### addLookup(onodeID: str, mnodeID: str) +#### *async* addLookup(onodeID: str, mnodeID: str) Adds node to lookup @@ -92,3 +92,11 @@ Import map from path #### render(pathPrefix: str = '') Render outer and inner network map to disk at the given path prefix + +#### syncaddLookup(onodeID: str, mnodeID: str) + +Adds node to lookup + +* **Parameters:** + * **onodeID** (*str*) – Internal nodeID + * **mnodeID** (*str*) – MeshTastic nodeID diff --git a/docs/Splash/serve.md b/docs/Splash/serve.md index f9215aae..1a0c5b56 100644 --- a/docs/Splash/serve.md +++ b/docs/Splash/serve.md @@ -2,22 +2,18 @@ # serve: Web UI server -### *class* Splash.serve.Server(transceiver, catch, onodeID, network, cLog) +### *class* Splash.serve.Server(transceiver, catch, onodeID, network, cryptographyInfo, remoteCatchIndex, cache) Web server that serves the web ui and provides web to node communication [πŸ”— Source](https://git.utopic.work/PierMesh/piermesh/src/branch/main/src/Sponge/serve.py) -#### cLog +#### transceiver -Reference to run.Node.cLog for logging - -#### transmitter - -Reference to our Transmission.transmission.Transmitter instance +Reference to our Transceiver.Transceiver.Transceiver instance * **Type:** - Transmission.transmission.Transmitter + [Transceiver.Transceiver.Transceiver](/PierMesh/piermesh/src/branch/main/docs/Transceiver/Transceiver.md#Transceiver.Transceiver.Transceiver) #### network @@ -51,6 +47,10 @@ Reference to our Catch Cache instance to pull from for serving Catchs * **Type:** [Daisy.Catch.Catch](/PierMesh/piermesh/src/branch/main/docs/Daisy/Catch.md#Daisy.Catch.Catch) -#### *async* sendToPeer(peerID: str, data: str) +#### *async* getPSKs() + +Get all PSKs for display + +#### *async* sendToPeer(peerID: str, data: str, target: str) Send data to Websocket of peer with peerID diff --git a/docs/Sponge/Protocols/Yellow.md b/docs/Sponge/Protocols/Yellow.md new file mode 100644 index 00000000..1b061c76 --- /dev/null +++ b/docs/Sponge/Protocols/Yellow.md @@ -0,0 +1,86 @@ + + +# Sponge.Protocols.Yellow + +### *class* Sponge.Protocols.Yellow.YCTX(packetsID, packetCount, pAction, todo, cryptographyInfo, sourceNode, subMessage=False, subMessages={}, submessagesIDs=[], eData=None) + +Context data structure for message parsing + +### *class* Sponge.Protocols.Yellow.Yellow(yctx: [YCTX](#Sponge.Protocols.Yellow.YCTX)) + +Message parser that’s subclassed to easily make parsers for specific protocols + +#### yctx + +Message parsing context + +* **Type:** + [YCTX](#Sponge.Protocols.Yellow.YCTX) + +#### message + +#### submessages + +Dictionary of submessages + +* **Type:** + dict + +#### submessagesIDs + +List of required submessages + +* **Type:** + list + +#### finishedSubmessages + +Dictionary of finished submessages + +* **Type:** + dict + +#### dataOrder + +List that maps packets based on their received order + +* **Type:** + list + +#### data + +Data of primary message + +* **Type:** + list + +#### nonce + +Cryptography artifact for decrypting message + +#### tag + +Cryptography artifact for decrypting message + +#### gotHead + +Whether we’ve gotten the head/header packet + +* **Type:** + bool + +#### todo + +Queue of actions to execute in the main loop + +#### checkComplete() + +#### *async* doAct(setpAction=False, repeatDataOnActions=[], subMessage=False) + +#### *async* dump() + +#### *async* id() + +#### pActions *= []* + +#### *async* processPacket(p, subMessage=False, rdoaoc=[]) diff --git a/docs/Sponge/Protocols/bubble.md b/docs/Sponge/Protocols/bubble.md new file mode 100644 index 00000000..1dd9ec0b --- /dev/null +++ b/docs/Sponge/Protocols/bubble.md @@ -0,0 +1,11 @@ + + +# Sponge.Protocols.bubble + +### *class* Sponge.Protocols.bubble.Bubble(yctx: [YCTX](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/Yellow.md#Sponge.Protocols.Yellow.YCTX)) + +Peer to peer protol + +[πŸ”— Source](https://git.utopic.work/PierMesh/piermesh/src/branch/main/src/Sponge/Protocols/bubble.py) + +#### pActions *= ['sendToPeer']* diff --git a/docs/Sponge/Protocols/catch.md b/docs/Sponge/Protocols/catch.md new file mode 100644 index 00000000..f04f2ee2 --- /dev/null +++ b/docs/Sponge/Protocols/catch.md @@ -0,0 +1,11 @@ + + +# Sponge.Protocols.catch + +### *class* Sponge.Protocols.catch.Catch(yctx: [YCTX](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/Yellow.md#Sponge.Protocols.Yellow.YCTX)) + +Catch exchange protocol + +[πŸ”— Source](https://git.utopic.work/PierMesh/piermesh/src/branch/main/Sponge/Protocols/catch.py) + +#### pActions *= ['sendCatch', 'routeCatch', 'syncIndex']* diff --git a/docs/Sponge/Protocols/cryptography.md b/docs/Sponge/Protocols/cryptography.md new file mode 100644 index 00000000..19592ee3 --- /dev/null +++ b/docs/Sponge/Protocols/cryptography.md @@ -0,0 +1,11 @@ + + +# Sponge.Protocols.cryptography + +### *class* Sponge.Protocols.cryptography.CryptographyFilter(yctx: [YCTX](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/Yellow.md#Sponge.Protocols.Yellow.YCTX)) + +Cryptographic operations protocol + +[πŸ”— Source](https://git.utopic.work/PierMesh/piermesh/src/branch/main/Sponge/Protocols/cryptography.py) + +#### pActions *= ['initCryptography']* diff --git a/docs/Sponge/Protocols/hopper.md b/docs/Sponge/Protocols/hopper.md new file mode 100644 index 00000000..b3b5dd23 --- /dev/null +++ b/docs/Sponge/Protocols/hopper.md @@ -0,0 +1,11 @@ + + +# Sponge.Protocols.hopper + +### *class* Sponge.Protocols.hopper.Hopper(yctx: [YCTX](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/Yellow.md#Sponge.Protocols.Yellow.YCTX)) + +Internet inter(h)op protocol + +[πŸ”— Source](https://git.utopic.work/PierMesh/piermesh/src/branch/main/Sponge/Protocols/hopper.py) + +#### pActions *= ['hop', 'routeHop']* diff --git a/docs/Sponge/Protocols/map.md b/docs/Sponge/Protocols/map.md new file mode 100644 index 00000000..ac98ed54 --- /dev/null +++ b/docs/Sponge/Protocols/map.md @@ -0,0 +1,13 @@ + + +# Sponge.Protocols.map + +### *class* Sponge.Protocols.map.Map(yctx: [YCTX](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/Yellow.md#Sponge.Protocols.Yellow.YCTX)) + +Network mapping protocol + +[πŸ”— Source](https://git.utopic.work/PierMesh/piermesh/src/branch/main/Sponge/Protocols/map.py) + +#### pActions *= ['map', 'initCryptography']* + +#### process(message, isSubMessage=False) diff --git a/docs/Sponge/base.md b/docs/Sponge/base.md index cbe96fc5..eb6e106e 100644 --- a/docs/Sponge/base.md +++ b/docs/Sponge/base.md @@ -2,43 +2,63 @@ # base: Primary filtering functionality -### *class* Sponge.base.Filter(cache, onodeID, todo, cLog) +### *class* Sponge.base.Filter(cache, onodeID, todo, cryptographyInfo) Packet filtering orchestration [πŸ”— Source](https://git.utopic.work/PierMesh/piermesh/src/branch/main/Sponge/base.py) -cLog -: Reference to run.Node.cLog for logging - -cache: Daisy.Cache.Cache -: Reference to our Daisy Cache instance - -completed: list -: List of completed messages IDs - -todo -: Reference to list of actions to do in the Node - -onodeID -: PierMesh node ID - #### cache -Messages is temporary storage for unfinished messages +Reference to our Daisy Cache instance + +* **Type:** + [Daisy.Cache.Cache](/PierMesh/piermesh/src/branch/main/docs/Daisy/Cache.md#Daisy.Cache.Cache) + +#### onodeID + +PierMesh node ID + +#### todo + +Reference to list of actions to do in the Node + +#### cryptographyInfo + +Cryptography instance for encrypting message + +* **Type:** + [Cryptography.WhaleSong.Transport](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.Transport) + +#### messages + +Temporary storage for unfinished messages + +* **Type:** + dict + +#### submessages + +Temporary storage for unfinished submessages + +* **Type:** + dict + +#### completed + +List of finished message ids so we don’t reprocess messages + +* **Type:** + list #### mCheck(payload: bytes) Check if payload bytes are msgpack encoded, otherwise skip -#### *async* protoMap(protocolID: int) +#### *async* protoMap(protocolID: int, packetsID, packetCount, sourceNode, submessagesIDs=[], pAction=None) Get protocol from protocol ID using the mlookup table -#### *async* protoRoute(completeMessage: dict) - -Route message to proper protocol handler - #### selfCheck(packet) Check if this is a self packet, if so skip @@ -46,31 +66,3 @@ Check if this is a self packet, if so skip #### *async* sieve(packet) Base filtering logic, takes a single MeshTastic packet - - - -# Protocols - -#### *async* bubble.filter(recipient, recipientNode, onodeID, todo) - -Peer to peer protol - -[πŸ”— Source](https://git.utopic.work/PierMesh/piermesh/src/branch/main/src/Sponge/Protocols/bubble.py) - -#### *async* catch.filter(recipient, recipientNode, todo) - -Catch exchange protocol - -[πŸ”— Source](https://git.utopic.work/PierMesh/piermesh/src/branch/main/Sponge/Protocols/catch.py) - -#### *async* cryptography.filter(recipientNode, todo) - -Cryptographic operations protocol - -[πŸ”— Source](https://git.utopic.work/PierMesh/piermesh/src/branch/main/Sponge/Protocols/cryptography.py) - -#### *async* map.filter(todo) - -Network mapping protocol - -[πŸ”— Source](https://git.utopic.work/PierMesh/piermesh/src/branch/main/Sponge/Protocols/map.py) diff --git a/docs/Transceiver/Transceiver.md b/docs/Transceiver/Transceiver.md index 4e06d37f..542c312d 100644 --- a/docs/Transceiver/Transceiver.md +++ b/docs/Transceiver/Transceiver.md @@ -2,7 +2,7 @@ # Transceiver: Layer 0 data transceiving -### *class* Transceiver.Transceiver.Transceiver(device, filter, onodeID, cache, catch, cryptographyInfo, cLog) +### *class* Transceiver.Transceiver.Transceiver(device, filter, onodeID, cache, catch, cryptographyInfo, network) Handling LoRa transceiving @@ -17,7 +17,7 @@ Reference to run.Node.cLog for logging Cryptography instance for encrypting transmissions * **Type:** - [Cryptography.WhaleSong.DHEFern](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.DHEFern) + Cryptography.WhaleSong.DHEFern #### filter @@ -114,6 +114,7 @@ Checks if acknowldgement was received per packet and if not resends #### responseCheck(packet) On acknowldgement response set acks based on response +TODO: Stop this being sent to sieve #### send(packet, recipientNode=False) @@ -122,6 +123,8 @@ Send individual packet * **Parameters:** **recipientNode** – If set send to specified node -#### *async* sendAnnounce() +#### *async* sendAnnounce(dontRespond=False) Send an announce packet (contains basic network mapping information) every so often so new nodes autoconnect + +#### *async* sendMessage(message: Message) diff --git a/docs/readme.md b/docs/readme.md index ada8c20e..7f5d7c16 100644 --- a/docs/readme.md +++ b/docs/readme.md @@ -11,7 +11,6 @@ sphinx-quickstart on Fri Jul 26 23:30:55 2024. --> * [run: PierMesh service runner](/PierMesh/piermesh/src/branch/main/docs/run.md) * [`Node`](/PierMesh/piermesh/src/branch/main/docs/run.md#run.Node) - * [`Node.toLog`](/PierMesh/piermesh/src/branch/main/docs/run.md#run.Node.toLog) * [`Node.actions`](/PierMesh/piermesh/src/branch/main/docs/run.md#run.Node.actions) * [`Node.todo`](/PierMesh/piermesh/src/branch/main/docs/run.md#run.Node.todo) * [`Node.network`](/PierMesh/piermesh/src/branch/main/docs/run.md#run.Node.network) @@ -20,29 +19,28 @@ sphinx-quickstart on Fri Jul 26 23:30:55 2024. --> * [`Node.nodeInfo`](/PierMesh/piermesh/src/branch/main/docs/run.md#run.Node.nodeInfo) * [`Node.onodeID`](/PierMesh/piermesh/src/branch/main/docs/run.md#run.Node.onodeID) * [`Node.oTransceiver`](/PierMesh/piermesh/src/branch/main/docs/run.md#run.Node.oTransceiver) - * [`Node.processed`](/PierMesh/piermesh/src/branch/main/docs/run.md#run.Node.processed) * [`Node.proc`](/PierMesh/piermesh/src/branch/main/docs/run.md#run.Node.proc) * [`Node.mTasks`](/PierMesh/piermesh/src/branch/main/docs/run.md#run.Node.mTasks) - * [`Node.action_initNodeDH()`](/PierMesh/piermesh/src/branch/main/docs/run.md#run.Node.action_initNodeDH) - * [`Node.action_keyDeriveDH()`](/PierMesh/piermesh/src/branch/main/docs/run.md#run.Node.action_keyDeriveDH) + * [`Node.action_addPSK()`](/PierMesh/piermesh/src/branch/main/docs/run.md#run.Node.action_addPSK) + * [`Node.action_hop()`](/PierMesh/piermesh/src/branch/main/docs/run.md#run.Node.action_hop) + * [`Node.action_initCryptography()`](/PierMesh/piermesh/src/branch/main/docs/run.md#run.Node.action_initCryptography) * [`Node.action_map()`](/PierMesh/piermesh/src/branch/main/docs/run.md#run.Node.action_map) + * [`Node.action_routeCatch()`](/PierMesh/piermesh/src/branch/main/docs/run.md#run.Node.action_routeCatch) + * [`Node.action_routeHop()`](/PierMesh/piermesh/src/branch/main/docs/run.md#run.Node.action_routeHop) * [`Node.action_sendCatch()`](/PierMesh/piermesh/src/branch/main/docs/run.md#run.Node.action_sendCatch) * [`Node.action_sendToPeer()`](/PierMesh/piermesh/src/branch/main/docs/run.md#run.Node.action_sendToPeer) - * [`Node.cLog()`](/PierMesh/piermesh/src/branch/main/docs/run.md#run.Node.cLog) + * [`Node.action_syncIndex()`](/PierMesh/piermesh/src/branch/main/docs/run.md#run.Node.action_syncIndex) + * [`Node.fsInit()`](/PierMesh/piermesh/src/branch/main/docs/run.md#run.Node.fsInit) + * [`Node.main()`](/PierMesh/piermesh/src/branch/main/docs/run.md#run.Node.main) + * [`Node.monitor()`](/PierMesh/piermesh/src/branch/main/docs/run.md#run.Node.monitor) * [`Node.spongeListen()`](/PierMesh/piermesh/src/branch/main/docs/run.md#run.Node.spongeListen) -* [ui: TUI application](/PierMesh/piermesh/src/branch/main/docs/ui.md) - * [`TUI`](/PierMesh/piermesh/src/branch/main/docs/ui.md#ui.TUI) - * [`TUI.visibleLogo`](/PierMesh/piermesh/src/branch/main/docs/ui.md#ui.TUI.visibleLogo) - * [`TUI.nodeOb`](/PierMesh/piermesh/src/branch/main/docs/ui.md#ui.TUI.nodeOb) - * [`TUI.done`](/PierMesh/piermesh/src/branch/main/docs/ui.md#ui.TUI.done) - * [`TUI.CSS_PATH`](/PierMesh/piermesh/src/branch/main/docs/ui.md#ui.TUI.CSS_PATH) - * [`TUI.action_quitFull()`](/PierMesh/piermesh/src/branch/main/docs/ui.md#ui.TUI.action_quitFull) - * [`TUI.action_toggleFullscreen()`](/PierMesh/piermesh/src/branch/main/docs/ui.md#ui.TUI.action_toggleFullscreen) - * [`TUI.compose()`](/PierMesh/piermesh/src/branch/main/docs/ui.md#ui.TUI.compose) - * [`TUI.do_set_cpu_percent()`](/PierMesh/piermesh/src/branch/main/docs/ui.md#ui.TUI.do_set_cpu_percent) - * [`TUI.do_set_mem()`](/PierMesh/piermesh/src/branch/main/docs/ui.md#ui.TUI.do_set_mem) - * [`TUI.do_write_line()`](/PierMesh/piermesh/src/branch/main/docs/ui.md#ui.TUI.do_write_line) - * [`TUI.on_mount()`](/PierMesh/piermesh/src/branch/main/docs/ui.md#ui.TUI.on_mount) +* [tlog](/PierMesh/piermesh/src/branch/main/docs/tlog.md) + * [`VHandler`](/PierMesh/piermesh/src/branch/main/docs/tlog.md#tlog.VHandler) + * [`VHandler.tolog`](/PierMesh/piermesh/src/branch/main/docs/tlog.md#tlog.VHandler.tolog) + * [`VHandler.emit()`](/PierMesh/piermesh/src/branch/main/docs/tlog.md#tlog.VHandler.emit) + * [`VHandler.tolog`](/PierMesh/piermesh/src/branch/main/docs/tlog.md#id0) + * [`logUI()`](/PierMesh/piermesh/src/branch/main/docs/tlog.md#tlog.logUI) + * [`runLogUI()`](/PierMesh/piermesh/src/branch/main/docs/tlog.md#tlog.runLogUI) * [Network: Network map representation](/PierMesh/piermesh/src/branch/main/docs/Siph/map.md) * [`Network`](/PierMesh/piermesh/src/branch/main/docs/Siph/map.md#Siph.map.Network) * [`Network.omap`](/PierMesh/piermesh/src/branch/main/docs/Siph/map.md#Siph.map.Network.omap) @@ -60,7 +58,9 @@ sphinx-quickstart on Fri Jul 26 23:30:55 2024. --> * [`Network.getRoute()`](/PierMesh/piermesh/src/branch/main/docs/Siph/map.md#Siph.map.Network.getRoute) * [`Network.mimport()`](/PierMesh/piermesh/src/branch/main/docs/Siph/map.md#Siph.map.Network.mimport) * [`Network.render()`](/PierMesh/piermesh/src/branch/main/docs/Siph/map.md#Siph.map.Network.render) + * [`Network.syncaddLookup()`](/PierMesh/piermesh/src/branch/main/docs/Siph/map.md#Siph.map.Network.syncaddLookup) * [hopper: Small internet interop utilities](/PierMesh/piermesh/src/branch/main/docs/Components/hopper.md) + * [`downloadFile()`](/PierMesh/piermesh/src/branch/main/docs/Components/hopper.md#Components.hopper.downloadFile) * [`get()`](/PierMesh/piermesh/src/branch/main/docs/Components/hopper.md#Components.hopper.get) * [`post()`](/PierMesh/piermesh/src/branch/main/docs/Components/hopper.md#Components.hopper.post) * [Daisy based cache](/PierMesh/piermesh/src/branch/main/docs/Daisy/Cache.md) @@ -73,55 +73,102 @@ sphinx-quickstart on Fri Jul 26 23:30:55 2024. --> * [`Catch`](/PierMesh/piermesh/src/branch/main/docs/Daisy/Catch.md#Daisy.Catch.Catch) * [`Catch.get()`](/PierMesh/piermesh/src/branch/main/docs/Daisy/Catch.md#Daisy.Catch.Catch.get) * [`Catch.sget()`](/PierMesh/piermesh/src/branch/main/docs/Daisy/Catch.md#Daisy.Catch.Catch.sget) +* [Credential](/PierMesh/piermesh/src/branch/main/docs/Daisy/Credential.md) + * [`Credential`](/PierMesh/piermesh/src/branch/main/docs/Daisy/Credential.md#Daisy.Credential.Credential) +* [CryptographyUtil](/PierMesh/piermesh/src/branch/main/docs/Daisy/CryptographyUtil.md) + * [`SteelPetal`](/PierMesh/piermesh/src/branch/main/docs/Daisy/CryptographyUtil.md#Daisy.CryptographyUtil.SteelPetal) + * [`SteelPetal.decrypt()`](/PierMesh/piermesh/src/branch/main/docs/Daisy/CryptographyUtil.md#Daisy.CryptographyUtil.SteelPetal.decrypt) + * [`SteelPetal.encrypt()`](/PierMesh/piermesh/src/branch/main/docs/Daisy/CryptographyUtil.md#Daisy.CryptographyUtil.SteelPetal.encrypt) + * [`SteelPetal.pad()`](/PierMesh/piermesh/src/branch/main/docs/Daisy/CryptographyUtil.md#Daisy.CryptographyUtil.SteelPetal.pad) * [Daisy](/PierMesh/piermesh/src/branch/main/docs/Daisy/Daisy.md) * [`Daisy`](/PierMesh/piermesh/src/branch/main/docs/Daisy/Daisy.md#Daisy.Daisy.Daisy) * [`Daisy.filepath`](/PierMesh/piermesh/src/branch/main/docs/Daisy/Daisy.md#Daisy.Daisy.Daisy.filepath) * [`Daisy.msg`](/PierMesh/piermesh/src/branch/main/docs/Daisy/Daisy.md#Daisy.Daisy.Daisy.msg) * [`Daisy.get()`](/PierMesh/piermesh/src/branch/main/docs/Daisy/Daisy.md#Daisy.Daisy.Daisy.get) + * [`Daisy.json_to_msg()`](/PierMesh/piermesh/src/branch/main/docs/Daisy/Daisy.md#Daisy.Daisy.Daisy.json_to_msg) * [`Daisy.read()`](/PierMesh/piermesh/src/branch/main/docs/Daisy/Daisy.md#Daisy.Daisy.Daisy.read) * [`Daisy.sublist()`](/PierMesh/piermesh/src/branch/main/docs/Daisy/Daisy.md#Daisy.Daisy.Daisy.sublist) * [`Daisy.write()`](/PierMesh/piermesh/src/branch/main/docs/Daisy/Daisy.md#Daisy.Daisy.Daisy.write) -* [Soil: Daisy signal management](/PierMesh/piermesh/src/branch/main/docs/Daisy/Soil.md) - * [`Compound`](/PierMesh/piermesh/src/branch/main/docs/Daisy/Soil.md#Daisy.Soil.Compound) - * [`Compound.on_any_event()`](/PierMesh/piermesh/src/branch/main/docs/Daisy/Soil.md#Daisy.Soil.Compound.on_any_event) +* [Index](/PierMesh/piermesh/src/branch/main/docs/Daisy/Index.md) + * [`Index`](/PierMesh/piermesh/src/branch/main/docs/Daisy/Index.md#Daisy.Index.Index) + * [`Index.addEntry()`](/PierMesh/piermesh/src/branch/main/docs/Daisy/Index.md#Daisy.Index.Index.addEntry) + * [`Index.search()`](/PierMesh/piermesh/src/branch/main/docs/Daisy/Index.md#Daisy.Index.Index.search) +* [Ref](/PierMesh/piermesh/src/branch/main/docs/Daisy/Ref.md) + * [`Ref`](/PierMesh/piermesh/src/branch/main/docs/Daisy/Ref.md#Daisy.Ref.Ref) * [Store: Daisy key value store](/PierMesh/piermesh/src/branch/main/docs/Daisy/Store.md) * [`Store`](/PierMesh/piermesh/src/branch/main/docs/Daisy/Store.md#Daisy.Store.Store) + * [`Store.epehemeral`](/PierMesh/piermesh/src/branch/main/docs/Daisy/Store.md#Daisy.Store.Store.epehemeral) + * [`Store.createEmpty()`](/PierMesh/piermesh/src/branch/main/docs/Daisy/Store.md#Daisy.Store.Store.createEmpty) * [`Store.getRecord()`](/PierMesh/piermesh/src/branch/main/docs/Daisy/Store.md#Daisy.Store.Store.getRecord) * [`Store.update()`](/PierMesh/piermesh/src/branch/main/docs/Daisy/Store.md#Daisy.Store.Store.update) -* [WhaleSong: Diffie hellman ephemeral Fernet based encryption](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md) - * [`DHEFern`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.DHEFern) - * [`DHEFern.cLog`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.DHEFern.cLog) - * [`DHEFern.loadedParams`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.DHEFern.loadedParams) - * [`DHEFern.loadedKeys`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.DHEFern.loadedKeys) - * [`DHEFern.nodeNickname`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.DHEFern.nodeNickname) - * [`DHEFern.cache`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.DHEFern.cache) - * [`DHEFern.publicKey`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.DHEFern.publicKey) - * [`DHEFern.privateKey`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.DHEFern.privateKey) - * [`DHEFern.checkInMem()`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.DHEFern.checkInMem) - * [`DHEFern.decrypt()`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.DHEFern.decrypt) - * [`DHEFern.encrypt()`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.DHEFern.encrypt) - * [`DHEFern.genKeyPair()`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.DHEFern.genKeyPair) - * [`DHEFern.genParams()`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.DHEFern.genParams) - * [`DHEFern.getParamsBytes()`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.DHEFern.getParamsBytes) - * [`DHEFern.getRecord()`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.DHEFern.getRecord) - * [`DHEFern.getSalt()`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.DHEFern.getSalt) - * [`DHEFern.initStore()`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.DHEFern.initStore) - * [`DHEFern.keyDerive()`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.DHEFern.keyDerive) - * [`DHEFern.loadParamBytes()`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.DHEFern.loadParamBytes) - * [`DHEFern.loadRecordToMem()`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.DHEFern.loadRecordToMem) +* [WhaleSong](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md) + * [`Transport`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.Transport) + * [`Transport.loadedKeys`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.Transport.loadedKeys) + * [`Transport.nodeNickname`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.Transport.nodeNickname) + * [`Transport.cache`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.Transport.cache) + * [`Transport.publicKey`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.Transport.publicKey) + * [`Transport.privateKey`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.Transport.privateKey) + * [`Transport.daisyCryptography`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.Transport.daisyCryptography) + * [`Transport.addPeerEphemeralKey()`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.Transport.addPeerEphemeralKey) + * [`Transport.addPublickey()`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.Transport.addPublickey) + * [`Transport.decrypt()`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.Transport.decrypt) + * [`Transport.encrypt()`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.Transport.encrypt) + * [`Transport.genOurEphemeralKey()`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.Transport.genOurEphemeralKey) + * [`Transport.genStaticKey()`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.Transport.genStaticKey) + * [`Transport.generateSessionKey()`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.Transport.generateSessionKey) + * [`Transport.getRecord()`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.Transport.getRecord) + * [`Transport.initStore()`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.Transport.initStore) + * [`Transport.kdf()`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.Transport.kdf) + * [`Transport.sessionSetup()`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.Transport.sessionSetup) * [base: Primary filtering functionality](/PierMesh/piermesh/src/branch/main/docs/Sponge/base.md) * [`Filter`](/PierMesh/piermesh/src/branch/main/docs/Sponge/base.md#Sponge.base.Filter) * [`Filter.cache`](/PierMesh/piermesh/src/branch/main/docs/Sponge/base.md#Sponge.base.Filter.cache) + * [`Filter.onodeID`](/PierMesh/piermesh/src/branch/main/docs/Sponge/base.md#Sponge.base.Filter.onodeID) + * [`Filter.todo`](/PierMesh/piermesh/src/branch/main/docs/Sponge/base.md#Sponge.base.Filter.todo) + * [`Filter.cryptographyInfo`](/PierMesh/piermesh/src/branch/main/docs/Sponge/base.md#Sponge.base.Filter.cryptographyInfo) + * [`Filter.messages`](/PierMesh/piermesh/src/branch/main/docs/Sponge/base.md#Sponge.base.Filter.messages) + * [`Filter.submessages`](/PierMesh/piermesh/src/branch/main/docs/Sponge/base.md#Sponge.base.Filter.submessages) + * [`Filter.completed`](/PierMesh/piermesh/src/branch/main/docs/Sponge/base.md#Sponge.base.Filter.completed) * [`Filter.mCheck()`](/PierMesh/piermesh/src/branch/main/docs/Sponge/base.md#Sponge.base.Filter.mCheck) * [`Filter.protoMap()`](/PierMesh/piermesh/src/branch/main/docs/Sponge/base.md#Sponge.base.Filter.protoMap) - * [`Filter.protoRoute()`](/PierMesh/piermesh/src/branch/main/docs/Sponge/base.md#Sponge.base.Filter.protoRoute) * [`Filter.selfCheck()`](/PierMesh/piermesh/src/branch/main/docs/Sponge/base.md#Sponge.base.Filter.selfCheck) * [`Filter.sieve()`](/PierMesh/piermesh/src/branch/main/docs/Sponge/base.md#Sponge.base.Filter.sieve) -* [Protocols](/PierMesh/piermesh/src/branch/main/docs/Sponge/base.md#protocols) - * [`bubble.filter()`](/PierMesh/piermesh/src/branch/main/docs/Sponge/base.md#Sponge.Protocols.bubble.filter) - * [`catch.filter()`](/PierMesh/piermesh/src/branch/main/docs/Sponge/base.md#Sponge.Protocols.catch.filter) - * [`cryptography.filter()`](/PierMesh/piermesh/src/branch/main/docs/Sponge/base.md#Sponge.Protocols.cryptography.filter) - * [`map.filter()`](/PierMesh/piermesh/src/branch/main/docs/Sponge/base.md#Sponge.Protocols.map.filter) +* [Sponge.Protocols.Yellow](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/Yellow.md) + * [`YCTX`](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/Yellow.md#Sponge.Protocols.Yellow.YCTX) + * [`Yellow`](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/Yellow.md#Sponge.Protocols.Yellow.Yellow) + * [`Yellow.yctx`](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/Yellow.md#Sponge.Protocols.Yellow.Yellow.yctx) + * [`Yellow.message`](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/Yellow.md#Sponge.Protocols.Yellow.Yellow.message) + * [`Yellow.submessages`](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/Yellow.md#Sponge.Protocols.Yellow.Yellow.submessages) + * [`Yellow.submessagesIDs`](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/Yellow.md#Sponge.Protocols.Yellow.Yellow.submessagesIDs) + * [`Yellow.finishedSubmessages`](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/Yellow.md#Sponge.Protocols.Yellow.Yellow.finishedSubmessages) + * [`Yellow.dataOrder`](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/Yellow.md#Sponge.Protocols.Yellow.Yellow.dataOrder) + * [`Yellow.data`](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/Yellow.md#Sponge.Protocols.Yellow.Yellow.data) + * [`Yellow.nonce`](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/Yellow.md#Sponge.Protocols.Yellow.Yellow.nonce) + * [`Yellow.tag`](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/Yellow.md#Sponge.Protocols.Yellow.Yellow.tag) + * [`Yellow.gotHead`](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/Yellow.md#Sponge.Protocols.Yellow.Yellow.gotHead) + * [`Yellow.todo`](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/Yellow.md#Sponge.Protocols.Yellow.Yellow.todo) + * [`Yellow.checkComplete()`](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/Yellow.md#Sponge.Protocols.Yellow.Yellow.checkComplete) + * [`Yellow.doAct()`](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/Yellow.md#Sponge.Protocols.Yellow.Yellow.doAct) + * [`Yellow.dump()`](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/Yellow.md#Sponge.Protocols.Yellow.Yellow.dump) + * [`Yellow.id()`](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/Yellow.md#Sponge.Protocols.Yellow.Yellow.id) + * [`Yellow.pActions`](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/Yellow.md#Sponge.Protocols.Yellow.Yellow.pActions) + * [`Yellow.processPacket()`](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/Yellow.md#Sponge.Protocols.Yellow.Yellow.processPacket) +* [Sponge.Protocols.bubble](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/bubble.md) + * [`Bubble`](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/bubble.md#Sponge.Protocols.bubble.Bubble) + * [`Bubble.pActions`](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/bubble.md#Sponge.Protocols.bubble.Bubble.pActions) +* [Sponge.Protocols.catch](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/catch.md) + * [`Catch`](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/catch.md#Sponge.Protocols.catch.Catch) + * [`Catch.pActions`](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/catch.md#Sponge.Protocols.catch.Catch.pActions) +* [Sponge.Protocols.cryptography](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/cryptography.md) + * [`CryptographyFilter`](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/cryptography.md#Sponge.Protocols.cryptography.CryptographyFilter) + * [`CryptographyFilter.pActions`](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/cryptography.md#Sponge.Protocols.cryptography.CryptographyFilter.pActions) +* [Sponge.Protocols.hopper](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/hopper.md) + * [`Hopper`](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/hopper.md#Sponge.Protocols.hopper.Hopper) + * [`Hopper.pActions`](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/hopper.md#Sponge.Protocols.hopper.Hopper.pActions) +* [Sponge.Protocols.map](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/map.md) + * [`Map`](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/map.md#Sponge.Protocols.map.Map) + * [`Map.pActions`](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/map.md#Sponge.Protocols.map.Map.pActions) + * [`Map.process()`](/PierMesh/piermesh/src/branch/main/docs/Sponge/Protocols/map.md#Sponge.Protocols.map.Map.process) * [Header packet: Metadata packet](/PierMesh/piermesh/src/branch/main/docs/Packets/HeaderPacket.md) * [`Header`](/PierMesh/piermesh/src/branch/main/docs/Packets/HeaderPacket.md#Packets.HeaderPacket.Header) * [`Header.sender`](/PierMesh/piermesh/src/branch/main/docs/Packets/HeaderPacket.md#Packets.HeaderPacket.Header.sender) @@ -147,7 +194,23 @@ sphinx-quickstart on Fri Jul 26 23:30:55 2024. --> * [SubMessage: Additional data for larger messages](/PierMesh/piermesh/src/branch/main/docs/Packets/SubMessage.md) * [`SubMessage`](/PierMesh/piermesh/src/branch/main/docs/Packets/SubMessage.md#Packets.SubMessage.SubMessage) * [SubPacket: Packets for submessages](/PierMesh/piermesh/src/branch/main/docs/Packets/SubPacket.md) - * [`SubPacket`](/PierMesh/piermesh/src/branch/main/docs/Packets/SubPacket.md#Packets.SubPacket.SubPacket) + * [`SubMessage`](/PierMesh/piermesh/src/branch/main/docs/Packets/SubPacket.md#Packets.SubMessage.SubMessage) +* [bubble.Bubble](/PierMesh/piermesh/src/branch/main/docs/Packets/Messages/Protocols/bubble/Bubble.md) + * [`Bubble`](/PierMesh/piermesh/src/branch/main/docs/Packets/Messages/Protocols/bubble/Bubble.md#Packets.Messages.Protocols.bubble.Bubble.Bubble) +* [catch.IndexSync](/PierMesh/piermesh/src/branch/main/docs/Packets/Messages/Protocols/catch/IndexSync.md) + * [`IndexSync`](/PierMesh/piermesh/src/branch/main/docs/Packets/Messages/Protocols/catch/IndexSync.md#Packets.Messages.Protocols.catch.IndexSync.IndexSync) +* [catch.Request](/PierMesh/piermesh/src/branch/main/docs/Packets/Messages/Protocols/catch/Request.md) + * [`CatchRequest`](/PierMesh/piermesh/src/branch/main/docs/Packets/Messages/Protocols/catch/Request.md#Packets.Messages.Protocols.catch.Request.CatchRequest) +* [catch.Response](/PierMesh/piermesh/src/branch/main/docs/Packets/Messages/Protocols/catch/Response.md) + * [`CatchResponse`](/PierMesh/piermesh/src/branch/main/docs/Packets/Messages/Protocols/catch/Response.md#Packets.Messages.Protocols.catch.Response.CatchResponse) +* [cryptography.Handshake](/PierMesh/piermesh/src/branch/main/docs/Packets/Messages/Protocols/cryptography/Handshake.md) + * [`Handshake`](/PierMesh/piermesh/src/branch/main/docs/Packets/Messages/Protocols/cryptography/Handshake.md#Packets.Messages.Protocols.cryptography.Handshake.Handshake) +* [hopper.Request](/PierMesh/piermesh/src/branch/main/docs/Packets/Messages/Protocols/hopper/Request.md) + * [`HopperRequest`](/PierMesh/piermesh/src/branch/main/docs/Packets/Messages/Protocols/hopper/Request.md#Packets.Messages.Protocols.hopper.Request.HopperRequest) +* [hopper.Response](/PierMesh/piermesh/src/branch/main/docs/Packets/Messages/Protocols/hopper/Response.md) + * [`HopperResponse`](/PierMesh/piermesh/src/branch/main/docs/Packets/Messages/Protocols/hopper/Response.md#Packets.Messages.Protocols.hopper.Response.HopperResponse) +* [map.Announce](/PierMesh/piermesh/src/branch/main/docs/Packets/Messages/Protocols/map/Announce.md) + * [`AnnounceMessage`](/PierMesh/piermesh/src/branch/main/docs/Packets/Messages/Protocols/map/Announce.md#Packets.Messages.Protocols.map.Announce.AnnounceMessage) * [Transceiver: Layer 0 data transceiving](/PierMesh/piermesh/src/branch/main/docs/Transceiver/Transceiver.md) * [`Transceiver`](/PierMesh/piermesh/src/branch/main/docs/Transceiver/Transceiver.md#Transceiver.Transceiver.Transceiver) * [`Transceiver.cLog`](/PierMesh/piermesh/src/branch/main/docs/Transceiver/Transceiver.md#Transceiver.Transceiver.Transceiver.cLog) @@ -170,17 +233,32 @@ sphinx-quickstart on Fri Jul 26 23:30:55 2024. --> * [`Transceiver.responseCheck()`](/PierMesh/piermesh/src/branch/main/docs/Transceiver/Transceiver.md#Transceiver.Transceiver.Transceiver.responseCheck) * [`Transceiver.send()`](/PierMesh/piermesh/src/branch/main/docs/Transceiver/Transceiver.md#Transceiver.Transceiver.Transceiver.send) * [`Transceiver.sendAnnounce()`](/PierMesh/piermesh/src/branch/main/docs/Transceiver/Transceiver.md#Transceiver.Transceiver.Transceiver.sendAnnounce) + * [`Transceiver.sendMessage()`](/PierMesh/piermesh/src/branch/main/docs/Transceiver/Transceiver.md#Transceiver.Transceiver.Transceiver.sendMessage) * [serve: Web UI server](/PierMesh/piermesh/src/branch/main/docs/Splash/serve.md) * [`Server`](/PierMesh/piermesh/src/branch/main/docs/Splash/serve.md#Splash.serve.Server) - * [`Server.cLog`](/PierMesh/piermesh/src/branch/main/docs/Splash/serve.md#Splash.serve.Server.cLog) - * [`Server.transmitter`](/PierMesh/piermesh/src/branch/main/docs/Splash/serve.md#Splash.serve.Server.transmitter) + * [`Server.transceiver`](/PierMesh/piermesh/src/branch/main/docs/Splash/serve.md#Splash.serve.Server.transceiver) * [`Server.network`](/PierMesh/piermesh/src/branch/main/docs/Splash/serve.md#Splash.serve.Server.network) * [`Server.nodeID`](/PierMesh/piermesh/src/branch/main/docs/Splash/serve.md#Splash.serve.Server.nodeID) * [`Server.peerIDs`](/PierMesh/piermesh/src/branch/main/docs/Splash/serve.md#Splash.serve.Server.peerIDs) * [`Server.app`](/PierMesh/piermesh/src/branch/main/docs/Splash/serve.md#Splash.serve.Server.app) * [`Server.catch`](/PierMesh/piermesh/src/branch/main/docs/Splash/serve.md#Splash.serve.Server.catch) + * [`Server.getPSKs()`](/PierMesh/piermesh/src/branch/main/docs/Splash/serve.md#Splash.serve.Server.getPSKs) * [`Server.sendToPeer()`](/PierMesh/piermesh/src/branch/main/docs/Splash/serve.md#Splash.serve.Server.sendToPeer) - +* [Context](/PierMesh/piermesh/src/branch/main/docs/Config/Context.md) + * [`Context`](/PierMesh/piermesh/src/branch/main/docs/Config/Context.md#Config.Context.Context) + * [`Context.ctx`](/PierMesh/piermesh/src/branch/main/docs/Config/Context.md#Config.Context.Context.ctx) +* [Services.Action](/PierMesh/piermesh/src/branch/main/docs/Services/Action.md) + * [`Action`](/PierMesh/piermesh/src/branch/main/docs/Services/Action.md#Services.Action.Action) + * [`Action.action`](/PierMesh/piermesh/src/branch/main/docs/Services/Action.md#Services.Action.Action.action) + * [`Action.data`](/PierMesh/piermesh/src/branch/main/docs/Services/Action.md#Services.Action.Action.data) + * [`Action.sender`](/PierMesh/piermesh/src/branch/main/docs/Services/Action.md#Services.Action.Action.sender) + * [`Action.senderID`](/PierMesh/piermesh/src/branch/main/docs/Services/Action.md#Services.Action.Action.senderID) + * [`Action.sourceNode`](/PierMesh/piermesh/src/branch/main/docs/Services/Action.md#Services.Action.Action.sourceNode) + * [`Action.recipient`](/PierMesh/piermesh/src/branch/main/docs/Services/Action.md#Services.Action.Action.recipient) + * [`Action.recipientNode`](/PierMesh/piermesh/src/branch/main/docs/Services/Action.md#Services.Action.Action.recipientNode) + * [`Action.getAction()`](/PierMesh/piermesh/src/branch/main/docs/Services/Action.md#Services.Action.Action.getAction) + * [`Action.getData()`](/PierMesh/piermesh/src/branch/main/docs/Services/Action.md#Services.Action.Action.getData) +$ # System Overview diff --git a/docs/run.md b/docs/run.md index c6bfadfb..811f8bbc 100644 --- a/docs/run.md +++ b/docs/run.md @@ -8,13 +8,6 @@ Class that handles most of the PierMesh data [πŸ”— Source](https://git.utopic.work/PierMesh/piermesh/src/branch/main/src/run.py) -#### toLog - -We store logs to be processed here - -* **Type:** - list - #### actions Dictionary mapping methods with the action prefix to the method name after action dynamically to be called through Sponge (Sponge.base) filtering @@ -71,13 +64,6 @@ LoRa transceiver Transceiver * **Type:** [Transceiver](/PierMesh/piermesh/src/branch/main/docs/Transceiver/Transceiver.md#Transceiver.Transceiver.Transceiver) -#### processed - -List of IDs of already completed messages so that we don’t reprocess messages - -* **Type:** - list - #### proc This process (psutil.Process), used for managing and monitoring PierMesh @@ -92,29 +78,37 @@ Dictionary of PierMesh service tasks * **Type:** dict -#### SEE ALSO -`logPassLoop` -: Loop to handle logging to file and TUI +#### *async* action_addPSK(data) -#### *async* action_initNodeDH(data: dict) +Action to add PSK for specific node, currently unused -Initialize diffie hellman key exchange +#### *async* action_hop(data) + +Proxy a request to the main internet (in the future cross protocol/link) + +#### *async* action_initCryptography(data: dict) + +Initialize AES-GCM encrypted transport session #### SEE ALSO -`Cryptography.DHEFern.DHEFern` +[`Cryptography.WhaleSong.Transport`](/PierMesh/piermesh/src/branch/main/docs/Cryptography/WhaleSong.md#Cryptography.WhaleSong.Transport) : End to end encryption functionality -#### *async* action_keyDeriveDH(data: dict) - -Derive key via diffie hellman key exchange - #### *async* action_map(data: dict) Map new network data to internal network map #### SEE ALSO `Siph.network.Network` -: Layered graph etwork representation +: Layered graph network representation + +#### *async* action_routeCatch(data: dict) + +Route received catch to peer who requested it + +#### *async* action_routeHop(data: dict) + +Return proxy request results to requester #### *async* action_sendCatch(data: dict) @@ -131,21 +125,27 @@ Send data to a peer connected to the server `Sponge.Protocols` : Protocol based packet filtering -`webui.serve.Server` +[`Splash.serve.Server`](/PierMesh/piermesh/src/branch/main/docs/Splash/serve.md#Splash.serve.Server) : Runs a light Microdot web server with http/s and websocket functionality -`webui.serve.Server.sendToPeer` +[`Splash.serve.Server.sendToPeer`](/PierMesh/piermesh/src/branch/main/docs/Splash/serve.md#Splash.serve.Server.sendToPeer) : Function to actually execute the action -#### cLog(priority: int, message: str) +#### *async* action_syncIndex(data: dict) -Convenience function that logs to the ui and log files +Add received index entries to Catch via the a remote Catch index -* **Parameters:** - * **priority** (*int*) – Priority of message to be passed to logging - * **message** (*str*) – Message to log -* **Return type:** - None +#### *async* fsInit() + +Initialize the file system for use + +#### *async* main() + +Main loop, sets up the message listening, system monitoring and server running loops + +#### *async* monitor() + +Monitor and log ram and cpu usage #### *async* spongeListen() diff --git a/docs/stale/Soil.md b/docs/stale/Soil.md new file mode 100644 index 00000000..987f50c2 --- /dev/null +++ b/docs/stale/Soil.md @@ -0,0 +1,3 @@ + + +# Soil: Daisy signal management diff --git a/docs/stale/ui.md b/docs/stale/ui.md new file mode 100644 index 00000000..ba08fd05 --- /dev/null +++ b/docs/stale/ui.md @@ -0,0 +1,3 @@ + + +# ui: TUI application diff --git a/docs/tlog.md b/docs/tlog.md new file mode 100644 index 00000000..c18ef839 --- /dev/null +++ b/docs/tlog.md @@ -0,0 +1,35 @@ + + + + +# tlog + +### *class* tlog.VHandler(level, tolog) + +Custom log handler to push logs into a thread-safe queue so the TUI can read them + +[πŸ”— Source](https://git.utopic.work/PierMesh/piermesh/src/branch/main/src/tlog.py) + +#### tolog + +Thread-safe log queue + +* **Type:** + Queue.queue + +#### emit(record) + +Do whatever it takes to actually log the specified logging record. + +This version is intended to be implemented by subclasses and so +raises a NotImplementedError. + +#### tolog *= * + +### tlog.logUI(stdscr, tolog, nodeNickname) + +TUI loop + +### tlog.runLogUI(tolog, nodeNickname) + +Some required kludge diff --git a/docs/ui.md b/docs/ui.md deleted file mode 100644 index da7ecc20..00000000 --- a/docs/ui.md +++ /dev/null @@ -1,74 +0,0 @@ - - -# ui: TUI application - -### *class* ui.TUI(driver_class: Type[Driver] | None = None, css_path: str | PurePath | List[str | PurePath] | None = None, watch_css: bool = False) - -TUI for PierMesh - -[πŸ”— Source](https://git.utopic.work/PierMesh/piermesh/src/branch/main/src/ui.py) - -#### visibleLogo - -Whether the logo is visible or not, used in toggling visibility - -* **Type:** - bool - -#### nodeOb - -Reference to the Node running the PierMesh service - -* **Type:** - [Node](/PierMesh/piermesh/src/branch/main/docs/run.md#run.Node) - -#### done - -Whether the TUI has been killed - -* **Type:** - bool - -#### CSS_PATH *: ClassVar[CSSPathType | None]* *= 'ui.tcss'* - -File paths to load CSS from. - -#### action_quitFull() - -Kill the whole stack by setting self to done and terminating the thread. We check in run.monitor later and kill the rest of the stack then with psutil - -#### SEE ALSO -`run.monitor` - -#### action_toggleFullscreen() - -Toggle fullscreen logs by either collapsing width or setting it to it’s original size - -#### compose() - -Build the TUI - -#### do_set_cpu_percent(percent: float) - -Set CPU percent in the label and progress bar - -* **Parameters:** - **percent** (*float*) – Percent of the cpu PierMesh is using - -#### do_set_mem(memmb: float) - -Set memory usage label in the ui - -* **Parameters:** - **memmb** (*float*) – Memory usage of PierMesh in megabytes - -#### do_write_line(logLine: str) - -Write line to the logs panel - -* **Parameters:** - **logLine** (*str*) – Line to log - -#### on_mount() - -Called at set up, configures the title and the progess bar diff --git a/src/.piermesh b/src/.piermesh index 4e2e5242..616426e2 100644 --- a/src/.piermesh +++ b/src/.piermesh @@ -1,9 +1,9 @@ # DONT TOUCH THIS SECTION UNLESS YOU KNOW WHAT YOURE DOING [DEFAULT] -Nickname = node00 +Nickname = node1 StartupDelay = 0 WebUIPort = 5000 -ShowTUI = True +ShowTUI = False [OPERATOR_REQUIRED] TransceiverPort = /dev/ttyACM0 @@ -12,3 +12,4 @@ PSK = jgf765!FS0+6 # DO YOUR NON REQUIRED SETTINGS HERE [OPERATOR_OVERRIDES] +ShowTUI = False diff --git a/src/Actions/Actions.rst b/src/Actions/Actions.rst new file mode 100644 index 00000000..e69de29b diff --git a/src/Components/hopper.py b/src/Components/hopper.py index 9896404a..d27fee87 100755 --- a/src/Components/hopper.py +++ b/src/Components/hopper.py @@ -1,19 +1,21 @@ -from bs4 import BeautifulSoup - +# NOTE: Used for requesting web pages import requests -import msgpack +# NOTE: Used for parsing web pages +from bs4 import BeautifulSoup -import lzma +# NOTE: Generic imports import base64 import mimetypes import logging -from Packets.Messages.Protocols.hopper.Response import HopperResponse - logger = logging.getLogger("__main__." + __name__) + def downloadFile(url, text=True, mimeType=None): + """ + Download resource from url and convert it to text or a data url + """ fbytes = b"" with requests.get(url, stream=True) as r: r.raise_for_status() @@ -25,7 +27,7 @@ def downloadFile(url, text=True, mimeType=None): if mimeType == None: mimeType, encoding = mimetypes.guess_type(url) if mimeType == None: - raise Error( + raise ValueError( "Couldnt guess mime type and none was supplied, cant encode to data url" ) b64str = base64.b64encode(fbytes).decode("utf-8") diff --git a/src/Config/Args.rst b/src/Config/Args.rst new file mode 100644 index 00000000..e69de29b diff --git a/src/Config/Context.py b/src/Config/Context.py index 9cff904b..e82dd00a 100644 --- a/src/Config/Context.py +++ b/src/Config/Context.py @@ -1,8 +1,25 @@ - class Context: + """ + Generic context data structure, current subclassed for use in filters, see Sponge/Protocols/Yellow.py + + `πŸ”— Source `__ + + Attributes + ---------- + ctx: dict + Dictionary of context values + + """ def __init__(self, subsets: dict={}, **kwargs): - # Subsets should be a dict of list of value keys + """ + Parameters + ---------- + subsets: dict + Keys mapped to lists of keys for grouping values (unused currently) + kwargs: kwargs + key word arguments to map to the context as key/val pairs + """ self.ctx = {} self.subsets = subsets for key, val in kwargs.items(): diff --git a/src/Config/Context.rst b/src/Config/Context.rst new file mode 100644 index 00000000..181bb066 --- /dev/null +++ b/src/Config/Context.rst @@ -0,0 +1,5 @@ +Context +======= + +.. autoclass:: Config.Context.Context + :members: diff --git a/src/Cryptography/WhaleSong.py b/src/Cryptography/WhaleSong.py index 96606cb5..58134dfa 100755 --- a/src/Cryptography/WhaleSong.py +++ b/src/Cryptography/WhaleSong.py @@ -1,24 +1,25 @@ -import base64 +# NOTE: Generic imports import os import lzma import logging +import traceback +# NOTE: Import for handling message data import msgpack +# NOTE: Cryptography imports from Crypto.PublicKey import ECC from Crypto.Hash import SHAKE128 from Crypto.Protocol.DH import key_agreement from Crypto.Cipher import AES +# NOTE: Daisy database import from Daisy.Store import Store logger = logging.getLogger("__main__." + __name__) # TODO: Different store directories per node -# TODO: First time psk transport initiation -# Add this credential manually, its picked up and used when the two nodes try to communicate before the session is encrypted - class Transport: """ @@ -27,11 +28,6 @@ class Transport: Attributes ---------- - cLog - Method reference to `run.Node.cLog` so we can log to the ui from here - - loadedParams: dict - In memory representations of cryptography parameters loadedKeys: dict In memory representations of cryptography keys @@ -39,7 +35,7 @@ class Transport: nodeNickname: str Name of node for isolating configs when running multiple nodes - cache: Components.daisy.Cache + cache: Daisy.Cache.Cache Daisy cache for use in storing cryptography information publicKey @@ -47,39 +43,43 @@ class Transport: privateKey Private key for node - """ + daisyCryptography: Daisy.CryptographyUtil.SteelPetal + Record cryptography reference + """ def __init__(self, cache, nodeNickname, daisyCryptography, psk): """ Parameters ---------- - cache: Components.daisy.Cache + cache: Daisy.Cache.Cache Reference to the node instances Daisy cache nodeNickname: str Node nickname for record storage - cLog - Reference to `run.Node.cLog` + daisyCryptography: Daisy.CryptographyUtil.SteelPetal + Record cryptography reference + psk: str + Plaintext pre shared key """ self.stores = {} self.nodeNickname = nodeNickname self.cache = cache self.daisyCryptography = daisyCryptography - if os.path.exists("daisy/cryptography/{0}/key".format(nodeNickname)) == False: - logger.log(20, "Key store DNE, initializing") + if not os.path.exists("{0}/daisy/cryptography/key".format(nodeNickname)): + logger.info("Key store DNE, initializing") self.initStore("key") else: - logger.log(20, "Key store exists, loading") - self.stores["key"] = Store( - "key", "cryptography", nodeNickname, daisyCryptography - ) - logger.log(20, "Store loaded") - logger.log(20, "Key store initialized") + logger.info("Key store exists, loading") + self.stores["key"] = Store( + "key", "cryptography", nodeNickname, daisyCryptography + ) + logger.info("Key store initialized") srecord = self.getRecord("key", "self") if srecord == False: self.stores["key"].createEmpty("self") + # TODO: Note that this happens in the docs self.stores["key"].update( "self", {"PSK": self.daisyCryptography.pad(psk).encode("utf-8")}, @@ -95,56 +95,12 @@ class Transport: }, write=False) def kdf(self, bytesX): + """ + Key derivation function + """ + # TODO: Better explanation return SHAKE128.new(bytesX).read(32) - def checkInMem(self, store: str, nodeID: str, checkFieldsExist=[]): - """ - Check if parameters or keys are loaded for node of nodeID - - Parameters - ---------- - store: str - Whether to check loaded keys or parameters - - """ - if store == "param": - return nodeID in self.loadedParams.keys() - elif store == "key": - record = self.getRecord("key", nodeID) - if record != False: - for field in checkFieldsExist: - if not (field in record.keys()): - if field == "staticKey": - self.genStaticKey(nodeID) - elif field == "ourEphemeralKey": - self.genOurEphemeralKey(nodeID) - - def loadRecordToMem(self, store: str, nodeID: str): - """ - Load record of nodeID from store to either keys or pameters - """ - r = self.getRecord(store, nodeID) - if r == False: - logger.log( - 30, "Tried to load nonexistent {0} for node {1}".format(store, nodeID) - ) - return False - elif self.checkInMem(store, nodeID): - logger.log(10, "{0}s already deserialized, skipping".format(store)) - else: - if store == "param": - self.loadedParams[nodeID] = self.loadParamBytes(r) - """ - elif store == "key": - self.loadedKeys[nodeID] = { - "publicKey": Serialization.load_pem_public_key(r["publicKey"]), - "privateKey": Serialization.load_pem_private_key( - r["privateKey"], None - ), - } - """ - return True - def getRecord(self, store: str, key: str, ephemeral=False): """ Get record from store: store with key: key @@ -156,7 +112,6 @@ class Transport: else: return r - # TODO: Fix stores, URGENT def initStore(self, store: str): """ Initialize store: store @@ -164,23 +119,14 @@ class Transport: self.stores[store] = Store( store, "cryptography", self.nodeNickname, self.daisyCryptography ) - if store == "param": - self.genParams() - self.stores[store].update("self", self.getParamsBytes(), recur=False) - elif store == "key": + if store == "key": self.stores[store].update("self", {}, recur=False) else: - logger.log(30, "Store not defined") + logger.warning("Store not defined") - def genStaticKey(self, onodeID, paramsOverride=False): + def genStaticKey(self, onodeID): """ - Generate public and private keys from self.params (TODO: Gen from passed params) - - paramsOverride - False or parameters to use (TODO) - - setSelf: bool - Whether to set self.privateKey and self.publicKey + Generate static key for session encryption with given node """ staticKey = ECC.generate(curve="p256") self.stores["key"].update( @@ -194,10 +140,25 @@ class Transport: self.stores["key"].update(onodeID, {"staticKey": staticKey}, write=False) def genOurEphemeralKey(self, onodeID): + """ + Generate epehemeral key for session encryption with given node + """ ourEphemeralKey = ECC.generate(curve="p256") self.stores["key"].update(onodeID, {"ourEphemeralKey": ourEphemeralKey}, write=False) - def addPublickey(self, onodeID, publicKey, forSelf=False): + def addPublickey(self, onodeID, publicKey, forSelf: bool = False): + """ + Add a public key for a given node including this one + + onodeID + Node identifier + + publicKey + Public key to add + + forSelf: bool + Whether to add key for this node + """ if forSelf: publicKey = ECC.generate(curve="p256") self.stores["key"].update("self", { @@ -210,30 +171,48 @@ class Transport: }, write=False) else: - # TODO: Fix stores - # self.stores["key"].update(onodeID, {"publicKey": publicKey}) - logger.log(20, "Importing keys") + logger.info("Importing keys") record = self.getRecord("key", onodeID) if record == False: self.stores["key"].createEmpty(onodeID) self.stores["key"].update(onodeID, {"publicKey": publicKey}) self.stores["key"].update(onodeID, {"publicKey": ECC.import_key(publicKey)}, write=False) - def addPeerEphemeralKey(self, onodeID, peerEphemeralKey): + def addPeerEphemeralKey(self, onodeID, peerEphemeralKey: bytes): + """ + Add a peer node's epehemeral key for session encryption + + onodeID + Node identifier + + peerEphemeralKey: bytes + Serialized ephemeral key + """ self.stores["key"].update(onodeID, {"peerEphemeralKey": ECC.import_key(peerEphemeralKey)}, write=False) - def sessionSetup(self, onodeID, peerEphemeralKey): - # TODO: Deeper checking before loading - # TODO: Loading existing records - if self.getRecord("key", onodeID) == False: - logger.log(30, "No record, waiting for announce") + def sessionSetup(self, onodeID, peerEphemeralKey: bytes): + """ + Set up transport encryption session + + onodeID + Node identifier + + peerEphemeralKey: bytes + Serialized ephemeral key + """ + if not self.getRecord("key", onodeID): + logger.warning("No record, waiting for announce") else: self.addPeerEphemeralKey(onodeID, peerEphemeralKey) self.generateSessionKey(onodeID) def generateSessionKey(self, onodeID): - # TODO: Gen static key if not exists - # TODO: Gen our ephemeral key if not exists + """ + Generate session key for transport encryption + + onodeID + Node identifier + """ keysOb = self.getRecord("key", onodeID, ephemeral=True) if ("publicKey" not in keysOb) or ("staticKey" not in keysOb): dkeysOb = self.getRecord("key", onodeID) @@ -273,16 +252,18 @@ class Transport: self.stores["key"].update(onodeID, {"sessionKey": sessionKey}, write=False) return sessionKey - # TODO: Build in transport security (node/node) def encrypt(self, data, nodeID: str, isDict: bool = True, pskEncrypt=False): """ - Do Fernet encryption + Encrypt given data with AES GCM data Either bytes or dict to encrypt isDict: bool Whether data is a dictionary + + pskEncrypt: bool + Whether to encrypt with pre-shared key """ if nodeID == "-00001" or pskEncrypt: cipher = AES.new(self.getRecord("key", "self", ephemeral=True)["PSK"], AES.MODE_GCM) @@ -317,24 +298,33 @@ class Transport: else: logger.log(20, "Node {0} does not have session key".format(nodeID)) - def decrypt(self, data, nodeID: str, nonce, tag): + def decrypt(self, data, onodeID: str, nonce, tag): """ - Decrypt bytes and return either str or dict (TODO: Check whether to msgpack load) + Decrypt bytes and return either str or dict depending on result + + onodeID: str + Node identifier + + nonce + Encryption nonce + + tag + Encryption tag """ - # TODO: Handling existing record - record = self.getRecord("key", nodeID, ephemeral=True) + record = self.getRecord("key", onodeID, ephemeral=True) if (record == False) or ("sessionKey" not in record.keys()): cipher = AES.new(self.getRecord("key", "self", ephemeral=True)["PSK"], AES.MODE_GCM, nonce=nonce) data = cipher.decrypt(data) - logger.log(10, data) - #data = msgpack.loads(data) - data = msgpack.loads(lzma.decompress(data)) - logger.log(10, "Decrypt/deserialize output") - logger.log(10, data) + # logger.debug(data) + try: + data = msgpack.loads(lzma.decompress(data)) + except Exception: + logger.error(traceback.format_exc()) + return False + # logger.debug("Decrypt/deserialize output") + # logger.debug(data) return data - # logger.log(20, "Node {0} not in keychain".format(nodeID)) - # return False else: if "sessionKey" in record.keys(): sessionKey = record["sessionKey"] @@ -352,5 +342,5 @@ class Transport: return data else: - logger.log(20, "Node {0} does not have session key".format(nodeID)) + logger.warning("Node {0} does not have session key".format(onodeID)) return False diff --git a/src/Cryptography/WhaleSong.rst b/src/Cryptography/WhaleSong.rst index cd10fad5..6818f48c 100644 --- a/src/Cryptography/WhaleSong.rst +++ b/src/Cryptography/WhaleSong.rst @@ -1,7 +1,7 @@ -WhaleSong: Diffie hellman ephemeral Fernet based encryption -=========================================================== +WhaleSong +========= -.. autoclass:: Cryptography.WhaleSong.DHEFern +.. autoclass:: Cryptography.WhaleSong.Transport :members: :undoc-members: diff --git a/src/Daisy/Cache.py b/src/Daisy/Cache.py index 7029e986..051aaf51 100755 --- a/src/Daisy/Cache.py +++ b/src/Daisy/Cache.py @@ -1,17 +1,18 @@ from Daisy.Daisy import Daisy import os +import logging import msgpack -from watchdog.observers import Observer - # TODO: Dumping to cacheFile +logger = logging.getLogger("__main__." + __name__) + class Cache: """ - In memory collection of Daisy records + In memory collection of Daisy records, provides a search functionality currently utilized by `Daisy.Catch.Catch` `πŸ”— Source `__ """ @@ -22,26 +23,25 @@ class Cache: filepaths=None, cacheFile=None, path: str = "daisy", - walk: bool = False, - isCatch: bool = False, + walk: bool = False ): """ Parameters ---------- + daisyCryptography: Daisy.CryptographyUtil.SteelPetal + Record cryptography reference + filepaths Either a list of filepaths to load or None cacheFile - Path to a cache file which is a collection of paths to load + Path to a cache file which is a collection of paths to load or None path: str Path prefix to load records from walk: bool Whether to automatically walk the path and load records - - isCatch: bool - Whether this cache is for catchs """ self.daisyCryptography = daisyCryptography self.data = {} @@ -50,12 +50,13 @@ class Cache: if not os.path.exists(self.path): os.makedirs(self.path) - if filepaths != None: + if filepaths is not None: for fp in filepaths: fp = path + "/" + fp if os.path.isfile(fp): self.data[fp] = Daisy(fp, daisyCryptography) - elif cacheFile != None: + elif cacheFile is not None: + self.cacheFile = cacheFile with open(cacheFile, "r") as f: for fp in f.read().split("\n"): self.data[fp] = Daisy(fp, daisyCryptography) @@ -78,17 +79,23 @@ class Cache: data: dict Data to populate record with + + remote: bool + Whether this is a reference to a distributed file (not implemented yet) """ - if remote == False: + if not remote: with open(self.path + "/" + path, "wb") as f: f.write(msgpack.dumps(data)) - # logging.log(10, "Done creating record") + logger.debug("Done creating record") self.data[path] = Daisy(self.path + "/" + path, self.daisyCryptography) - # logging.log(10, "Done loading to Daisy") + logger.debug("Done loading to Daisy") return self.data[path] else: - self.data[path] = Ref(path, remote) - return self.data[path] + logger.debug("Not that (you shouldn't be here yet, remote Daisy links aren't ready yet)") + # TODO: Full remote path functionality + pass + # self.data[path] = Ref(path, remote) + # return self.data[path] def get(self, path: str): """ @@ -104,7 +111,8 @@ class Cache: self.data[path] = Daisy(self.path + "/" + path, self.daisyCryptography) return self.data[path] else: - # logging.log(10, "File does not exist") + path = self.path + "/" + path + logger.debug(f"File {path} does not exist") return False def refresh(self): @@ -116,18 +124,18 @@ class Cache: def search(self, keydict: dict, strict: bool = True): """ - Search cache for record for records with values + Search cache for record for records with keys and values matching those + in the keydict keydict: dict - Values to search for strict: bool - Whether to require values match + Whether to require all keys/values match """ results = [] for key, val in self.data.items(): val = val.get() - if strict and type(val) != str: + if strict and type(val) is not str: addcheck = False for k, v in keydict.items(): if k in val.keys(): @@ -138,7 +146,7 @@ class Cache: break if addcheck: results.append([key, val]) - elif type(val) != str: + elif type(val) is not str: for k, v in keydict.items(): if k in val.keys(): if v in val[k]: diff --git a/src/Daisy/Catch.py b/src/Daisy/Catch.py index aa72d104..353e5d78 100755 --- a/src/Daisy/Catch.py +++ b/src/Daisy/Catch.py @@ -1,9 +1,10 @@ +# NOTE: Local imports from Daisy.Cache import Cache -from Daisy.Ref import Ref +# from Daisy.Ref import Ref +# NOTE: Generic imports import os import random -import uuid class Catch(Cache): @@ -26,7 +27,22 @@ class Catch(Cache): walk: bool = False, ): """ - Basically the same initialization parameters as Catch + Parameters + ---------- + path: str + Path prefix to load records from + + daisyCryptography: Daisy.CryptographyUtil.SteelPetal + Record cryptography reference + + filepaths + Either a list of filepaths to load or None + + cacheFile + Path to a cache file which is a collection of paths to load or None + + walk: bool + Whether to automatically walk the path and load records """ super().__init__( daisyCryptography, @@ -34,7 +50,6 @@ class Catch(Cache): cacheFile=catchFile, path=path, walk=walk, - isCatch=True, ) # TODO: Fins @@ -62,7 +77,7 @@ class Catch(Cache): List of (maximum 8 characters) strings at the end of the catch oe None if none """ r = "" - if fins != None and fins != "": + if fins is not None and fins != "": r = self.search({"head": head, "body": body, "fins": fins}) else: r = self.search({"head": head, "body": body}) @@ -73,16 +88,16 @@ class Catch(Cache): def addc(self, peer, node, seperator, head, body, data, fins=None, remote=False): tnpath = f"catch/{node}" - if os.path.exists(self.path + "/" + tnpath) != True: + if not os.path.exists(self.path + "/" + tnpath): os.makedirs(self.path + "/" + tnpath) tppath = tnpath + "/" + peer - if os.path.exists(self.path + "/" + tppath) != True: + if not os.path.exists(self.path + "/" + tppath): os.makedirs(self.path + "/" + tppath) sid = str(random.randrange(0, 999999)).zfill(6) data["seperator"] = seperator data["head"] = head data["body"] = body - if fins != None: + if fins is not None: data["fins"] = fins res = self.create("{0}/{1}".format(tppath, sid), data, remote=remote) return [sid, res] @@ -91,7 +106,7 @@ class Catch(Cache): dirList = [] for k, v in self.data.items(): curCatch = {"remoteNode": onodeID} - if type(v.msg) != str: + if type(v.msg) is not str: curCatch = curCatch | v.msg del curCatch["html"] dirList.append(curCatch) diff --git a/src/Daisy/Credential.py b/src/Daisy/Credential.py index 64a32ceb..99f38100 100644 --- a/src/Daisy/Credential.py +++ b/src/Daisy/Credential.py @@ -2,6 +2,9 @@ from Daisy.Daisy import Daisy class Credential(Daisy): + """ + Currently unused credential class, will be fleshed out for credentialed access to the web ui + """ def __init__(self, nodeNickname, credentialName, extension, daisyCryptography): fname = "data/{0}/{1}.{2}".format(nodeNickname, credentialName, extension) super().__init__( diff --git a/src/Daisy/Credential.rst b/src/Daisy/Credential.rst new file mode 100644 index 00000000..87dd9948 --- /dev/null +++ b/src/Daisy/Credential.rst @@ -0,0 +1,6 @@ +Credential +========== + +.. autoclass:: Daisy.Credential.Credential + :members: + :undoc-members: diff --git a/src/Daisy/CryptographyUtil.py b/src/Daisy/CryptographyUtil.py index a9423dd6..c20436f8 100644 --- a/src/Daisy/CryptographyUtil.py +++ b/src/Daisy/CryptographyUtil.py @@ -1,4 +1,7 @@ +# NOTE: Cryptography import from Crypto.Cipher import AES + +# NOTE: Generic imports import traceback import logging @@ -7,9 +10,22 @@ logger = logging.getLogger("__main__." + __name__) class SteelPetal: - def __init__(self, key, nonce=None, testData=None): + """ + Cryptography utility for encrypting files + """ + + def __init__(self, key: str, nonce=None, testData=None): + """ + Parameters + ---------- + key: str + User's plaintext key + + nonce + Cryptographic artifact we can use to reinitialize cryptographic operations + """ try: - if nonce == None: + if nonce is None: self.cipher = AES.new(self.pad(key).encode("utf-8"), AES.MODE_GCM) self.nonce = self.cipher.nonce else: @@ -17,30 +33,48 @@ class SteelPetal: self.pad(key).encode("utf-8"), AES.MODE_GCM, nonce=nonce ) self.nonce = nonce - if testData != None: + if testData is not None: try: self.cipher.decrypt(testData) - except: - logger.log(30, traceback.format_exc()) + except Exception: + logger.error(traceback.format_exc()) return False - except: - logger.log(30, traceback.format_exc()) + except Exception: + logger.error(traceback.format_exc()) - def pad(self, key): + def pad(self, key: str): + """ + Pad key to make it usable + + key: str + User's plain text key + """ BS = AES.block_size key = key + (BS - len(key) % BS) * chr(BS - len(key) % BS) return key - def encrypt(self, data): + def encrypt(self, data: bytes): + """ + Encrypt binary data + + data: bytes + Data to encrypt + """ try: return self.cipher.encrypt_and_digest(data) - except: - logger.log(20, traceback.format_exec()) + except Exception: + logger.error(traceback.format_exec()) return False - def decrypt(self, data): + def decrypt(self, data: bytes): + """ + Decrypt encrypted binary data + + data: bytes + Data to decrypt + """ try: return self.cipher.decrypt(data) - except: - logger.log(20, traceback.format_exec()) + except Exception: + logger.error(traceback.format_exec()) return False diff --git a/src/Daisy/CryptographyUtil.rst b/src/Daisy/CryptographyUtil.rst new file mode 100644 index 00000000..76d87544 --- /dev/null +++ b/src/Daisy/CryptographyUtil.rst @@ -0,0 +1,6 @@ +CryptographyUtil +================ + +.. autoclass:: Daisy.CryptographyUtil.SteelPetal + :members: + :undoc-members: diff --git a/src/Daisy/Daisy.py b/src/Daisy/Daisy.py index 3b3ce53d..14d9a58c 100755 --- a/src/Daisy/Daisy.py +++ b/src/Daisy/Daisy.py @@ -4,9 +4,6 @@ import msgpack import logging # TODO: delete -# TODO: propagate json changes to msgpack automatically -# TODO: propagate msgpack changes to cache automatically -# TODO: Indexing logger = logging.getLogger("__main__." + __name__) @@ -42,7 +39,6 @@ class Daisy: In memory representation """ - # TODO: Strong encrypt def __init__( self, filepath: str, @@ -64,30 +60,31 @@ class Daisy: template: bool Which template to Use - prefillDict: bool - Whether to fill the record with a template + prefillDict: dict + Data to prefill record with """ + # TODO: Finish remote implementation self.remote = False self.filepath = filepath - if remote != False: + if remote: self.remote = True self.remoteNodeID = remote else: - if os.path.exists(filepath) != True: + if not os.path.exists(filepath): with open(filepath, "wb") as f: - if template != False: + if template: if template in templates.keys(): t = templates[template].get() - if prefillDict != False: + if prefillDict: for k in prefillDict.keys(): t[k] = prefillDict[k] f.write(msgpack.dumps(t)) self.msg = t else: - logger.log(20, "No such template as: " + template) + logger.error("No such template as: " + template) else: t = {} - if prefillDict != False: + if prefillDict: for k in prefillDict.keys(): t[k] = prefillDict[k] f.write(msgpack.dumps(t)) @@ -98,8 +95,6 @@ class Daisy: with open(filepath, "rb") as f: self.msg = msgpack.loads(f.read()) - # Use override for updating - def write( self, override=False, @@ -108,7 +103,7 @@ class Daisy: recur: bool = False, ): """ - Write record to disk + Write record to disk, note: use override with updated record to update record Parameters ---------- @@ -124,11 +119,11 @@ class Daisy: recur: bool Whether to recursively handle keys """ - if override != False: + if override: for key in override.keys(): # TODO: Deeper recursion if recur: - if not key in self.msg.keys(): + if key not in self.msg.keys(): self.msg[key] = {} for ikey in override[key].keys(): self.msg[key][ikey] = override[key][ikey] diff --git a/src/Daisy/Index.py b/src/Daisy/Index.py index 6ac820db..36210187 100644 --- a/src/Daisy/Index.py +++ b/src/Daisy/Index.py @@ -2,15 +2,37 @@ from Daisy.Daisy import Daisy class Index(Daisy): + """ + A searchable index of records, this is currently only half implemented + but works enough to hold our remote catch index + """ + def __init__( self, - nodeNickname, + nodeNickname: str, daisyCryptography, - prefill=[], - indexedFields=[], - autoIndex=True, + prefill: list = [], + indexedFields: list = [], + autoIndex: bool = True, ): - # TODO: Load from disk + """ + Attributes + ---------- + nodeNickname: str + Node nickname for record storage + + daisyCryptography: Daisy.CryptographyUtil.SteelPetal + Record cryptography reference + + prefill: list[dict] + List of objects to prefill the index with + + indexedFields: list[str] + List of fields to index (what not to drop from a full record) + + autoIndex: bool + Whether to automatically build the list of indexed fields present in the prefill objects + """ if autoIndex: if prefill != []: if indexedFields == []: @@ -21,29 +43,36 @@ class Index(Daisy): indexedFields.append(k) indexedFields = list(set(indexedFields)) super().__init__( - nodeNickname + ".index", + f"{nodeNickname}/daisy/{nodeNickname}.index", daisyCryptography, prefillDict={"_index": prefill, "_fields": indexedFields}, ) - def addEntry(self, entry): + def addEntry(self, entry: dict): + """ + Add a record to the index + + entry: dict + Record to add to the index + """ + # TODO: Filter entry for only indexed fields index = self.msg["_index"] index.append(entry) self.write(override={"_index": index}) def search(self, keydict: dict, strict: bool = True): """ - Search cache for record for records with values + Search index for record for records with values keydict: dict - Values to search for + Keys/Values to search for strict: bool - Whether to require values match + Whether to require all keys/values match """ results = [] for ob in self.msg["_index"]: - if strict and type(ob) != str: + if strict and type(ob) is not str: addcheck = False for k, v in keydict.items(): if k in ob.keys(): diff --git a/src/Daisy/Index.rst b/src/Daisy/Index.rst new file mode 100644 index 00000000..adb6ac96 --- /dev/null +++ b/src/Daisy/Index.rst @@ -0,0 +1,6 @@ +Index +===== + +.. autoclass:: Daisy.Index.Index + :members: + :undoc-members: diff --git a/src/Daisy/Ref.py b/src/Daisy/Ref.py index 4ec77ca2..cfaabfba 100644 --- a/src/Daisy/Ref.py +++ b/src/Daisy/Ref.py @@ -2,5 +2,15 @@ from Daisy.Daisy import Daisy class Ref(Daisy): - def __init__(self, path, remoteNodeID): - super().__init__(path, remote=remoteNodeID) + """ + Reference to a remote record + + metadata: dict + Data to fill record with, should only be metadata + + path: str + Where to store data locally + """ + + def __init__(self, metadata: dict, path: str, remoteNodeID: str): + super().__init__(path, remote=remoteNodeID, prefillDict=metadata) diff --git a/src/Daisy/Ref.rst b/src/Daisy/Ref.rst new file mode 100644 index 00000000..b2629025 --- /dev/null +++ b/src/Daisy/Ref.rst @@ -0,0 +1,6 @@ +Ref +=== + +.. autoclass:: Daisy.Ref.Ref + :members: + :undoc-members: diff --git a/src/Daisy/Soil.py b/src/Daisy/Soil.py deleted file mode 100644 index 9975d792..00000000 --- a/src/Daisy/Soil.py +++ /dev/null @@ -1,49 +0,0 @@ -from watchdog.observers import Observer -from watchdog.events import FileSystemEventHandler - -global garden -""" -Map of instances to list of signals -to be processed -""" -garden = {} - - -class Compound(FileSystemEventHandler): - """ - File system watcher to propagate disk changes - - `πŸ”— Source `__ - """ - - def __init__(self, cache, isCatch: bool = False): - """ - Parameters - ---------- - cache: Cache - Daisy cache to update - - isCatch: bool - Is the cache for catchs - """ - self.cache = cache - self.isCatch = isCatch - super().__init__() - - def on_any_event(self, event): - """ - Called when a CRUD operation is performed on a record file - - Parameters - ---------- - event - Event object provided by watchdog - """ - if not (".json" in event.src_path): - if not (".md" in event.src_path): - tpath = "/".join(event.src_path.split("/")[1:]) - if tpath != "": - if self.isCatch: - self.cache.sget(tpath) - else: - self.cache.get(tpath).get() diff --git a/src/Daisy/Soil.rst b/src/Daisy/Soil.rst deleted file mode 100644 index 7005bebc..00000000 --- a/src/Daisy/Soil.rst +++ /dev/null @@ -1,6 +0,0 @@ -Soil: Daisy signal management -============================= - -.. autoclass:: Daisy.Soil.Compound - :members: - :undoc-members: diff --git a/src/Daisy/Store.py b/src/Daisy/Store.py index cf9b78cb..8c9bb750 100755 --- a/src/Daisy/Store.py +++ b/src/Daisy/Store.py @@ -6,28 +6,71 @@ import traceback logger = logging.getLogger("__main__." + __name__) -# TODO: Higher priority erros class Store(Daisy): """ Key value store - `πŸ”— Source `__ - """ + `πŸ”— Source `_ + Attributes + ---------- + + epehemeral: dict + Memory only records + """ def __init__(self, store: str, path: str, nodeNickname: str, daisyCryptography): - fpath = "daisy/{0}/{1}".format(path, nodeNickname) - cpath = "{0}/{1}/{2}".format(path, nodeNickname, store) + """ + Parameters + ---------- + store: str + Store name/filename + + path: str + Folder record should be in + + nodeNickname: str + Node nickname for record storage + + daisyCryptography: Daisy.CryptographyUtil.SteelPetal + Record cryptography reference + """ + fpath = f"{nodeNickname}/daisy/{path}" + cpath = f"{fpath}/{store}" if not os.path.exists(fpath): os.mkdir(fpath) - super().__init__("daisy/" + cpath, daisyCryptography) + super().__init__(cpath, daisyCryptography) self.ephemeral = {} - def createEmpty(self, key): + def createEmpty(self, key: str): + """ + Parameters + ---------- + Create empty record at the given key + + key: str + Key to create empty record at + """ self.msg[key] = {} - # TODO: Update usages of update where necessary to keep out of mem def update(self, entry: str, data, recur: bool = True, write=True): + """ + Update given record + + Parameters + ---------- + entry: str + Key to update record of + + data + Data to update record with + + recur: bool + Whether to iterate over data + + write: bool + Whether record is ephemeral + """ if write: if recur: if entry not in self.msg.keys(): @@ -47,19 +90,29 @@ class Store(Daisy): self.ephemeral[entry] = data def getRecord(self, key: str, ephemeral=False): - logger.log(30, key) + """ + Get record at key + + Parameters + ---------- + key: str + + ephemeral: bool + Whether key is only in memory, used for session cryptography credentials currently + """ + logger.debug(key) try: if ephemeral: if key in self.ephemeral.keys(): return self.ephemeral[key] else: - logger.log(20, "Record does not exist") + logger.info("Record does not exist") return False else: if key in self.get().keys(): return self.get()[key] else: - logger.log(20, "Record does not exist") + logger.info("Record does not exist") return False except Exception: - logger.log(30, traceback.format_exc()) + logger.warning(traceback.format_exc()) diff --git a/src/Packets/HeaderPacket.py b/src/Packets/HeaderPacket.py index 2fb6f36c..48f7b830 100755 --- a/src/Packets/HeaderPacket.py +++ b/src/Packets/HeaderPacket.py @@ -30,7 +30,7 @@ class Header(Packet): Whether a response should be sent when the message completes reception (TODO) pAction: int - 3 digit (maximum) pAction ID for mapping precise actions within a protocol (TODO) + 3 digit (maximum) pAction ID for mapping precise actions within a protocol """ def __init__( @@ -42,12 +42,23 @@ class Header(Packet): sourceNode: int, recipient: int, recipientNode: int, - subpacket: bool = False, wantFullResponse: bool = False, packetsClass: int = 0, pAction: int = -1, target=True, ): + """ + Arguments + --------- + sourceNode: int + Source of request + + packetsClass: int + Integer ID matching the class of the message + + target + Whether the message is being sent to a target, if so, where + """ super().__init__( b"", packetsID=packetsID, packetCount=packetCount, packetsClass=packetsClass ) @@ -60,7 +71,6 @@ class Header(Packet): else: self.recipient = -1 self.recipientNode = -1 - # TODO: Populating with submessage ids self.submessages = [] self.wantFullResponse = wantFullResponse self.pAction = pAction @@ -69,7 +79,7 @@ class Header(Packet): def usePreset(self, path: str, daisyCryptography): """ - Add preset fields to the packet + Add preset fields to the packet, currently unused """ preset = Daisy(path, daisyCryptography) for key in preset.get().keys(): diff --git a/src/Packets/Message.py b/src/Packets/Message.py index 10ca8aee..c324b28c 100755 --- a/src/Packets/Message.py +++ b/src/Packets/Message.py @@ -44,7 +44,6 @@ class Message: primaryMessage=None, pskEncrypt=False ): - # TODO: PSK for usage prior to credentials """ Parameters ---------- @@ -57,20 +56,41 @@ class Message: senderDisplayName: int 3 digit (maximum) ID for mapping display names to a given user + sourceNode: int + Source of request + recipient: int 6 digit (maximum) node or peer ID recipientNode: int 6 digit (maximum) node ID to route the packet to + cryptographyInfo: Cryptography.WhaleSong.Transport + Cryptography instance for encrypting message + + packetsClass: int + Which protocol the packets are using + + pAction: int + 3 digit (maximum) pAction ID for mapping precise actions within a protocol + dataSize: int Size to cut the bytesObject into per packet wantFullResponse: bool Whether to send a response when the message has completed reception (TODO: Kill all retries for associated packets when received) - packetsClass: int - Which protocol the packets are using + target + Whether the message is being sent to a target, if so, where + + subMessage: bool + Whether this is a submessage + + primaryMessage + Primary message this is a submessage to, if this is a submessage + + pskEncrypt: bool + Whether to encrypt the message with the pre shared key """ self.recipientNode = recipientNode self.target = target @@ -90,28 +110,25 @@ class Message: ) self.packets = packets else: - # Data passed in by peers should already have been e2ee encrypted by SubtleCrypto - # Transport encryption - # bytesObject = lzma.compress(bytesObject, str(recipientNode).zfill(6), isDict=False) + # TODO: Data passed in by peers should already have been e2ee encrypted by SubtleCrypto if subMessage == False: bytesObject, nonce, tag = cryptographyInfo.encrypt( bytesObject, str(recipientNode).zfill(6), isDict=False, pskEncrypt=pskEncrypt ) - logger.log(10, bytesObject) + # logger.debug(bytesObject) self.nonce = nonce self.tag = tag packets = [] self.packetsID = random.randrange(0, 999999) pnum = 1 - # if subMessage: dataSize = 80 blen = math.ceil(len(bytesObject) / dataSize) tb = b"" for it in range(blen): if it >= (blen - 1): - b = bytesObject[it * dataSize :] + b = bytesObject[it * dataSize:] else: - b = bytesObject[it * dataSize : (it * dataSize + dataSize)] + b = bytesObject[it * dataSize: (it * dataSize + dataSize)] if subMessage: packets.append( p.Packet( @@ -152,7 +169,7 @@ class Message: if subMessage: pnum -= 1 - + for it in range(pnum): packet = msgpack.loads(packets[it].dump()) packet["packetCount"] = pnum @@ -168,31 +185,45 @@ class Message: def reassemble(self, completedMessage: dict, cryptographyInfo, subMessage=False, yctx=None, packetCount=None): """ - Reassemble packets from a completed message in `Sponge.base` + Reassemble packets from a completed message in `Sponge.base`, meant to be used without instantiation + + Arguments + --------- + completedMessage: dict + All parts of the message and submessage + + cryptographyInfo: Cryptography.WhaleSong.Transport + Cryptography instance for encrypting message + + subMessage: bool + Whether this is a submessage + + yctx + Message parsing context + + packetCount + Number of packets """ data = b"" sourceNode = None - # TODO: Fix reassembly for primary if subMessage: sourceNode = yctx["sourceNode"]["val"] for it in range(1, packetCount+1): data += completedMessage["data"][completedMessage["dataOrder"].index(it)] data = msgpack.loads(lzma.decompress(data)) - logger.log(10, data) - logger.log(10, completedMessage["data"]) - logger.log(10, completedMessage["dataOrder"]) + # logger.debug(data) + # logger.debug(completedMessage["data"]) + # logger.debug(completedMessage["dataOrder"]) else: - # TODO: Cryptography setup packetCount = int(completedMessage.yctx["packetCount"]["val"]) sourceNode = completedMessage.yctx["sourceNode"]["val"] - logger.log(10, completedMessage.data) + # logger.debug(completedMessage.data) for it in range(1, packetCount): if it in completedMessage.dataOrder: data += completedMessage.data[completedMessage.dataOrder.index(it)] - logger.log(10, "pre decrypt") - logger.log(10, data) + # logger.debug("pre decrypt") + # logger.debug(data) data = cryptographyInfo.decrypt( data, sourceNode, completedMessage.nonce, completedMessage.tag ) - # data = msgpack.loads(lzma.decompress(data)) return data diff --git a/src/Packets/Messages/Protocols/bubble/Bubble.py b/src/Packets/Messages/Protocols/bubble/Bubble.py index 3347a948..b13bc8cd 100644 --- a/src/Packets/Messages/Protocols/bubble/Bubble.py +++ b/src/Packets/Messages/Protocols/bubble/Bubble.py @@ -3,6 +3,10 @@ import Packets.Message class Bubble(Message): + """ + Send data from peer to peer + """ + def __init__( self, sender, @@ -13,6 +17,30 @@ class Bubble(Message): cryptographyInfo, data, ): + """ + Parameters + ---------- + sender + 6 digit (maximum) node or peer ID + + senderID + 3 digit (maximum) ID for mapping display names to a given user + + sourceNode: int + Source of request + + recipient + 6 digit (maximum) recipient peer ID + + recipientNode + 6 digit (maxmium) recipient node ID + + cryptographyInfo: Cryptography.WhaleSong.Transport + Cryptography instance for encrypting message + + data + Data to send to peer + """ bytesOb = Packets.Message.dict2bytes({"data": data, "recipient": recipient, "target": "bubble"}) super().__init__( bytesOb, diff --git a/src/Packets/Messages/Protocols/bubble/Bubble.rst b/src/Packets/Messages/Protocols/bubble/Bubble.rst new file mode 100644 index 00000000..e4d7290a --- /dev/null +++ b/src/Packets/Messages/Protocols/bubble/Bubble.rst @@ -0,0 +1,6 @@ +bubble.Bubble +============= + +.. autoclass:: Packets.Messages.Protocols.bubble.Bubble.Bubble + :members: + :undoc-members: diff --git a/src/Packets/Messages/Protocols/catch/IndexSync.py b/src/Packets/Messages/Protocols/catch/IndexSync.py index e67d93bc..0aac92ed 100644 --- a/src/Packets/Messages/Protocols/catch/IndexSync.py +++ b/src/Packets/Messages/Protocols/catch/IndexSync.py @@ -3,6 +3,9 @@ import Packets.Message class IndexSync(Message): + """ + Sync indices of Catchs across nodes + """ def __init__( self, sender, @@ -14,6 +17,33 @@ class IndexSync(Message): index, target=False ): + """ + Parameters + ---------- + sender + 6 digit (maximum) node or peer ID + + senderID + 3 digit (maximum) ID for mapping display names to a given user + + sourceNode: int + Source of request + + recipient + 6 digit (maximum) recipient peer ID + + recipientNode + 6 digit (maxmium) recipient node ID + + cryptographyInfo: Cryptography.WhaleSong.Transport + Cryptography instance for encrypting message + + index: dict + Index of catch's to sync across nodes + + target: bool + Whether to send this to a specific target (str) or just broadcast (False) + """ bytesOb = Packets.Message.dict2bytes({"index": index}) super().__init__( bytesOb, diff --git a/src/Packets/Messages/Protocols/catch/IndexSync.rst b/src/Packets/Messages/Protocols/catch/IndexSync.rst new file mode 100644 index 00000000..c1d8b966 --- /dev/null +++ b/src/Packets/Messages/Protocols/catch/IndexSync.rst @@ -0,0 +1,6 @@ +catch.IndexSync +=============== + +.. autoclass:: Packets.Messages.Protocols.catch.IndexSync.IndexSync + :members: + :undoc-members: diff --git a/src/Packets/Messages/Protocols/catch/Request.py b/src/Packets/Messages/Protocols/catch/Request.py index d9ab13fd..509f4189 100644 --- a/src/Packets/Messages/Protocols/catch/Request.py +++ b/src/Packets/Messages/Protocols/catch/Request.py @@ -3,6 +3,10 @@ import Packets.Message class CatchRequest(Message): + """ + Request Catch (website) from another node + """ + def __init__( self, sender, @@ -16,6 +20,39 @@ class CatchRequest(Message): fins, pskEncrypt=False ): + """ + Parameters + ---------- + sender + 6 digit (maximum) node or peer ID + + senderID + 3 digit (maximum) ID for mapping display names to a given user + + sourceNode: int + Source of request + + recipient + 6 digit (maximum) recipient peer ID + + recipientNode + 6 digit (maxmium) recipient node ID + + cryptographyInfo: Cryptography.WhaleSong.Transport + Cryptography instance for encrypting message + + head: str + First part of the Catch (4 characters max) + + body: str + Second part of the Catch (8 characters max) + + fins: list[str] + Last part of the Catch (6 characters max each) + + pskEncrypt: bool + Whether to encrypt with PSK + """ bytesOb = Packets.Message.dict2bytes({"head": head, "body": body, "fins": fins, "recipient": sender, "recipientNode": sourceNode}) super().__init__( bytesOb, diff --git a/src/Packets/Messages/Protocols/catch/Request.rst b/src/Packets/Messages/Protocols/catch/Request.rst new file mode 100644 index 00000000..ab6d5619 --- /dev/null +++ b/src/Packets/Messages/Protocols/catch/Request.rst @@ -0,0 +1,6 @@ +catch.Request +============= + +.. autoclass:: Packets.Messages.Protocols.catch.Request.CatchRequest + :members: + :undoc-members: diff --git a/src/Packets/Messages/Protocols/catch/Response.py b/src/Packets/Messages/Protocols/catch/Response.py index d31fc51a..97c27c0d 100644 --- a/src/Packets/Messages/Protocols/catch/Response.py +++ b/src/Packets/Messages/Protocols/catch/Response.py @@ -3,6 +3,10 @@ import Packets.Message class CatchResponse(Message): + """ + Send local Catch (website) to user who requested it + """ + def __init__( self, sender, @@ -14,6 +18,33 @@ class CatchResponse(Message): html, pskEncrypt=False ): + """ + Parameters + ---------- + sender + 6 digit (maximum) node or peer ID + + senderID + 3 digit (maximum) ID for mapping display names to a given user + + sourceNode: int + Source of request + + recipient + 6 digit (maximum) recipient peer ID + + recipientNode + 6 digit (maxmium) recipient node ID + + cryptographyInfo: Cryptography.WhaleSong.Transport + Cryptography instance for encrypting message + + html + Contents of Catch to send back + + pskEncrypt: + Whether to encrypt with PSK + """ bytesOb = Packets.Message.dict2bytes({"html": html, "recipient": recipient, "target": "catch"}) super().__init__( bytesOb, diff --git a/src/Packets/Messages/Protocols/catch/Response.rst b/src/Packets/Messages/Protocols/catch/Response.rst new file mode 100644 index 00000000..b68d82cb --- /dev/null +++ b/src/Packets/Messages/Protocols/catch/Response.rst @@ -0,0 +1,6 @@ +catch.Response +============== + +.. autoclass:: Packets.Messages.Protocols.catch.Response.CatchResponse + :members: + :undoc-members: diff --git a/src/Packets/Messages/Protocols/cryptography/Handshake.py b/src/Packets/Messages/Protocols/cryptography/Handshake.py index ce847a19..f3d3b019 100644 --- a/src/Packets/Messages/Protocols/cryptography/Handshake.py +++ b/src/Packets/Messages/Protocols/cryptography/Handshake.py @@ -1,20 +1,45 @@ from Packets.Message import Message import Packets.Message -# TODO: Send with psk encryption class Handshake(Message): + """ + Provides the ephemeral key for session encryption + """ + def __init__( self, sender, senderID, recipient, recipientNode, cryptographyInfo, onodeID, sourceNode ): - publicKey = None + """ + Parameters + ---------- + sender + 6 digit (maximum) node or peer ID + + senderID + 3 digit (maximum) ID for mapping display names to a given user + + sourceNode: int + Source of request + + recipient + 6 digit (maximum) recipient peer ID + + recipientNode + 6 digit (maxmium) recipient node ID + + cryptographyInfo: Cryptography.WhaleSong.Transport + Cryptography instance for encrypting message + + onodeID + Node identifier for current node + + sourceNode + Source of request + """ ephemeralKey = None record = cryptographyInfo.getRecord("key", "self") - if record != False: - if "publicKey" in record.keys(): - publicKey = record["publicKey"] - else: - raise Exception("Public key missing for node") + if record: if "ourEphemeralKey" in record.keys(): ephemeralKey = record["ourEphemeralKey"] else: diff --git a/src/Packets/Messages/Protocols/cryptography/Handshake.rst b/src/Packets/Messages/Protocols/cryptography/Handshake.rst new file mode 100644 index 00000000..14d12040 --- /dev/null +++ b/src/Packets/Messages/Protocols/cryptography/Handshake.rst @@ -0,0 +1,6 @@ +cryptography.Handshake +====================== + +.. autoclass:: Packets.Messages.Protocols.cryptography.Handshake.Handshake + :members: + :undoc-members: diff --git a/src/Packets/Messages/Protocols/hopper/Request.py b/src/Packets/Messages/Protocols/hopper/Request.py index d8e6db7b..e3b2d8f9 100644 --- a/src/Packets/Messages/Protocols/hopper/Request.py +++ b/src/Packets/Messages/Protocols/hopper/Request.py @@ -3,6 +3,10 @@ import Packets.Message class HopperRequest(Message): + """ + Proxy request to main internet from remote node + """ + def __init__( self, sender, @@ -10,11 +14,42 @@ class HopperRequest(Message): sourceNode, recipient, recipientNode, - url, - params, - method, + url: str, + params: dict, + method: str, cryptographyInfo, ): + """ + Parameters + ---------- + sender + 6 digit (maximum) node or peer ID + + senderID + 3 digit (maximum) ID for mapping display names to a given user + + sourceNode: int + Source of request + + recipient + 6 digit (maximum) recipient peer ID + + recipientNode + 6 digit (maxmium) recipient node ID + + url: str + URL to parse + + params: dict + Parameters to add to the request for the URL + + method: str + Method to use for request (GET/POST currently) + + cryptographyInfo: Cryptography.WhaleSong.Transport + Cryptography instance for encrypting message + + """ bytesOb = Packets.Message.dict2bytes({"url": url, "parameters": params, "method": method, "recipient": sender, "recipientNode": sourceNode}) super().__init__( bytesOb, diff --git a/src/Packets/Messages/Protocols/hopper/Request.rst b/src/Packets/Messages/Protocols/hopper/Request.rst new file mode 100644 index 00000000..7839b08c --- /dev/null +++ b/src/Packets/Messages/Protocols/hopper/Request.rst @@ -0,0 +1,6 @@ +hopper.Request +============== + +.. autoclass:: Packets.Messages.Protocols.hopper.Request.HopperRequest + :members: + :undoc-members: diff --git a/src/Packets/Messages/Protocols/hopper/Response.py b/src/Packets/Messages/Protocols/hopper/Response.py index ad219506..6c00ffc4 100644 --- a/src/Packets/Messages/Protocols/hopper/Response.py +++ b/src/Packets/Messages/Protocols/hopper/Response.py @@ -3,12 +3,38 @@ import Packets.Message class HopperResponse(Message): + """ + Send proxied request back to requester + """ + def __init__( self, sender, senderID, sourceNode, recipient, recipientNode, response, cryptographyInfo ): - bytesOb = Packets.Message.dict2bytes({"res": response, "recipient": recipient, "target": "hopper"}) + """ + Parameters + ---------- + sender + 6 digit (maximum) node or peer ID - # bytesOb = cryptographyInfo.encrypt(bytesOb, recipientNode) + senderID + 3 digit (maximum) ID for mapping display names to a given user + + sourceNode: int + Source of request + + recipient + 6 digit (maximum) recipient peer ID + + recipientNode + 6 digit (maxmium) recipient node ID + + response + Data from proxied request + + cryptographyInfo: Cryptography.WhaleSong.Transport + Cryptography instance for encrypting message + """ + bytesOb = Packets.Message.dict2bytes({"res": response, "recipient": recipient, "target": "hopper"}) super().__init__( bytesOb, diff --git a/src/Packets/Messages/Protocols/hopper/Response.rst b/src/Packets/Messages/Protocols/hopper/Response.rst new file mode 100644 index 00000000..4d6df376 --- /dev/null +++ b/src/Packets/Messages/Protocols/hopper/Response.rst @@ -0,0 +1,6 @@ +hopper.Response +=============== + +.. autoclass:: Packets.Messages.Protocols.hopper.Response.HopperResponse + :members: + :undoc-members: diff --git a/src/Packets/Messages/Protocols/map/Announce.py b/src/Packets/Messages/Protocols/map/Announce.py index 9683743d..bac0754b 100644 --- a/src/Packets/Messages/Protocols/map/Announce.py +++ b/src/Packets/Messages/Protocols/map/Announce.py @@ -5,9 +5,12 @@ import logging logger = logging.getLogger("__main__." + __name__) -# TODO: Add public key class AnnounceMessage(Message): + """ + Announce the network map details and public key of the node for discovery + """ + def __init__( self, sender, @@ -16,12 +19,30 @@ class AnnounceMessage(Message): cryptographyInfo, mapping, ): + """ + Parameters + ---------- + sender + 6 digit (maximum) node or peer ID + + senderID + 3 digit (maximum) ID for mapping display names to a given user + + sourceNode: int + Source of request + + cryptographyInfo: Cryptography.WhaleSong.Transport + Cryptography instance for encrypting message + + mapping: dict + Network map + """ mapping["publicKey"] = cryptographyInfo.getRecord("key", "self")["publicKey"] recipient = -1 recipientNode = -1 bytesOb = Packets.Message.dict2bytes(mapping) - logger.log(10, "Mapping bytes") - logger.log(10, bytesOb) + # logger.debug(10, "Mapping bytes") + # logger.debug(10, bytesOb) super().__init__( bytesOb, sender, diff --git a/src/Packets/Messages/Protocols/map/Announce.rst b/src/Packets/Messages/Protocols/map/Announce.rst new file mode 100644 index 00000000..f088b875 --- /dev/null +++ b/src/Packets/Messages/Protocols/map/Announce.rst @@ -0,0 +1,6 @@ +map.Announce +============ + +.. autoclass:: Packets.Messages.Protocols.map.Announce.AnnounceMessage + :members: + :undoc-members: diff --git a/src/Packets/Packet.py b/src/Packets/Packet.py index c114c312..2a3681ff 100755 --- a/src/Packets/Packet.py +++ b/src/Packets/Packet.py @@ -34,7 +34,7 @@ class Packet: packetsClass: int = -1, primaryMessage=None, ): - if packetsID == False: + if not packetsID: self.packetsID, self.packetNumber, self.data, self.packetsClass = ( self.parsePayload(data) ) @@ -69,8 +69,7 @@ class Packet: } if res["data"] == "": res.pop("data") - if self.primaryMessage != None: + if self.primaryMessage is not None: res["primaryMessage"] = self.primaryMessage ores = msgpack.dumps(res) - # logging.log(20, "Packet size: " + str(sys.getsizeof(ores))) return ores diff --git a/src/Packets/SinglePacket.py b/src/Packets/SinglePacket.py index a7defd33..bb177c43 100755 --- a/src/Packets/SinglePacket.py +++ b/src/Packets/SinglePacket.py @@ -2,9 +2,6 @@ from .Packet import Packet import msgpack import lzma -# TODO: Instantiation -# TODO: Packet template loading - class SinglePacket(Packet): """ diff --git a/src/Packets/SubMessage.py b/src/Packets/SubMessage.py index 186bd13d..726d8014 100644 --- a/src/Packets/SubMessage.py +++ b/src/Packets/SubMessage.py @@ -5,7 +5,12 @@ import logging logger = logging.getLogger("__main__." + __name__) + class SubMessage(Message): + """ + SubMessage to a primary message, enables us to send more/dynamic data + """ + def __init__( self, sender, @@ -20,9 +25,54 @@ class SubMessage(Message): target=True, primaryMessage=None ): + """ + Parameters + ---------- + bytesObject: bytes + Bytes to split into packets + + sender: int + 6 digit (maximum) node or peer ID + + senderDisplayName: int + 3 digit (maximum) ID for mapping display names to a given user + + sourceNode: int + Source of request + + recipient: int + 6 digit (maximum) node or peer ID + + recipientNode: int + 6 digit (maximum) node ID to route the packet to + + cryptographyInfo: Cryptography.WhaleSong.DHEFern + Cryptography instance for encrypting message + + packetsClass: int + Which protocol the packets are using + + pAction: int + 3 digit (maximum) pAction ID for mapping precise actions within a protocol + + dataSize: int + Size to cut the bytesObject into per packet + + wantFullResponse: bool + Whether to send a response when the message has completed reception (TODO: Kill all retries for associated packets when received) + + target + Whether the message is being sent to a target, if so, where + + subMessage: bool + Whether this is a submessage + + primaryMessage + Primary message this is a submessage to, if this is a submessage + """ bytesOb = Packets.Message.dict2bytes(data) - logger.log(10, "Submessage bytes") - logger.log(10, bytesOb) + # logger.debug("Submessage bytes") + # logger.debug(bytesOb) super().__init__( bytesOb, sender, diff --git a/src/Packets/SubPacket.rst b/src/Packets/SubPacket.rst index 1eb6f265..f86b496f 100644 --- a/src/Packets/SubPacket.rst +++ b/src/Packets/SubPacket.rst @@ -1,5 +1,5 @@ SubPacket: Packets for submessages ================================== -.. autoclass:: Packets.SubPacket.SubPacket +.. autoclass:: Packets.SubMessage.SubMessage :members: diff --git a/src/Services/Action.py b/src/Services/Action.py index eb54d1f8..9a1bf36e 100644 --- a/src/Services/Action.py +++ b/src/Services/Action.py @@ -1,4 +1,35 @@ class Action: + """ + Generic action class for triggering actions from sub processes on the main thread + + `πŸ”— Source `_ + + Attributes + ---------- + + action: str + Action to run + + data: dict + Data to pass to action + + sender: str + Sender identifier + + senderID: str + Sender second level identifier + + sourceNode: str + Sending node + + recipient: str + Peer identifier to route to + + recipientNode: + Intended destination node identifier + """ + # TODO: Utilize the attributes here to skip over doing it manually + def __init__( self, action, @@ -11,19 +42,19 @@ class Action: ): self.action = action self.data = data - if sender != None: + if sender is not None: self.data["sender"] = sender - if senderID != None: + if senderID is not None: self.data["senderID"] = senderID - if sourceNode != None: + if sourceNode is not None: self.data["sourceNode"] = sourceNode - if recipient != None: + if recipient is not None: self.data["recipient"] = recipient - if recipientNode != None: + if recipientNode is not None: self.data["recipientNode"] = recipientNode def getAction(self): diff --git a/src/Services/Action.rst b/src/Services/Action.rst new file mode 100644 index 00000000..0645a59a --- /dev/null +++ b/src/Services/Action.rst @@ -0,0 +1,6 @@ +Services.Action +=============== + +.. autoclass:: Services.Action.Action + :members: + :undoc-members: diff --git a/src/Splash/serve.py b/src/Splash/serve.py index 7b05263b..22e38f1f 100755 --- a/src/Splash/serve.py +++ b/src/Splash/serve.py @@ -1,10 +1,11 @@ -from uuid import uuid4 +# NOTE: Local imports import Components.hopper as hopper from Packets.Messages.Protocols.catch.Request import CatchRequest from Packets.Messages.Protocols.catch.IndexSync import IndexSync from Packets.Messages.Protocols.hopper.Request import HopperRequest from Packets.Messages.Protocols.bubble.Bubble import Bubble +# NOTE: Server imports from microdot import Microdot from microdot import send_file from microdot.websocket import with_websocket, WebSocketError @@ -12,6 +13,7 @@ from microdot import Request from microdot.jinja import Template from microdot.session import Session, with_session +# NOTE: Generic imports import random import json import time @@ -19,13 +21,11 @@ import logging import traceback import uuid import re - -import msgpack - +from uuid import uuid4 logger = logging.getLogger("__main__." + __name__) -# Enable 500 kB files in the webui +# Enable 1 mB files in the webui Request.max_content_length = 1024 * 1024 * 0.5 Request.max_body_length = 1024 * 1024 * 0.5 Request.max_readline = 1024 * 1024 @@ -39,11 +39,8 @@ class Server: Attributes ---------- - cLog - Reference to `run.Node.cLog` for logging - - transmitter: Transmission.transmission.Transmitter - Reference to our `Transmission.transmission.Transmitter` instance + transceiver: Transceiver.Transceiver.Transceiver + Reference to our `Transceiver.Transceiver.Transceiver` instance network: Siph.Network.Network Reference to our `Siph.Network.Network` @@ -71,6 +68,30 @@ class Server: remoteCatchIndex, cache, ): + """ + Parameters + ---------- + transceiver: Transceiver.Transceiver.Transceiver + Reference to our `Transceiver.Transceiver.Transceiver` instance + + catch: Daisy.Catch.Catch + Reference to our Catch Cache instance to pull from for serving Catchs + + nodeID: str + String converted PierMesh node ID + + network: Siph.Network.Network + Reference to our `Siph.Network.Network` + + cryptographyInfo: Cryptography.WhaleSong.Transport + Cryptography instance for encrypting messagei + + remoteCacheIndex: Daisy.Index.Index + Reference to our index of remote Catch's to pull from + + cache: Daisy.Cache.Cache + Reference to our on disk files + """ self.transceiver = transceiver self.network = network self.network.syncaddLookup(onodeID, self.transceiver.interface.localNode.nodeNum) @@ -92,7 +113,7 @@ class Server: Static resources endpoint """ if ".." in path: - # directory traversal is not allowed + # NOTE: Directory traversal is not allowed return "Not found", 404 return send_file("Splash/build/res/" + path, max_age=86400) @@ -103,42 +124,49 @@ class Server: """ try: return send_file("Splash/build/index/index.html") - except Exception as e: + except Exception: logger.error(traceback.format_exc()) return "Server error", 500 @self.app.route("/hop/") async def hop(request, tmpResourceID): + """ + Static handler to serve files from Hopper requests temporarily + """ try: return self.cache.get("tmp/hopper/" + tmpResourceID).get()["html"] - except Exception as e: + except Exception: logger.error(traceback.format_exc()) return "Server error", 500 @self.app.route("/api/json") async def api(request): + """ + Currently just a test json api endpoint that returns {"hello": "world"} + """ try: return {"hello": "world"} - except Exception as e: + except Exception: logger.error(traceback.format_exc()) return "Server error", 500 @self.app.route("/admin") @with_session async def admin(request): + """ + Static endpoint for the Admin interface, currently only displays our PSKs + """ try: return Template("Splash/admin/admin.html").render(psks=self.getPSKs()) - except Exception as e: + except Exception: logger.error(traceback.format_exc()) return "Server error", 500 - - @self.app.route("/bubble") @with_websocket async def bubble(request, ws): """ - Websocket handler that bridges HTMX to our transmitter + Websocket handler that bridges HTMX to our transmitter for client side PierMesh operations Notes ----- @@ -148,7 +176,7 @@ class Server: try: r = await ws.receive() message = json.loads(r) - #logger.debug(json.dumps(message, indent=4)) + # logger.debug(json.dumps(message, indent=4)) trigger = message["HEADERS"]["HX-Trigger"] logger.debug(f"Trigger: {trigger}") # TODO: Drop old id from cache on regen @@ -216,7 +244,7 @@ class Server: message["body"], fins=message["finsStr"].split(","), ) - if res == False: + if not res: await ws.send( '
{0}
'.format( "Searching PierMesh for Catch please wait..." @@ -235,7 +263,7 @@ class Server: fins = message["fins"] # TODO: Handling multiple results q = self.remoteCatchIndex.search(q)[0] - if q != False: + if q: m = CatchRequest( peerID, 000000, @@ -347,24 +375,26 @@ class Server: await ws.send( """
hi
""" ) - except WebSocketError as e: + except WebSocketError: pass - + # Uncomment below for WebSocket debugging logger.debug(traceback.format_exc()) return "Server error", 500 - except Exception as e: + except Exception: logger.error(traceback.format_exc()) return "Server error", 500 - + async def getPSKs(self): + """ + Get all PSKs for display + """ psks = [ {"psk": v["PSK"], "nodeID": k} for k, v in self.cryptographyInfo["msg"].items() ] return psks - # TODO: Send catch to catch display async def sendToPeer(self, peerID: str, data: str, target: str): """ Send data to Websocket of peer with peerID diff --git a/src/Splash/serve.rst b/src/Splash/serve.rst index 7dd60b4e..3aec0790 100644 --- a/src/Splash/serve.rst +++ b/src/Splash/serve.rst @@ -1,5 +1,5 @@ serve: Web UI server -============================ +==================== .. autoclass:: Splash.serve.Server :members: diff --git a/src/Sponge/Protocols/Yellow.py b/src/Sponge/Protocols/Yellow.py index af57e8d8..51f82b05 100644 --- a/src/Sponge/Protocols/Yellow.py +++ b/src/Sponge/Protocols/Yellow.py @@ -8,6 +8,10 @@ logger = logging.getLogger("__main__." + __name__) class YCTX(Context): + """ + Context data structure for message parsing + """ + def __init__( self, packetsID, @@ -21,6 +25,37 @@ class YCTX(Context): submessagesIDs=[], eData=None, ): + """ + packetsID: int + Identifier for message + + packetCount: int + Number of packets in message + + pAction: str + Action to execute after parsing + + todo + Queue of actions to execute in the main loop + + cryptographyInfo: Cryptography.WhaleSong.Transport + Cryptography instance for encrypting message + + sourceNode: int + Source of request + + subMessage: bool + Whether this is a submessage + + subMessages: dict + Dict of parsed submessages of primary message + + submessagesIDs: list + List of required submessages to complete full message + + eData + Any extra data + """ super().__init__( sourceNode=sourceNode, packetsID=packetsID, @@ -36,13 +71,56 @@ class YCTX(Context): class Yellow: - # TODO: Submessage completion actions + """ + Message parser that's subclassed to easily make parsers for specific protocols + + Attributes + ---------- + yctx: YCTX + Message parsing context + + message + + submessages: dict + Dictionary of submessages + + submessagesIDs: list + List of required submessages + + finishedSubmessages: dict + Dictionary of finished submessages + + dataOrder: list + List that maps packets based on their received order + + data: list + Data of primary message + + nonce + Cryptography artifact for decrypting message + + tag + Cryptography artifact for decrypting message + + gotHead: bool + Whether we've gotten the head/header packet + + todo + Queue of actions to execute in the main loop + + """ pActions = [] def __init__( self, yctx: YCTX, ): + """ + Parameters + ---------- + yctx: YCTX + Message parsing context + """ self.yctx = yctx self.message = None self.submessages = yctx["subMessages"]["val"] @@ -54,7 +132,7 @@ class Yellow: self.tag = None self.gotHead = False self.todo = yctx.todo - if yctx["eData"]["val"] != None: + if yctx["eData"]["val"] is not None: self.dataOrder = yctx["eData"]["val"]["dataOrder"] self.data = yctx["eData"]["val"]["data"] diff --git a/src/Sponge/Protocols/Yellow.rst b/src/Sponge/Protocols/Yellow.rst new file mode 100644 index 00000000..6d9aad7f --- /dev/null +++ b/src/Sponge/Protocols/Yellow.rst @@ -0,0 +1,10 @@ +Sponge.Protocols.Yellow +======================= + +.. autoclass:: Sponge.Protocols.Yellow.YCTX + :members: + :undoc-members: + +.. autoclass:: Sponge.Protocols.Yellow.Yellow + :members: + :undoc-members: diff --git a/src/Sponge/Protocols/bubble.py b/src/Sponge/Protocols/bubble.py index bda2610f..e9623a45 100644 --- a/src/Sponge/Protocols/bubble.py +++ b/src/Sponge/Protocols/bubble.py @@ -3,6 +3,7 @@ from Sponge.Protocols.Yellow import Yellow # TODO: Forwarding message to next node # TODO: Method to get next node in path to recipient node + class Bubble(Yellow): """ Peer to peer protol diff --git a/src/Sponge/Protocols/bubble.rst b/src/Sponge/Protocols/bubble.rst new file mode 100644 index 00000000..5f234ac1 --- /dev/null +++ b/src/Sponge/Protocols/bubble.rst @@ -0,0 +1,6 @@ +Sponge.Protocols.bubble +======================= + +.. autoclass:: Sponge.Protocols.bubble.Bubble + :members: + :undoc-members: diff --git a/src/Sponge/Protocols/catch.py b/src/Sponge/Protocols/catch.py index 2b5131bb..562947d0 100644 --- a/src/Sponge/Protocols/catch.py +++ b/src/Sponge/Protocols/catch.py @@ -7,5 +7,5 @@ class Catch(Yellow): `πŸ”— Source `__ """ - + pActions = ["sendCatch", "routeCatch", "syncIndex"] diff --git a/src/Sponge/Protocols/catch.rst b/src/Sponge/Protocols/catch.rst new file mode 100644 index 00000000..283ff364 --- /dev/null +++ b/src/Sponge/Protocols/catch.rst @@ -0,0 +1,6 @@ +Sponge.Protocols.catch +======================= + +.. autoclass:: Sponge.Protocols.catch.Catch + :members: + :undoc-members: diff --git a/src/Sponge/Protocols/cryptography.rst b/src/Sponge/Protocols/cryptography.rst new file mode 100644 index 00000000..a64f3ba9 --- /dev/null +++ b/src/Sponge/Protocols/cryptography.rst @@ -0,0 +1,6 @@ +Sponge.Protocols.cryptography +============================= + +.. autoclass:: Sponge.Protocols.cryptography.CryptographyFilter + :members: + :undoc-members: diff --git a/src/Sponge/Protocols/daisy.py b/src/Sponge/Protocols/daisy.py index 33979d6b..3cb10e43 100644 --- a/src/Sponge/Protocols/daisy.py +++ b/src/Sponge/Protocols/daisy.py @@ -1,5 +1,6 @@ from Sponge.Protocols.Yellow import Yellow +# NOTE: Placeholder class Daisy(Yellow): """ diff --git a/src/Sponge/Protocols/hopper.py b/src/Sponge/Protocols/hopper.py index 67a35a42..d1b0be54 100644 --- a/src/Sponge/Protocols/hopper.py +++ b/src/Sponge/Protocols/hopper.py @@ -8,4 +8,3 @@ class Hopper(Yellow): `πŸ”— Source `__ """ pActions = ["hop", "routeHop"] - diff --git a/src/Sponge/Protocols/hopper.rst b/src/Sponge/Protocols/hopper.rst new file mode 100644 index 00000000..b9c0b46c --- /dev/null +++ b/src/Sponge/Protocols/hopper.rst @@ -0,0 +1,6 @@ +Sponge.Protocols.hopper +======================= + +.. autoclass:: Sponge.Protocols.hopper.Hopper + :members: + :undoc-members: diff --git a/src/Sponge/Protocols/map.rst b/src/Sponge/Protocols/map.rst new file mode 100644 index 00000000..c97f2d34 --- /dev/null +++ b/src/Sponge/Protocols/map.rst @@ -0,0 +1,6 @@ +Sponge.Protocols.map +======================= + +.. autoclass:: Sponge.Protocols.map.Map + :members: + :undoc-members: diff --git a/src/Sponge/base.py b/src/Sponge/base.py index 31f8a1a2..601e0101 100644 --- a/src/Sponge/base.py +++ b/src/Sponge/base.py @@ -1,42 +1,67 @@ +# NOTE: Generic imports import logging - -import msgpack import traceback -from Packets.Message import Message +# NOTE: Messsage encoding import +import msgpack + +# NOTE: Local import import Sponge.Protocols as Protocols logger = logging.getLogger("__main__." + __name__) + class Filter: """ Packet filtering orchestration `πŸ”— Source `__ - cLog - Reference to `run.Node.cLog` for logging + Attributes + ---------- cache: Daisy.Cache.Cache Reference to our Daisy Cache instance - completed: list - List of completed messages IDs + onodeID + PierMesh node ID todo Reference to list of actions to do in the Node - onodeID - PierMesh node ID + cryptographyInfo: Cryptography.WhaleSong.Transport + Cryptography instance for encrypting message + + messages: dict + Temporary storage for unfinished messages + + submessages: dict + Temporary storage for unfinished submessages + + completed: list + List of finished message ids so we don't reprocess messages """ def __init__(self, cache, onodeID, todo, cryptographyInfo): + """ + Parameters + ---------- + + cache: Daisy.Cache.Cache + Reference to our Daisy Cache instance + + onodeID + PierMesh node ID + + todo + Reference to list of actions to do in the Node + + cryptographyInfo: Cryptography.WhaleSong.Transport + Cryptography instance for encrypting message + """ self.cache = cache self.cryptographyInfo = cryptographyInfo - """ - Messages is temporary storage for unfinished messages - """ self.messages = {} self.submessages = {} self.completed = [] @@ -50,7 +75,7 @@ class Filter: try: msgpack.loads(payload) return True - except Exception as e: + except Exception: logger.debug("Not msgpack encoded, skipping") return False @@ -89,54 +114,19 @@ class Filter: ) return cf - # async def protoRoute(self, completeMessage: dict): - # """ - # Route message to proper protocol handler - # """ - # m = completeMessage - # """ - # Shorthand reference completeMessage for ease - # """ - # sender = m["sender"] - # senderDisplay = m["senderDisplayName"] - # recipient = m["recipient"] - # recipientNode = m["recipientNode"] - # protocol = await self.protoMap(m["packetsClass"]) - # logger.log(20, "Protocol: " + protocol) - # if protocol == "bubble": - # await Sponge.Protocols.bubble.filter( - # m, recipient, recipientNode, self.onodeID, self.todo - # ) - # elif protocol == "map": - # await Sponge.Protocols.map.filter(m, self.todo) - # elif protocol == "catch": - # await Sponge.Protocols.catch.filter(m, recipient, recipientNode, self.todo) - # elif protocol == "cryptography": - # await Sponge.Protocols.cryptography.filter( - # completeMessage, recipientNode, self.todo - # ) - # elif protocol == "hopper": - # await Sponge.Protocols.hopper.filter( - # completeMessage, self.todo, recipient, recipientNode - # ) - # else: - # logger.log(30, "Cant route, no protocol") - async def sieve(self, packet): """ Base filtering logic, takes a single MeshTastic packet """ - # TODO: Instantiating the protocol on submessage p = packet["decoded"]["payload"] if self.selfCheck(packet) and self.mCheck(p): try: p = msgpack.loads(p) logger.log(20, p) packetsID = p["packetsID"] - packetsClass = p["packetsClass"] if packetsID == 0: - logger.log(20, "Single packet") - # Do sp handling + logger.info("Single packet") + # TODO: Do sp handling pass if packetsID in self.completed: raise ValueError("Message already completed") @@ -145,8 +135,6 @@ class Filter: ): if "primaryMessage" in p.keys(): if p["primaryMessage"] in self.messages.keys(): - # TODO: Temporary store for submessages if main hasnt arrived - # TODO: Check for submessages when instantiating await self.messages[p["primaryMessage"]].processPacket(p, subMessage=True) else: if not str(p["packetsID"]) in self.submessages.keys(): @@ -156,7 +144,6 @@ class Filter: } self.submessages[str(p["packetsID"])]["data"].append(p["data"]) self.submessages[str(p["packetsID"])]["dataOrder"].append(p["packetNumber"]) - #await self.messages[p["primaryMessage"]].processPacket(p, subMessage=True) else: self.messages[packetsID] = await self.protoMap( @@ -167,7 +154,6 @@ class Filter: if packetsID in self.messages.keys(): self.messages[packetsID].yctx["pAction"]["val"] = p["pAction"] self.messages[packetsID].yctx["sourceNode"]["val"] = p["sourceNode"] - # self.messages[packetsID].submessageIDs = p["submessages"] await self.messages[packetsID].processPacket(p) else: self.messages[packetsID] = await self.protoMap( @@ -183,4 +169,4 @@ class Filter: await self.messages[packetsID].processPacket(p) except Exception: - logger.log(30, traceback.format_exc()) + logger.error(traceback.format_exc()) diff --git a/src/Sponge/base.rst b/src/Sponge/base.rst index b75ca237..81972b1d 100644 --- a/src/Sponge/base.rst +++ b/src/Sponge/base.rst @@ -4,15 +4,3 @@ base: Primary filtering functionality .. autoclass:: Sponge.base.Filter :members: :undoc-members: - -Protocols -========= - -.. automethod:: Sponge.Protocols.bubble.filter - -.. automethod:: Sponge.Protocols.catch.filter - -.. automethod:: Sponge.Protocols.cryptography.filter - -.. automethod:: Sponge.Protocols.map.filter - diff --git a/src/Transceiver/Transceiver.py b/src/Transceiver/Transceiver.py index fb64f9c1..b4943a83 100644 --- a/src/Transceiver/Transceiver.py +++ b/src/Transceiver/Transceiver.py @@ -397,4 +397,5 @@ class Transceiver: await asyncio.sleep(2) await asyncio.sleep(5) logger.info("Announce") - await self.sendAnnounce() + await asyncio.create_task(self.sendAnnounce()) + await asyncio.sleep(1) diff --git a/src/index.rst b/src/index.rst index 17a20d50..546f855d 100644 --- a/src/index.rst +++ b/src/index.rst @@ -9,12 +9,21 @@ PierMesh documentation :glob: run - ui + tlog ../Siph/* ../Components/* ../Daisy/* ../Cryptography/* ../Sponge/* + ../Sponge/Protocols/* ../Packets/* + ../Packets/Messages/Protocols/bubble/* + ../Packets/Messages/Protocols/catch/* + ../Packets/Messages/Protocols/cryptography/* + ../Packets/Messages/Protocols/hopper/* + ../Packets/Messages/Protocols/map/* ../Transceiver/* ../Splash/* + ../Config/* + ../Actions/* + ../Services/* diff --git a/src/mlookup.json b/src/mlookup.json new file mode 100644 index 00000000..1697ad41 --- /dev/null +++ b/src/mlookup.json @@ -0,0 +1,8 @@ +{ + "000000": "map", + "000001": "catch", + "000002": "bubble", + "000003": "cryptography", + "000004": "daisy", + "000005": "hopper" +} diff --git a/src/run.py b/src/run.py index a3c2f7b8..1b8cb4f8 100755 --- a/src/run.py +++ b/src/run.py @@ -1,6 +1,7 @@ # PierMesh libraries from Sponge.base import Filter from Siph.map import Network +from Daisy import Daisy as du from Daisy.Catch import Catch from Daisy.Cache import Cache from Daisy.Index import Index @@ -8,12 +9,10 @@ from Daisy.CryptographyUtil import SteelPetal from Splash.serve import Server from Transceiver.Transceiver import Transceiver from Cryptography.WhaleSong import Transport - import Components.hopper as hopper from Packets.Messages.Protocols.hopper.Response import HopperResponse from Packets.Messages.Protocols.catch.Response import CatchResponse from Packets.Messages.Protocols.cryptography.Handshake import Handshake - import tlog from tlog import VHandler @@ -31,35 +30,44 @@ import configparser import shutil import queue import json + # Process management library import psutil +# Faster asyncio drop-in import uvloop # TODO: Switch config to toml # TODO: Better protocol management # TODO: Dry run - +# TODO: Fix scripts/marcille +# TODO: Manually trigger announce +# TODO: Run announce at initialization if __name__ == "__main__": """ Global objects for the PierMesh service and the TUI so we can terminate the associated processes later and configuration objects for the command line and config (.piermesh) """ logger = "" passkey = input("Enter node decryption key: ") - #psk = input("Enter psk: ") + # TODO: Check for PSK + # psk = input("Enter psk: ") nodeOb = None - parser = argparse.ArgumentParser() - parser.add_argument("-d", "--device", help="Set transceiver device path") - parser.add_argument("-p", "--port", help="Web UI server port") - parser.add_argument("-n", "--nickname", help="Node nickname") + # NOTE: Command line argument parser, example of a proper configuration + # is in scripts/falin + parser = argparse.ArgumentParser() + parser.add_argument("-d", "--device", help="Set transceiver device path (ex. /dev/ttyACM0)") + parser.add_argument("-p", "--port", help="Web UI server port, default: 5000") + # NOTE: Detetmines where your node's data is stored oj disk + parser.add_argument("-n", "--nickname", help="Node nickname (sets node data path)") parser.add_argument( "-s", "--startupDelay", help="Startup delay (useful for testing)" ) parser.add_argument( - "-o", "--override", help="Whether to override config", default=False + "-o", "--override", help="Whether to override config file", default=False ) + # TODO: Fix parser.add_argument("-x", "--showTUI", - help="Whether to show TUI", default=True) + help="Whether to show TUI (needs fix)", default=True) parser.add_argument( "-l", "--confList", @@ -74,6 +82,7 @@ if __name__ == "__main__": ) argConfig = parser.parse_args() + # NOTE: .piermesh conf parser, proper example is in .piermesh.example config = configparser.ConfigParser() if argConfig.confList != False: pass @@ -123,16 +132,17 @@ if __name__ == "__main__": showTUI = bool(showTUI) psk = False if argConfig.override: - if argConfig.PSK == False: + if not argConfig.PSK: print("No PSK set quitting...") exit(0) else: psk = argConfig.PSK - #else: + # else: if "PSK" in config["OPERATOR_REQUIRED"]: psk = config["OPERATOR_REQUIRED"]["PSK"] - print("Staggering {0} seconds please wait".format(delay)) - time.sleep(delay) + if delay > 0: + print("Staggering {0} seconds please wait".format(delay)) + time.sleep(delay) class Node: @@ -143,8 +153,6 @@ class Node: Attributes ---------- - toLog: list - We store logs to be processed here actions: dict Dictionary mapping methods with the action prefix to the method name after action dynamically to be called through Sponge (`Sponge.base`) filtering @@ -170,20 +178,14 @@ class Node: oTransceiver: Transceiver LoRa transceiver `Transceiver` - processed: list - List of IDs of already completed messages so that we don't reprocess messages - proc: psutil.Process This process (`psutil.Process`), used for managing and monitoring PierMesh mTasks: dict Dictionary of PierMesh service tasks - See Also - -------- - logPassLoop: Loop to handle logging to file and TUI """ - + # isDone = False def __init__(self): actionsList = [f for f in dir(self) if "action" in f] self.actions = {} @@ -194,10 +196,10 @@ class Node: self.network = Network() self.daisyCryptography = SteelPetal(passkey) # TODO: Better directory structure - self.catch = Catch(self.daisyCryptography, walk=True, path=nodeNickname) - - self.cache = Cache(self.daisyCryptography, walk=True) - self.cache.get("mlookup").json_to_msg("daisy/mlookup") + self.catch = Catch(self.daisyCryptography, walk=True, path=f"{nodeNickname}/daisy/catch") + self.cache = Cache(self.daisyCryptography, walk=True, path=f"{nodeNickname}/daisy/cache") + du._json_to_msg("mlookup") + shutil.copy("mlookup", f"{nodeNickname}/daisy/cache/mlookup") logger.info("Protocols:\n" + json.dumps(self.cache.get("mlookup").get(), indent=4)) self.remoteCatchIndex = Index(nodeNickname, self.daisyCryptography) serverInfoFile = nodeNickname + ".info" @@ -221,8 +223,7 @@ class Node: self.sponge = Filter(self.cache, self.onodeID, self.todo, self.cryptographyInfo) logger.log(20, "Filter initialized") - - # TODO: Run in different thread, dispatch through queue + # TODO: Run in thread self.oTransceiver = Transceiver( device, self.sponge, @@ -241,11 +242,13 @@ class Node: self.remoteCatchIndex, self.cache, ) - self.processed = [] self.proc = psutil.Process(os.getpid()) self.mTasks = {} async def main(self): + """ + Main loop, sets up the message listening, system monitoring and server running loops + """ self.mTasks["list"] = asyncio.create_task(self.spongeListen()) await asyncio.sleep(1) # self.mTasks["pct"] = asyncio.create_task(self.oTransceiver.progressCheck()) @@ -257,31 +260,10 @@ class Node: await asyncio.sleep(1) await self.server.app.start_server(port=int(webPort)) - def cLog(self, priority: int, message: str): - logger = logging.getLogger(__name__) - """ - Convenience function that logs to the ui and log files - - Parameters - ---------- - - priority: int - Priority of message to be passed to logging - - message: str - Message to log - - Returns - ------- - None - """ - - logger.log(priority, message) - # pass - # logging.log(priority, message) - # self.toLog.append("[{0}]:\n{1}".format(datetime.datetime.now(), message)) - async def fsInit(self): + """ + Initialize the file system for use + """ # TODO: Flesh out and properly link everything if not os.path.exists("data"): os.makedirs("data") @@ -289,10 +271,10 @@ class Node: os.makedirs("data/" + nodeNickname) async def monitor(self): - global tuiOb """ Monitor and log ram and cpu usage """ + global tuiOb while True: if tuiOb != None: await asyncio.sleep(10) @@ -325,8 +307,6 @@ class Node: We use a common technique here that calls the function from our preloaded actions via dictionary entry """ while True: - # logger.log(10, "Sponge listening...") - # logger.log(10, self.todo) while (len(self.todo) >= 1): try: logger.log(10, "In todo") @@ -355,12 +335,13 @@ class Node: -------- Sponge.Protocols: Protocol based packet filtering - webui.serve.Server: Runs a light Microdot web server with http/s and websocket functionality + Splash.serve.Server: Runs a light Microdot web server with http/s and websocket functionality - webui.serve.Server.sendToPeer: Function to actually execute the action + Splash.serve.Server.sendToPeer: Function to actually execute the action """ logger.info(data) - await self.server.sendToPeer(data["recipient"], data["data"], data["target"]) + await asyncio.create_task(self.server.sendToPeer(data["recipient"], data["data"], data["target"])) + await asyncio.sleep(1) async def action_sendCatch(self, data: dict): """ @@ -372,7 +353,6 @@ class Node: else: res = self.catch.get(data["head"], data["body"]) logger.info(res) - # TODO: Manually connect two nodes r = CatchResponse( self.onodeID, 000000, @@ -383,15 +363,21 @@ class Node: res, pskEncrypt=True ) - await self.oTransceiver.sendMessage(r) - - async def action_cLog(self, data: dict): - logger.log(data["priority"], data["message"]) + + await asyncio.create_task(self.oTransceiver.sendMessage(r)) + await asyncio.sleep(1) async def action_routeCatch(self, data: dict): - await self.server.sendToPeer(data["recipient"], data["html"], data["target"]) + """ + Route received catch to peer who requested it + """ + await asyncio.create_task(self.server.sendToPeer(data["recipient"], data["html"], data["target"])) + await asyncio.sleep(1) async def action_syncIndex(self, data: dict): + """ + Add received index entries to Catch via the a remote Catch index + """ for entry in data["index"]: self.remoteCatchIndex.addEntry(entry) @@ -406,7 +392,8 @@ class Node: """ if self.cryptographyInfo.getRecord("key", data["onodeID"]) == False: logger.log(20, "In map") - await self.network.addLookup(data["onodeID"], data["mnodeID"]) + await asyncio.create_task(self.network.addLookup(data["onodeID"], data["mnodeID"])) + await asyncio.sleep(1) logger.log(20, "After add lookup") logger.log(20, "Adding public key") self.cryptographyInfo.addPublickey(data["onodeID"], data["publicKey"]) @@ -417,28 +404,27 @@ class Node: logger.log(20, "Node already exists skipping addition to map") if "sessionKey" not in self.cryptographyInfo.getRecord("key", data["onodeID"]).keys(): if not data["dontRespond"]: - #await self.oTransceiver.sendAnnounce(dontRespond=True) - #await asyncio.sleep(180) h = Handshake(self.nodeInfo.get()["nodeID"], self.nodeInfo.get()["nodeID"], data["onodeID"], data["onodeID"], self.cryptographyInfo, data["onodeID"], self.onodeID) - await self.oTransceiver.sendMessage(h) + await asyncio.create_task(self.oTransceiver.sendMessage(h)) + await asyncio.sleep(1) async def action_initCryptography(self, data: dict): """ - Initialize diffie hellman key exchange + Initialize AES-GCM encrypted transport session See Also -------- - Cryptography.DHEFern.DHEFern: End to end encryption functionality + Cryptography.WhaleSong.Transport: End to end encryption functionality """ - # TODO: Response message - # TODO: Initialize if not initialized + # TODO: Expiration self.cryptographyInfo.sessionSetup(data["yctx"]["sourceNode"]["val"], data["ephemeralKey"]) logger.log(20, "Cryptography handshake complete") - # TODO: Now encrypt all communications node to node with session encryption - # TODO: Expiration? async def action_hop(self, data): + """ + Proxy a request to the main internet (in the future cross protocol/link) + """ try: r = None if data["method"] == "get": @@ -459,16 +445,23 @@ class Node: r, self.cryptographyInfo, ) - await self.oTransceiver.sendMessage(r) + await asyncio.create_task(self.oTransceiver.sendMessage(r)) + await asyncio.sleep(1) except: logger.log(30, traceback.format_exc()) async def action_routeHop(self, data: dict): - await self.server.sendToPeer(data["recipient"], data["res"], data["target"]) + """ + Return proxy request results to requester + """ + await asyncio.create_task(self.server.sendToPeer(data["recipient"], data["res"], data["target"])) + await asyncio.sleep(1) async def action_addPSK(self, data): - # TODO: Switch to credential + """ + Action to add PSK for specific node, currently unused + """ self.cryptographyInfo.createEmpty(data["nodeID"]) self.cryptographyInfo.update(data["nodeID"], {"PSK": data["PSK"]}) @@ -480,14 +473,18 @@ async def main(): try: nodeOb = Node() await nodeOb.main() + nodeOb.isDone = True except: logger.log(20, traceback.format_exc()) -def initLogger(nodeName, tolog): +def initLogger(nodeName, tolog, tui=True): + """ + Initialize our logging setup, we use a custom logger when running the TUI to consume the logs + """ global logger logger = logging.getLogger(__name__) - logger.propagate = False + logger.propagate = not tui logger.setLevel(logging.DEBUG) vh = VHandler(logging.DEBUG, tolog) @@ -517,6 +514,7 @@ if __name__ == "__main__": # TODO: Logging to screen when no tui if showTUI: tlog.runLogUI(tolog, nodeNickname) - except: + # else: + except Exception: print(traceback.format_exc()) os._exit(1) diff --git a/src/run.rst b/src/run.rst index a0a74389..fbee8f8e 100644 --- a/src/run.rst +++ b/src/run.rst @@ -3,3 +3,4 @@ run: PierMesh service runner .. autoclass:: run.Node :members: + :undoc-members: diff --git a/src/tlog.py b/src/tlog.py index 6b7c7550..06b33ccb 100644 --- a/src/tlog.py +++ b/src/tlog.py @@ -1,19 +1,33 @@ +# TUI library import curses from curses import wrapper +# Generic imports import os import logging import queue import shutil import time +# System monitoring library import psutil # TODO: Multi node mode # TODO: Tab per node - class VHandler(logging.Handler): + """ + Custom log handler to push logs into a thread-safe queue so the TUI can read them + + `πŸ”— Source `_ + + Attributes + ---------- + + tolog: Queue.queue + Thread-safe log queue + + """ tolog = queue.Queue() def __init__(self, level, tolog): @@ -24,34 +38,10 @@ class VHandler(logging.Handler): r = self.format(record) self.tolog.put([r, record.levelno]) -""" -def initLogger(nodeName, tolog): - logger = logging.getLogger(__name__) - - logger.setLevel(logging.DEBUG) - # Is tolog variable creating an error that cant be logged - # and logging is defaulting - # Check if this reference is causing an issue - vh = VHandler(logging.DEBUG, tolog) - - formatter = logging.Formatter( - '%(asctime)s - %(name)s - %(levelname)s - %(message)s') - - vh.setFormatter(formatter) - - logger.addHandler(vh) - - dt = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") - - fh = logging.FileHandler(f"logs/{nodeName}_{dt}.log") - fh.setFormatter(formatter) - - logger.addHandler(fh) - return logger -""" - - def logUI(stdscr, tolog, nodeNickname): + """ + TUI loop + """ logcache = [] height, width = shutil.get_terminal_size((49, 27)) if height < 28 or height < 28: @@ -171,4 +161,7 @@ def logUI(stdscr, tolog, nodeNickname): def runLogUI(tolog, nodeNickname): + """ + Some required kludge + """ wrapper(logUI, tolog, nodeNickname) diff --git a/src/tlog.rst b/src/tlog.rst new file mode 100644 index 00000000..4dbe8374 --- /dev/null +++ b/src/tlog.rst @@ -0,0 +1,6 @@ +tlog +==== + +.. automodule:: tlog + :members: + :undoc-members: diff --git a/src/ui.py b/src/ui.py deleted file mode 100644 index aea40436..00000000 --- a/src/ui.py +++ /dev/null @@ -1,150 +0,0 @@ -# TODO: DEPRECATE -from textual.app import App, ComposeResult -from textual.widgets import Log, Label, Footer, Header, ProgressBar, Input, Button -from textual.binding import Binding -from textual.containers import Horizontal, Vertical -import sys, os - -global nodeOb -nodeOb = None - - -class TUI(App): - """ - TUI for PierMesh - - `πŸ”— Source `_ - - Attributes - ---------- - visibleLogo: bool - Whether the logo is visible or not, used in toggling visibility - nodeOb: Node - Reference to the Node running the PierMesh service - done: bool - Whether the TUI has been killed - """ - - todo = [] - visibleLogo = True - nodeOb = None - done = False - CSS_PATH = "ui.tcss" - BINDINGS = [ - Binding(key="q", action="quitFull", description="Quit the app", show=True), - Binding( - key="f", - action="toggleFullscreen", - description="Full screen the logs", - show=True, - ), - ] - - def action_toggleFullscreen(self): - """ - Toggle fullscreen logs by either collapsing width or setting it to it's original size - """ - if self.visibleLogo: - self.query_one("#logo").styles.width = 0 - else: - self.query_one("#logo").styles.width = "50%" - self.visibleLogo = not self.visibleLogo - - def action_quitFull(self): - """ - Kill the whole stack by setting self to done and terminating the thread. We check in run.monitor later and kill the rest of the stack then with psutil - - See Also - -------- - run.monitor - """ - self.done = True - # sys.exit("Terminating TUI...") - - def compose(self): - """ - Build the TUI - """ - ascii = "" - with open("piermesh-mini.ascii", "r") as f: - ascii = f.read() - """ - Load the ascii art for display on the left label - """ - yield Header(icon="P") - yield Vertical( - Label(ascii, classes="largeLabel", name="logo", id="logo"), - Label("Add/set pre shared key for node\n"), - Label("Node ID:"), - Input(placeholder="000000", type="integer", max_length=6, name="pskNodeID", id="pskNodeID"), - Label("PSK:"), - Input(type="text", max_length=6, name="psk", id="psk"), - Button("Add/set PSK", name="addPSK", id="addPSK"), - ) - yield Vertical( - Log(auto_scroll=True, classes="baseLog"), - Label("CPU usage:", name="cpul", id="cpul"), - ProgressBar(show_eta=False, show_percentage=True), - Label("MEM usage: ", name="meml", id="meml"), - ) - yield Footer() - - def on_button_pressed(self, event: Button.Pressed) -> None: - if event.button.id == "addPSK": - self.todo.append({ - "action": "addPSK", - "data": { - "nodeID": self.query_one("#pskNodeID").value.zpad(6), - "PSK": self.query_one("#PSK").value - } - }) - - def do_write_line(self, logLine: str): - """ - Write line to the logs panel - - Parameters - ---------- - logLine: str - Line to log - """ - log = self.query_one(Log) - log.write_line(logLine) - - def do_set_cpu_percent(self, percent: float): - """ - Set CPU percent in the label and progress bar - - Parameters - ---------- - percent: float - Percent of the cpu PierMesh is using - """ - self.query_one("#cpul").update("CPU usage: {0} %".format(str(percent))) - pbar = self.query_one(ProgressBar) - pbar.progress = percent - - def do_set_mem(self, memmb: float): - """ - Set memory usage label in the ui - - Parameters - ---------- - memmb: float - Memory usage of PierMesh in megabytes - """ - self.query_one("#meml").update("MEM usage: {0} mB".format(str(memmb))) - - def on_mount(self): - """ - Called at set up, configures the title and the progess bar - """ - self.title = "PierMesh TUI" - self.query_one(ProgressBar).update(total=100) - self.query_one(ProgressBar).update(progress=0) - - -if __name__ == "__main__": - app = TUI() - app.run() - diff --git a/src/ui.rst b/src/ui.rst deleted file mode 100644 index 19fcc6ca..00000000 --- a/src/ui.rst +++ /dev/null @@ -1,5 +0,0 @@ -ui: TUI application -========================== - -.. autoclass:: ui.TUI - :members: diff --git a/src/ui.tcss b/src/ui.tcss deleted file mode 100644 index b594124c..00000000 --- a/src/ui.tcss +++ /dev/null @@ -1,23 +0,0 @@ -Screen { - layout: horizontal; - scrollbar-size: 0 0; -} - -.largeLabel { - width: 40%; -} - -.baseLog { - height: 80%; - scrollbar-background: $primary-background; - scrollbar-corner-color: $primary-background; - scrollbar-color: green; - scrollbar-size: 0 1; - } - -ProgressBar { - width: 50%; - } -Bar > .bar--bar { - color: green; -}