Readme fix

.gitignore

@@ -0,0 +1,5 @@
+**/__pycache__/
+bin/
+lib*
+share/
+pyvenv.cfg

README.md

@@ -1,3 +1,14 @@
 # Daisy
 
-Lightweight msgpack based schemaless local and distributed database
+Lightweight msgpack based schemaless local and distributed database
+
+# License
+
+```
+This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with this program. If not, see https://www.gnu.org/licenses/. 
+
+```

requirements.txt

@@ -0,0 +1,2 @@
+msgpack
+watchdog

src/Cache.py

@@ -0,0 +1,141 @@
+from Daisy.Daisy import Daisy
+
+import os
+
+import msgpack
+
+from watchdog.observers import Observer
+
+# TODO: Dumping to cacheFile
+
+
+class Cache:
+    """
+    In memory collection of Daisy records
+
+    `🔗 Source <https://git.utopic.work/PierMesh/piermesh/src/branch/main/Daisy/Cache.py>`__
+    """
+
+    def __init__(
+        self,
+        filepaths=None,
+        cacheFile=None,
+        path: str = "daisy",
+        walk: bool = False,
+        isCatch: bool = False,
+    ):
+        """
+        Parameters
+        ----------
+        filepaths
+            Either a list of filepaths to load or None
+
+        cacheFile
+            Path to a cache file which is a collection of paths to load
+
+        path: str
+            Path prefix to load records from
+
+        walk: bool
+            Whether to automatically walk the path and load records
+
+        isCatch: bool
+            Whether this cache is for catchs
+        """
+        self.data = {}
+        self.path = path
+
+        if filepaths != None:
+            for fp in filepaths:
+                fp = path + "/" + fp
+                if os.path.isfile(fp):
+                    self.data[fp] = Daisy(fp)
+        elif cacheFile != None:
+            with open(cacheFile, "r") as f:
+                for fp in f.read().split("\n"):
+                    self.data[fp] = Daisy(fp)
+        elif walk:
+            for root, dirs, files in os.walk(self.path):
+                for p in dirs + files:
+                    if not (".json" in p):
+                        if not (".md" in p):
+                            tpath = root + "/" + p
+                            self.data[tpath] = Daisy(tpath)
+
+    def create(self, path: str, data: dict, remote=False):
+        """
+        Create new record
+
+        Parameters
+        ----------
+        path: str
+            Path to create record at
+
+        data: dict
+            Data to populate record with
+        """
+        if remote == False:
+            with open(self.path + "/" + path, "wb") as f:
+                f.write(msgpack.dumps(data))
+            # logging.log(10, "Done creating record")
+            self.data[path] = Daisy(self.path + "/" + path)
+            # logging.log(10, "Done loading to Daisy")
+            return self.data[path]
+        else:
+            self.data[path] = Ref(path, remote)
+            return self.data[path]
+
+    def get(self, path: str):
+        """
+        Get record at path, else return False
+
+        path: str
+            Path of record
+        """
+        if path in self.data.keys():
+            return self.data[path]
+        else:
+            if os.path.exists(self.path + "/" + path):
+                self.data[path] = Daisy(self.path + "/" + path)
+                return self.data[path]
+            else:
+                # logging.log(10, "File does not exist")
+                return False
+
+    def refresh(self):
+        """
+        Reload from disk to memory
+        """
+        for key in self.data.keys():
+            self.data[key].read()
+
+    def search(self, keydict: dict, strict: bool = True):
+        """
+        Search cache for record for records with values
+
+        keydict: dict
+            Values to search for
+
+        strict: bool
+            Whether to require values match
+        """
+        results = []
+        for key, val in self.data.items():
+            val = val.get()
+            if strict and type(val) != str:
+                addcheck = False
+                for k, v in keydict.items():
+                    if k in val.keys():
+                        if v in val[k]:
+                            addcheck = True
+                        else:
+                            addcheck = False
+                            break
+                if addcheck:
+                    results.append([key, val])
+            elif type(val) != str:
+                for k, v in keydict.items():
+                    if k in val.keys():
+                        if v in val[k]:
+                            results.append([key, val])
+        return results

src/Cache.rst

@@ -0,0 +1,6 @@
+Daisy based cache
+=================
+
+.. autoclass:: Daisy.Cache.Cache
+      :members:
+      :undoc-members:

src/Catch.py

@@ -0,0 +1,74 @@
+from Daisy.Cache import Cache
+from Daisy.Ref import Ref
+
+import os
+import random
+
+
+class Catch(Cache):
+    """
+    Sub class of Cache for handling catchs
+
+    .. image:: https://git.utopic.work/PierMesh/piermesh/raw/branch/main/imgs/catchdisplay.png
+
+    `🔗 Source <https://git.utopic.work/PierMesh/piermesh/src/branch/main/Daisy/Catch.py>`__
+    """
+
+    catches = {}
+    remoteCatchesMap = {}
+
+    def __init__(
+        self, path: str = "catch", filepaths=None, catchFile=None, walk: bool = False
+    ):
+        """
+        Basically the same initialization parameters as Catch
+        """
+        super().__init__(
+            filepaths=filepaths, cacheFile=catchFile, path=path, walk=walk, isCatch=True
+        )
+
+    # TODO: Fins
+
+    def sget(self, path: str):
+        """
+        Call Cache's get to get record
+        """
+        return super().get(path)
+
+    # TODO: Rename
+    def get(self, head: str, tail: str, fins=None):
+        """
+        Get catch by pieces
+
+        Parameters
+        ----------
+        head: str
+            First part of catch (maximum: 4 characters)
+
+        tail: str
+            Second part of catch (maximum: 16 characters)
+
+        fins
+            List of (maximum 8 characters) strings at the end of the catch oe None if none
+        """
+        r = self.search({"head": head, "tail": tail})
+        return r[0][1]["html"]
+
+    def addc(self, peer, node, seperator, head, tail, data, fins=None, remote=False):
+        tnpath = "catch/" + node
+        if os.path.exists(tnpath) != True:
+            os.makedirs(tnpath)
+        tppath = tnpath + "/" + peer
+        if os.path.exists(tppath) != True:
+            os.makedirs(tppath)
+        sid = str(random.randrange(0, 999999)).zfill(6)
+        data["seperator"] = seperator
+        data["head"] = head
+        data["tail"] = tail
+        if fins != None:
+            data["fins"] = fins
+        res = self.create("{0}/{1}/{2}".format(node, peer, sid), data, remote=remote)
+        return [sid, res]
+
+    def exportDirectoryListing(self):
+        return [k for k in self.data.keys]

src/Catch.rst

@@ -0,0 +1,5 @@
+Daisy Catch cache
+=================
+
+.. autoclass:: Daisy.Catch.Catch
+      :members:

src/Daisy.py

@@ -0,0 +1,195 @@
+import os
+import json
+import msgpack
+
+# TODO: delete
+# TODO: propagate json changes to msgpack automatically
+# TODO: propagate msgpack changes to cache automatically
+# TODO: Indexing
+
+
+def _json_to_msg(path: str):
+    """
+    Convert json at the path plus .json to a msgpack binary
+
+    Parameters
+    ----------
+    path: str
+        Path to json minus the extension
+    """
+    rpath = path + ".json"
+    res = b""
+    with open(rpath) as f:
+        res = msgpack.dumps(json.load(f))
+    with open(path, "wb") as f:
+        f.write(res)
+
+
+class Daisy:
+    """
+    Base class for Daisy data representation
+
+    `🔗 Source <https://git.utopic.work/PierMesh/piermesh/src/branch/main/Components/daisy.py>`__
+
+    Attributes
+    ----------
+    filepath: str
+        Path to file representation on disk
+
+    msg: dict
+        In memory representation
+    """
+
+    def __init__(
+        self,
+        filepath: str,
+        templates: dict = {},
+        template: bool = False,
+        prefillDict: bool = False,
+        remote=False,
+    ):
+        """
+        Parameters
+        ----------
+        filepath: str
+            Path to disk location
+
+        templates: dict
+            Dictionary of templates to Use
+
+        template: bool
+            Which template to Use
+
+        prefillDict: bool
+            Whether to fill the record with a template
+        """
+        self.remote = False
+        self.filepath = filepath
+        if remote != False:
+            self.remote = True
+            self.remoteNodeID = remote
+        else:
+            if os.path.exists(filepath) != True:
+                with open(filepath, "wb") as f:
+                    if template != False:
+                        if template in templates.keys():
+                            t = templates[template].get()
+                            if prefillDict != False:
+                                for k in prefillDict.keys():
+                                    t[k] = prefillDict[k]
+                            f.write(msgpack.dumps(t))
+                            self.msg = t
+                        else:
+                            print("No such template as: " + template)
+                    else:
+                        f.write(msgpack.dumps({}))
+                        self.msg = {}
+            elif os.path.isdir(filepath):
+                self.msg = "directory"
+            else:
+                with open(filepath, "rb") as f:
+                    self.msg = msgpack.loads(f.read())
+
+    # Use override for updating
+
+    def write(
+        self,
+        override=False,
+        encrypt: bool = False,
+        encryptKey=None,
+        recur: bool = False,
+    ):
+        """
+        Write record to disk
+
+        Parameters
+        ----------
+        override
+            Either false or a dictionary of values to set on the record
+
+        encrypt: bool
+            Whether to encrypt the record (TODO)
+
+        encryptKey
+            Key to encrypt record with, or None if not set
+
+        recur: bool
+            Whether to recursively handle keys
+        """
+        if override != False:
+            for key in override.keys():
+                # TODO: Deeper recursion
+                if recur:
+                    if not key in self.msg.keys():
+                        self.msg[key] = {}
+                    for ikey in override[key].keys():
+                        self.msg[key][ikey] = override[key][ikey]
+                else:
+                    self.msg[key] = override[key]
+        data = msgpack.dumps(self.msg)
+        with open(self.filepath, "wb") as f:
+            f.write(data)
+
+    # Use for refreshing
+
+    def read(self, decrypt: bool = False, decryptKey=False):
+        """
+        Read record from disk to memory
+
+        Parameters
+        ----------
+        decrypt: bool
+            Whether to decrypt record
+
+        decryptKey
+            Key to decrypt record
+        """
+        if os.path.isdir(self.filepath):
+            self.msg = "directory"
+        else:
+            with open(self.filepath, "rb") as f:
+                self.msg = msgpack.loads(f.read())
+
+    def get(self):
+        """
+        Get record dictionary from memory
+
+        Returns
+        -------
+        self.msg: dict
+        """
+        return self.msg
+
+    def sublist(self):
+        """
+        Lists contents of directory if object is a directory, otherwise return None
+        """
+        fpath = self.filepath
+        if os.path.isdir(fpath):
+            return ["messages/" + x for x in os.listdir(fpath)]
+        else:
+            return None
+
+
+def loadTemplates(templatePath: str = "templates"):
+    """Load templates for prefilling records
+
+    Parameters
+    ----------
+    templatePath: str
+        Path to templates
+    """
+    templates = {}
+    for p in os.listdir(templatePath):
+        p = templatePath + "/" + p
+        if os.path.isdir(p):
+            for ip in os.listdir(p):
+                ip = p + "/" + ip
+                if os.path.isdir(ip):
+                    print("Too deep, skipping: " + ip)
+                else:
+                    templates[ip] = Daisy(ip)
+        else:
+            templates[p] = Daisy(p)
+    self.templates = templates
+    return templates

src/Daisy.rst

@@ -0,0 +1,6 @@
+Daisy
+=====
+
+.. autoclass:: Daisy.Daisy.Daisy
+      :members:
+      :undoc-members:

src/Ref.py

@@ -0,0 +1,6 @@
+from Daisy.Daisy import Daisy
+
+
+class Ref(Daisy):
+    def __init__(self, path, remoteNodeID):
+        super().__init__(path, remote=remoteNodeID)

src/Soil.py

@@ -0,0 +1,49 @@
+from watchdog.observers import Observer
+from watchdog.events import FileSystemEventHandler
+
+global garden
+"""
+Map of instances to list of signals
+to be processed
+"""
+garden = {}
+
+
+class Compound(FileSystemEventHandler):
+    """
+    File system watcher to propagate disk changes
+
+    `🔗 Source <https://git.utopic.work/PierMesh/piermesh/src/branch/main/Daisy/Soil.py>`__
+    """
+
+    def __init__(self, cache, isCatch: bool = False):
+        """
+        Parameters
+        ----------
+        cache: Cache
+            Daisy cache to update
+
+        isCatch: bool
+            Is the cache for catchs
+        """
+        self.cache = cache
+        self.isCatch = isCatch
+        super().__init__()
+
+    def on_any_event(self, event):
+        """
+        Called when a CRUD operation is performed on a record file
+
+        Parameters
+        ----------
+        event
+            Event object provided by watchdog
+        """
+        if not (".json" in event.src_path):
+            if not (".md" in event.src_path):
+                tpath = "/".join(event.src_path.split("/")[1:])
+                if tpath != "":
+                    if self.isCatch:
+                        self.cache.sget(tpath)
+                    else:
+                        self.cache.get(tpath).get()

src/Soil.rst

@@ -0,0 +1,6 @@
+Soil: Daisy signal management
+=============================
+
+.. autoclass:: Daisy.Soil.Compound
+      :members:
+      :undoc-members:

src/Store.py

@@ -0,0 +1,33 @@
+from Daisy.Daisy import Daisy
+
+import os
+
+
+class Store(Daisy):
+    """
+    Key value store
+
+    `🔗 Source <https://git.utopic.work/PierMesh/piermesh/src/branch/main/Daisy/Store.py>`__
+    """
+
+    def __init__(self, store: str, path: str, nodeNickname: str):
+        fpath = "daisy/{0}/{1}".format(path, nodeNickname)
+        cpath = "{0}/{1}/{2}".format(path, nodeNickname, store)
+        if not os.path.exists(fpath):
+            os.mkdir(fpath)
+        super().__init__("daisy/" + cpath)
+
+    def update(self, entry: str, data, recur: bool = True):
+        if recur:
+            for key in data.keys():
+                self.msg[entry][key] = data[key]
+        else:
+            self.msg[entry] = data
+        self.write()
+
+    def getRecord(self, key: str):
+        if key in self.get().keys():
+            return self.get()[key]
+        else:
+            self.cLog(20, "Record does not exist")
+            return False

src/Store.rst

@@ -0,0 +1,6 @@
+Store: Daisy key value store
+============================
+
+.. autoclass:: Daisy.Store.Store
+      :members:
+      :undoc-members: