PierMesh
/
Daisy
1
0
Fork 0
Code Issues 3 Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: PierMesh:dev
Branches Tags
PierMesh:main
PierMesh:dev
..
compare: PierMesh:main
Branches Tags
PierMesh:dev
PierMesh:main

No commits in common. "dev" and "main" have entirely different histories.
dev ... main

14 changed files with 1 additions and 546 deletions
Show Stats Expand all files Collapse all files

5
.gitignore vendored
View File

@ -1,5 +0,0 @@
**/__pycache__/
bin/
lib*
share/
pyvenv.cfg

11
README.md
View File

@ -1,14 +1,3 @@
# Daisy
Lightweight msgpack based schemaless local and distributed database
# License
```
This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with this program. If not, see https://www.gnu.org/licenses/.
```

2
requirements.txt
View File

@ -1,2 +0,0 @@
msgpack
watchdog

141
src/Cache.py
View File

@ -1,141 +0,0 @@
from Daisy.Daisy import Daisy
import os
import msgpack
from watchdog.observers import Observer
# TODO: Dumping to cacheFile
class Cache:
"""
In memory collection of Daisy records
`🔗 Source <https://git.utopic.work/PierMesh/piermesh/src/branch/main/Daisy/Cache.py>`__
"""
def __init__(
self,
filepaths=None,
cacheFile=None,
path: str = "daisy",
walk: bool = False,
isCatch: bool = False,
):
"""
Parameters
----------
filepaths
Either a list of filepaths to load or None
cacheFile
Path to a cache file which is a collection of paths to load
path: str
Path prefix to load records from
walk: bool
Whether to automatically walk the path and load records
isCatch: bool
Whether this cache is for catchs
"""
self.data = {}
self.path = path
if filepaths != None:
for fp in filepaths:
fp = path + "/" + fp
if os.path.isfile(fp):
self.data[fp] = Daisy(fp)
elif cacheFile != None:
with open(cacheFile, "r") as f:
for fp in f.read().split("\n"):
self.data[fp] = Daisy(fp)
elif walk:
for root, dirs, files in os.walk(self.path):
for p in dirs + files:
if not (".json" in p):
if not (".md" in p):
tpath = root + "/" + p
self.data[tpath] = Daisy(tpath)
def create(self, path: str, data: dict, remote=False):
"""
Create new record
Parameters
----------
path: str
Path to create record at
data: dict
Data to populate record with
"""
if remote == False:
with open(self.path + "/" + path, "wb") as f:
f.write(msgpack.dumps(data))
# logging.log(10, "Done creating record")
self.data[path] = Daisy(self.path + "/" + path)
# logging.log(10, "Done loading to Daisy")
return self.data[path]
else:
self.data[path] = Ref(path, remote)
return self.data[path]
def get(self, path: str):
"""
Get record at path, else return False
path: str
Path of record
"""
if path in self.data.keys():
return self.data[path]
else:
if os.path.exists(self.path + "/" + path):
self.data[path] = Daisy(self.path + "/" + path)
return self.data[path]
else:
# logging.log(10, "File does not exist")
return False
def refresh(self):
"""
Reload from disk to memory
"""
for key in self.data.keys():
self.data[key].read()
def search(self, keydict: dict, strict: bool = True):
"""
Search cache for record for records with values
keydict: dict
Values to search for
strict: bool
Whether to require values match
"""
results = []
for key, val in self.data.items():
val = val.get()
if strict and type(val) != str:
addcheck = False
for k, v in keydict.items():
if k in val.keys():
if v in val[k]:
addcheck = True
else:
addcheck = False
break
if addcheck:
results.append([key, val])
elif type(val) != str:
for k, v in keydict.items():
if k in val.keys():
if v in val[k]:
results.append([key, val])
return results

6
src/Cache.rst
View File

@ -1,6 +0,0 @@
Daisy based cache
=================
.. autoclass:: Daisy.Cache.Cache
:members:
:undoc-members:

74
src/Catch.py
View File

@ -1,74 +0,0 @@
from Daisy.Cache import Cache
from Daisy.Ref import Ref
import os
import random
class Catch(Cache):
"""
Sub class of Cache for handling catchs
.. image:: https://git.utopic.work/PierMesh/piermesh/raw/branch/main/imgs/catchdisplay.png
`🔗 Source <https://git.utopic.work/PierMesh/piermesh/src/branch/main/Daisy/Catch.py>`__
"""
catches = {}
remoteCatchesMap = {}
def __init__(
self, path: str = "catch", filepaths=None, catchFile=None, walk: bool = False
):
"""
Basically the same initialization parameters as Catch
"""
super().__init__(
filepaths=filepaths, cacheFile=catchFile, path=path, walk=walk, isCatch=True
)
# TODO: Fins
def sget(self, path: str):
"""
Call Cache's get to get record
"""
return super().get(path)
# TODO: Rename
def get(self, head: str, tail: str, fins=None):
"""
Get catch by pieces
Parameters
----------
head: str
First part of catch (maximum: 4 characters)
tail: str
Second part of catch (maximum: 16 characters)
fins
List of (maximum 8 characters) strings at the end of the catch oe None if none
"""
r = self.search({"head": head, "tail": tail})
return r[0][1]["html"]
def addc(self, peer, node, seperator, head, tail, data, fins=None, remote=False):
tnpath = "catch/" + node
if os.path.exists(tnpath) != True:
os.makedirs(tnpath)
tppath = tnpath + "/" + peer
if os.path.exists(tppath) != True:
os.makedirs(tppath)
sid = str(random.randrange(0, 999999)).zfill(6)
data["seperator"] = seperator
data["head"] = head
data["tail"] = tail
if fins != None:
data["fins"] = fins
res = self.create("{0}/{1}/{2}".format(node, peer, sid), data, remote=remote)
return [sid, res]
def exportDirectoryListing(self):
return [k for k in self.data.keys]

5
src/Catch.rst
View File

@ -1,5 +0,0 @@
Daisy Catch cache
=================
.. autoclass:: Daisy.Catch.Catch
:members:

195
src/Daisy.py
View File

@ -1,195 +0,0 @@
import os
import json
import msgpack
# TODO: delete
# TODO: propagate json changes to msgpack automatically
# TODO: propagate msgpack changes to cache automatically
# TODO: Indexing
def _json_to_msg(path: str):
"""
Convert json at the path plus .json to a msgpack binary
Parameters
----------
path: str
Path to json minus the extension
"""
rpath = path + ".json"
res = b""
with open(rpath) as f:
res = msgpack.dumps(json.load(f))
with open(path, "wb") as f:
f.write(res)
class Daisy:
"""
Base class for Daisy data representation
`🔗 Source <https://git.utopic.work/PierMesh/piermesh/src/branch/main/Components/daisy.py>`__
Attributes
----------
filepath: str
Path to file representation on disk
msg: dict
In memory representation
"""
def __init__(
self,
filepath: str,
templates: dict = {},
template: bool = False,
prefillDict: bool = False,
remote=False,
):
"""
Parameters
----------
filepath: str
Path to disk location
templates: dict
Dictionary of templates to Use
template: bool
Which template to Use
prefillDict: bool
Whether to fill the record with a template
"""
self.remote = False
self.filepath = filepath
if remote != False:
self.remote = True
self.remoteNodeID = remote
else:
if os.path.exists(filepath) != True:
with open(filepath, "wb") as f:
if template != False:
if template in templates.keys():
t = templates[template].get()
if prefillDict != False:
for k in prefillDict.keys():
t[k] = prefillDict[k]
f.write(msgpack.dumps(t))
self.msg = t
else:
print("No such template as: " + template)
else:
f.write(msgpack.dumps({}))
self.msg = {}
elif os.path.isdir(filepath):
self.msg = "directory"
else:
with open(filepath, "rb") as f:
self.msg = msgpack.loads(f.read())
# Use override for updating
def write(
self,
override=False,
encrypt: bool = False,
encryptKey=None,
recur: bool = False,
):
"""
Write record to disk
Parameters
----------
override
Either false or a dictionary of values to set on the record
encrypt: bool
Whether to encrypt the record (TODO)
encryptKey
Key to encrypt record with, or None if not set
recur: bool
Whether to recursively handle keys
"""
if override != False:
for key in override.keys():
# TODO: Deeper recursion
if recur:
if not key in self.msg.keys():
self.msg[key] = {}
for ikey in override[key].keys():
self.msg[key][ikey] = override[key][ikey]
else:
self.msg[key] = override[key]
data = msgpack.dumps(self.msg)
with open(self.filepath, "wb") as f:
f.write(data)
# Use for refreshing
def read(self, decrypt: bool = False, decryptKey=False):
"""
Read record from disk to memory
Parameters
----------
decrypt: bool
Whether to decrypt record
decryptKey
Key to decrypt record
"""
if os.path.isdir(self.filepath):
self.msg = "directory"
else:
with open(self.filepath, "rb") as f:
self.msg = msgpack.loads(f.read())
def get(self):
"""
Get record dictionary from memory
Returns
-------
self.msg: dict
"""
return self.msg
def sublist(self):
"""
Lists contents of directory if object is a directory, otherwise return None
"""
fpath = self.filepath
if os.path.isdir(fpath):
return ["messages/" + x for x in os.listdir(fpath)]
else:
return None
def loadTemplates(templatePath: str = "templates"):
"""Load templates for prefilling records
Parameters
----------
templatePath: str
Path to templates
"""
templates = {}
for p in os.listdir(templatePath):
p = templatePath + "/" + p
if os.path.isdir(p):
for ip in os.listdir(p):
ip = p + "/" + ip
if os.path.isdir(ip):
print("Too deep, skipping: " + ip)
else:
templates[ip] = Daisy(ip)
else:
templates[p] = Daisy(p)
self.templates = templates
return templates

6
src/Daisy.rst
View File

@ -1,6 +0,0 @@
Daisy
=====
.. autoclass:: Daisy.Daisy.Daisy
:members:
:undoc-members:

6
src/Ref.py
View File

@ -1,6 +0,0 @@
from Daisy.Daisy import Daisy
class Ref(Daisy):
def __init__(self, path, remoteNodeID):
super().__init__(path, remote=remoteNodeID)

49
src/Soil.py
View File

@ -1,49 +0,0 @@
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
global garden
"""
Map of instances to list of signals
to be processed
"""
garden = {}
class Compound(FileSystemEventHandler):
"""
File system watcher to propagate disk changes
`🔗 Source <https://git.utopic.work/PierMesh/piermesh/src/branch/main/Daisy/Soil.py>`__
"""
def __init__(self, cache, isCatch: bool = False):
"""
Parameters
----------
cache: Cache
Daisy cache to update
isCatch: bool
Is the cache for catchs
"""
self.cache = cache
self.isCatch = isCatch
super().__init__()
def on_any_event(self, event):
"""
Called when a CRUD operation is performed on a record file
Parameters
----------
event
Event object provided by watchdog
"""
if not (".json" in event.src_path):
if not (".md" in event.src_path):
tpath = "/".join(event.src_path.split("/")[1:])
if tpath != "":
if self.isCatch:
self.cache.sget(tpath)
else:
self.cache.get(tpath).get()

6
src/Soil.rst
View File

@ -1,6 +0,0 @@
Soil: Daisy signal management
=============================
.. autoclass:: Daisy.Soil.Compound
:members:
:undoc-members:

33
src/Store.py
View File

@ -1,33 +0,0 @@
from Daisy.Daisy import Daisy
import os
class Store(Daisy):
"""
Key value store
`🔗 Source <https://git.utopic.work/PierMesh/piermesh/src/branch/main/Daisy/Store.py>`__
"""
def __init__(self, store: str, path: str, nodeNickname: str):
fpath = "daisy/{0}/{1}".format(path, nodeNickname)
cpath = "{0}/{1}/{2}".format(path, nodeNickname, store)
if not os.path.exists(fpath):
os.mkdir(fpath)
super().__init__("daisy/" + cpath)
def update(self, entry: str, data, recur: bool = True):
if recur:
for key in data.keys():
self.msg[entry][key] = data[key]
else:
self.msg[entry] = data
self.write()
def getRecord(self, key: str):
if key in self.get().keys():
return self.get()[key]
else:
self.cLog(20, "Record does not exist")
return False

6
src/Store.rst
View File

@ -1,6 +0,0 @@
Store: Daisy key value store
============================
.. autoclass:: Daisy.Store.Store
:members:
:undoc-members: