dynamo store的设计使dynamodb中的多个分片数据存储无缝
dynamo-store的Python项目详细描述
发电机商店
dynamo存储是为了使dynamodb中的多个分片数据存储无缝。
它支持开箱即用：
- 将子词典自动分片到其他表中（对于获得大约400KB的dyanmodb限制很有用）
- 使用128位aes_cbc加密静止数据
- 对象到表的自动逻辑序列化
文档
营养不良
dystrore表示dynamodb表。表的结构应该用dystrore对象表示，包括到每个表中碎片的路径。当序列化时，dystrore对象在表中存储了足够的信息，允许重新创建dystrore对象，以便在某些情况下使生活更轻松。这意味着一旦一个对象被写入一个dystore，它就可以从表中存储的元数据中重建。
动态对象
dyobject表示dystrore中的一个对象。它依赖于jsonmodels来控制对象将如何序列化为json，而json最终将出现在dynamodb表中。
继承dyobject时，根和shard dyobjects应实现以下类变量：
- table\u name=要将此碎片保存到aws中的“shard1”表
- 区域名='us-east-2''region to sav eto in aws
- 要使用的主键名
- ignore\u list=
[]序列化期间要忽略的变量名- config_loader=config_loader（）类级配置加载程序，以防止必须传递到
save()和load()。 否则上述类变量都是可选的。 - ignore\u list=
配置加载
config_loader(config, **kwargs)是一个回调，用于dystore和dyobject调用，以控制dynamo存储的某些操作。第一个参数config指定查询哪个配置项，其余的关键字参数是调用的上下文相关数据。
参数config将是dystore或dyobject中定义的CONFIG_LOADER_*类变量之一，它们在下面的api文档中有记录。
dyobject既支持传入save()和load()的配置加载器，也支持使用配置加载器类变量的类级配置加载器。传入的配置加载器接受类变量实例的优先级。
示例dystore用法
from dynamo_store.store import DyStore
def root_store():
shards = [DyStore('Shard1', 'IDX', path='$.birth_details'),
DyStore('Shard2', 'IDX', path='$.location')]
return DyStore('Root', 'ID', shards=shards)
def loader(config, **kwargs):
if config == DyStore.CONFIG_LOADER_LOAD_KEY:
encrypted_paths = ['birth_details.hospital', 'birth_details.dob', 'location.city', 'location.country', 'firstname', 'lastname']
path = kwargs['path']
if path in encrypted_paths:
return 'somekey'
elif config == DyStore.CONFIG_LOADER_KEEP_METADATA:
return False
return None
d = {'firstname': 'john',
'lastname': 'smith',
'location': {'city': 'Osaka',
'country': 'Japan'},
'birth_details': {'hospital': 'Kosei Nenkin',
'dob': '12/2/1995'}}
# Write object to store
key = root_store().write(d)
# Update individual path
root_store().write_path(key, "location.city", "New York")
# Read back object from store
success, obj = root_store().read(key)
# Read individual path
path = root_store().read_path(key, "location.country")
# Delete object from store
root_store().delete(key)
dyobject用法示例
from dynamo_store.object import DyObject
from dynamo_store.store import DyStore
from jsonmodels import fields
class BirthDetails(DyObject):
TABLE_NAME = 'Shard1' # Table to save this shard to in AWS
REGION_NAME = 'us-east-2' # Region to sav eto in AWS
PRIMARY_KEY_NAME = 'IDX' # Primary key name to use
hospital = fields.StringField()
dob = fields.StringField()
class Location(DyObject):
TABLE_NAME = 'Shard2' # Table to save this shard to in AWS
REGION_NAME = 'us-east-2' # Region to save to in AWS
PRIMARY_KEY_NAME = 'IDX' # Primary key name to use
city = fields.StringField()
country = fields.StringField()
geolocation = fields.EmbeddedField(GeoLocation)
class Root(DyObject):
TABLE_NAME = 'Root' # Table to save to in AWS
REGION_NAME = 'us-east-2' # Region to save to in AWS
PRIMARY_KEY_NAME = 'ID' # Primary key name to use
firstname = fields.StringField()
lastname = fields.StringField()
location = fields.EmbeddedField(Location)
birth_details = fields.EmbeddedField(BirthDetails)
def loader(config, **kwargs):
if config == DyStore.CONFIG_LOADER_LOAD_KEY:
encrypted_paths = ['birth_details.hospital.value', 'birth_details.dob.value', 'location.city.value', 'location.country.value', 'firstname.value', 'lastname.value']
path = kwargs['path']
if path in encrypted_paths:
return 'somekey'
elif config == DyStore.CONFIG_LOADER_KEEP_METADATA:
return False
elif config == DyObject.CONFIG_LOADER_DICT_TO_CLASS:
# In this case we can just use the dict key to determine what kind of object it is
# other cases might require examining the value
key = kwargs['key']
if key == 'location':
return Location
elif key == 'birth_details':
return BirthDetails
elif key == 'geolocation':
return GeoLocation
return None
# Setup and save object
orig = Root()
orig.firstname = 'john'
orig.lastname = 'smith'
orig.location.city = 'Osaka'
orig.location.country = 'Kewpie'
orig.birth_details.dob = '15/03/1980'
orig.birth_details.hospital = 'Good one'
key = orig.save(config_loader=loader)
# Load object using key obtained from save() call
o = Root.load(key, config_loader=loader)
print(o.firstname)
print(o.lastname)
print(o.location.city)
print(o.location.country)
print(o.birth_details.dob)
print(o.birth_details.hospital)
迪索尔API
class DyStore(object):
"""
Invoked on writes to allow control of primary key.
config_loader(DyStore.CONFIG_LOADER_GENERATE_PK, data=item)
:param config: DyStore.CONFIG_LOADER_GENERATE_PK
:param data: item being written
:returns: string containing primary key to use, None to use uuid4()
"""
CONFIG_LOADER_GENERATE_PK = 'pk'
"""
Invoked on decryption/encryption to control key used.
config_loader(DyStore.CONFIG_LOADER_LOAD_KEY, path=path, data=root)
:param config: DyStore.CONFIG_LOADER_LOAD_KEY
:param path: json path of item being encrypted/decrypted
:param data: root object being encrypted/decrypted
:returns: string containing key to use, None to ignore decryption/encryption
"""
CONFIG_LOADER_LOAD_KEY = 'key'
"""
Invoked on read/write to control if DyStore metadata should be kept in object.
config_loader(DyStore.CONFIG_LOADER_KEEP_METADATA, data=item)
:param config: DyStore.CONFIG_LOADER_KEEP_METADATA
:param data: item being read/written
:returns: bool controlling if metadata should be kept or not
"""
CONFIG_LOADER_KEEP_METADATA = 'meta'
def __init__(self, table_name=None, primary_key_name=None, path=None, shards=[], region='us-east-2'):
"""
:param table_name: Name of DynamoDB table this object will access
:param primary_key_name: Primary key name in DynamoDB
:param path: JSON Path of this object when it is used in a shard, note: see jsonpath-ng for documentation on jsonpath.
:param shards: Items to shard out to other tables in this object.
:param region: AWS region for table.
:param ignore_paths: Paths to ignore during encryption/decryption, can be regexes
"""
def read_path(self, primary_key, path, config_loader=None):
"""
Reads a path from an object from this store.
:param primary_key: Primary key of object to read.
:param path: JSON path of object to read (can reside in a shard).
:param config_loader: Config loader to be used: config_loader(config, data) returns setting
:returns: list of values on success, None otherwise
"""
def read(self, primary_key, resolve_shards=True, config_loader=None):
"""
Reads an object from this store.
:param primary_key: Primary key of object to read.
:param resolve_shards: Boolean to control whether shards are read.
:param config_loader: Config loader to be used: config_loader(config, data) returns setting
:returns: success, value
"""
def write_path(self, primary_key, path, value, config_loader=None):
"""
Writes a path in an object to this store.
:param primary_key: Primary key of object to read.
:param path: JSON path of object to read (can reside in a shard).
:param value: Value to write at the JSON path.
:param config_loader: Config loader to be used: config_loader(config, data) returns setting
:param root_object: Internal parameter used for proper path resolution in config load calls.
:returns: True if successful, False otherwise
"""
def write(self, data, primary_key=None, save_shards=True, config_loader=None):
"""
Writes an object to this store.
:param primary_key: Primary key of object to write.
:param save_shards: Boolean to control whether shards are saved.
:param config_loader: Config loader to be used: config_loader(config, data) returns setting
:returns: primary_key used/generated
"""
def delete(self, primary_key, delete_shards=True, config_loader=None):
"""
Deletes an object from this store.
:param primary_key: Primary key of object to delete.
:param delete_shards: Boolean to control whether shards are deleted.
:param config_loader: Config loader to be used: config_loader(config, data) returns setting
:returns: True if successful, False otherwise
"""
动态对象API
class DyObject(object):
"""
Name of table in AWS to save this object to.
"""
TABLE_NAME = None
"""
Region in AWS to save this object to.
"""
REGION_NAME = None
"""
Name of primary key to use for this object.
"""
PRIMARY_KEY_NAME = None
"""
Config loader callable to use when config queries are made
"""
CONFIG_LOADER = None
"""
Variable names to ignore during serialization
"""
IGNORE_LIST = []
"""
Variable names to ignore during encryption/decryption
"""
CRYPTO_IGNORE_PATHS = []
"""
Invoked on object load when class cant be determined.
config_loader(DyObject.CONFIG_LOADER_DICT_TO_KEY, key=key, value=value)
:param config: DyObject.CONFIG_LOADER_DICT_TO_CLASS
:param key: key in parent object
:param value: value of dict in object
:returns: Class to instantiate, None if to keep as dict
"""
CONFIG_LOADER_DICT_TO_CLASS = 'dict'
def delete(self, primary_key=None, config_loader=None):
"""
Delete an object from the store.
:param primary_key: Primary key to use, (optional: value passed in will be stored in instance for future use).
:param config_loader: Config loader to be used: config_loader(config, data) returns setting
:returns: True if successful, False otherwise
"""
def save(self, primary_key=None, config_loader=None):
"""
Saves this object to the store.
:param primary_key: Primary key to use.
:param config_loader: Config loader to be used: config_loader(config, data) returns setting
A class wide config loader can also be set, however the passed in config loader takes preference.
:returns: key of object written
"""
@classmethod
def load(cls, primary_key, config_loader=None):
"""
Loads an object from the store.
:param cls: Class to instantiate
:param primary_key: Primary key of object to load.
:param config_loader: Config loader to be used: config_loader(config, data) returns setting
:param validate: Enable JSON Models field validation
A class wide config loader can also be set, however the passed in config loader takes preference.
:returns: cls object
"""
from dynamo_store.object import DyObject
from dynamo_store.store import DyStore
from jsonmodels import fields
class BirthDetails(DyObject):
TABLE_NAME = 'Shard1' # Table to save this shard to in AWS
REGION_NAME = 'us-east-2' # Region to sav eto in AWS
PRIMARY_KEY_NAME = 'IDX' # Primary key name to use
hospital = fields.StringField()
dob = fields.StringField()
class Location(DyObject):
TABLE_NAME = 'Shard2' # Table to save this shard to in AWS
REGION_NAME = 'us-east-2' # Region to save to in AWS
PRIMARY_KEY_NAME = 'IDX' # Primary key name to use
city = fields.StringField()
country = fields.StringField()
geolocation = fields.EmbeddedField(GeoLocation)
class Root(DyObject):
TABLE_NAME = 'Root' # Table to save to in AWS
REGION_NAME = 'us-east-2' # Region to save to in AWS
PRIMARY_KEY_NAME = 'ID' # Primary key name to use
firstname = fields.StringField()
lastname = fields.StringField()
location = fields.EmbeddedField(Location)
birth_details = fields.EmbeddedField(BirthDetails)
def loader(config, **kwargs):
if config == DyStore.CONFIG_LOADER_LOAD_KEY:
encrypted_paths = ['birth_details.hospital.value', 'birth_details.dob.value', 'location.city.value', 'location.country.value', 'firstname.value', 'lastname.value']
path = kwargs['path']
if path in encrypted_paths:
return 'somekey'
elif config == DyStore.CONFIG_LOADER_KEEP_METADATA:
return False
elif config == DyObject.CONFIG_LOADER_DICT_TO_CLASS:
# In this case we can just use the dict key to determine what kind of object it is
# other cases might require examining the value
key = kwargs['key']
if key == 'location':
return Location
elif key == 'birth_details':
return BirthDetails
elif key == 'geolocation':
return GeoLocation
return None
# Setup and save object
orig = Root()
orig.firstname = 'john'
orig.lastname = 'smith'
orig.location.city = 'Osaka'
orig.location.country = 'Kewpie'
orig.birth_details.dob = '15/03/1980'
orig.birth_details.hospital = 'Good one'
key = orig.save(config_loader=loader)
# Load object using key obtained from save() call
o = Root.load(key, config_loader=loader)
print(o.firstname)
print(o.lastname)
print(o.location.city)
print(o.location.country)
print(o.birth_details.dob)
print(o.birth_details.hospital)
class DyStore(object):
"""
Invoked on writes to allow control of primary key.
config_loader(DyStore.CONFIG_LOADER_GENERATE_PK, data=item)
:param config: DyStore.CONFIG_LOADER_GENERATE_PK
:param data: item being written
:returns: string containing primary key to use, None to use uuid4()
"""
CONFIG_LOADER_GENERATE_PK = 'pk'
"""
Invoked on decryption/encryption to control key used.
config_loader(DyStore.CONFIG_LOADER_LOAD_KEY, path=path, data=root)
:param config: DyStore.CONFIG_LOADER_LOAD_KEY
:param path: json path of item being encrypted/decrypted
:param data: root object being encrypted/decrypted
:returns: string containing key to use, None to ignore decryption/encryption
"""
CONFIG_LOADER_LOAD_KEY = 'key'
"""
Invoked on read/write to control if DyStore metadata should be kept in object.
config_loader(DyStore.CONFIG_LOADER_KEEP_METADATA, data=item)
:param config: DyStore.CONFIG_LOADER_KEEP_METADATA
:param data: item being read/written
:returns: bool controlling if metadata should be kept or not
"""
CONFIG_LOADER_KEEP_METADATA = 'meta'
def __init__(self, table_name=None, primary_key_name=None, path=None, shards=[], region='us-east-2'):
"""
:param table_name: Name of DynamoDB table this object will access
:param primary_key_name: Primary key name in DynamoDB
:param path: JSON Path of this object when it is used in a shard, note: see jsonpath-ng for documentation on jsonpath.
:param shards: Items to shard out to other tables in this object.
:param region: AWS region for table.
:param ignore_paths: Paths to ignore during encryption/decryption, can be regexes
"""
def read_path(self, primary_key, path, config_loader=None):
"""
Reads a path from an object from this store.
:param primary_key: Primary key of object to read.
:param path: JSON path of object to read (can reside in a shard).
:param config_loader: Config loader to be used: config_loader(config, data) returns setting
:returns: list of values on success, None otherwise
"""
def read(self, primary_key, resolve_shards=True, config_loader=None):
"""
Reads an object from this store.
:param primary_key: Primary key of object to read.
:param resolve_shards: Boolean to control whether shards are read.
:param config_loader: Config loader to be used: config_loader(config, data) returns setting
:returns: success, value
"""
def write_path(self, primary_key, path, value, config_loader=None):
"""
Writes a path in an object to this store.
:param primary_key: Primary key of object to read.
:param path: JSON path of object to read (can reside in a shard).
:param value: Value to write at the JSON path.
:param config_loader: Config loader to be used: config_loader(config, data) returns setting
:param root_object: Internal parameter used for proper path resolution in config load calls.
:returns: True if successful, False otherwise
"""
def write(self, data, primary_key=None, save_shards=True, config_loader=None):
"""
Writes an object to this store.
:param primary_key: Primary key of object to write.
:param save_shards: Boolean to control whether shards are saved.
:param config_loader: Config loader to be used: config_loader(config, data) returns setting
:returns: primary_key used/generated
"""
def delete(self, primary_key, delete_shards=True, config_loader=None):
"""
Deletes an object from this store.
:param primary_key: Primary key of object to delete.
:param delete_shards: Boolean to control whether shards are deleted.
:param config_loader: Config loader to be used: config_loader(config, data) returns setting
:returns: True if successful, False otherwise
"""
class DyObject(object):
"""
Name of table in AWS to save this object to.
"""
TABLE_NAME = None
"""
Region in AWS to save this object to.
"""
REGION_NAME = None
"""
Name of primary key to use for this object.
"""
PRIMARY_KEY_NAME = None
"""
Config loader callable to use when config queries are made
"""
CONFIG_LOADER = None
"""
Variable names to ignore during serialization
"""
IGNORE_LIST = []
"""
Variable names to ignore during encryption/decryption
"""
CRYPTO_IGNORE_PATHS = []
"""
Invoked on object load when class cant be determined.
config_loader(DyObject.CONFIG_LOADER_DICT_TO_KEY, key=key, value=value)
:param config: DyObject.CONFIG_LOADER_DICT_TO_CLASS
:param key: key in parent object
:param value: value of dict in object
:returns: Class to instantiate, None if to keep as dict
"""
CONFIG_LOADER_DICT_TO_CLASS = 'dict'
def delete(self, primary_key=None, config_loader=None):
"""
Delete an object from the store.
:param primary_key: Primary key to use, (optional: value passed in will be stored in instance for future use).
:param config_loader: Config loader to be used: config_loader(config, data) returns setting
:returns: True if successful, False otherwise
"""
def save(self, primary_key=None, config_loader=None):
"""
Saves this object to the store.
:param primary_key: Primary key to use.
:param config_loader: Config loader to be used: config_loader(config, data) returns setting
A class wide config loader can also be set, however the passed in config loader takes preference.
:returns: key of object written
"""
@classmethod
def load(cls, primary_key, config_loader=None):
"""
Loads an object from the store.
:param cls: Class to instantiate
:param primary_key: Primary key of object to load.
:param config_loader: Config loader to be used: config_loader(config, data) returns setting
:param validate: Enable JSON Models field validation
A class wide config loader can also be set, however the passed in config loader takes preference.
:returns: cls object
"""
欢迎加入QQ群-->： 979659372
