1
0
mirror of https://github.com/kataras/iris.git synced 2025-12-22 04:17:03 +00:00

Update to 8.2.0 | BoltDB session database, fix file sessiondb, faster, simpler and improvement Session Database API

Former-commit-id: 4034737a65b78a77277e4283fd9289c17f4a452e
This commit is contained in:
kataras
2017-08-07 06:04:35 +03:00
parent 36e6fb37b8
commit 48e352e1df
24 changed files with 1128 additions and 430 deletions

View File

@@ -0,0 +1,223 @@
package boltdb
import (
"bytes"
"os"
"path/filepath"
"time"
"github.com/boltdb/bolt"
"github.com/kataras/golog"
"github.com/kataras/iris/core/errors"
"github.com/kataras/iris/sessions"
)
// DefaultFileMode used as the default database's "fileMode"
// for creating the sessions directory path, opening and write
// the session boltdb(file-based) storage.
var (
DefaultFileMode = 0666
)
// Database the BoltDB(file-based) session storage.
type Database struct {
path string // path included the name, i.e sessions/store.db
fileMode os.FileMode // defaults to 0666.
table []byte
Service *bolt.DB // `New` sets it but it can be override exactly after `New`, use with caution.
async bool
}
var (
// ErrOptionsMissing returned on `New` when path or tableName are empty.
ErrOptionsMissing = errors.New("required options are missing")
)
// New creates and returns a new BoltDB(file-based) storage
// instance based on the "path".
// Path should include the filename and the directory(aka fullpath), i.e sessions/store.db.
//
// It will remove any old session files.
func New(path string, fileMode os.FileMode, bucketName string) (*Database, error) {
if path == "" || bucketName == "" {
return nil, ErrOptionsMissing
}
if fileMode <= 0 {
fileMode = os.FileMode(DefaultFileMode)
}
// create directories if necessary
if err := os.MkdirAll(filepath.Dir(path), fileMode); err != nil {
golog.Errorf("error while trying to create the necessary directories for %s: %v", path, err)
return nil, err
}
service, err := bolt.Open(path, 0600,
&bolt.Options{Timeout: 15 * time.Second},
)
bucket := []byte(bucketName)
if err != nil {
golog.Errorf("unable to initialize the BoltDB-based session database: %v", err)
return nil, err
}
service.Update(func(tx *bolt.Tx) (err error) {
_, err = tx.CreateBucketIfNotExists(bucket)
return
})
db := &Database{path: path, fileMode: fileMode,
table: bucket, Service: service,
}
return db, db.Cleanup()
}
// Cleanup removes any invalid(have expired) session entries,
// it's being called automatically on `New` as well.
func (db *Database) Cleanup() error {
err := db.Service.Update(func(tx *bolt.Tx) error {
b := db.getBucket(tx)
c := b.Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
if len(k) == 0 { // empty key, continue to the next pair
continue
}
storeDB, err := sessions.DecodeRemoteStore(v)
if err != nil {
continue
}
if storeDB.Lifetime.HasExpired() {
if err := c.Delete(); err != nil {
golog.Warnf("troubles when cleanup a session remote store from BoltDB: %v", err)
}
}
}
return nil
})
return err
}
// Async if true passed then it will use different
// go routines to update the BoltDB(file-based) storage.
func (db *Database) Async(useGoRoutines bool) *Database {
db.async = useGoRoutines
return db
}
// Load loads the sessions from the BoltDB(file-based) session storage.
func (db *Database) Load(sid string) (storeDB sessions.RemoteStore) {
bsid := []byte(sid)
err := db.Service.View(func(tx *bolt.Tx) (err error) {
// db.getSessBucket(tx, sid)
b := db.getBucket(tx)
c := b.Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
if len(k) == 0 { // empty key, continue to the next pair
continue
}
if bytes.Equal(k, bsid) { // session id should be the name of the key-value pair
storeDB, err = sessions.DecodeRemoteStore(v) // decode the whole value, as a remote store
break
}
}
return
})
if err != nil {
golog.Errorf("error while trying to load from the remote store: %v", err)
}
return
}
// Sync syncs the database with the session's (memory) store.
func (db *Database) Sync(p sessions.SyncPayload) {
if db.async {
go db.sync(p)
} else {
db.sync(p)
}
}
func (db *Database) sync(p sessions.SyncPayload) {
bsid := []byte(p.SessionID)
if p.Action == sessions.ActionDestroy {
if err := db.destroy(bsid); err != nil {
golog.Errorf("error while destroying a session(%s) from boltdb: %v",
p.SessionID, err)
}
return
}
s, err := p.Store.Serialize()
if err != nil {
golog.Errorf("error while serializing the remote store: %v", err)
}
err = db.Service.Update(func(tx *bolt.Tx) error {
return db.getBucket(tx).Put(bsid, s)
})
if err != nil {
golog.Errorf("error while writing the session bucket: %v", err)
}
}
func (db *Database) destroy(bsid []byte) error {
return db.Service.Update(func(tx *bolt.Tx) error {
return db.getBucket(tx).Delete(bsid)
})
}
// we store the whole data to the key-value pair of the root bucket
// so we don't need a separate bucket for each session
// this method could be faster if we had large data to store
// but with sessions we recommend small ammount of data, so the method finally choosen
// is faster (decode/encode the whole store + lifetime and return it as it's)
//
// func (db *Database) getSessBucket(tx *bolt.Tx, sid string) (*bolt.Bucket, error) {
// table, err := db.getBucket(tx).CreateBucketIfNotExists([]byte(sid))
// return table, err
// }
func (db *Database) getBucket(tx *bolt.Tx) *bolt.Bucket {
return tx.Bucket(db.table)
}
// Len reports the number of sessions that are stored to the this BoltDB table.
func (db *Database) Len() (num int) {
db.Service.View(func(tx *bolt.Tx) error {
// Assume bucket exists and has keys
b := db.getBucket(tx)
if b == nil {
return nil
}
b.ForEach(func([]byte, []byte) error {
num++
return nil
})
return nil
})
return
}
// Close shutdowns the BoltDB connection.
func (db *Database) Close() error {
err := db.Service.Close()
if err != nil {
golog.Warnf("closing the BoltDB connection: %v", err)
}
return err
}

View File

@@ -1,128 +1,214 @@
package file
import (
"bytes"
"encoding/gob"
"io/ioutil"
"os"
"path/filepath"
"time"
"github.com/kataras/golog"
"github.com/kataras/iris/core/errors"
"github.com/kataras/iris/sessions"
)
// DefaultFileMode used as the default database's "fileMode"
// for creating the sessions directory path, opening and write the session file.
var (
// PathFileMode for creating the sessions directory path, opening and write the session file.
// Defaults to 0666.
PathFileMode uint32 = 0666
DefaultFileMode = 0666
)
// Database is the basic file-storage session database.
//
// What it does
// It removes old(expired) session files, at init (`Cleanup`).
// It creates a session file on the first inserted key-value session data.
// It removes a session file on destroy.
// It sync the session file to the session's memstore on any other action (insert, delete, clear).
// It automatically remove the session files on runtime when a session is expired.
//
// Remember: sessions are not a storage for large data, everywhere: on any platform on any programming language.
type Database struct {
path string
dir string
fileMode os.FileMode // defaults to 0666 if missing.
// if true then it will use go routines to:
// append or re-write a file
// create a file
// remove a file
async bool
}
// New returns a new file-storage database instance based on the "path".
func New(path string) *Database {
lindex := path[len(path)-1]
// New creates and returns a new file-storage database instance based on the "path".
// It will remove any old session files.
func New(directoryPath string, fileMode os.FileMode) (*Database, error) {
lindex := directoryPath[len(directoryPath)-1]
if lindex != os.PathSeparator && lindex != '/' {
path += string(os.PathSeparator)
directoryPath += string(os.PathSeparator)
}
if fileMode <= 0 {
fileMode = os.FileMode(DefaultFileMode)
}
// create directories if necessary
os.MkdirAll(path, os.FileMode(PathFileMode))
return &Database{path: path}
if err := os.MkdirAll(directoryPath, fileMode); err != nil {
return nil, err
}
db := &Database{dir: directoryPath, fileMode: fileMode}
return db, db.Cleanup()
}
func (d *Database) sessPath(sid string) string {
return filepath.Join(d.path, sid)
// Cleanup removes any invalid(have expired) session files, it's being called automatically on `New` as well.
func (db *Database) Cleanup() error {
return filepath.Walk(db.dir, func(path string, info os.FileInfo, err error) error {
if info.IsDir() {
return nil
}
sessPath := path
storeDB, _ := db.load(sessPath) // we don't care about errors here, the file may be not a session a file at all.
if storeDB.Lifetime.HasExpired() {
os.Remove(path)
}
return nil
})
}
// Load loads the values to the underline
func (d *Database) Load(sid string) (values map[string]interface{}, expireDate *time.Time) {
sessPath := d.sessPath(sid)
f, err := os.OpenFile(sessPath, os.O_RDONLY, os.FileMode(PathFileMode))
// FileMode for creating the sessions directory path, opening and write the session file.
//
// Defaults to 0666.
func (db *Database) FileMode(fileMode uint32) *Database {
db.fileMode = os.FileMode(fileMode)
return db
}
// Async if true passed then it will use go routines to:
// append or re-write a file
// create a file
// remove a file.
//
// Defaults to false.
func (db *Database) Async(useGoRoutines bool) *Database {
db.async = useGoRoutines
return db
}
func (db *Database) sessPath(sid string) string {
return filepath.Join(db.dir, sid)
}
// Load loads the values from the storage and returns them
func (db *Database) Load(sid string) sessions.RemoteStore {
sessPath := db.sessPath(sid)
store, err := db.load(sessPath)
if err != nil {
golog.Error(err.Error())
}
return store
}
func (db *Database) load(fileName string) (storeDB sessions.RemoteStore, loadErr error) {
f, err := os.OpenFile(fileName, os.O_RDONLY, db.fileMode)
if err != nil {
// we don't care if filepath doesn't exists yet, it will be created on Update.
// we don't care if filepath doesn't exists yet, it will be created later on.
return
}
defer f.Close()
val, err := ioutil.ReadAll(f)
contents, err := ioutil.ReadAll(f)
if err != nil {
// we don't care if filepath doesn't exists yet, it will be created on Update.
golog.Errorf("error while reading the session file's data: %v", err)
loadErr = errors.New("error while reading the session file's data: %v").Format(err)
return
}
if err == nil {
err = DeserializeBytes(val, &values)
if err != nil { // we care for this error only
golog.Errorf("load error: %v", err)
}
}
storeDB, err = sessions.DecodeRemoteStore(contents)
return // no expiration
}
// serialize the values to be stored as strings inside the session file-storage.
func serialize(values map[string]interface{}) []byte {
val, err := SerializeBytes(values)
if err != nil {
golog.Errorf("serialize error: %v", err)
}
return val
}
func (d *Database) expireSess(sid string) {
go os.Remove(d.sessPath(sid))
}
// Update updates the session file-storage.
func (d *Database) Update(sid string, newValues map[string]interface{}, expireDate *time.Time) {
if len(newValues) == 0 { // means delete by call
d.expireSess(sid)
if err != nil { // we care for this error only
loadErr = errors.New("load error: %v").Format(err)
return
}
// delete the file on expiration
if expireDate != nil && !expireDate.IsZero() {
now := time.Now()
return
}
if expireDate.Before(now) {
// already expirated, delete it now and return.
d.expireSess(sid)
return
// Sync syncs the database.
func (db *Database) Sync(p sessions.SyncPayload) {
if db.async {
go db.sync(p)
} else {
db.sync(p)
}
}
func (db *Database) sync(p sessions.SyncPayload) {
// if destroy then remove the file from the disk
if p.Action == sessions.ActionDestroy {
if err := db.destroy(p.SessionID); err != nil {
golog.Errorf("error while destroying and removing the session file: %v", err)
}
// otherwise set a timer to delete the file automatically
afterDur := expireDate.Sub(now)
time.AfterFunc(afterDur, func() {
d.expireSess(sid)
})
return
}
if err := ioutil.WriteFile(d.sessPath(sid), serialize(newValues), os.FileMode(PathFileMode)); err != nil {
golog.Errorf("error while writing the session to the file: %v", err)
if err := db.override(p.SessionID, p.Store); err != nil {
golog.Errorf("error while writing the session file: %v", err)
}
}
// SerializeBytes serializes the "m" into bytes using gob encoder and returns the result.
func SerializeBytes(m interface{}) ([]byte, error) {
buf := new(bytes.Buffer)
enc := gob.NewEncoder(buf)
err := enc.Encode(m)
if err == nil {
return buf.Bytes(), nil
// good idea but doesn't work, it is not just an array of entries
// which can be appended with the gob...anyway session data should be small so we don't have problem
// with that:
// on insert new data, it appends to the file
// func (db *Database) insert(sid string, entry memstore.Entry) error {
// f, err := os.OpenFile(
// db.sessPath(sid),
// os.O_WRONLY|os.O_CREATE|os.O_RDWR|os.O_APPEND,
// db.fileMode,
// )
// if err != nil {
// return err
// }
// if _, err := f.Write(serializeEntry(entry)); err != nil {
// f.Close()
// return err
// }
// return f.Close()
// }
// removes all entries but keeps the file.
// func (db *Database) clearAll(sid string) error {
// return ioutil.WriteFile(
// db.sessPath(sid),
// []byte{},
// db.fileMode,
// )
// }
// on update, remove and clear, it re-writes the file to the current values(may empty).
func (db *Database) override(sid string, store sessions.RemoteStore) error {
s, err := store.Serialize()
if err != nil {
return err
}
return nil, err
return ioutil.WriteFile(
db.sessPath(sid),
s,
db.fileMode,
)
}
// DeserializeBytes converts the bytes to a go value and puts that to "m" using the gob decoder.
func DeserializeBytes(b []byte, m interface{}) error {
dec := gob.NewDecoder(bytes.NewBuffer(b))
return dec.Decode(m) //no reference here otherwise doesn't work because of go remote object
// on destroy, it removes the file
func (db *Database) destroy(sid string) error {
return db.expireSess(sid)
}
func (db *Database) expireSess(sid string) error {
sessPath := db.sessPath(sid)
return os.Remove(sessPath)
}

View File

@@ -1,103 +1,87 @@
package redis
import (
"bytes"
"encoding/gob"
"time"
"github.com/kataras/golog"
"github.com/kataras/iris/sessions"
"github.com/kataras/iris/sessions/sessiondb/redis/service"
)
// Database the redis back-end session database for the sessions.
type Database struct {
redis *service.Service
async bool
}
// New returns a new redis database.
func New(cfg ...service.Config) *Database {
return &Database{redis: service.New(cfg...)}
// Note: no need to clean up here, the redis should handle these automatically because of the "SETEX"
// but that expiration doesn't depend on the session, instead it depends on the `MaxAgeSeconds`
// of the redis database configuration.
}
// Config returns the configuration for the redis server bridge, you can change them.
func (d *Database) Config() *service.Config {
return d.redis.Config
func (db *Database) Config() *service.Config {
return db.redis.Config
}
// Async if true passed then it will use different
// go routines to update the redis storage.
func (db *Database) Async(useGoRoutines bool) *Database {
db.async = useGoRoutines
return db
}
// Load loads the values to the underline.
func (d *Database) Load(sid string) (datas map[string]interface{}, expireDate *time.Time) {
values := make(map[string]interface{})
func (db *Database) Load(sid string) (storeDB sessions.RemoteStore) {
// values := make(map[string]interface{})
if !d.redis.Connected { //yes, check every first time's session for valid redis connection
d.redis.Connect()
_, err := d.redis.PingPong()
if !db.redis.Connected { //yes, check every first time's session for valid redis connection
db.redis.Connect()
_, err := db.redis.PingPong()
if err != nil {
golog.Errorf("redis database error on connect: %v", err)
return
}
}
//fetch the values from this session id and copy-> store them
val, err := d.redis.GetBytes(sid)
if err == nil {
// err removed because of previous TODO
DeserializeBytes(val, &values)
}
datas, _ = values["session-data"].(map[string]interface{})
dbExpireDateValue, exists := values["expire-date"]
if !exists {
return
}
expireDateValue, ok := dbExpireDateValue.(time.Time)
if !ok {
return
}
return datas, &expireDateValue
}
// serialize the values to be stored as strings inside the Redis, we panic at any serialization error here
func serialize(values map[string]interface{}) []byte {
val, err := SerializeBytes(values)
// fetch the values from this session id and copy-> store them
storeMaybe, err := db.redis.Get(sid)
if err != nil {
return nil
golog.Errorf("error while trying to load session values(%s) from redis: %v", sid, err)
return
}
return val
storeDB, ok := storeMaybe.(sessions.RemoteStore)
if !ok {
golog.Errorf(`error while trying to load session values(%s) from redis:
the retrieved value is not a sessions.RemoteStore type, please report that as bug, it should never occur`,
sid)
return
}
return
}
// Update updates the real redis store
func (d *Database) Update(sid string, newValues map[string]interface{}, expireDate *time.Time) {
if len(newValues) == 0 {
go d.redis.Delete(sid)
// Sync syncs the database.
func (db *Database) Sync(p sessions.SyncPayload) {
if db.async {
go db.sync(p)
} else {
datas := map[string]interface{}{"session-data": newValues}
if expireDate != nil {
datas["expire-date"] = *expireDate
}
db.sync(p)
}
}
//set/update all the values
go d.redis.Set(sid, serialize(datas))
func (db *Database) sync(p sessions.SyncPayload) {
if p.Action == sessions.ActionDestroy {
db.redis.Delete(p.SessionID)
return
}
storeB, err := p.Store.Serialize()
if err != nil {
golog.Error("error while encoding the remote session store")
return
}
}
// SerializeBytes serialize the "m" into bytes using the gob encoder and returns the result.
func SerializeBytes(m interface{}) ([]byte, error) {
buf := new(bytes.Buffer)
enc := gob.NewEncoder(buf)
err := enc.Encode(m)
if err == nil {
return buf.Bytes(), nil
}
return nil, err
}
// DeserializeBytes converts the bytes to a go value and puts that to "m" using the gob decoder.
func DeserializeBytes(b []byte, m interface{}) error {
dec := gob.NewDecoder(bytes.NewBuffer(b))
return dec.Decode(m) //no reference here otherwise doesn't work because of go remote object
db.redis.Set(p.SessionID, storeB)
}

View File

@@ -42,21 +42,21 @@ func (r *Service) CloseConnection() error {
return ErrRedisClosed
}
// Set sets to the redis
// key string, value string, you can use utils.Serialize(&myobject{}) to convert an object to []byte
func (r *Service) Set(key string, value []byte) (err error) { // map[interface{}]interface{}) (err error) {
// Set sets a key-value to the redis store.
// The expiration is setted by the MaxAgeSeconds.
func (r *Service) Set(key string, value interface{}) error {
c := r.pool.Get()
defer c.Close()
if err = c.Err(); err != nil {
return
if c.Err() != nil {
return c.Err()
}
_, err = c.Do("SETEX", r.Config.Prefix+key, r.Config.MaxAgeSeconds, value)
return
_, err := c.Do("SETEX", r.Config.Prefix+key, r.Config.MaxAgeSeconds, value)
return err
}
// Get returns value, err by its key
// you can use utils.Deserialize((.Get("yourkey"),&theobject{})
//returns nil and a filled error if something wrong happens
//returns nil and a filled error if something bad happened.
func (r *Service) Get(key string) (interface{}, error) {
c := r.pool.Get()
defer c.Close()
@@ -75,6 +75,27 @@ func (r *Service) Get(key string) (interface{}, error) {
return redisVal, nil
}
// GetAll returns all redis entries using the "SCAN" command (2.8+).
func (r *Service) GetAll() (interface{}, error) {
c := r.pool.Get()
defer c.Close()
if err := c.Err(); err != nil {
return nil, err
}
redisVal, err := c.Do("SCAN", 0) // 0 -> cursor
if err != nil {
return nil, err
}
if redisVal == nil {
return nil, err
}
return redisVal, nil
}
// GetBytes returns value, err by its key
// you can use utils.Deserialize((.GetBytes("yourkey"),&theobject{})
//returns nil and a filled error if something wrong happens
@@ -97,95 +118,6 @@ func (r *Service) GetBytes(key string) ([]byte, error) {
return redis.Bytes(redisVal, err)
}
// GetString returns value, err by its key
// you can use utils.Deserialize((.GetString("yourkey"),&theobject{})
//returns empty string and a filled error if something wrong happens
func (r *Service) GetString(key string) (string, error) {
redisVal, err := r.Get(key)
if redisVal == nil {
return "", ErrKeyNotFound.Format(key)
}
sVal, err := redis.String(redisVal, err)
if err != nil {
return "", err
}
return sVal, nil
}
// GetInt returns value, err by its key
// you can use utils.Deserialize((.GetInt("yourkey"),&theobject{})
//returns -1 int and a filled error if something wrong happens
func (r *Service) GetInt(key string) (int, error) {
redisVal, err := r.Get(key)
if redisVal == nil {
return -1, ErrKeyNotFound.Format(key)
}
intVal, err := redis.Int(redisVal, err)
if err != nil {
return -1, err
}
return intVal, nil
}
// GetStringMap returns map[string]string, err by its key
//returns nil and a filled error if something wrong happens
func (r *Service) GetStringMap(key string) (map[string]string, error) {
redisVal, err := r.Get(key)
if redisVal == nil {
return nil, ErrKeyNotFound.Format(key)
}
_map, err := redis.StringMap(redisVal, err)
if err != nil {
return nil, err
}
return _map, nil
}
// GetAll returns all keys and their values from a specific key (map[string]string)
// returns a filled error if something bad happened
func (r *Service) GetAll(key string) (map[string]string, error) {
c := r.pool.Get()
defer c.Close()
if err := c.Err(); err != nil {
return nil, err
}
reply, err := c.Do("HGETALL", r.Config.Prefix+key)
if err != nil {
return nil, err
}
if reply == nil {
return nil, ErrKeyNotFound.Format(key)
}
return redis.StringMap(reply, err)
}
// GetAllKeysByPrefix returns all []string keys by a key prefix from the redis
func (r *Service) GetAllKeysByPrefix(prefix string) ([]string, error) {
c := r.pool.Get()
defer c.Close()
if err := c.Err(); err != nil {
return nil, err
}
reply, err := c.Do("KEYS", r.Config.Prefix+prefix)
if err != nil {
return nil, err
}
if reply == nil {
return nil, ErrKeyNotFound.Format(prefix)
}
return redis.Strings(reply, err)
}
// Delete removes redis entry by specific key
func (r *Service) Delete(key string) error {
c := r.pool.Get()