package dataprovider import ( "encoding/binary" "encoding/json" "errors" "fmt" "path/filepath" "time" "github.com/drakkan/sftpgo/logger" "github.com/drakkan/sftpgo/utils" bolt "go.etcd.io/bbolt" ) const ( boltDatabaseVersion = 3 ) var ( usersBucket = []byte("users") usersIDIdxBucket = []byte("users_id_idx") dbVersionBucket = []byte("db_version") dbVersionKey = []byte("version") ) // BoltProvider auth provider for bolt key/value store type BoltProvider struct { dbHandle *bolt.DB } type compatUserV2 struct { ID int64 `json:"id"` Username string `json:"username"` Password string `json:"password,omitempty"` PublicKeys []string `json:"public_keys,omitempty"` HomeDir string `json:"home_dir"` UID int `json:"uid"` GID int `json:"gid"` MaxSessions int `json:"max_sessions"` QuotaSize int64 `json:"quota_size"` QuotaFiles int `json:"quota_files"` Permissions []string `json:"permissions"` UsedQuotaSize int64 `json:"used_quota_size"` UsedQuotaFiles int `json:"used_quota_files"` LastQuotaUpdate int64 `json:"last_quota_update"` UploadBandwidth int64 `json:"upload_bandwidth"` DownloadBandwidth int64 `json:"download_bandwidth"` ExpirationDate int64 `json:"expiration_date"` LastLogin int64 `json:"last_login"` Status int `json:"status"` } func initializeBoltProvider(basePath string) error { var err error logSender = fmt.Sprintf("dataprovider_%v", BoltDataProviderName) dbPath := config.Name if !utils.IsFileInputValid(dbPath) { return fmt.Errorf("Invalid database path: %#v", dbPath) } if !filepath.IsAbs(dbPath) { dbPath = filepath.Join(basePath, dbPath) } dbHandle, err := bolt.Open(dbPath, 0600, &bolt.Options{ NoGrowSync: false, FreelistType: bolt.FreelistArrayType, Timeout: 5 * time.Second}) if err == nil { providerLog(logger.LevelDebug, "bolt key store handle created") err = dbHandle.Update(func(tx *bolt.Tx) error { _, e := tx.CreateBucketIfNotExists(usersBucket) return e }) if err != nil { providerLog(logger.LevelWarn, "error creating users bucket: %v", err) return err } err = dbHandle.Update(func(tx *bolt.Tx) error { _, e := tx.CreateBucketIfNotExists(usersIDIdxBucket) return e }) if err != nil { providerLog(logger.LevelWarn, "error creating username idx bucket: %v", err) return err } err = dbHandle.Update(func(tx *bolt.Tx) error { _, e := tx.CreateBucketIfNotExists(dbVersionBucket) return e }) if err != nil { providerLog(logger.LevelWarn, "error creating database version bucket: %v", err) return err } provider = BoltProvider{dbHandle: dbHandle} } else { providerLog(logger.LevelWarn, "error creating bolt key/value store handler: %v", err) } return err } func (p BoltProvider) checkAvailability() error { _, err := p.getUsers(1, 0, "ASC", "") return err } func (p BoltProvider) validateUserAndPass(username string, password string) (User, error) { var user User if len(password) == 0 { return user, errors.New("Credentials cannot be null or empty") } user, err := p.userExists(username) if err != nil { providerLog(logger.LevelWarn, "error authenticating user: %v, error: %v", username, err) return user, err } return checkUserAndPass(user, password) } func (p BoltProvider) validateUserAndPubKey(username string, pubKey string) (User, string, error) { var user User if len(pubKey) == 0 { return user, "", errors.New("Credentials cannot be null or empty") } user, err := p.userExists(username) if err != nil { providerLog(logger.LevelWarn, "error authenticating user: %v, error: %v", username, err) return user, "", err } return checkUserAndPubKey(user, pubKey) } func (p BoltProvider) getUserByID(ID int64) (User, error) { var user User err := p.dbHandle.View(func(tx *bolt.Tx) error { bucket, idxBucket, err := getBuckets(tx) if err != nil { return err } userIDAsBytes := itob(ID) username := idxBucket.Get(userIDAsBytes) if username == nil { return &RecordNotFoundError{err: fmt.Sprintf("user with ID %v does not exist", ID)} } u := bucket.Get(username) if u == nil { return &RecordNotFoundError{err: fmt.Sprintf("username %#v and ID: %v does not exist", string(username), ID)} } return json.Unmarshal(u, &user) }) return user, err } func (p BoltProvider) updateLastLogin(username string) error { return p.dbHandle.Update(func(tx *bolt.Tx) error { bucket, _, err := getBuckets(tx) if err != nil { return err } var u []byte if u = bucket.Get([]byte(username)); u == nil { return &RecordNotFoundError{err: fmt.Sprintf("username %#v does not exist, unable to update last login", username)} } var user User err = json.Unmarshal(u, &user) if err != nil { return err } user.LastLogin = utils.GetTimeAsMsSinceEpoch(time.Now()) buf, err := json.Marshal(user) if err != nil { return err } return bucket.Put([]byte(username), buf) }) } func (p BoltProvider) updateQuota(username string, filesAdd int, sizeAdd int64, reset bool) error { return p.dbHandle.Update(func(tx *bolt.Tx) error { bucket, _, err := getBuckets(tx) if err != nil { return err } var u []byte if u = bucket.Get([]byte(username)); u == nil { return &RecordNotFoundError{err: fmt.Sprintf("username %#v does not exist, unable to update quota", username)} } var user User err = json.Unmarshal(u, &user) if err != nil { return err } if reset { user.UsedQuotaSize = sizeAdd user.UsedQuotaFiles = filesAdd } else { user.UsedQuotaSize += sizeAdd user.UsedQuotaFiles += filesAdd } user.LastQuotaUpdate = utils.GetTimeAsMsSinceEpoch(time.Now()) buf, err := json.Marshal(user) if err != nil { return err } return bucket.Put([]byte(username), buf) }) } func (p BoltProvider) getUsedQuota(username string) (int, int64, error) { user, err := p.userExists(username) if err != nil { providerLog(logger.LevelWarn, "unable to get quota for user %v error: %v", username, err) return 0, 0, err } return user.UsedQuotaFiles, user.UsedQuotaSize, err } func (p BoltProvider) userExists(username string) (User, error) { var user User err := p.dbHandle.View(func(tx *bolt.Tx) error { bucket, _, err := getBuckets(tx) if err != nil { return err } u := bucket.Get([]byte(username)) if u == nil { return &RecordNotFoundError{err: fmt.Sprintf("username %v does not exist", username)} } return json.Unmarshal(u, &user) }) return user, err } func (p BoltProvider) addUser(user User) error { err := validateUser(&user) if err != nil { return err } return p.dbHandle.Update(func(tx *bolt.Tx) error { bucket, idxBucket, err := getBuckets(tx) if err != nil { return err } if u := bucket.Get([]byte(user.Username)); u != nil { return fmt.Errorf("username %v already exists", user.Username) } id, err := bucket.NextSequence() if err != nil { return err } user.ID = int64(id) buf, err := json.Marshal(user) if err != nil { return err } userIDAsBytes := itob(user.ID) err = bucket.Put([]byte(user.Username), buf) if err != nil { return err } return idxBucket.Put(userIDAsBytes, []byte(user.Username)) }) } func (p BoltProvider) updateUser(user User) error { err := validateUser(&user) if err != nil { return err } return p.dbHandle.Update(func(tx *bolt.Tx) error { bucket, _, err := getBuckets(tx) if err != nil { return err } if u := bucket.Get([]byte(user.Username)); u == nil { return &RecordNotFoundError{err: fmt.Sprintf("username %v does not exist", user.Username)} } buf, err := json.Marshal(user) if err != nil { return err } return bucket.Put([]byte(user.Username), buf) }) } func (p BoltProvider) deleteUser(user User) error { return p.dbHandle.Update(func(tx *bolt.Tx) error { bucket, idxBucket, err := getBuckets(tx) if err != nil { return err } userIDAsBytes := itob(user.ID) userName := idxBucket.Get(userIDAsBytes) if userName == nil { return &RecordNotFoundError{err: fmt.Sprintf("user with id %v does not exist", user.ID)} } err = bucket.Delete(userName) if err != nil { return err } return idxBucket.Delete(userIDAsBytes) }) } func (p BoltProvider) dumpUsers() ([]User, error) { users := []User{} var err error err = p.dbHandle.View(func(tx *bolt.Tx) error { bucket, _, err := getBuckets(tx) if err != nil { return err } cursor := bucket.Cursor() for k, v := cursor.First(); k != nil; k, v = cursor.Next() { var user User err = json.Unmarshal(v, &user) if err != nil { return err } err = addCredentialsToUser(&user) if err != nil { return err } users = append(users, user) } return err }) return users, err } func (p BoltProvider) getUserWithUsername(username string) ([]User, error) { users := []User{} var user User user, err := p.userExists(username) if err == nil { users = append(users, HideUserSensitiveData(&user)) return users, nil } if _, ok := err.(*RecordNotFoundError); ok { err = nil } return users, err } func (p BoltProvider) getUsers(limit int, offset int, order string, username string) ([]User, error) { users := []User{} var err error if limit <= 0 { return users, err } if len(username) > 0 { if offset == 0 { return p.getUserWithUsername(username) } return users, err } err = p.dbHandle.View(func(tx *bolt.Tx) error { bucket, _, err := getBuckets(tx) if err != nil { return err } cursor := bucket.Cursor() itNum := 0 if order == "ASC" { for k, v := cursor.First(); k != nil; k, v = cursor.Next() { itNum++ if itNum <= offset { continue } var user User err = json.Unmarshal(v, &user) if err == nil { users = append(users, HideUserSensitiveData(&user)) } if len(users) >= limit { break } } } else { for k, v := cursor.Last(); k != nil; k, v = cursor.Prev() { itNum++ if itNum <= offset { continue } var user User err = json.Unmarshal(v, &user) if err == nil { users = append(users, HideUserSensitiveData(&user)) } if len(users) >= limit { break } } } return err }) return users, err } func (p BoltProvider) close() error { return p.dbHandle.Close() } func (p BoltProvider) reloadConfig() error { return nil } // initializeDatabase does nothing, no initilization is needed for bolt provider func (p BoltProvider) initializeDatabase() error { return errNoInitRequired } func (p BoltProvider) migrateDatabase() error { dbVersion, err := getBoltDatabaseVersion(p.dbHandle) if err != nil { return err } if dbVersion.Version == boltDatabaseVersion { providerLog(logger.LevelDebug, "bolt database is updated, current version: %v", dbVersion.Version) return nil } if dbVersion.Version == 1 { err = updateDatabaseFrom1To2(p.dbHandle) if err != nil { return err } return updateDatabaseFrom2To3(p.dbHandle) } else if dbVersion.Version == 2 { return updateDatabaseFrom2To3(p.dbHandle) } return nil } // itob returns an 8-byte big endian representation of v. func itob(v int64) []byte { b := make([]byte, 8) binary.BigEndian.PutUint64(b, uint64(v)) return b } func getBuckets(tx *bolt.Tx) (*bolt.Bucket, *bolt.Bucket, error) { var err error bucket := tx.Bucket(usersBucket) idxBucket := tx.Bucket(usersIDIdxBucket) if bucket == nil || idxBucket == nil { err = fmt.Errorf("unable to find required buckets, bolt database structure not correcly defined") } return bucket, idxBucket, err } func updateDatabaseFrom1To2(dbHandle *bolt.DB) error { providerLog(logger.LevelInfo, "updating bolt database version: 1 -> 2") usernames, err := getBoltAvailableUsernames(dbHandle) if err != nil { return err } for _, u := range usernames { user, err := provider.userExists(u) if err != nil { return err } user.Status = 1 err = provider.updateUser(user) if err != nil { return err } providerLog(logger.LevelInfo, "user %#v updated, \"status\" setted to 1", user.Username) } return updateBoltDatabaseVersion(dbHandle, 2) } func updateDatabaseFrom2To3(dbHandle *bolt.DB) error { providerLog(logger.LevelInfo, "updating bolt database version: 2 -> 3") users := []User{} err := dbHandle.View(func(tx *bolt.Tx) error { bucket, _, err := getBuckets(tx) if err != nil { return err } cursor := bucket.Cursor() for k, v := cursor.First(); k != nil; k, v = cursor.Next() { var compatUser compatUserV2 err = json.Unmarshal(v, &compatUser) if err == nil { user := User{} user.ID = compatUser.ID user.Username = compatUser.Username user.Password = compatUser.Password user.PublicKeys = compatUser.PublicKeys user.HomeDir = compatUser.HomeDir user.UID = compatUser.UID user.GID = compatUser.GID user.MaxSessions = compatUser.MaxSessions user.QuotaSize = compatUser.QuotaSize user.QuotaFiles = compatUser.QuotaFiles user.Permissions = make(map[string][]string) user.Permissions["/"] = compatUser.Permissions user.UsedQuotaSize = compatUser.UsedQuotaSize user.UsedQuotaFiles = compatUser.UsedQuotaFiles user.LastQuotaUpdate = compatUser.LastQuotaUpdate user.UploadBandwidth = compatUser.UploadBandwidth user.DownloadBandwidth = compatUser.DownloadBandwidth user.ExpirationDate = compatUser.ExpirationDate user.LastLogin = compatUser.LastLogin user.Status = compatUser.Status users = append(users, user) } } return err }) if err != nil { return err } for _, user := range users { err = provider.updateUser(user) if err != nil { return err } providerLog(logger.LevelInfo, "user %#v updated, \"permissions\" setted to %+v", user.Username, user.Permissions) } return updateBoltDatabaseVersion(dbHandle, 3) } func getBoltAvailableUsernames(dbHandle *bolt.DB) ([]string, error) { usernames := []string{} err := dbHandle.View(func(tx *bolt.Tx) error { _, idxBucket, err := getBuckets(tx) if err != nil { return err } cursor := idxBucket.Cursor() for k, v := cursor.First(); k != nil; k, v = cursor.Next() { usernames = append(usernames, string(v)) } return nil }) return usernames, err } func getBoltDatabaseVersion(dbHandle *bolt.DB) (schemaVersion, error) { var dbVersion schemaVersion err := dbHandle.View(func(tx *bolt.Tx) error { bucket := tx.Bucket(dbVersionBucket) if bucket == nil { return fmt.Errorf("unable to find database version bucket") } v := bucket.Get(dbVersionKey) if v == nil { dbVersion = schemaVersion{ Version: 1, } return nil } return json.Unmarshal(v, &dbVersion) }) return dbVersion, err } func updateBoltDatabaseVersion(dbHandle *bolt.DB, version int) error { err := dbHandle.Update(func(tx *bolt.Tx) error { bucket := tx.Bucket(dbVersionBucket) if bucket == nil { return fmt.Errorf("unable to find database version bucket") } newDbVersion := schemaVersion{ Version: version, } buf, err := json.Marshal(newDbVersion) if err != nil { return err } return bucket.Put(dbVersionKey, buf) }) return err }