liliesh/scabiosa
Archived
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Merge pull request #16 from netbenix/develop

Browse source 
SQL Changes
This commit is contained in:
netbenix 2021-12-23 11:20:18 +01:00 • committed by GitHub
commit 5dbd1e4179
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 95 additions and 49 deletions
Download patch file Download diff file Expand all files Collapse all files

6
Compressor/Compression.go
View file

@ -39,7 +39,7 @@ func compress(fileToWrite *os.File, folderPath string, backupName string){
zr, _ := gzip.NewWriterLevel(fileToWrite, flate.BestCompression) zr, _ := gzip.NewWriterLevel(fileToWrite, flate.BestCompression)
tw := tar.NewWriter(zr) tw := tar.NewWriter(zr)
go fmt.Printf("[%s] Start compression...\n", filepath.Base(folderPath)) go fmt.Printf("[%s] Start compression...\n", backupName)
SQL.NewLogEntry(SQL.GetSQLInstance(), uuid.New(), SQL.LogInfo, backupName, SQL.SQLStage_Compress, SQL.REMOTE_NONE, "Start compression", time.Now()) SQL.NewLogEntry(SQL.GetSQLInstance(), uuid.New(), SQL.LogInfo, backupName, SQL.SQLStage_Compress, SQL.REMOTE_NONE, "Start compression", time.Now())
filepath.Walk(folderPath, func(file string, fi os.FileInfo, err error) error { filepath.Walk(folderPath, func(file string, fi os.FileInfo, err error) error {
header, err := tar.FileInfoHeader(fi, file) header, err := tar.FileInfoHeader(fi, file)
@ -60,7 +60,7 @@ func compress(fileToWrite *os.File, folderPath string, backupName string){
logger.Fatal(err) logger.Fatal(err)
} }
go fmt.Printf("[%s] Compressing: %s (%d bytes)\n", filepath.Base(folderPath) ,relPath, fi.Size()) go fmt.Printf("[%s] Compressing: %s (%d bytes)\n", backupName, relPath, fi.Size())
if _, err := io.Copy(tw, data); err != nil { if _, err := io.Copy(tw, data); err != nil {
logger.Fatal(err) logger.Fatal(err)
} }
@ -78,6 +78,6 @@ func compress(fileToWrite *os.File, folderPath string, backupName string){
} }
go fmt.Printf("[%s] Compression Done.\n", filepath.Base(folderPath)) go fmt.Printf("[%s] Compression Done.\n", backupName)
SQL.NewLogEntry(SQL.GetSQLInstance(), uuid.New(), SQL.LogInfo, backupName, SQL.SQLStage_Compress, SQL.REMOTE_NONE, "Compression complete.", time.Now()) SQL.NewLogEntry(SQL.GetSQLInstance(), uuid.New(), SQL.LogInfo, backupName, SQL.SQLStage_Compress, SQL.REMOTE_NONE, "Compression complete.", time.Now())
} }

42
README.md
View file

@ -9,6 +9,10 @@ Trello board: [Click me!](https://trello.com/b/6zWLE6Jm)
- Log the Backup progress to a database - Log the Backup progress to a database
- Upload the files to a remote storage of your choice (see [Storage Types](#storage-types)) - Upload the files to a remote storage of your choice (see [Storage Types](#storage-types))
## Planned features for the Future!
- Backup restore
- Service for scheduled updates
- (Maybe) a web interface
## Database Types ## Database Types
- MariaDB - MariaDB
@ -21,9 +25,9 @@ Trello board: [Click me!](https://trello.com/b/6zWLE6Jm)
## Storage types ## Storage types
- Local storage (soon) - Local storage
- Azure Blob Storage (planned)
- Azure File Share - Azure File Share
- Azure Blob Storage (planned)
- S3 Bucket (far future) - S3 Bucket (far future)
- Dropbox (far future) - Dropbox (far future)
- OneDrive (far future) - OneDrive (far future)
@ -32,24 +36,26 @@ Trello board: [Click me!](https://trello.com/b/6zWLE6Jm)
| Storage Type | Config Type | | Storage Type | Config Type |
|-------------------------|--------------------------| |-------------------------|--------------------------|
| Azure File Share | azure-fileshare | | Azure File Share | azure-fileshare |
| Local Storage | none |
## Config Explaination ## Config Explaination
### config.json ### config.json
| Field | Type | Description | | Field | Type | Description |
|---------------------|:----------------:|------------------------------------------------| |---------------------------|:----------------:|------------------------------------------------|
| localBackupPath | string | Path where local backups are stored | | localBackupPath | string | Path where local backups are stored |
| **sqlConfig** | ---------------- | ---------------------------------------------- | | **sqlConfig** | ---------------- | ---------------------------------------------- |
| enableSQL | boolean | Enable/Disables the SQL entries | enableSQL | boolean | Enable/Disables the SQL entries
| sqlType | string | See [DatabaseTypes](#database-types) | | sqlType | string | See [DatabaseTypes](#database-types) |
| sql-address | string | Address to the SQL Server | | sql-address | string | Address to the SQL Server |
| sql-port | uint16 | SQL Server Port | | sql-port | uint16 | SQL Server Port |
| database | string | Database name | | database | string | Database name |
| db-user | string | SQL username from user which should be used | | db-user | string | SQL username from user which should be used |
| db-password | string | SQL password from user which should be used | | db-password | string | SQL password from user which should be used |
| **foldersToBackup** | ---------------- | ---------------------------------------------- | | **foldersToBackup** | ---------------- | ---------------------------------------------- |
| backupName | string | .bak file name | | backupName | string | .bak file name |
| folderPath | string | Path to folder which should be backed up | | folderPath | string | Path to folder which should be backed up |
| storageType | string | See [StorageTypes](#storage-types) | | remoteStorageType | string | See [StorageTypes](#storage-types) |
| createLocalBackup | boolean | Sets if .bak file should also be saved locally | | targetPath | string | Sets the targetPath for local backups |
| createLocalBackup | boolean | Sets if .bak file should also be saved locally |

27
SQL/MariaDBConnector.go
View file

@ -4,6 +4,7 @@ import (
"database/sql" "database/sql"
_ "github.com/go-sql-driver/mysql" _ "github.com/go-sql-driver/mysql"
"github.com/google/uuid" "github.com/google/uuid"
"os"
"scabiosa/Logging" "scabiosa/Logging"
"scabiosa/Tools" "scabiosa/Tools"
"strconv" "strconv"
@ -30,7 +31,6 @@ func GetMariaDBInstance(config Tools.Config) MariaDBConnector {
return mariadb return mariadb
} }
func checkIfEventLogTableExist(db *sql.DB, mariadb MariaDBConnector) bool { func checkIfEventLogTableExist(db *sql.DB, mariadb MariaDBConnector) bool {
rows, _ := db.Query("SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_NAME = 'EventLog';", mariadb.Database) rows, _ := db.Query("SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_NAME = 'EventLog';", mariadb.Database)
if !rows.Next(){ return false } if !rows.Next(){ return false }
@ -43,8 +43,8 @@ func checkIfBackupTableExist(db *sql.DB, mariadb MariaDBConnector) bool {
return true return true
} }
func checkIfBackupEntryExist(db *sql.DB, mariadb MariaDBConnector, backupName string) bool { func checkIfBackupEntryExist(db *sql.DB, mariadb MariaDBConnector, backupName string, hostname string) bool {
rows, _ := db.Query("SELECT * FROM `" + mariadb.Database + "`.Backups WHERE BackupName = '" + backupName + "';") rows, _ := db.Query("SELECT * FROM `" + mariadb.Database + "`.Backups WHERE Hostname = ? AND BackupName = ?;", hostname, backupName)
if !rows.Next(){ return false; } if !rows.Next(){ return false; }
return true return true
} }
@ -61,9 +61,8 @@ func createMariaDBConnection(mariadb MariaDBConnector) *sql.DB{
func (mariadb MariaDBConnector) createDefaultTables(){ func (mariadb MariaDBConnector) createDefaultTables(){
logger := Logging.DetailedLogger("MariaDB", "createDefaultTables") logger := Logging.DetailedLogger("MariaDB", "createDefaultTables")
eventLogSQL := "create table " + mariadb.Database +".EventLog\n(\n UUID text null,\n LogType enum ('INFO', 'WARNING', 'ERROR', 'FATAL') null,\n BackupName varchar(256) null,\n Stage enum ('COMPRESS', 'UPLOAD', 'DELETE TMP') null,\n RemoteStorage enum ('AZURE-FILE', 'AZURE-BLOB', 'NONE') null,\n Description text null,\n Timestamp datetime null\n);" eventLogSQL := "create table `" + mariadb.Database +"`.EventLog(UUID text null, LogType enum ('INFO', 'WARNING', 'ERROR', 'FATAL') null, Hostname varchar(256) null,BackupName varchar(256) null, Stage enum ('COMPRESS', 'UPLOAD', 'DELETE TMP') null, RemoteStorage enum ('AZURE-FILE', 'AZURE-BLOB', 'NONE') null, Description text null, Timestamp datetime null);"
backupSQL := "create table " + mariadb.Database +".Backups\n(\n UUID text null,\n BackupName varchar(256) null,\n LastBackup datetime null,\n LocalBackup tinyint(1) null,\n FilePath varchar(256) null,\n RemoteStorage enum ('AZURE-FILE', 'AZURE-BLOB', 'NONE') null,\n RemotePath varchar(256) null\n);\n\n" backupSQL := "create table `" + mariadb.Database +"`.Backups(UUID text null, Hostname varchar(256) null, BackupName varchar(256) null, LastBackup datetime null, LocalBackup tinyint(1) null, FilePath varchar(256) null, RemoteStorage enum ('AZURE-FILE', 'AZURE-BLOB', 'NONE') null, RemotePath varchar(256) null, LocalPath varchar(256) null);"
db := createMariaDBConnection(mariadb) db := createMariaDBConnection(mariadb)
@ -88,26 +87,28 @@ func (mariadb MariaDBConnector) newLogEntry(uuid uuid.UUID, logType LogType, bac
logger := Logging.DetailedLogger("MariaDB", "newLogEntry") logger := Logging.DetailedLogger("MariaDB", "newLogEntry")
db := createMariaDBConnection(mariadb) db := createMariaDBConnection(mariadb)
_, err := db.Query("INSERT INTO `" + mariadb.Database + "`.EventLog VALUES (?, ?, ?, ?, ?, ?, ?);", uuid.String(), strconv.FormatInt(int64(logType), 10), backupName, stage, strconv.FormatInt(int64(storageType), 10), description ,timestamp) hostname, _ := os.Hostname()
_, err := db.Query("INSERT INTO `" + mariadb.Database + "`.EventLog VALUES (?, ?, ?, ?, ?, ?, ?, ?);", uuid.String(), strconv.FormatInt(int64(logType), 10), hostname, backupName, stage, strconv.FormatInt(int64(storageType), 10), description ,timestamp)
if err != nil{ if err != nil{
logger.Fatal(err) logger.Fatal(err)
} }
} }
func (mariadb MariaDBConnector) newBackupEntry(backupName string, lastBackup time.Time, localBackup bool, filePath string, storageType RemoteStorageType, remotePath string, localPath string){
func (mariadb MariaDBConnector) newBackupEntry(backupName string, lastBackup time.Time, localBackup bool, filePath string, storageType RemoteStorageType, remotePath string){
logger := Logging.DetailedLogger("MariaDB", "newBackupEntry") logger := Logging.DetailedLogger("MariaDB", "newBackupEntry")
db := createMariaDBConnection(mariadb) db := createMariaDBConnection(mariadb)
if checkIfBackupEntryExist(db, mariadb, backupName){ hostname, _ := os.Hostname()
_, err := db.Query("UPDATE `" + mariadb.Database + "`.Backups SET LastBackup = ? WHERE BackupName = ?;", lastBackup, backupName)
if checkIfBackupEntryExist(db, mariadb, backupName, hostname){
_, err := db.Query("UPDATE `" + mariadb.Database + "`.Backups SET LastBackup = ?, LocalBackup = ?, RemoteStorage = ?, RemotePath = ?, LocalPath = ? WHERE Hostname = ? AND BackupName = ?;",lastBackup, localBackup, strconv.FormatInt(int64(storageType), 10), remotePath, localPath, hostname, backupName)
if err != nil { if err != nil {
logger.Fatal(err) logger.Fatal(err)
} }
} else { } else {
_, err := db.Query("INSERT INTO `" + mariadb.Database + "`.Backups VALUES (?, ?, ?, ?, ?, ?, ?);", uuid.New(), backupName, lastBackup, localBackup, filePath, strconv.FormatInt(int64(storageType), 10), remotePath) _, err := db.Query("INSERT INTO `" + mariadb.Database + "`.Backups VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?);", uuid.New(), hostname, backupName, lastBackup, localBackup, filePath, strconv.FormatInt(int64(storageType), 10), remotePath, localPath)
if err != nil { if err != nil {
logger.Fatal(err) logger.Fatal(err)
} }

6
SQL/SQLInterface.go
View file

@ -9,7 +9,7 @@ import (
type SQLService interface { type SQLService interface {
createDefaultTables() createDefaultTables()
newLogEntry(uuid uuid.UUID, logType LogType, backupName string, stage SQLStage, storageType RemoteStorageType, description string, timestamp time.Time) newLogEntry(uuid uuid.UUID, logType LogType, backupName string, stage SQLStage, storageType RemoteStorageType, description string, timestamp time.Time)
newBackupEntry(backupName string, lastBackup time.Time, localBackup bool, filePath string, storageType RemoteStorageType, remotePath string) newBackupEntry(backupName string, lastBackup time.Time, localBackup bool, filePath string, storageType RemoteStorageType, remotePath string, localPath string)
} }
func CreateDefaultTables(sqlService SQLService){ func CreateDefaultTables(sqlService SQLService){
@ -26,10 +26,10 @@ func NewLogEntry(sqlService SQLService, uuid uuid.UUID, logType LogType, backupN
} }
} }
func NewBackupEntry(sqlService SQLService, backupName string, lastBackup time.Time, localBackup bool, filePath string, storageType RemoteStorageType, remotePath string){ func NewBackupEntry(sqlService SQLService, backupName string, lastBackup time.Time, localBackup bool, filePath string, storageType RemoteStorageType, remotePath string, localPath string){
config := Tools.GetConfig() config := Tools.GetConfig()
if config.SQLConfig.EnableSQL{ if config.SQLConfig.EnableSQL{
sqlService.newBackupEntry(backupName, lastBackup, localBackup, filePath, storageType, remotePath) sqlService.newBackupEntry(backupName, lastBackup, localBackup, filePath, storageType, remotePath, localPath)
} }
} }

38
Tools/Config.go
View file

@ -21,8 +21,9 @@ type Config struct {
BackupName string `json:"backupName"` BackupName string `json:"backupName"`
FolderPath string `json:"folderPath"` FolderPath string `json:"folderPath"`
RemoteStorageType string `json:"remoteStorageType"` RemoteStorageType string `json:"remoteStorageType"`
TargetPath string `json:"targetPath"` RemoteTargetPath string `json:"remoteTargetPath"`
CreateLocalBackup bool `json:"createLocalBackup"` CreateLocalBackup bool `json:"createLocalBackup"`
LocalTargetPath string `json:"LocalTargetPath"`
} `json:"foldersToBackup"` } `json:"foldersToBackup"`
} }
@ -36,9 +37,40 @@ func readConfig() []byte {
return file return file
} }
func CheckIfConfigExists(){
logger := Logging.DetailedLogger("ConfigHandler", "CheckIfConfigExists")
if _, err := os.Stat("config/config.json"); os.IsNotExist(err){
_, fileErr := os.OpenFile("config/config.json", os.O_CREATE, 0775)
if fileErr != nil{
logger.Fatal(fileErr)
}
generateDefaultConfig()
}
}
func generateDefaultConfig() {
logger := Logging.DetailedLogger("ConfigHandler", "GenerateDefaultConfig")
var config Config
var conf []byte
conf, err := json.MarshalIndent(config, "", "\t")
//conf, err := json.Marshal(config)
if err != nil {
logger.Fatal(err)
}
err = os.WriteFile("config/config.json", conf, 0755)
if err != nil {
logger.Fatal(err)
}
}
func GetConfig() Config { func GetConfig() Config {
logger := Logging.DetailedLogger("ConfigHandler", "GetConfig()")
logger := Logging.DetailedLogger("ConfigHandler", "GetConfig")
var config Config var config Config
err := json.Unmarshal(readConfig(), &config) err := json.Unmarshal(readConfig(), &config)

7
config/config.json
View file

@ -12,9 +12,10 @@
{ {
"backupName": "", "backupName": "",
"folderPath": "", "folderPath": "",
"storageType": "", "remoteStorageType": "",
"targetPath": "", "remoteTargetPath": "",
"createLocalBackup": false "createLocalBackup": false,
"localTargetPath": ""
} }
] ]
} }

18
main.go
View file

@ -12,6 +12,7 @@ import (
) )
func main(){ func main(){
Tools.CheckIfConfigExists()
config := Tools.GetConfig() config := Tools.GetConfig()
SQL.CreateDefaultTables(SQL.GetSQLInstance()) SQL.CreateDefaultTables(SQL.GetSQLInstance())
@ -23,22 +24,27 @@ func main(){
if backupItem.RemoteStorageType != "none"{ if backupItem.RemoteStorageType != "none"{
storage = StorageTypes.CheckStorageType(backupItem.RemoteStorageType) storage = StorageTypes.CheckStorageType(backupItem.RemoteStorageType)
destPath = checkTmpPath(backupItem.CreateLocalBackup, backupItem.TargetPath) destPath = checkTmpPath(backupItem.CreateLocalBackup, backupItem.LocalTargetPath)
} else { } else {
destPath = backupItem.TargetPath destPath = backupItem.LocalTargetPath
} }
bakFile := Compressor.CreateBakFile(backupItem.BackupName + getTimeSuffix(), backupItem.FolderPath, destPath, backupItem.BackupName) bakFile := Compressor.CreateBakFile(backupItem.BackupName + getTimeSuffix(), backupItem.FolderPath, destPath, backupItem.BackupName)
if backupItem.RemoteStorageType != "none"{ if backupItem.RemoteStorageType != "none"{
StorageTypes.UploadFile(storage, bakFile, backupItem.BackupName, backupItem.TargetPath) StorageTypes.UploadFile(storage, bakFile, backupItem.BackupName, backupItem.RemoteTargetPath)
} }
if !backupItem.CreateLocalBackup && backupItem.RemoteStorageType != "none"{ if !backupItem.CreateLocalBackup && backupItem.RemoteStorageType != "none"{
_ = os.Remove(bakFile) _ = os.Remove(bakFile)
SQL.NewLogEntry(SQL.GetSQLInstance(), uuid.New(), SQL.LogInfo, backupItem.BackupName, SQL.SQLStage_DeleteTmp, SQL.REMOTE_NONE, "Deleted tmp file" ,time.Now()) SQL.NewLogEntry(SQL.GetSQLInstance(), uuid.New(), SQL.LogInfo, backupItem.BackupName, SQL.SQLStage_DeleteTmp, SQL.REMOTE_NONE, "Deleted tmp file" ,time.Now())
} }
SQL.NewBackupEntry(SQL.GetSQLInstance(), backupItem.BackupName, time.Now(), backupItem.CreateLocalBackup, backupItem.FolderPath, StorageTypes.CheckRemoteStorageType(backupItem.RemoteStorageType), StorageTypes.GetAzureStorage().TargetDirectory)
if backupItem.RemoteStorageType == "none" {
backupItem.CreateLocalBackup = true
backupItem.RemoteTargetPath = "NONE"
}
SQL.NewBackupEntry(SQL.GetSQLInstance(), backupItem.BackupName, time.Now(), backupItem.CreateLocalBackup, backupItem.FolderPath, StorageTypes.CheckRemoteStorageType(backupItem.RemoteStorageType), backupItem.RemoteTargetPath, backupItem.LocalTargetPath)
} }
} }