liliesh/scabiosa
Archived
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Merge pull request #3 from netbenix/develop

Browse source 
Implemented SQL Interface
This commit is contained in:
netbenix 2021-11-22 21:30:19 +01:00 • committed by GitHub
commit 842028afce
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 182 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

12
Compression.go
View file

@ -6,10 +6,13 @@ import (
"compress/flate"
"compress/gzip"
"fmt"
"github.com/google/uuid"
"io"
"os"
"path/filepath"
"scabiosa/Logging"
"scabiosa/SQL"
"time"
)
func CreateBakFile(filename string, folderPath string, destinationPath string) string {
@ -29,6 +32,10 @@ func CreateBakFile(filename string, folderPath string, destinationPath string) s
logger.Fatal(err)
}
//TODO Remove Hardcoded SQL Instance
SQL.NewLogEntry(SQL.GetMariaDBInstance(), uuid.New(), SQL.LogInfo, filepath.Base(folderPath), SQL.SQLStage_Compress, SQL.REMOTE_NONE, "File successfully written.", time.Now())
return fileName
}
@ -40,7 +47,8 @@ func compress(folderPath string, buf io.Writer){
tw := tar.NewWriter(zr)
fmt.Printf("[%s] Start compression...\n", filepath.Base(folderPath))
//TODO Remove Hardcoded SQL Instance
SQL.NewLogEntry(SQL.GetMariaDBInstance(), uuid.New(), SQL.LogInfo, filepath.Base(folderPath), SQL.SQLStage_Compress, SQL.REMOTE_NONE, "Start compression", time.Now())
filepath.Walk(folderPath, func(file string, fi os.FileInfo, err error) error {
header, err := tar.FileInfoHeader(fi, file)
if err != nil {
@ -78,4 +86,6 @@ func compress(folderPath string, buf io.Writer){
fmt.Printf("[%s] Compression Done.\n", filepath.Base(folderPath))
//TODO Remove Hardcoded SQL Instance
SQL.NewLogEntry(SQL.GetMariaDBInstance(), uuid.New(), SQL.LogInfo, filepath.Base(folderPath), SQL.SQLStage_Compress, SQL.REMOTE_NONE, "Compression complete.", time.Now())
}

10
SQL/LogTypes.go Normal file
View file

@ -0,0 +1,10 @@
package SQL
type LogType int64
const(
LogInfo LogType = 1
LogWarning = 2
LogError = 3
LogFatal = 4
)

93
SQL/MariaDB.go Normal file
View file

@ -0,0 +1,93 @@
package SQL
import (
"database/sql"
_ "github.com/go-sql-driver/mysql"
"github.com/google/uuid"
"scabiosa/Logging"
"scabiosa/Tools"
"strconv"
"time"
)
type MariaDBConnector struct {
Address string
Port uint16
Database string
DbUser string
DbPassword string
}
func GetMariaDBInstance() MariaDBConnector{
config := Tools.GetConfig()
var mariadb MariaDBConnector
mariadb.Address = config.SQLConfig.SqlAddress
mariadb.Port = config.SQLConfig.SqlPort
mariadb.Database = config.SQLConfig.Database
mariadb.DbUser = config.SQLConfig.DbUser
mariadb.DbPassword = config.SQLConfig.DbPassword
return mariadb
}
func checkIfEventLogTableExist(db *sql.DB, mariadb MariaDBConnector) bool {
rows, _ := db.Query("SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_NAME = 'EventLog';", mariadb.Database)
if !rows.Next(){ return false }
return true
}
func checkIfBackupTableExist(db *sql.DB, mariadb MariaDBConnector) bool {
rows, _ := db.Query("SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_NAME = 'Backups';", mariadb.Database)
if !rows.Next(){ return false }
return true
}
func createMariaDBConnection(mariadb MariaDBConnector) *sql.DB{
logger := Logging.DetailedLogger("MariaDB", "createConnection")
db, err := sql.Open("mysql", mariadb.DbUser + ":" + mariadb.DbPassword + "@(" + mariadb.Address + ":" +strconv.Itoa(int(mariadb.Port))+ ")/" + mariadb.Database)
if err != nil{
logger.Fatal(err)
}
return db
}
func (mariadb MariaDBConnector) createDefaultTables(){
logger := Logging.DetailedLogger("MariaDB", "createDefaultTables")
eventLogSQL := "create table " + mariadb.Database +".EventLog\n(\n UUID text null,\n LogType enum ('INFO', 'WARNING', 'ERROR', 'FATAL') null,\n BackupName varchar(256) null,\n Stage enum ('COMPRESS', 'UPLOAD', 'DELETE TMP') null,\n RemoteStorage enum ('AZURE-FILE', 'AZURE-BLOB', 'NONE') null,\n Description text null,\n Timestamp datetime null\n);"
backupSQL := "create table " + mariadb.Database +".Backups\n(\n UUID text null,\n BackupName varchar(256) null,\n LastBackup datetime null,\n LocalBackup tinyint(1) null,\n FilePath varchar(256) null,\n RemoteStorage enum ('AZURE-FILE', 'AZURE-BLOB', 'NONE') null,\n RemotePath varchar(256) null,\n `DurationToBackup (s)` double null,\n HadErrors tinyint(1) null\n);\n\n"
db := createMariaDBConnection(mariadb)
if !checkIfBackupTableExist(db, mariadb){
_, err := db.Exec(backupSQL)
if err != nil{
logger.Fatal(err)
}
}
if !checkIfEventLogTableExist(db, mariadb){
_, err := db.Exec(eventLogSQL)
if err != nil{
logger.Fatal(err)
}
}
_ = db.Close()
}
func (mariadb MariaDBConnector) newLogEntry(uuid uuid.UUID, logType LogType, backupName string, stage SQLStage, storageType RemoteStorageType, description string, timestamp time.Time){
logger := Logging.DetailedLogger("MariaDB", "newLogEntry")
db := createMariaDBConnection(mariadb)
_, err := db.Query("INSERT INTO `" + mariadb.Database + "`.EventLog VALUES (?, ?, ?, ?, ?, ?, ?);", uuid.String(), strconv.FormatInt(int64(logType), 10), backupName, stage, strconv.FormatInt(int64(storageType), 10), description ,timestamp)
if err != nil{
logger.Fatal(err)
}
}
func (mariadb MariaDBConnector) newBackupEntry(uuid uuid.UUID, backupName string, lastBackup time.Time, localBackup bool, filePath string, storageType RemoteStorageType, remotePath string, durationToBackup time.Duration, hadErrors bool){}

9
SQL/RemoteStorage.go Normal file
View file

@ -0,0 +1,9 @@
package SQL
type RemoteStorageType int64
const(
REMOTE_AZURE_FILE = 1
REMOTE_AZURE_BLOB = 2
REMOTE_NONE = 3
)

24
SQL/SQLInterface.go Normal file
View file

@ -0,0 +1,24 @@
package SQL
import (
"github.com/google/uuid"
"time"
)
type SQLService interface {
createDefaultTables()
newLogEntry(uuid uuid.UUID, logType LogType, backupName string, stage SQLStage, storageType RemoteStorageType, description string, timestamp time.Time)
newBackupEntry(uuid uuid.UUID, backupName string, lastBackup time.Time, localBackup bool, filePath string, storageType RemoteStorageType, remotePath string, durationToBackup time.Duration, hadErrors bool)
}
func CreateDefaultTables(sqlService SQLService){
sqlService.createDefaultTables()
}
func NewLogEntry(sqlService SQLService, uuid uuid.UUID, logType LogType, backupName string, stage SQLStage, storageType RemoteStorageType, description string, timestamp time.Time){
sqlService.newLogEntry(uuid, logType, backupName, stage, storageType, description, timestamp)
}
func NewBackupEntry(sqlService SQLService, uuid uuid.UUID, backupName string, lastBackup time.Time, localBackup bool, filePath string, storageType RemoteStorageType, remotePath string, durationToBackup time.Duration, hadErrors bool){
sqlService.newBackupEntry(uuid, backupName, lastBackup, localBackup, filePath, storageType, remotePath, durationToBackup, hadErrors)
}

9
SQL/SQLStages.go Normal file
View file

@ -0,0 +1,9 @@
package SQL
type SQLStage int64
const(
SQLStage_Compress = 1
SQLStage_Upload = 2
SQLStage_DeleteTmp = 3
)

7
StorageTypes/AzureFileStorage.go
View file

@ -5,11 +5,14 @@ import (
"encoding/json"
"fmt"
"github.com/Azure/azure-storage-file-go/azfile"
"github.com/google/uuid"
"net/url"
"os"
"path/filepath"
"scabiosa/Logging"
"scabiosa/SQL"
"strings"
"time"
)
type AzureFileStorage struct{
@ -46,6 +49,8 @@ func (azure AzureFileStorage) upload(fileName string){
ctx := context.Background()
fmt.Printf("[%s] Starting upload to Azure File Share...\n", strings.Trim(filepath.Base(fileName), ".bak"))
//TODO Remove Hardcoded SQL Instance
SQL.NewLogEntry(SQL.GetMariaDBInstance(), uuid.New(), SQL.LogInfo, filepath.Base(fileName), SQL.SQLStage_Upload, SQL.REMOTE_AZURE_FILE, "Starting upload.", time.Now())
err = azfile.UploadFileToAzureFile(ctx, file, fileURL,
azfile.UploadToAzureFileOptions{
@ -58,6 +63,8 @@ func (azure AzureFileStorage) upload(fileName string){
}})
fmt.Printf("[%s] Upload finished.\n", strings.Trim(filepath.Base(fileName), ".bak"))
//TODO Remove Hardcoded SQL Instance
SQL.NewLogEntry(SQL.GetMariaDBInstance(), uuid.New(), SQL.LogInfo, filepath.Base(fileName), SQL.SQLStage_Upload, SQL.REMOTE_AZURE_FILE, "Finished upload.", time.Now())
}
func readConfig() []byte {

2
Config.go → Tools/Config.go
View file

@ -1,4 +1,4 @@
package main
package Tools
import (
"encoding/json"

6
go.mod
View file

@ -4,7 +4,11 @@ go 1.17
require github.com/sirupsen/logrus v1.8.1
require github.com/Azure/azure-storage-file-go v0.8.0
require (
github.com/Azure/azure-storage-file-go v0.8.0
github.com/go-sql-driver/mysql v1.6.0
github.com/google/uuid v1.3.0
)
require (
github.com/Azure/azure-pipeline-go v0.2.1 // indirect

4
go.sum
View file

@ -4,6 +4,10 @@ github.com/Azure/azure-storage-file-go v0.8.0 h1:OX8DGsleWLUE6Mw4R/OeWEZMvsTIpwN
github.com/Azure/azure-storage-file-go v0.8.0/go.mod h1:3w3mufGcMjcOJ3w+4Gs+5wsSgkT7xDwWWqMMIrXtW4c=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=

12
main.go
View file

@ -1,25 +1,31 @@
package main
import (
"github.com/google/uuid"
"os"
"scabiosa/Logging"
"scabiosa/SQL"
"scabiosa/StorageTypes"
"scabiosa/Tools"
"time"
)
func main(){
config := GetConfig()
config := Tools.GetConfig()
SQL.CreateDefaultTables(SQL.GetMariaDBInstance())
for _, backupItem := range config.FolderToBackup{
storage := StorageTypes.CheckStorageType(backupItem.StorageType)
destPath := checkTmpPath(config, backupItem.CreateLocalBackup)
bakFile := CreateBakFile(backupItem.BackupName + getTimeSuffix(), backupItem.FolderPath, destPath)
StorageTypes.UploadFile(storage, destPath + string(os.PathSeparator) + bakFile)
if !backupItem.CreateLocalBackup {
_ = os.Remove(destPath + string(os.PathSeparator) + bakFile)
//TODO Remove Hardcoded SQL Instance
SQL.NewLogEntry(SQL.GetMariaDBInstance(), uuid.New(), SQL.LogInfo, backupItem.BackupName, SQL.SQLStage_DeleteTmp, SQL.REMOTE_NONE, "Deleted tmp file" ,time.Now())
}
}
@ -33,7 +39,7 @@ func getTimeSuffix() string{
return "_" + currTime.Format("02-01-2006_15-04")
}
func checkTmpPath(config Config, createLocalBackup bool) string{
func checkTmpPath(config Tools.Config, createLocalBackup bool) string{
logger := Logging.DetailedLogger("mainThread", "checkTmpPath")
if !createLocalBackup{
if _, err := os.Stat("tmp"); os.IsNotExist(err) {