diff --git a/Compression.go b/Compression.go index 17fdd5a..957d66d 100644 --- a/Compression.go +++ b/Compression.go @@ -6,10 +6,13 @@ import ( "compress/flate" "compress/gzip" "fmt" + "github.com/google/uuid" "io" "os" "path/filepath" "scabiosa/Logging" + "scabiosa/SQL" + "time" ) func CreateBakFile(filename string, folderPath string, destinationPath string) string { @@ -29,6 +32,10 @@ func CreateBakFile(filename string, folderPath string, destinationPath string) s logger.Fatal(err) } + //TODO Remove Hardcoded SQL Instance + SQL.NewLogEntry(SQL.GetMariaDBInstance(), uuid.New(), SQL.LogInfo, filepath.Base(folderPath), SQL.SQLStage_Compress, SQL.REMOTE_NONE, "File successfully written.", time.Now()) + + return fileName } @@ -40,7 +47,8 @@ func compress(folderPath string, buf io.Writer){ tw := tar.NewWriter(zr) fmt.Printf("[%s] Start compression...\n", filepath.Base(folderPath)) - + //TODO Remove Hardcoded SQL Instance + SQL.NewLogEntry(SQL.GetMariaDBInstance(), uuid.New(), SQL.LogInfo, filepath.Base(folderPath), SQL.SQLStage_Compress, SQL.REMOTE_NONE, "Start compression", time.Now()) filepath.Walk(folderPath, func(file string, fi os.FileInfo, err error) error { header, err := tar.FileInfoHeader(fi, file) if err != nil { @@ -78,4 +86,6 @@ func compress(folderPath string, buf io.Writer){ fmt.Printf("[%s] Compression Done.\n", filepath.Base(folderPath)) + //TODO Remove Hardcoded SQL Instance + SQL.NewLogEntry(SQL.GetMariaDBInstance(), uuid.New(), SQL.LogInfo, filepath.Base(folderPath), SQL.SQLStage_Compress, SQL.REMOTE_NONE, "Compression complete.", time.Now()) } \ No newline at end of file diff --git a/SQL/LogTypes.go b/SQL/LogTypes.go new file mode 100644 index 0000000..16f2ea0 --- /dev/null +++ b/SQL/LogTypes.go @@ -0,0 +1,10 @@ +package SQL + +type LogType int64 + +const( + LogInfo LogType = 1 + LogWarning = 2 + LogError = 3 + LogFatal = 4 +) diff --git a/SQL/MariaDB.go b/SQL/MariaDB.go new file mode 100644 index 0000000..905a4aa --- /dev/null +++ b/SQL/MariaDB.go @@ -0,0 +1,93 @@ +package SQL + +import ( + "database/sql" + _ "github.com/go-sql-driver/mysql" + "github.com/google/uuid" + "scabiosa/Logging" + "scabiosa/Tools" + "strconv" + "time" +) + +type MariaDBConnector struct { + Address string + Port uint16 + Database string + DbUser string + DbPassword string +} + +func GetMariaDBInstance() MariaDBConnector{ + config := Tools.GetConfig() + var mariadb MariaDBConnector + + mariadb.Address = config.SQLConfig.SqlAddress + mariadb.Port = config.SQLConfig.SqlPort + mariadb.Database = config.SQLConfig.Database + mariadb.DbUser = config.SQLConfig.DbUser + mariadb.DbPassword = config.SQLConfig.DbPassword + + return mariadb +} + + +func checkIfEventLogTableExist(db *sql.DB, mariadb MariaDBConnector) bool { + rows, _ := db.Query("SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_NAME = 'EventLog';", mariadb.Database) + if !rows.Next(){ return false } + return true +} + +func checkIfBackupTableExist(db *sql.DB, mariadb MariaDBConnector) bool { + rows, _ := db.Query("SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_NAME = 'Backups';", mariadb.Database) + if !rows.Next(){ return false } + return true +} + +func createMariaDBConnection(mariadb MariaDBConnector) *sql.DB{ + logger := Logging.DetailedLogger("MariaDB", "createConnection") + db, err := sql.Open("mysql", mariadb.DbUser + ":" + mariadb.DbPassword + "@(" + mariadb.Address + ":" +strconv.Itoa(int(mariadb.Port))+ ")/" + mariadb.Database) + if err != nil{ + logger.Fatal(err) + } + return db +} + +func (mariadb MariaDBConnector) createDefaultTables(){ + logger := Logging.DetailedLogger("MariaDB", "createDefaultTables") + + eventLogSQL := "create table " + mariadb.Database +".EventLog\n(\n UUID text null,\n LogType enum ('INFO', 'WARNING', 'ERROR', 'FATAL') null,\n BackupName varchar(256) null,\n Stage enum ('COMPRESS', 'UPLOAD', 'DELETE TMP') null,\n RemoteStorage enum ('AZURE-FILE', 'AZURE-BLOB', 'NONE') null,\n Description text null,\n Timestamp datetime null\n);" + backupSQL := "create table " + mariadb.Database +".Backups\n(\n UUID text null,\n BackupName varchar(256) null,\n LastBackup datetime null,\n LocalBackup tinyint(1) null,\n FilePath varchar(256) null,\n RemoteStorage enum ('AZURE-FILE', 'AZURE-BLOB', 'NONE') null,\n RemotePath varchar(256) null,\n `DurationToBackup (s)` double null,\n HadErrors tinyint(1) null\n);\n\n" + + + db := createMariaDBConnection(mariadb) + + if !checkIfBackupTableExist(db, mariadb){ + _, err := db.Exec(backupSQL) + if err != nil{ + logger.Fatal(err) + } + } + + if !checkIfEventLogTableExist(db, mariadb){ + _, err := db.Exec(eventLogSQL) + if err != nil{ + logger.Fatal(err) + } + } + + _ = db.Close() +} + +func (mariadb MariaDBConnector) newLogEntry(uuid uuid.UUID, logType LogType, backupName string, stage SQLStage, storageType RemoteStorageType, description string, timestamp time.Time){ + logger := Logging.DetailedLogger("MariaDB", "newLogEntry") + + db := createMariaDBConnection(mariadb) + + _, err := db.Query("INSERT INTO `" + mariadb.Database + "`.EventLog VALUES (?, ?, ?, ?, ?, ?, ?);", uuid.String(), strconv.FormatInt(int64(logType), 10), backupName, stage, strconv.FormatInt(int64(storageType), 10), description ,timestamp) + if err != nil{ + logger.Fatal(err) + } + +} +func (mariadb MariaDBConnector) newBackupEntry(uuid uuid.UUID, backupName string, lastBackup time.Time, localBackup bool, filePath string, storageType RemoteStorageType, remotePath string, durationToBackup time.Duration, hadErrors bool){} \ No newline at end of file diff --git a/SQL/RemoteStorage.go b/SQL/RemoteStorage.go new file mode 100644 index 0000000..dd835c0 --- /dev/null +++ b/SQL/RemoteStorage.go @@ -0,0 +1,9 @@ +package SQL + +type RemoteStorageType int64 + +const( + REMOTE_AZURE_FILE = 1 + REMOTE_AZURE_BLOB = 2 + REMOTE_NONE = 3 +) \ No newline at end of file diff --git a/SQL/SQLInterface.go b/SQL/SQLInterface.go new file mode 100644 index 0000000..79f69ee --- /dev/null +++ b/SQL/SQLInterface.go @@ -0,0 +1,24 @@ +package SQL + +import ( + "github.com/google/uuid" + "time" +) + +type SQLService interface { + createDefaultTables() + newLogEntry(uuid uuid.UUID, logType LogType, backupName string, stage SQLStage, storageType RemoteStorageType, description string, timestamp time.Time) + newBackupEntry(uuid uuid.UUID, backupName string, lastBackup time.Time, localBackup bool, filePath string, storageType RemoteStorageType, remotePath string, durationToBackup time.Duration, hadErrors bool) +} + +func CreateDefaultTables(sqlService SQLService){ + sqlService.createDefaultTables() +} + +func NewLogEntry(sqlService SQLService, uuid uuid.UUID, logType LogType, backupName string, stage SQLStage, storageType RemoteStorageType, description string, timestamp time.Time){ + sqlService.newLogEntry(uuid, logType, backupName, stage, storageType, description, timestamp) +} + +func NewBackupEntry(sqlService SQLService, uuid uuid.UUID, backupName string, lastBackup time.Time, localBackup bool, filePath string, storageType RemoteStorageType, remotePath string, durationToBackup time.Duration, hadErrors bool){ + sqlService.newBackupEntry(uuid, backupName, lastBackup, localBackup, filePath, storageType, remotePath, durationToBackup, hadErrors) +} \ No newline at end of file diff --git a/SQL/SQLStages.go b/SQL/SQLStages.go new file mode 100644 index 0000000..36c7441 --- /dev/null +++ b/SQL/SQLStages.go @@ -0,0 +1,9 @@ +package SQL + +type SQLStage int64 + +const( + SQLStage_Compress = 1 + SQLStage_Upload = 2 + SQLStage_DeleteTmp = 3 +) diff --git a/StorageTypes/AzureFileStorage.go b/StorageTypes/AzureFileStorage.go index c4a9387..bfcb84f 100644 --- a/StorageTypes/AzureFileStorage.go +++ b/StorageTypes/AzureFileStorage.go @@ -5,11 +5,14 @@ import ( "encoding/json" "fmt" "github.com/Azure/azure-storage-file-go/azfile" + "github.com/google/uuid" "net/url" "os" "path/filepath" "scabiosa/Logging" + "scabiosa/SQL" "strings" + "time" ) type AzureFileStorage struct{ @@ -46,6 +49,8 @@ func (azure AzureFileStorage) upload(fileName string){ ctx := context.Background() fmt.Printf("[%s] Starting upload to Azure File Share...\n", strings.Trim(filepath.Base(fileName), ".bak")) + //TODO Remove Hardcoded SQL Instance + SQL.NewLogEntry(SQL.GetMariaDBInstance(), uuid.New(), SQL.LogInfo, filepath.Base(fileName), SQL.SQLStage_Upload, SQL.REMOTE_AZURE_FILE, "Starting upload.", time.Now()) err = azfile.UploadFileToAzureFile(ctx, file, fileURL, azfile.UploadToAzureFileOptions{ @@ -58,6 +63,8 @@ func (azure AzureFileStorage) upload(fileName string){ }}) fmt.Printf("[%s] Upload finished.\n", strings.Trim(filepath.Base(fileName), ".bak")) + //TODO Remove Hardcoded SQL Instance + SQL.NewLogEntry(SQL.GetMariaDBInstance(), uuid.New(), SQL.LogInfo, filepath.Base(fileName), SQL.SQLStage_Upload, SQL.REMOTE_AZURE_FILE, "Finished upload.", time.Now()) } func readConfig() []byte { diff --git a/Config.go b/Tools/Config.go similarity index 98% rename from Config.go rename to Tools/Config.go index 0e0106f..216a48b 100644 --- a/Config.go +++ b/Tools/Config.go @@ -1,4 +1,4 @@ -package main +package Tools import ( "encoding/json" diff --git a/go.mod b/go.mod index a170343..cfa3a30 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,11 @@ go 1.17 require github.com/sirupsen/logrus v1.8.1 -require github.com/Azure/azure-storage-file-go v0.8.0 +require ( + github.com/Azure/azure-storage-file-go v0.8.0 + github.com/go-sql-driver/mysql v1.6.0 + github.com/google/uuid v1.3.0 +) require ( github.com/Azure/azure-pipeline-go v0.2.1 // indirect diff --git a/go.sum b/go.sum index ef59b4b..dc7cf76 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,10 @@ github.com/Azure/azure-storage-file-go v0.8.0 h1:OX8DGsleWLUE6Mw4R/OeWEZMvsTIpwN github.com/Azure/azure-storage-file-go v0.8.0/go.mod h1:3w3mufGcMjcOJ3w+4Gs+5wsSgkT7xDwWWqMMIrXtW4c= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= diff --git a/main.go b/main.go index c8c4a9a..3a82cbc 100644 --- a/main.go +++ b/main.go @@ -1,25 +1,31 @@ package main import ( + "github.com/google/uuid" "os" "scabiosa/Logging" + "scabiosa/SQL" "scabiosa/StorageTypes" + "scabiosa/Tools" "time" ) func main(){ - config := GetConfig() + config := Tools.GetConfig() + + SQL.CreateDefaultTables(SQL.GetMariaDBInstance()) for _, backupItem := range config.FolderToBackup{ storage := StorageTypes.CheckStorageType(backupItem.StorageType) destPath := checkTmpPath(config, backupItem.CreateLocalBackup) bakFile := CreateBakFile(backupItem.BackupName + getTimeSuffix(), backupItem.FolderPath, destPath) - StorageTypes.UploadFile(storage, destPath + string(os.PathSeparator) + bakFile) if !backupItem.CreateLocalBackup { _ = os.Remove(destPath + string(os.PathSeparator) + bakFile) + //TODO Remove Hardcoded SQL Instance + SQL.NewLogEntry(SQL.GetMariaDBInstance(), uuid.New(), SQL.LogInfo, backupItem.BackupName, SQL.SQLStage_DeleteTmp, SQL.REMOTE_NONE, "Deleted tmp file" ,time.Now()) } } @@ -33,7 +39,7 @@ func getTimeSuffix() string{ return "_" + currTime.Format("02-01-2006_15-04") } -func checkTmpPath(config Config, createLocalBackup bool) string{ +func checkTmpPath(config Tools.Config, createLocalBackup bool) string{ logger := Logging.DetailedLogger("mainThread", "checkTmpPath") if !createLocalBackup{ if _, err := os.Stat("tmp"); os.IsNotExist(err) {