From 85fcf0d52e2625fce26e1222f24a19205d2db02a Mon Sep 17 00:00:00 2001 From: ssongliu Date: Tue, 3 Mar 2026 09:59:06 +0800 Subject: [PATCH] fix: add PostgreSQL recreation and DB host env sync in app restore --- agent/app/service/backup_app.go | 237 ++++++++++++++++++++--------- agent/utils/mysql/client/local.go | 3 - agent/utils/mysql/client/remote.go | 4 - 3 files changed, 169 insertions(+), 75 deletions(-) diff --git a/agent/app/service/backup_app.go b/agent/app/service/backup_app.go index da5bb3d16f76..06ed5bcff04e 100644 --- a/agent/app/service/backup_app.go +++ b/agent/app/service/backup_app.go @@ -11,6 +11,7 @@ import ( "time" "github.com/1Panel-dev/1Panel/agent/app/repo" + "github.com/subosito/gotenv" "github.com/1Panel-dev/1Panel/agent/app/task" "github.com/1Panel-dev/1Panel/agent/i18n" @@ -179,6 +180,9 @@ func handleAppRecover(install *model.AppInstall, parentTask *task.Task, recoverF return err } tmpPath := strings.ReplaceAll(recoverFile, ".tar.gz", "") + if err := fileOp.TarGzExtractPro(tmpPath+"/app.tar.gz", tmpPath, ""); err != nil { + return err + } defer func() { _, _ = compose.Up(install.GetComposePath()) _ = os.RemoveAll(strings.ReplaceAll(recoverFile, ".tar.gz", "")) @@ -187,22 +191,26 @@ func handleAppRecover(install *model.AppInstall, parentTask *task.Task, recoverF if !fileOp.Stat(tmpPath+"/app.json") || !fileOp.Stat(tmpPath+"/app.tar.gz") { return errors.New(i18n.GetMsgByKey("AppBackupFileIncomplete")) } - var oldInstall model.AppInstall + var backupInstall model.AppInstall appJson, err := os.ReadFile(tmpPath + "/app.json") if err != nil { return err } - newEnv, err := os.ReadFile(install.GetEnvPath()) - if err != nil { - return err - } - if err := json.Unmarshal(appJson, &oldInstall); err != nil { + if err := json.Unmarshal(appJson, &backupInstall); err != nil { return fmt.Errorf("unmarshal app.json failed, err: %v", err) } - if oldInstall.App.Key != install.App.Key || oldInstall.Name != install.Name { + if backupInstall.App.Key != install.App.Key || backupInstall.Name != install.Name { return errors.New(i18n.GetMsgByKey("AppAttributesNotMatch")) } - + backupEnvMap, err := getEnvMapByPath(path.Join(tmpPath, install.Name, ".env")) + if err != nil { + return err + } + installedEnvMap, err := getEnvMapByPath(install.GetEnvPath()) + if err != nil { + return err + } + var mergedEnvContent string resources, _ := appInstallResourceRepo.GetBy(appInstallResourceRepo.WithAppInstallId(install.ID)) for _, resource := range resources { var database model.Database @@ -228,11 +236,20 @@ func handleAppRecover(install *model.AppInstall, parentTask *task.Task, recoverF if err != nil { return err } + newDB, err := reCreatePostgresqlDB(db.ID, database, backupEnvMap) + if err != nil { + return err + } + backupInstall.Env, mergedEnvContent, err = buildRecoverEnv(backupInstall.Env, backupEnvMap, installedEnvMap) + if err != nil { + return err + } + _ = appInstallResourceRepo.BatchUpdateBy(map[string]interface{}{"resource_id": newDB.ID}, repo.WithByID(resource.ID)) taskName := task.GetTaskName(db.Name, task.TaskRecover, task.TaskScopeDatabase) t.LogStart(taskName) if err := handlePostgresqlRecover(dto.CommonRecover{ - Name: database.Name, - DetailName: db.Name, + Name: newDB.PostgresqlName, + DetailName: newDB.Name, File: fmt.Sprintf("%s/%s.sql.gz", tmpPath, install.Name), }, parentTask, true); err != nil { t.LogFailedWithErr(taskName, err) @@ -244,11 +261,20 @@ func handleAppRecover(install *model.AppInstall, parentTask *task.Task, recoverF if err != nil { return err } + newDB, err := reCreateDB(db.ID, database, backupEnvMap) + if err != nil { + return err + } + backupInstall.Env, mergedEnvContent, err = buildRecoverEnv(backupInstall.Env, backupEnvMap, installedEnvMap) + if err != nil { + return err + } + _ = appInstallResourceRepo.BatchUpdateBy(map[string]interface{}{"resource_id": newDB.ID}, repo.WithByID(resource.ID)) taskName := task.GetTaskName(db.Name, task.TaskRecover, task.TaskScopeDatabase) t.LogStart(taskName) if err := handleMysqlRecover(dto.CommonRecover{ - Name: database.Name, - DetailName: db.Name, + Name: newDB.MysqlName, + DetailName: newDB.Name, File: fmt.Sprintf("%s/%s.sql.gz", tmpPath, install.Name), }, parentTask, true); err != nil { t.LogFailedWithErr(taskName, err) @@ -274,20 +300,22 @@ func handleAppRecover(install *model.AppInstall, parentTask *task.Task, recoverF t.LogSuccess(deCompressName) _ = fileOp.DeleteDir(backPath) - oldEnv, err := os.ReadFile(install.GetEnvPath()) - if err != nil { - return err - } - if err := handleRecoverEnv(install, string(newEnv), string(oldEnv)); err != nil { - return err + if len(mergedEnvContent) != 0 { + envPath := fmt.Sprintf("%s/%s/.env", install.GetAppPath(), install.Name) + file, err := os.OpenFile(envPath, os.O_WRONLY|os.O_TRUNC, 0640) + if err != nil { + return err + } + defer file.Close() + _, _ = file.WriteString(mergedEnvContent) } - oldInstall.ID = install.ID - oldInstall.Status = constant.StatusRunning - oldInstall.AppId = install.AppId - oldInstall.AppDetailId = install.AppDetailId - oldInstall.App.ID = install.AppId - if err := appInstallRepo.Save(context.Background(), &oldInstall); err != nil { + backupInstall.ID = install.ID + backupInstall.Status = constant.StatusRunning + backupInstall.AppId = install.AppId + backupInstall.AppDetailId = install.AppDetailId + backupInstall.App.ID = install.AppId + if err := appInstallRepo.Save(context.Background(), &backupInstall); err != nil { global.LOG.Errorf("save db app install failed, err: %v", err) return err } @@ -361,58 +389,131 @@ func doAppBackup(install *model.AppInstall, parentTask *task.Task, backupDir, fi return nil } -func handleRecoverEnv(install *model.AppInstall, newEnvFile string, oldEnvFile string) error { - oldEnvInDB := make(map[string]interface{}) - if err := json.Unmarshal([]byte(install.Env), &oldEnvInDB); err != nil { - return err +func reCreateDB(dbID uint, database model.Database, envMap map[string]interface{}) (*model.DatabaseMysql, error) { + mysqlService := NewIMysqlService() + ctx := context.Background() + _ = mysqlService.Delete(ctx, dto.MysqlDBDelete{ID: dbID, Database: database.Name, Type: database.Type, DeleteBackup: false, ForceDelete: true}) + + dbInfo := getDBCreateInfoFromEnv(envMap, "utf8mb4") + createDB, err := mysqlService.Create(context.Background(), dto.MysqlDBCreate{ + Name: dbInfo.Name, + From: database.From, + Database: database.Name, + Format: dbInfo.Format, + Username: dbInfo.User, + Password: dbInfo.Password, + Permission: "%", + }) + if err != nil { + return nil, err } + updateCronjobsDBRef(dbID, createDB.ID) + return createDB, nil +} + +func reCreatePostgresqlDB(dbID uint, database model.Database, envMap map[string]interface{}) (*model.DatabasePostgresql, error) { + postgresqlService := NewIPostgresqlService() + _ = postgresqlService.Delete(context.Background(), dto.PostgresqlDBDelete{ + ID: dbID, + Type: database.Type, + Database: database.Name, + DeleteBackup: false, + ForceDelete: true, + }) + + dbInfo := getDBCreateInfoFromEnv(envMap, "UTF8") + createDB, err := postgresqlService.Create(context.Background(), dto.PostgresqlDBCreate{ + Name: dbInfo.Name, + From: database.From, + Database: database.Name, + Format: dbInfo.Format, + Username: dbInfo.User, + Password: dbInfo.Password, + SuperUser: true, + }) + if err != nil { + return nil, err + } + updateCronjobsDBRef(dbID, createDB.ID) + return createDB, nil +} + +type dbRecreateInfo struct { + Name string + User string + Password string + Format string +} - lines := strings.Split(newEnvFile, "\n") - oldEnvMap := make(map[string]string) - oldLines := strings.Split(oldEnvFile, "\n") - for _, line := range oldLines { - if strings.TrimSpace(line) == "" || strings.HasPrefix(line, "#") { +func getDBCreateInfoFromEnv(envMap map[string]interface{}, defaultFormat string) dbRecreateInfo { + name, _ := envMap["PANEL_DB_NAME"].(string) + user, _ := envMap["PANEL_DB_USER"].(string) + password, _ := envMap["PANEL_DB_USER_PASSWORD"].(string) + format, _ := envMap["format"].(string) + if len(format) == 0 { + format = defaultFormat + } + return dbRecreateInfo{ + Name: name, + User: user, + Password: password, + Format: format, + } +} + +func updateCronjobsDBRef(oldDBID, newDBID uint) { + cronjobs, _ := cronjobRepo.List(cronjobRepo.WithByDbName(fmt.Sprintf("%v", oldDBID))) + for _, job := range cronjobs { + _ = cronjobRepo.Update(job.ID, map[string]interface{}{"db_name": fmt.Sprintf("%v", newDBID)}) + } +} + +func buildRecoverEnv(appEnv string, backupEnvMap, installedEnvMap map[string]interface{}) (string, string, error) { + currentHostVal, hasCurrentHost := installedEnvMap["PANEL_DB_HOST"] + if hasCurrentHost && fmt.Sprintf("%v", currentHostVal) != "" { + backupHost := fmt.Sprintf("\"PANEL_DB_HOST\":\"%v\"", backupEnvMap["PANEL_DB_HOST"]) + currentHost := fmt.Sprintf("\"PANEL_DB_HOST\":\"%v\"", currentHostVal) + appEnv = strings.ReplaceAll(appEnv, backupHost, currentHost) + if _, ok := backupEnvMap["CASDOOR_DATASOURCE_NAME"]; ok { + backupEnvMap["CASDOOR_DATASOURCE_NAME"] = strings.ReplaceAll( + fmt.Sprintf("%v", backupEnvMap["CASDOOR_DATASOURCE_NAME"]), + fmt.Sprintf("%v", backupEnvMap["PANEL_DB_HOST"]), + fmt.Sprintf("%v", currentHostVal), + ) + } + backupEnvMap["PANEL_DB_HOST"] = currentHostVal + } + newEnvMap := make(map[string]string, len(backupEnvMap)) + handleMap(backupEnvMap, newEnvMap) + mergedEnvContent, err := gotenv.Marshal(newEnvMap) + if err != nil { + return appEnv, "", err + } + return appEnv, mergedEnvContent, nil +} + +func getEnvMapByPath(envPath string) (map[string]interface{}, error) { + envMap := make(map[string]interface{}) + envFile, err := os.ReadFile(envPath) + if err != nil { + return envMap, err + } + lines := strings.Split(string(envFile), "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { continue } parts := strings.SplitN(line, "=", 2) if len(parts) == 2 { - oldEnvMap[parts[0]] = parts[1] - } - } - - keysToKeep := []string{ - "PANEL_DB_PORT", - "PANEL_DB_NAME", - "PANEL_DB_USER", - "PANEL_DB_USER_PASSWORD", - "PANEL_DB_TYPE", - "PANEL_DB_HOST", - "DATABASE_NAME", - "PANEL_DB_HOST_NAME", - "CASDOOR_DRIVER_NAME", - "CASDOOR_DATASOURCE_NAME", - } - - for i := 0; i < len(lines); i++ { - for _, key := range keysToKeep { - prefix := key + "=" - if strings.HasPrefix(lines[i], prefix) { - if val, ok := oldEnvMap[key]; ok { - lines[i] = fmt.Sprintf("%s=%v", key, val) - oldEnvInDB[key] = val + value := strings.TrimSpace(parts[1]) + if len(value) >= 2 { + if (value[0] == '"' && value[len(value)-1] == '"') || (value[0] == '\'' && value[len(value)-1] == '\'') { + value = value[1 : len(value)-1] } } + envMap[parts[0]] = value } } - newEnvFile = strings.Join(lines, "\n") - file, err := os.OpenFile(install.GetEnvPath(), os.O_WRONLY|os.O_TRUNC, 0640) - if err != nil { - return err - } - defer file.Close() - _, _ = file.WriteString(newEnvFile) - - newEnvToDB, _ := json.Marshal(oldEnvInDB) - install.Env = string(newEnvToDB) - return nil + return envMap, nil } diff --git a/agent/utils/mysql/client/local.go b/agent/utils/mysql/client/local.go index 6b9b29470ecd..a9466b3e0cfa 100644 --- a/agent/utils/mysql/client/local.go +++ b/agent/utils/mysql/client/local.go @@ -274,9 +274,6 @@ func (r *Local) Recover(info RecoverInfo) error { mysqlCli = "mysql" } - if err := r.ExecSQL(fmt.Sprintf("drop database if exists `%s`", info.Name), 300); err != nil { - return fmt.Errorf("drop database failed, err: %v", err) - } cmd := exec.Command("docker", "exec", "-i", r.ContainerName, mysqlCli, "-uroot", "-p"+r.Password, "--default-character-set="+info.Format, info.Name) if strings.HasSuffix(info.SourceFile, ".gz") { gzipFile, err := os.Open(info.SourceFile) diff --git a/agent/utils/mysql/client/remote.go b/agent/utils/mysql/client/remote.go index 16a79bce425f..f3501484dede 100644 --- a/agent/utils/mysql/client/remote.go +++ b/agent/utils/mysql/client/remote.go @@ -300,10 +300,6 @@ func (r *Remote) Recover(info RecoverInfo) error { return err } - if err := r.ExecSQL(fmt.Sprintf("drop database if exists `%s`", info.Name), 300); err != nil { - return fmt.Errorf("drop database failed, err: %v", err) - } - recoverCmd := fmt.Sprintf("docker run --rm --net=host -i %s /bin/bash -c '%s -h %s -P %d -u%s -p%s %s --default-character-set=%s %s'", image, r.Type, r.Address, r.Port, r.User, r.Password, sslSkip(info.Version, r.Type), info.Format, info.Name)