Enhance ChunkedBackupService and ConcurrentUploadManager with improved error handling, database connection detection, and file path management. Introduce temporary config file for secure password handling in backups and streamline retention policy for successful backups.

Author: dgtlss

composer.json

@@ -3,7 +3,7 @@
     "description": "A Laravel package for backing up databases and files to external sources with notifications",
     "type": "library",
     "license": "MIT",
-    "version": "0.1.1",
+    "version": "0.1.5",
     "authors": [{
         "name": "Nathan Langer",
         "email": "nathanlanger@googlemail.com"

src/Services/ChunkedBackupService.php

@@ -158,8 +158,14 @@ protected function createChunkedBackup(): void
 
     protected function createChunkedDatabaseBackup(string $timestamp): void
     {
-        $connections = config('capsule.database.connections', 'default');
-        $connections = is_string($connections) ? [$connections] : $connections;
+        $connections = config('capsule.database.connections');
+        
+        // Auto-detect current database if not specified
+        if ($connections === null) {
+            $connections = [config('database.default')];
+        } else {
+            $connections = is_string($connections) ? [$connections] : $connections;
+        }
 
         foreach ($connections as $connection) {
             $this->log("Streaming database '{$connection}' to chunks...");
@@ -192,16 +198,28 @@ protected function streamDatabaseToChunks(string $connection, string $timestamp)
 
     protected function streamMysqlToChunks(array $config, string $connection, string $timestamp, int $chunkSize, string $tempPrefix): void
     {
+        // Create a temporary config file for secure password handling
+        $configFile = tempnam(sys_get_temp_dir(), 'mysql_config_');
+        $configContent = sprintf(
+            "[mysqldump]\nuser=%s\npassword=%s\nhost=%s\nport=%s\n",
+            $config['username'],
+            $config['password'],
+            $config['host'],
+            $config['port'] ?? 3306
+        );
+        file_put_contents($configFile, $configContent);
+        chmod($configFile, 0600);
+
         $command = sprintf(
-            'mysqldump --host=%s --port=%s --user=%s --password=%s --single-transaction --routines --triggers %s',
-            escapeshellarg($config['host']),
-            escapeshellarg($config['port'] ?? 3306),
-            escapeshellarg($config['username']),
-            escapeshellarg($config['password']),
+            'mysqldump --defaults-extra-file=%s --single-transaction --routines --triggers %s',
+            escapeshellarg($configFile),
             escapeshellarg($config['database'])
         );
 
         $this->streamCommandToChunks($command, "db_{$connection}_{$timestamp}", $chunkSize, $tempPrefix);
+
+        // Clean up the temporary config file
+        @unlink($configFile);
     }
 
     protected function streamPostgresToChunks(array $config, string $connection, string $timestamp, int $chunkSize, string $tempPrefix): void
@@ -399,7 +417,7 @@ protected function uploadChunksConcurrently(): array
             'stats' => $stats,
             'backup_id' => $this->backupId,
         ]);
-        $this->log("Concurrent upload complete. Time taken: {$stats['total_time']}s");
+        $this->log("Concurrent upload complete. Total: {$stats['total']}, Successful: {$stats['successful']}, Failed: {$stats['failed']}");
 
         // Check if any uploads failed
         $failedUploads = $this->uploadManager->getFailedUploads();
@@ -481,32 +499,29 @@ protected function cleanup(): void
             $retentionDays = config('capsule.retention.days', 30);
             $retentionCount = config('capsule.retention.count', 10);
 
-            BackupLog::where('status', 'success')
-                ->where('created_at', '<', now()->subDays($retentionDays))
-                ->orWhere(function ($query) use ($retentionCount) {
-                    $query->where('status', 'success')
-                        ->whereNotIn('id', function ($subQuery) use ($retentionCount) {
-                            $subQuery->select('id')
-                                ->from('backup_logs')
-                                ->where('status', 'success')
-                                ->orderBy('created_at', 'desc')
-                                ->limit($retentionCount);
-                        });
-                })
-                ->each(function (BackupLog $backup) {
-                    if ($backup->file_path) {
-                        if (isset($backup->metadata['chunked']) && $backup->metadata['chunked']) {
-                            // For chunked backups, delete from remote storage
-                            $fileName = basename($backup->file_path);
-                            $this->storageManager->delete($fileName);
-                        } else {
-                            // For regular backups, use storage manager for consistent handling
-                            $fileName = basename($backup->file_path);
-                            $this->storageManager->delete($fileName);
-                        }
-                    }
-                    $backup->delete();
-                });
+            // Get IDs of backups to keep (latest N successful backups)
+            $keepIds = BackupLog::where('status', 'success')
+                ->orderBy('created_at', 'desc')
+                ->limit($retentionCount)
+                ->pluck('id')
+                ->toArray();
+
+            // Find backups to delete
+            $query = BackupLog::where('status', 'success')
+                ->where('created_at', '<', now()->subDays($retentionDays));
+
+            if (!empty($keepIds)) {
+                $query->whereNotIn('id', $keepIds);
+            }
+
+            // Use eachById to avoid memory issues with large result sets
+            $query->eachById(function (BackupLog $backup) {
+                if ($backup->file_path) {
+                    $fileName = basename($backup->file_path);
+                    $this->storageManager->delete($fileName);
+                }
+                $backup->delete();
+            });
         }
     }
 }

src/Services/ConcurrentUploadManager.php

@@ -69,16 +69,22 @@ protected function createUploadProcess(array $chunk): array
             'completed' => false,
             'success' => false,
             'error' => null,
+            'pid' => null,
+            'error_log' => $tempScript . '.error.log', // Add an error log file
         ];
 
-        // Start the background process
+        // Start the background process and redirect stderr
         $command = sprintf(
-            'php %s > /dev/null 2>&1 & echo $!',
-            escapeshellarg($tempScript)
+            '(php %s > /dev/null 2> %s) & echo $!',
+            escapeshellarg($tempScript),
+            escapeshellarg($process['error_log'])
         );
 
         $process['handle'] = popen($command, 'r');
-        $process['pid'] = trim(fgets($process['handle']));
+        $pid = trim(fgets($process['handle']));
+        if (!empty($pid)) {
+            $process['pid'] = $pid;
+        }
         pclose($process['handle']);
 
         return $process;
@@ -87,6 +93,8 @@ protected function createUploadProcess(array $chunk): array
     protected function createTempUploadScript(array $chunk): string
     {
         $tempFile = tempnam(sys_get_temp_dir(), 'capsule_upload_');
+        $dataFile = $tempFile . '.data';
+        file_put_contents($dataFile, $chunk['data']);
 
         $script = '<?php
 require_once "' . base_path('vendor/autoload.php') . '";
@@ -97,7 +105,7 @@ protected function createTempUploadScript(array $chunk): string
     
     $storageManager = app(\Dgtlss\Capsule\Storage\StorageManager::class);
     
-    $chunkData = base64_decode("' . base64_encode($chunk['data']) . '");
+    $chunkData = file_get_contents("' . $dataFile . '");
     $chunkName = "' . $chunk['name'] . '";
     
     $tempStream = fopen("php://temp", "r+");
@@ -110,6 +118,9 @@ protected function createTempUploadScript(array $chunk): string
     file_put_contents("' . $tempFile . '.success", "SUCCESS");
 } catch (Exception $e) {
     file_put_contents("' . $tempFile . '.error", $e->getMessage());
+} finally {
+    // Clean up data file
+    @unlink("' . $dataFile . '");
 }
 ';
 
@@ -132,23 +143,33 @@ protected function processActiveUploads(): void
         }
     }
 
-    protected function checkUploadComplete(array $upload): bool
+    protected function checkUploadComplete(array &$upload): bool
     {
         $script = $upload['process']['script'];
+        $errorLog = $upload['process']['error_log'];
 
-        // Check if success or error file exists
+        // Check if success file exists
         if (file_exists($script . '.success')) {
             $upload['process']['completed'] = true;
             $upload['process']['success'] = true;
             return true;
         }
 
+        // Check if an error was logged by the script
         if (file_exists($script . '.error')) {
             $upload['process']['completed'] = true;
             $upload['process']['success'] = false;
             $upload['process']['error'] = file_get_contents($script . '.error');
             return true;
         }
+        
+        // Check if the stderr log has content
+        if (file_exists($errorLog) && filesize($errorLog) > 0) {
+            $upload['process']['completed'] = true;
+            $upload['process']['success'] = false;
+            $upload['process']['error'] = file_get_contents($errorLog);
+            return true;
+        }
 
         // Check if process is still running (timeout after 60 seconds)
         if (microtime(true) - $upload['started_at'] > 60) {
@@ -170,6 +191,7 @@ protected function completeUpload(string $uploadId, array $upload): void
         @unlink($process['script']);
         @unlink($process['script'] . '.success');
         @unlink($process['script'] . '.error');
+        @unlink($process['error_log']);
 
         $this->results[$uploadId] = [
             'chunk' => $chunk,

src/Storage/StorageManager.php

@@ -22,7 +22,7 @@ public function __construct()
     public function store(string $filePath): string
     {
         $fileName = basename($filePath);
-        $remotePath = $this->backupPath . '/' . $fileName;
+        $remotePath = $this->getPrefixedPath($fileName);
 
         $this->disk->putFileAs($this->backupPath, $filePath, $fileName);
 
@@ -31,7 +31,7 @@ public function store(string $filePath): string
 
     public function storeStream($stream, string $fileName): string
     {
-        $remotePath = $this->backupPath . '/' . $fileName;
+        $remotePath = $this->getPrefixedPath($fileName);
 
         $this->disk->put($remotePath, $stream);
 
@@ -40,21 +40,25 @@ public function storeStream($stream, string $fileName): string
 
     public function delete(string $fileName): bool
     {
-        $remotePath = $this->backupPath . '/' . $fileName;
+        $remotePath = $this->getPrefixedPath($fileName);
 
         return $this->disk->delete($remotePath);
     }
 
     public function exists(string $fileName): bool
     {
-        $remotePath = $this->backupPath . '/' . $fileName;
+        $remotePath = $this->getPrefixedPath($fileName);
 
         return $this->disk->exists($remotePath);
     }
 
     public function getFileSize(string $fileName): int
     {
-        $remotePath = $this->backupPath . '/' . $fileName;
+        $remotePath = $this->getPrefixedPath($fileName);
+
+        if (!$this->disk->exists($remotePath)) {
+            throw new Exception("Unable to retrieve the file_size for file at location: {$remotePath}.");
+        }
 
         return $this->disk->size($remotePath) ?? 0;
     }
@@ -76,14 +80,14 @@ public function listFiles(string $path = null): array
 
     public function size(string $fileName): int
     {
-        $remotePath = $this->backupPath . '/' . $fileName;
+        $remotePath = $this->getPrefixedPath($fileName);
 
         return $this->disk->size($remotePath) ?? 0;
     }
 
     public function collateChunks(array $chunkGroups, string $finalFileName): string
     {
-        $finalPath = $this->backupPath . '/' . $finalFileName;
+        $finalPath = $this->getPrefixedPath($finalFileName);
 
         // Create a temporary zip file
         $tempZip = tmpfile();
@@ -99,7 +103,7 @@ public function collateChunks(array $chunkGroups, string $finalFileName): string
             $combinedData = '';
 
             foreach ($chunks as $chunk) {
-                $chunkPath = $this->backupPath . '/' . $chunk['name'];
+                $chunkPath = $this->getPrefixedPath($chunk['name']);
 
                 if ($this->disk->exists($chunkPath)) {
                     $combinedData .= $this->disk->get($chunkPath);
@@ -129,4 +133,18 @@ public function getBackupPath(): string
     {
         return $this->backupPath;
     }
-}
+
+    protected function getPrefixedPath(string $fileName): string
+    {
+        // Normalize slashes and remove leading/trailing slashes
+        $backupPath = trim($this->backupPath, '/');
+        $fileName = trim($fileName, '/');
+        
+        // Check if the filename already starts with the backup path
+        if (strpos($fileName, $backupPath) === 0) {
+            return $fileName;
+        }
+
+        return $backupPath . '/' . $fileName;
+    }
+}