Working on the backup tool

src/main/scala/is/kow/backupcoordinator/HostBackup.scala

@@ -5,7 +5,7 @@ case class HostBackup(
                        hostname: String,
                        repoName: String,
                        backupPaths: List[String],
-                       preBackupCommand: Option[String] = None,
+                       preBackupScript: Option[String] = None,
                        postBackupCommand: Option[String] = None,
                        keepDaily: Int = 30,
                        keepMonthly: Int = 3

src/main/scala/is/kow/backupcoordinator/Main.scala

@@ -30,6 +30,28 @@ object Main extends App {
     HostBackup("scm.dark.kow.is",
       "gitrepos",
       List("/home/gitbucket/.gitbucket")
+    ),
+    HostBackup("sarafan.dark.kow.is",
+      "databases",
+      List("/srv/databases/newBackups"),
+      preBackupScript = Some(
+        """
+          |#!/bin/bash
+          |set -euxo pipefail
+          |
+          |MYSQL_DBS="weewx weewx_satx"
+          |
+          |mkdir -p /srv/databases/newBackups/mysql
+          |
+          |# clean out the backups from previous times
+          |rm /srv/databases/newBackups/mysql/*.sql
+          |
+          |for db in $MYSQL_DBS; do
+          |  echo "backing up mysql $db"
+          |  mysqldump --opt $db > /srv/databases/newBackups/mysql/$db.sql
+          |done
+        """.stripMargin
+      )
     )
   )
 
@@ -39,5 +61,6 @@ object Main extends App {
 
   val hostBackupActor = system.actorOf(HostBackupActor.props(smallerHostBackups.head, backupConfig))
 
+  //TODO: complete the entire system backup by rclone'ing into the cloud
 
 }

src/main/scala/is/kow/backupcoordinator/actors/ConsoleStatusActor.scala

+package is.kow.backupcoordinator.actors
+
+import akka.actor.{Actor, ActorLogging, Props}
+import is.kow.backupcoordinator.actors.StatusActorMessages.{FinishedTask, StartingTask, TaskUpdate}
+
+
+object ConsoleStatusActor {
+  def props() = Props[ConsoleStatusActor]
+}
+
+class ConsoleStatusActor extends Actor with ActorLogging {
+
+  def receive = {
+    case StartingTask(host, taskName, command) =>
+      ???
+    case TaskUpdate(host, taskName, currentPercentage) =>
+      ???
+    case FinishedTask(host, taskName, elapsedTime) =>
+      ???
+  }
+
+}

src/main/scala/is/kow/backupcoordinator/actors/HostBackupActor.scala

 package is.kow.backupcoordinator.actors
 
 import akka.actor.{Actor, ActorLogging, ActorRef, Props, ReceiveTimeout, Terminated}
-import is.kow.backupcoordinator.{BackupConfiguration, CommandWrapper, HostBackup}
 import is.kow.backupcoordinator.actors.HostBackupActor.EstablishSSHSession
 import is.kow.backupcoordinator.actors.SSHCommandActor.{ExecuteCommand, ExecutionCompleted}
 import is.kow.backupcoordinator.actors.SSHConnectionActor.Connect
+import is.kow.backupcoordinator.{BackupConfiguration, HostBackup}
 import net.schmizz.sshj.SSHClient
 
 object HostBackupActor {
@@ -92,7 +92,7 @@ class HostBackupActor(hostBackup: HostBackup, backupConfiguration: BackupConfigu
   def awaitingMountComplete(client: SSHClient, mountActor: ActorRef): Receive = {
     case ReceiveTimeout =>
       log.error("Unable to mount nfs in time! OH NOES")
-      throw new Exception("Did mount nfs in time!")
+      throw new Exception("Did not mount nfs in time!")
     //TODO: need to clean up behind me
     case t@Terminated(`mountActor`) =>
       log.error(s"there was an exception and mounting command died")
@@ -111,7 +111,7 @@ class HostBackupActor(hostBackup: HostBackup, backupConfiguration: BackupConfigu
       """.stripMargin.trim)
       }
       //Step four!
-      doBackup(client, hostBackup.preBackupCommand, hostBackup.backupPaths, hostBackup.postBackupCommand)
+      doBackup(client, hostBackup.preBackupScript, hostBackup.backupPaths, hostBackup.postBackupCommand)
 
   }
 
@@ -152,16 +152,16 @@ class HostBackupActor(hostBackup: HostBackup, backupConfiguration: BackupConfigu
     if (preBackup.nonEmpty) {
       //Do the pre-backup commands, and become awaitingPreBackup
       val preBackupCmd = context.actorOf(SSHCommandActor.props(client))
-      preBackupCmd ! ExecuteCommand(preBackup.get)
+      preBackupCmd ! ExecuteCommand(preBackup.get) //TODO: need to do this by dumping it into a shell script, and then executing it
       //TODO: what's a reasonable time for stopping? Should have some kind of tick to indicate processing
       context.become(awaitingPreBackup(client, preBackupCmd, backupPaths, postBackup))
-    }
-    else if (backupPaths.nonEmpty) {
+    } else if (backupPaths.nonEmpty) {
       //TODO: this assumes already initialized -- Need to add an initializer actor to handle that for me
       //TODO: can check initialization with `borg info`
       //have to build the backup command and issue one for each backup path
       val path = backupPaths.head
       val repoPath = s"/mnt/auto-backup/${hostBackup.repoName}"
+      //TODO: should remove any leading '.' because that's lame
       val backupName = path.split("/").last
       val backupCmd =
         s"""
@@ -179,10 +179,7 @@ class HostBackupActor(hostBackup: HostBackup, backupConfiguration: BackupConfigu
       val backupDirectoryActor = context.actorOf(SSHCommandActor.props(client))
       backupDirectoryActor ! ExecuteCommand(backupCmd)
       context.become(awaitingBackupDirectory(client, backupDirectoryActor, backupPaths, postBackup))
-
-    }
-
-    else if (postBackup.nonEmpty) {
+    } else if (postBackup.nonEmpty) {
       log.error("TODO Post backup stuff isn't actually ever run")
       //Backup paths should be empty at this stage
       doBackup(client, None, backupPaths, None)

src/main/scala/is/kow/backupcoordinator/actors/SSHCommandActor.scala

 package is.kow.backupcoordinator.actors
 
+import java.io.PrintWriter
+import java.util.UUID
 import java.util.concurrent.{Executors, TimeUnit}
 
 import akka.actor.{Actor, ActorLogging, Props}
-import is.kow.backupcoordinator.actors.SSHCommandActor.{ExecuteCommand, ExecutionCompleted, ExecutionFailedException}
+import is.kow.backupcoordinator.actors.SSHCommandActor.{ExecuteAsScript, ExecuteCommand, ExecutionCompleted, ExecutionFailedException}
 import net.schmizz.sshj.SSHClient
+import net.schmizz.sshj.sftp.{FileAttributes, OpenMode}
+import net.schmizz.sshj.xfer.FilePermission
 
 import scala.concurrent.{ExecutionContext, ExecutionContextExecutor, Future}
 import scala.io.Source
@@ -16,6 +20,11 @@ object SSHCommandActor {
 
   sealed trait SSHCommandActorMessages
 
+  case class ExecuteAsScript(contents: String,
+                             workingDir: Option[String] = None,
+                             environment: Map[String, String] = Map.empty[String, String]
+                            ) extends SSHCommandActorMessages
+
   case class ExecuteCommand(
                              command: String,
                              environment: Map[String, String] = Map.empty[String, String]
@@ -41,28 +50,46 @@ class SSHCommandActor(ssh: SSHClient) extends Actor with ActorLogging {
   }
 
   override def receive: Receive = {
+
+    case e: ExecuteAsScript =>
+      log.debug("dumping a script to be executed")
+      //Create a temporary file in the working directory if specified, otherwise use /tmp
+      //Put the contents into that file
+      //Make it executable (maybe just call bash on it?)
+      // Execute it
+      //Delete it (if it worked)
+      val sftp = ssh.newSFTPClient()
+
+      import scala.collection.JavaConverters._
+
+      val fileName = UUID.randomUUID().toString
+
+      val workingDir = e.workingDir.getOrElse("/tmp")
+      val fileMode = Set(OpenMode.WRITE, OpenMode.CREAT, OpenMode.TRUNC).asJava
+      val fileAttributes = new FileAttributes.Builder().withPermissions(Set(FilePermission.USR_RWX).asJava).build()
+
+      val fullPath = s"$workingDir/$fileName"
+
+      val remoteFile = sftp.open(fullPath, fileMode, fileAttributes)
+
+      val writer = new PrintWriter(new remoteFile.RemoteFileOutputStream())
+      try {
+        writer.print(e.contents)
+      } finally {
+        writer.close()
+      }
+      remoteFile.close()
+
+      //Execute the remote file
+      val command = ExecuteCommand(fullPath)
+      //TODO: Have to become something else to pay attention to the result coming back
+      //Could receive a successful execution, or a failure, and then we report back
+
+
     case e: ExecuteCommand =>
-      //This is really the only thing we're going to work on
-      log.debug(s"Starting to execute command || ${e.command} ||")
-      //TODO: also include environment
-      val command = session.exec(e.command)
       val replyTo = sender()
-      //This guy is fully synchronous and blocking so be sure to stick it into another thread
-      Future {
-        //Collect all the output
-        val stdOut = Source.fromInputStream(command.getInputStream).mkString
-        val stdErr = Source.fromInputStream(command.getErrorStream).mkString
-
-        command.join(5, TimeUnit.SECONDS)
-
-        val exitStatus = command.getExitStatus
-        //join on it, and then also get the exit status
-        log.debug("Command Execution completed!") //Has to be last thing!
-        if (exitStatus != 0) {
-          throw new ExecutionFailedException(e, stdOut, stdErr, exitStatus)
-        }
-        ExecutionCompleted(e, stdOut, stdErr, exitStatus)
-      }.onComplete {
+
+      executeCommand(e).onComplete {
         case Success(ec) =>
           //Send the completed message back to the sender and stop myself
           log.debug(s"Sending $ec to $replyTo")
@@ -76,4 +103,27 @@ class SSHCommandActor(ssh: SSHClient) extends Actor with ActorLogging {
           throw throwable
       }
   }
+
+  def executeCommand(e: ExecuteCommand): Future[ExecutionCompleted] = {
+    //This is really the only thing we're going to work on
+    log.debug(s"Starting to execute command || ${e.command} ||")
+    //TODO: also include environment
+    val command = session.exec(e.command)
+    //This guy is fully synchronous and blocking so be sure to stick it into another thread
+    Future {
+      //Collect all the output
+      val stdOut = Source.fromInputStream(command.getInputStream).mkString
+      val stdErr = Source.fromInputStream(command.getErrorStream).mkString
+
+      command.join(5, TimeUnit.SECONDS)
+
+      val exitStatus = command.getExitStatus
+      //join on it, and then also get the exit status
+      log.debug("Command Execution completed!") //Has to be last thing!
+      if (exitStatus != 0) {
+        throw new ExecutionFailedException(e, stdOut, stdErr, exitStatus)
+      }
+      ExecutionCompleted(e, stdOut, stdErr, exitStatus)
+    }
+  }
 }

src/main/scala/is/kow/backupcoordinator/actors/StatusActorMessages.scala

+package is.kow.backupcoordinator.actors
+
+object StatusActorMessages {
+
+  case class StartingTask(
+                           host: String,
+                           taskName: String,
+                           command: String
+                         )
+
+  case class TaskUpdate(
+                         host: String,
+                         taskName: String,
+                         currentPercentage: Int
+                       )
+
+  case class FinishedTask(
+                           host: String,
+                           taskName: String,
+                           elapsedTime: Long
+                         )
+
+}

src/main/scala/is/kow/backupcoordinator/experimentation/EUtils.scala → src/test/scala/is/kow/backupcoordinator/experimentation/experimentation/EUtils.scala

-package is.kow.backupcoordinator.experimentation
+package is.kow.backupcoordinator.experimentation.experimentation
 
 import java.nio.file.Paths
 

src/main/scala/is/kow/backupcoordinator/experimentation/SSHActorExperiment.scala → src/test/scala/is/kow/backupcoordinator/experimentation/experimentation/SSHActorExperiment.scala

-package is.kow.backupcoordinator.experimentation
+package is.kow.backupcoordinator.experimentation.experimentation
 
 import java.util.concurrent.TimeUnit
 

src/main/scala/is/kow/backupcoordinator/experimentation/SSHTest.scala → src/test/scala/is/kow/backupcoordinator/experimentation/experimentation/SSHTest.scala

-package is.kow.backupcoordinator.experimentation
+package is.kow.backupcoordinator.experimentation.experimentation
 
+import java.io.PrintWriter
 import java.security.Security
 import java.util.concurrent.TimeUnit
 
+import net.schmizz.sshj.sftp.{FileAttributes, OpenMode, RemoteFile}
+import net.schmizz.sshj.xfer.FilePermission
 import org.bouncycastle.jce.provider.BouncyCastleProvider
 
 import scala.io.Source
@@ -22,6 +25,28 @@ object SSHTest extends App with EUtils {
       val exitStatus = command.getExitStatus
 
       println(s"output:\n${output}\nExit Status: ${exitStatus}")
+
+
+      //Now let's write a file to the system.
+      //Establish an SFTP session, and remote file it directly
+      val sftpClient = ssh.newSFTPClient()
+
+      import scala.collection.JavaConverters._
+
+      val remoteFile = sftpClient.open("/tmp/testFile",
+        Set(OpenMode.WRITE, OpenMode.CREAT, OpenMode.TRUNC).asJava,
+        new FileAttributes.Builder().withPermissions(Set(FilePermission.USR_RWX).asJava).build()
+      )
+
+      val writer = new PrintWriter(new remoteFile.RemoteFileOutputStream())
+
+      writer.println("This is the content that I added to the file remotely")
+      writer.println("I should be able to put *anything* I want in here")
+
+      writer.close()
+
+      remoteFile.close()
+
     }
   }
 }