Commit 1f724dca authored by David's avatar David

Working on the backup tool

parent 8421ed76
......@@ -5,7 +5,7 @@ case class HostBackup(
hostname: String,
repoName: String,
backupPaths: List[String],
preBackupCommand: Option[String] = None,
preBackupScript: Option[String] = None,
postBackupCommand: Option[String] = None,
keepDaily: Int = 30,
keepMonthly: Int = 3
......
......@@ -30,6 +30,28 @@ object Main extends App {
HostBackup("scm.dark.kow.is",
"gitrepos",
List("/home/gitbucket/.gitbucket")
),
HostBackup("sarafan.dark.kow.is",
"databases",
List("/srv/databases/newBackups"),
preBackupScript = Some(
"""
|#!/bin/bash
|set -euxo pipefail
|
|MYSQL_DBS="weewx weewx_satx"
|
|mkdir -p /srv/databases/newBackups/mysql
|
|# clean out the backups from previous times
|rm /srv/databases/newBackups/mysql/*.sql
|
|for db in $MYSQL_DBS; do
| echo "backing up mysql $db"
| mysqldump --opt $db > /srv/databases/newBackups/mysql/$db.sql
|done
""".stripMargin
)
)
)
......@@ -39,5 +61,6 @@ object Main extends App {
val hostBackupActor = system.actorOf(HostBackupActor.props(smallerHostBackups.head, backupConfig))
//TODO: complete the entire system backup by rclone'ing into the cloud
}
package is.kow.backupcoordinator.actors
import akka.actor.{Actor, ActorLogging, Props}
import is.kow.backupcoordinator.actors.StatusActorMessages.{FinishedTask, StartingTask, TaskUpdate}
object ConsoleStatusActor {
def props() = Props[ConsoleStatusActor]
}
class ConsoleStatusActor extends Actor with ActorLogging {
def receive = {
case StartingTask(host, taskName, command) =>
???
case TaskUpdate(host, taskName, currentPercentage) =>
???
case FinishedTask(host, taskName, elapsedTime) =>
???
}
}
package is.kow.backupcoordinator.actors
import akka.actor.{Actor, ActorLogging, ActorRef, Props, ReceiveTimeout, Terminated}
import is.kow.backupcoordinator.{BackupConfiguration, CommandWrapper, HostBackup}
import is.kow.backupcoordinator.actors.HostBackupActor.EstablishSSHSession
import is.kow.backupcoordinator.actors.SSHCommandActor.{ExecuteCommand, ExecutionCompleted}
import is.kow.backupcoordinator.actors.SSHConnectionActor.Connect
import is.kow.backupcoordinator.{BackupConfiguration, HostBackup}
import net.schmizz.sshj.SSHClient
object HostBackupActor {
......@@ -92,7 +92,7 @@ class HostBackupActor(hostBackup: HostBackup, backupConfiguration: BackupConfigu
def awaitingMountComplete(client: SSHClient, mountActor: ActorRef): Receive = {
case ReceiveTimeout =>
log.error("Unable to mount nfs in time! OH NOES")
throw new Exception("Did mount nfs in time!")
throw new Exception("Did not mount nfs in time!")
//TODO: need to clean up behind me
case t@Terminated(`mountActor`) =>
log.error(s"there was an exception and mounting command died")
......@@ -111,7 +111,7 @@ class HostBackupActor(hostBackup: HostBackup, backupConfiguration: BackupConfigu
""".stripMargin.trim)
}
//Step four!
doBackup(client, hostBackup.preBackupCommand, hostBackup.backupPaths, hostBackup.postBackupCommand)
doBackup(client, hostBackup.preBackupScript, hostBackup.backupPaths, hostBackup.postBackupCommand)
}
......@@ -152,16 +152,16 @@ class HostBackupActor(hostBackup: HostBackup, backupConfiguration: BackupConfigu
if (preBackup.nonEmpty) {
//Do the pre-backup commands, and become awaitingPreBackup
val preBackupCmd = context.actorOf(SSHCommandActor.props(client))
preBackupCmd ! ExecuteCommand(preBackup.get)
preBackupCmd ! ExecuteCommand(preBackup.get) //TODO: need to do this by dumping it into a shell script, and then executing it
//TODO: what's a reasonable time for stopping? Should have some kind of tick to indicate processing
context.become(awaitingPreBackup(client, preBackupCmd, backupPaths, postBackup))
}
else if (backupPaths.nonEmpty) {
} else if (backupPaths.nonEmpty) {
//TODO: this assumes already initialized -- Need to add an initializer actor to handle that for me
//TODO: can check initialization with `borg info`
//have to build the backup command and issue one for each backup path
val path = backupPaths.head
val repoPath = s"/mnt/auto-backup/${hostBackup.repoName}"
//TODO: should remove any leading '.' because that's lame
val backupName = path.split("/").last
val backupCmd =
s"""
......@@ -179,10 +179,7 @@ class HostBackupActor(hostBackup: HostBackup, backupConfiguration: BackupConfigu
val backupDirectoryActor = context.actorOf(SSHCommandActor.props(client))
backupDirectoryActor ! ExecuteCommand(backupCmd)
context.become(awaitingBackupDirectory(client, backupDirectoryActor, backupPaths, postBackup))
}
else if (postBackup.nonEmpty) {
} else if (postBackup.nonEmpty) {
log.error("TODO Post backup stuff isn't actually ever run")
//Backup paths should be empty at this stage
doBackup(client, None, backupPaths, None)
......
package is.kow.backupcoordinator.actors
import java.io.PrintWriter
import java.util.UUID
import java.util.concurrent.{Executors, TimeUnit}
import akka.actor.{Actor, ActorLogging, Props}
import is.kow.backupcoordinator.actors.SSHCommandActor.{ExecuteCommand, ExecutionCompleted, ExecutionFailedException}
import is.kow.backupcoordinator.actors.SSHCommandActor.{ExecuteAsScript, ExecuteCommand, ExecutionCompleted, ExecutionFailedException}
import net.schmizz.sshj.SSHClient
import net.schmizz.sshj.sftp.{FileAttributes, OpenMode}
import net.schmizz.sshj.xfer.FilePermission
import scala.concurrent.{ExecutionContext, ExecutionContextExecutor, Future}
import scala.io.Source
......@@ -16,6 +20,11 @@ object SSHCommandActor {
sealed trait SSHCommandActorMessages
case class ExecuteAsScript(contents: String,
workingDir: Option[String] = None,
environment: Map[String, String] = Map.empty[String, String]
) extends SSHCommandActorMessages
case class ExecuteCommand(
command: String,
environment: Map[String, String] = Map.empty[String, String]
......@@ -41,28 +50,46 @@ class SSHCommandActor(ssh: SSHClient) extends Actor with ActorLogging {
}
override def receive: Receive = {
case e: ExecuteAsScript =>
log.debug("dumping a script to be executed")
//Create a temporary file in the working directory if specified, otherwise use /tmp
//Put the contents into that file
//Make it executable (maybe just call bash on it?)
// Execute it
//Delete it (if it worked)
val sftp = ssh.newSFTPClient()
import scala.collection.JavaConverters._
val fileName = UUID.randomUUID().toString
val workingDir = e.workingDir.getOrElse("/tmp")
val fileMode = Set(OpenMode.WRITE, OpenMode.CREAT, OpenMode.TRUNC).asJava
val fileAttributes = new FileAttributes.Builder().withPermissions(Set(FilePermission.USR_RWX).asJava).build()
val fullPath = s"$workingDir/$fileName"
val remoteFile = sftp.open(fullPath, fileMode, fileAttributes)
val writer = new PrintWriter(new remoteFile.RemoteFileOutputStream())
try {
writer.print(e.contents)
} finally {
writer.close()
}
remoteFile.close()
//Execute the remote file
val command = ExecuteCommand(fullPath)
//TODO: Have to become something else to pay attention to the result coming back
//Could receive a successful execution, or a failure, and then we report back
case e: ExecuteCommand =>
//This is really the only thing we're going to work on
log.debug(s"Starting to execute command || ${e.command} ||")
//TODO: also include environment
val command = session.exec(e.command)
val replyTo = sender()
//This guy is fully synchronous and blocking so be sure to stick it into another thread
Future {
//Collect all the output
val stdOut = Source.fromInputStream(command.getInputStream).mkString
val stdErr = Source.fromInputStream(command.getErrorStream).mkString
command.join(5, TimeUnit.SECONDS)
val exitStatus = command.getExitStatus
//join on it, and then also get the exit status
log.debug("Command Execution completed!") //Has to be last thing!
if (exitStatus != 0) {
throw new ExecutionFailedException(e, stdOut, stdErr, exitStatus)
}
ExecutionCompleted(e, stdOut, stdErr, exitStatus)
}.onComplete {
executeCommand(e).onComplete {
case Success(ec) =>
//Send the completed message back to the sender and stop myself
log.debug(s"Sending $ec to $replyTo")
......@@ -76,4 +103,27 @@ class SSHCommandActor(ssh: SSHClient) extends Actor with ActorLogging {
throw throwable
}
}
def executeCommand(e: ExecuteCommand): Future[ExecutionCompleted] = {
//This is really the only thing we're going to work on
log.debug(s"Starting to execute command || ${e.command} ||")
//TODO: also include environment
val command = session.exec(e.command)
//This guy is fully synchronous and blocking so be sure to stick it into another thread
Future {
//Collect all the output
val stdOut = Source.fromInputStream(command.getInputStream).mkString
val stdErr = Source.fromInputStream(command.getErrorStream).mkString
command.join(5, TimeUnit.SECONDS)
val exitStatus = command.getExitStatus
//join on it, and then also get the exit status
log.debug("Command Execution completed!") //Has to be last thing!
if (exitStatus != 0) {
throw new ExecutionFailedException(e, stdOut, stdErr, exitStatus)
}
ExecutionCompleted(e, stdOut, stdErr, exitStatus)
}
}
}
package is.kow.backupcoordinator.actors
object StatusActorMessages {
case class StartingTask(
host: String,
taskName: String,
command: String
)
case class TaskUpdate(
host: String,
taskName: String,
currentPercentage: Int
)
case class FinishedTask(
host: String,
taskName: String,
elapsedTime: Long
)
}
package is.kow.backupcoordinator.experimentation
package is.kow.backupcoordinator.experimentation.experimentation
import java.nio.file.Paths
......
package is.kow.backupcoordinator.experimentation
package is.kow.backupcoordinator.experimentation.experimentation
import java.util.concurrent.TimeUnit
......
package is.kow.backupcoordinator.experimentation
package is.kow.backupcoordinator.experimentation.experimentation
import java.io.PrintWriter
import java.security.Security
import java.util.concurrent.TimeUnit
import net.schmizz.sshj.sftp.{FileAttributes, OpenMode, RemoteFile}
import net.schmizz.sshj.xfer.FilePermission
import org.bouncycastle.jce.provider.BouncyCastleProvider
import scala.io.Source
......@@ -22,6 +25,28 @@ object SSHTest extends App with EUtils {
val exitStatus = command.getExitStatus
println(s"output:\n${output}\nExit Status: ${exitStatus}")
//Now let's write a file to the system.
//Establish an SFTP session, and remote file it directly
val sftpClient = ssh.newSFTPClient()
import scala.collection.JavaConverters._
val remoteFile = sftpClient.open("/tmp/testFile",
Set(OpenMode.WRITE, OpenMode.CREAT, OpenMode.TRUNC).asJava,
new FileAttributes.Builder().withPermissions(Set(FilePermission.USR_RWX).asJava).build()
)
val writer = new PrintWriter(new remoteFile.RemoteFileOutputStream())
writer.println("This is the content that I added to the file remotely")
writer.println("I should be able to put *anything* I want in here")
writer.close()
remoteFile.close()
}
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment