Commit 5778e9b9 authored by David's avatar David
Browse files

Removed java application

It's a pure angular app now, calling APIs from mayanEDMS to do the
actual work.
parent 161e3295
## Plan:
* OpenID Authentication via light.kow.is
* Staged documents API
Document IDs are UUID, so that they can be used in many locations
```text
GET /stage -- List of staged documents
GET /trash -- List of documents in the garbage
GET /documents -- List all documents, except staged ones
GET /documents/{uuid} -- specific document
GET /documents/byTag/{tag,tag,tag...} -- List documents by tags
POST /documents -- Create a new document
PUT /documents/{uuid} -- update a document (adding tags, changing metadata)
DELETE /documents/{uuid} -- trash document
GET /tags -- List of all tags
GET /tags/{tag} -- tag details
POST /tags -- create new tag
DELETE /tags/{tag} -- delete a tag
```
## Parameters:
* Staging directory
* Media storage directory
* Database credentials
## Database tables:
* documents
* document_tags
* tags
Special tags: `trash`, `new`
\ No newline at end of file
buildscript {
repositories {
mavenCentral()
}
dependencies {
classpath "org.hibernate:hibernate-gradle-plugin:5.4.3.Final"
}
}
plugins {
id "org.jetbrains.kotlin.jvm" version "1.3.50"
id "org.jetbrains.kotlin.kapt" version "1.3.50"
id "org.jetbrains.kotlin.plugin.allopen" version "1.3.50"
id "com.github.johnrengelman.shadow" version "5.0.0"
id "com.avast.gradle.docker-compose" version "0.10.7"
id "application"
}
apply plugin: 'org.hibernate.orm'
version "0.1"
group "example.micronaut"
repositories {
mavenCentral()
maven { url "https://jcenter.bintray.com" }
}
configurations {
// for dependencies that are needed for development only
developmentOnly
}
hibernate {
enhance {
enableLazyInitialization = true
enableDirtyTracking = true
enableAssociationManagement = true
}
}
dependencies {
implementation platform("io.micronaut:micronaut-bom:$micronautVersion")
implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${kotlinVersion}"
implementation "org.jetbrains.kotlin:kotlin-reflect:${kotlinVersion}"
implementation "io.micronaut:micronaut-runtime"
implementation "javax.annotation:javax.annotation-api"
implementation "io.micronaut:micronaut-http-server-netty"
implementation "io.micronaut:micronaut-http-client"
kapt "io.micronaut:micronaut-graal"
compileOnly "org.graalvm.nativeimage:svm"
kapt platform("io.micronaut:micronaut-bom:$micronautVersion")
kapt "io.micronaut:micronaut-inject-java"
kapt "io.micronaut:micronaut-validation"
kaptTest platform("io.micronaut:micronaut-bom:$micronautVersion")
kaptTest "io.micronaut:micronaut-inject-java"
runtimeOnly "com.fasterxml.jackson.module:jackson-module-kotlin:2.9.8"
runtimeOnly "ch.qos.logback:logback-classic:1.2.3"
testImplementation platform("io.micronaut:micronaut-bom:$micronautVersion")
testImplementation "io.mockk:mockk:1.9.3"
testImplementation "org.junit.jupiter:junit-jupiter-api"
testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine"
testImplementation 'com.willowtreeapps.assertk:assertk-jvm:0.20'
testImplementation ("org.spekframework.spek2:spek-dsl-jvm:$spekVersion") {
exclude group: 'org.jetbrains.kotlin'
}
testRuntimeOnly ("org.spekframework.spek2:spek-runner-junit5:$spekVersion") {
exclude group: 'org.junit.platform'
exclude group: 'org.jetbrains.kotlin'
}
testRuntimeOnly "org.jetbrains.kotlin:kotlin-reflect:$kotlinVersion"
implementation 'io.micronaut.configuration:micronaut-flyway'
kapt 'io.micronaut.data:micronaut-data-processor:1.0.0.M5'
implementation 'io.micronaut.data:micronaut-data-hibernate-jpa:1.0.0.M5', {
exclude group:'io.micronaut.configuration', module:'micronaut-hibernate-jpa-spring'
}
runtime "io.micronaut.configuration:micronaut-jdbc-hikari"
runtime group: 'org.postgresql', name: 'postgresql', version: '42.2.9'
kapt "io.micronaut.configuration:micronaut-openapi"
compile "io.swagger.core.v3:swagger-annotations"
// https://mvnrepository.com/artifact/com.j256.simplemagic/simplemagic
implementation group: 'com.j256.simplemagic', name: 'simplemagic', version: '1.16'
api("com.github.pozo:mapstruct-kotlin:1.3.1.0")
kapt("com.github.pozo:mapstruct-kotlin-processor:1.3.1.0")
kapt "org.mapstruct:mapstruct-processor:1.3.1.Final"
implementation "org.mapstruct:mapstruct:1.3.1.Final"
// webResources(project(path = ":admin-frontend", configuration = "webResources"))
}
test.classpath += configurations.developmentOnly
mainClassName = "is.kow.homeedms.Application"
dockerCompose.isRequiredBy(run)
def username = System.properties['user.name']
dockerCompose {
environment.put 'UID', ["id", "-u", username].execute().text.trim()
environment.put 'GID', ["id", "-g", username].execute().text.trim()
}
run.doFirst {
environment 'MICRONAUT_ENVIRONMENTS', 'local'
}
test {
useJUnitPlatform()
testLogging {
events 'FAILED', 'SKIPPED'
}
}
allOpen {
annotation("io.micronaut.aop.Around")
}
compileKotlin {
kotlinOptions {
jvmTarget = '1.8'
//Will retain parameter names for Java reflection
javaParameters = true
}
}
compileTestKotlin {
kotlinOptions {
jvmTarget = '1.8'
javaParameters = true
}
}
shadowJar {
mergeServiceFiles()
}
kapt {
useBuildCache = false
arguments {
arg("mapstruct.defaultComponentModel", "jsr330")
}
}
run.classpath += configurations.developmentOnly
run.jvmArgs('-noverify', '-XX:TieredStopAtLevel=1', '-Dcom.sun.management.jmxremote')
#!/bin/sh
docker build . -t homeedms
echo
echo
echo "To run the docker container execute:"
echo " $ docker run -p 8080:8080 homeedms"
version: '3.7'
services:
db:
image: postgres
user: "$UID:$GID"
ports:
- "5432:5432"
environment:
- POSTGRES_USER=homeedms
- POSTGRES_PASSWORD=homeedms
volumes:
- ./postgresql:/var/lib/postgresql
# This needs explicit mapping due to https://github.com/docker-library/postgres/blob/4e48e3228a30763913ece952c611e5e9b95c8759/Dockerfile.template#L52
- ./postgresql_data:/var/lib/postgresql/data
kapt.use.worker.api=true
micronautVersion=1.2.8
kotlinVersion=1.3.50
spekVersion=2.0.8
\ No newline at end of file
#!/bin/bash
set -euxo pipefail
export JENV_VERSION="graalvm64-11.0.5"
gu install native-image
# --initialize-at-run-time=io.netty.buffer.AbstractReferenceCountedByteBuf,io.netty.util.AbstractReferenceCounted \
native-image \
--no-server \
--initialize-at-build-time=org.jboss.logging,org.hibernate.internal.CoreMessageLogger_\$logger \
--static \
-cp build/libs/homeedms-*-all.jar \
-H:+TraceClassInitialization
profile: service
defaultPackage: example.micronaut
---
testFramework: kotlintest
sourceLanguage: kotlin
\ No newline at end of file
package `is`.kow.homeedms
import `is`.kow.homeedms.documents.StagingWatcher
import io.micronaut.core.annotation.TypeHint
import io.micronaut.runtime.Micronaut
import io.swagger.v3.oas.annotations.OpenAPIDefinition
import io.swagger.v3.oas.annotations.info.Contact
import io.swagger.v3.oas.annotations.info.Info
import io.swagger.v3.oas.annotations.info.License
import org.flywaydb.core.Flyway
import org.hibernate.dialect.PostgreSQL10Dialect
import org.slf4j.LoggerFactory
@OpenAPIDefinition(
info = Info(
title = "Document Filing",
version = "1.0",
description = "An api for simple home electronic document management",
license = License(name = "MIT", url = "https://gitlab.light.kow.is/dkowis/home-document-filer"),
contact = Contact(url = "https://david.kow.is", name = "David Kowis", email = "david@kow.is")
)
)
@TypeHint(typeNames = [
"org.postgresql.Driver",
"org.hibernate.dialect.PostgreSQL10Dialect"
])
object Application {
val logger = LoggerFactory.getLogger(Application::class.java)
@JvmStatic
fun main(args: Array<String>) {
//TODO: need to add CLI args for staging directory
val context = Micronaut.build()
.packages("is.kow.homeedms")
.mainClass(Application.javaClass)
.start()
val environment = context.getEnvironment()
logger.debug("Active Environments: ${environment.activeNames.joinToString(", ")}")
val stagingWatcher = context.getBean(StagingWatcher::class.java)
logger.debug("Got the staging watcher")
}
}
\ No newline at end of file
package `is`.kow.homeedms
import io.micronaut.http.MediaType
import io.micronaut.http.annotation.Controller
import io.micronaut.http.annotation.Get
import io.micronaut.http.annotation.Produces
@Controller("/hello")
class HelloController {
@Get("/")
@Produces(MediaType.TEXT_PLAIN)
fun index(): String {
return "Hello World!"
}
}
\ No newline at end of file
package `is`.kow.homeedms
import io.micronaut.context.annotation.ConfigurationProperties
import javax.validation.constraints.NotBlank
@ConfigurationProperties("app")
class HomeEdmsConfiguration {
@NotBlank
var pgUser: String = "homeedms"
@NotBlank
lateinit var pgPass: String
@NotBlank
var pgHost: String = "localhost"
@NotBlank
var pgPort: Int = 5432
@NotBlank
var pgDatabase: String = "homeedms"
@NotBlank
lateinit var watchDirectory: String
@NotBlank
lateinit var mediaDirectory: String
}
\ No newline at end of file
package `is`.kow.homeedms.documents
import `is`.kow.homeedms.dto.DocumentDto
import `is`.kow.homeedms.entity.DocumentEntity
import `is`.kow.homeedms.repository.DocumentRepository
import io.micronaut.http.MediaType
import io.micronaut.http.annotation.Controller
import io.micronaut.http.annotation.Get
import io.micronaut.http.annotation.PathVariable
import io.micronaut.http.annotation.Produces
import io.reactivex.Flowable
import org.slf4j.LoggerFactory
import java.util.*
@Controller("/documents")
class DocumentController(
private val docService: DocumentService
) {
val logger = LoggerFactory.getLogger(DocumentController::class.java)
@Get("/")
@Produces(MediaType.APPLICATION_JSON)
fun index(): Flowable<DocumentDto> {
return Flowable.fromPublisher(docService.allDocumentsWithTags());
}
@Get("/byTag/{tagList}")
@Produces(MediaType.APPLICATION_JSON)
fun documentsByTagList(@PathVariable tagList: String): List<DocumentEntity> {
return listOf()
}
@Get("/{uuid}")
@Produces(MediaType.APPLICATION_JSON)
fun documentDetails(@PathVariable uuid: UUID): DocumentEntity? {
return null;
}
@Get("/{uuid}.pdf")
@Produces(MediaType.APPLICATION_OCTET_STREAM)
fun documentContent(@PathVariable uuid: UUID): ByteArray? {
//TODO: how to stream the document back to them?
return null;
}
}
\ No newline at end of file
package `is`.kow.homeedms.documents
import `is`.kow.homeedms.HomeEdmsConfiguration
import `is`.kow.homeedms.entity.DocumentEntity
import `is`.kow.homeedms.repository.DocumentRepository
import `is`.kow.homeedms.repository.TagRepository
import com.j256.simplemagic.ContentInfoUtil
import com.j256.simplemagic.ContentType
import io.reactivex.Observer
import io.reactivex.Single
import io.reactivex.disposables.Disposable
import io.reactivex.schedulers.Schedulers
import io.reactivex.subjects.PublishSubject
import org.slf4j.LoggerFactory
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths
import java.util.Stack
import java.util.UUID
import javax.annotation.PostConstruct
import javax.inject.Singleton
@Singleton
class DocumentIntake(
private val docRepo: DocumentRepository,
private val tagRepo: TagRepository,
private val config: HomeEdmsConfiguration
) {
private val logger = LoggerFactory.getLogger(DocumentIntake::class.java)
private val incoming: PublishSubject<Path> = PublishSubject.create()
val submission = incoming as Observer<Path>
private val contentInfo = ContentInfoUtil()
private val subscriptions: Stack<Disposable> = Stack()
@PostConstruct
fun init() {
unsubAll()
subscriptions.push(incoming
.doOnNext {
logger.debug("potentially importing: $it")
}
.filter {
val info = contentInfo.findMatch(it.toFile().absoluteFile)
info.contentType == ContentType.PDF
}
.flatMap {
logger.debug("$it's a PDF!")
//Generate a UUID
// write the file into the media location at that UUID path
// import it into the database
// delete the original
importDocument(it).toObservable()
}
.retry()
.doOnError {
logger.warn("Unable to process file!", it.message)
}
.subscribe()
)
}
private fun unsubAll() {
while (subscriptions.isNotEmpty()) {
subscriptions.pop().dispose()
}
}
private fun importDocument(source: Path): Single<DocumentEntity> {
val uuid = UUID.randomUUID()
val dirs = uuid.toString().take(2)
val document = DocumentEntity()
document.name = source.toFile().name
document.uuid = uuid
return Single.just(document)
.observeOn(Schedulers.io())
.doOnSuccess {
//On successful processing of this, file it away
val storagePath = Paths.get(config.mediaDirectory, dirs[0].toString(), dirs[1].toString())
logger.debug("Creating storage directory for PDF: $storagePath")
storagePath.toFile().mkdirs()
val destination = Paths.get(storagePath.toFile().absolutePath, "$uuid.pdf")
Files.copy(source, destination)
//Delete it from the source directory
//Send it to the queue for OCR processing
}
.flatMap { doc ->
val newTag = tagRepo.findByName("new")
Single.fromPublisher(newTag)
.flatMap { tag ->
doc.tags = listOf(tag)
Single.fromPublisher(docRepo.save(doc))
}
}
}
}
\ No newline at end of file
package `is`.kow.homeedms.documents
import `is`.kow.homeedms.dto.DocumentDto
import `is`.kow.homeedms.dto.DocumentMapper
import `is`.kow.homeedms.repository.DocumentRepository
import io.reactivex.Flowable
import javax.inject.Singleton
@Singleton
class DocumentService(
private val documentRepository: DocumentRepository,
private val documentMapper: DocumentMapper
) {
fun allDocumentsWithTags(): Flowable<DocumentDto> {
return Flowable.fromPublisher(documentRepository.findAll())
.map {
documentMapper.convertToDto(it)
}
}
}
\ No newline at end of file
package `is`.kow.homeedms.documents
import `is`.kow.homeedms.HomeEdmsConfiguration
import io.reactivex.Observable
import io.reactivex.disposables.Disposable
import org.slf4j.LoggerFactory
import java.nio.file.FileSystems
import java.nio.file.Path
import java.nio.file.Paths
import java.nio.file.StandardWatchEventKinds
import java.nio.file.WatchService
import java.util.*
import java.util.concurrent.TimeUnit
import javax.annotation.PostConstruct
import javax.inject.Singleton
@Singleton
class StagingWatcher(
private val config: HomeEdmsConfiguration,
private val documentIntake: DocumentIntake
) {
val logger = LoggerFactory.getLogger(StagingWatcher::class.java)
val subscriptions: Stack<Disposable> = Stack()
var watchService: WatchService? = null
@PostConstruct
fun init() {
while (subscriptions.isNotEmpty()) {
subscriptions.pop().dispose() //turn them off!
}
watchService?.close() //close it if it's there
watchService = FileSystems.getDefault().newWatchService()
val path = Paths.get(config.watchDirectory)
if (!path.toFile().isDirectory) {
logger.info("Creating watching path: ${config.watchDirectory}")
path.toFile().mkdirs()
}
path.register(watchService, StandardWatchEventKinds.ENTRY_CREATE)
//First thing should always start with a list of the files in the directory
val watcher = Observable.interval(0, 500, TimeUnit.MILLISECONDS)
.map { count ->
watchService!!.poll().let { k ->
val things = if (k != null) {
val t = k.pollEvents().map { event ->
logger.debug("File Added: " + event.context())
logger.debug("context is a " + event.context().javaClass)
event.context() as Path
}
k.reset()
t
} else {
emptyList()
}
things
}
}
.startWith(existingFiles(config.watchDirectory))
.doOnError {
logger.error("Directory Watcher error: ${config.watchDirectory} ", it)
}
.filter { it.isNotEmpty() }
.doOnNext {
logger.debug("Documents to process: ${it.joinToString(", ")}")
it.forEach { file ->
documentIntake.submission.onNext(file)
}
}
.retry()
subscriptions.push(watcher.subscribe())
}
private fun existingFiles(path: String): List<Path> {
return Paths.get(path).toFile().listFiles().map {
it.toPath()