Commit b49e17a5 authored by Jorge Aguilera's avatar Jorge Aguilera 📚

Initial commit

parents
Thumbs.db
.DS_Store
.gradle
target/
out/
.idea
*.iml
*.ipr
*.iws
.project
.settings
.classpath
**/build/
**/gradle.properties
**/.gradle/
**/.idea/
= K8s BiblioMadrid
A Kubernetes example.
https://jorge.aguilera.soy/blog/prestamos-bibliotecas/k8s-1.html
plugins {
id 'org.asciidoctor.jvm.convert' version '3.0.0-alpha.3'
}
subprojects {
version "0.2"
group "com.puravida.biblios"
}
repositories {
jcenter()
maven { url 'http://rubygems-proxy.torquebox.org/releases' }
mavenCentral()
}
configurations {
asciidoctorExt
}
dependencies {
asciidoctorExt 'com.puravida.asciidoctor:asciidoctor-extensions:2.2.1'
asciidoctorExt group: 'net.sourceforge.plantuml', name: 'plantuml', version: '1.2019.5'
asciidoctorExt group: 'org.scilab.forge', name: 'jlatexmath', version: '1.0.7'
}
plugins {
id "groovy"
id "com.github.johnrengelman.shadow" version "5.0.0"
id "application"
id "com.google.cloud.tools.jib" version "1.8.0"
}
repositories {
mavenCentral()
maven { url "https://jcenter.bintray.com" }
}
configurations {
// for dependencies that are needed for development only
developmentOnly
}
dependencies {
compileOnly platform("io.micronaut:micronaut-bom:$micronautVersion")
compileOnly "io.micronaut:micronaut-inject-groovy"
implementation platform("io.micronaut:micronaut-bom:$micronautVersion")
implementation "io.micronaut:micronaut-http-client"
implementation "io.micronaut:micronaut-http-server-netty"
implementation "io.micronaut:micronaut-runtime-groovy"
implementation "io.micronaut:micronaut-validation"
implementation "javax.annotation:javax.annotation-api"
runtimeOnly "ch.qos.logback:logback-classic:1.2.3"
testCompileOnly platform("io.micronaut:micronaut-bom:$micronautVersion")
testImplementation platform("io.micronaut:micronaut-bom:$micronautVersion")
testImplementation("org.spockframework:spock-core") {
exclude group: "org.codehaus.groovy", module: "groovy-all"
}
testImplementation "io.micronaut:micronaut-inject-groovy"
testImplementation "io.micronaut.test:micronaut-test-spock"
testImplementation "io.micronaut.test:micronaut-test-junit5"
testRuntimeOnly "org.junit.vintage:junit-vintage-engine:5.5.0"
compileOnly 'io.micronaut.data:micronaut-data-processor:1.0.0.M5'
compileOnly 'jakarta.persistence:jakarta.persistence-api:2.2.2'
compile 'io.micronaut.data:micronaut-data-jdbc:1.0.0.M5'
runtime 'io.micronaut.configuration:micronaut-jdbc-hikari'
runtime 'com.h2database:h2'
runtime 'org.postgresql:postgresql:42.2.8'
compile project(':com-puravida-biblios-model')
}
test.classpath += configurations.developmentOnly
mainClassName = "com.puravida.biblios.Application"
shadowJar {
mergeServiceFiles()
}
run.classpath += configurations.developmentOnly
run.jvmArgs('-noverify', '-XX:TieredStopAtLevel=1', '-Dcom.sun.management.jmxremote')
tasks.withType(GroovyCompile) {
groovyOptions.forkOptions.jvmArgs.add('-Dgroovy.parameters=true')
}
jib{
from{
image = 'gcr.io/distroless/java:11-debug'
}
to{
image = "jagedn/k8s-bibliomadrid-api:${project.version}"
}
}
jibDockerBuild.dependsOn build
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-5.3-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
profile: service
defaultPackage: com.puravida.biblios
---
testFramework: spock
sourceLanguage: groovy
\ No newline at end of file
package com.puravida.biblios
import io.micronaut.runtime.Micronaut
import groovy.transform.CompileStatic
@CompileStatic
class Application {
static void main(String[] args) {
Micronaut.run(Application)
}
}
\ No newline at end of file
package com.puravida.biblios
import com.puravida.biblios.model.Prestamo
import com.puravida.biblios.model.PrestamoRepository
import io.micronaut.data.model.Page
import io.micronaut.data.model.Pageable
import io.micronaut.data.model.Slice
import io.micronaut.http.annotation.Controller
import io.micronaut.http.annotation.Get
import io.reactivex.Maybe
@Controller('/prestamos')
class PrestamoController {
PrestamoRepository prestamoRepository
PrestamoController(PrestamoRepository prestamoRepository){
this.prestamoRepository = prestamoRepository
}
@Get('/{?requestPage*}')
Page<Prestamo> index(final RequestPage requestPage){
prestamoRepository.list(Pageable.from(requestPage.offset, requestPage.size ?: 10))
}
@Get('/title/{title}{?requestPage*}')
Page<Prestamo> findByTitle(final String title, final RequestPage requestPage){
prestamoRepository.findByTitituIlike("%$title%",Pageable.from(requestPage.offset, requestPage.size ?: 10))
}
@Get('/author/{author}{?requestPage*}')
Page<Prestamo> findByAuthor(final String author, final RequestPage requestPage){
prestamoRepository.findByTiautoIlike("%$author%",Pageable.from(requestPage.offset, requestPage.size ?: 10))
}
}
package com.puravida.biblios
class RequestPage {
int offset
int size
}
micronaut:
application:
name: api
datasources:
default:
url: jdbc:postgresql://${POSTGRESQL_SERVICE_HOST}/${POSTGRES_DB}
driverClassName: org.postgresql.Driver
username: ${POSTGRES_USER}
password: ${POSTGRES_PASSWORD}
dialect: POSTGRES
<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<withJansi>true</withJansi>
<!-- encoders are assigned the type
ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
<encoder>
<pattern>%cyan(%d{HH:mm:ss.SSS}) %gray([%thread]) %highlight(%-5level) %magenta(%logger{36}) - %msg%n</pattern>
</encoder>
</appender>
<root level="info">
<appender-ref ref="STDOUT" />
</root>
</configuration>
plugins {
id "groovy"
id "com.github.johnrengelman.shadow" version "5.0.0"
id "application"
id 'com.google.cloud.tools.jib' version '1.8.0'
}
repositories {
mavenCentral()
maven { url "https://jcenter.bintray.com" }
}
configurations {
// for dependencies that are needed for development only
developmentOnly
}
dependencies {
annotationProcessor "io.micronaut.configuration:micronaut-picocli:1.2.1"
compileOnly platform("io.micronaut:micronaut-bom:$micronautVersion")
compileOnly "io.micronaut:micronaut-inject-groovy"
implementation platform("io.micronaut:micronaut-bom:$micronautVersion")
implementation "io.micronaut:micronaut-runtime"
implementation "info.picocli:picocli"
implementation "io.micronaut.configuration:micronaut-picocli"
implementation "io.micronaut:micronaut-runtime-groovy"
implementation "io.micronaut:micronaut-validation"
runtimeOnly "ch.qos.logback:logback-classic:1.2.3"
testAnnotationProcessor "io.micronaut:micronaut-inject-java:1.2.6"
testCompileOnly platform("io.micronaut:micronaut-bom:$micronautVersion")
testImplementation platform("io.micronaut:micronaut-bom:$micronautVersion")
testImplementation("org.spockframework:spock-core") {
exclude group: "org.codehaus.groovy", module: "groovy-all"
}
testImplementation "io.micronaut:micronaut-inject-groovy"
testImplementation "io.micronaut.test:micronaut-test-spock"
testImplementation "io.micronaut.test:micronaut-test-junit5"
testImplementation "org.junit.jupiter:junit-jupiter-api"
testRuntimeOnly "org.junit.vintage:junit-vintage-engine:5.5.0"
testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine"
compileOnly 'io.micronaut.data:micronaut-data-processor:1.0.0.M5'
compileOnly 'jakarta.persistence:jakarta.persistence-api:2.2.2'
compile 'io.micronaut.data:micronaut-data-jdbc:1.0.0.M5'
runtime 'io.micronaut.configuration:micronaut-jdbc-hikari'
runtime 'com.h2database:h2'
runtime 'org.postgresql:postgresql:42.2.8'
compile project(':com-puravida-biblios-model')
}
test.classpath += configurations.developmentOnly
mainClassName = "com.puravida.biblios.ComPuravidaBibliotecasEtlCommand"
jar {
manifest {
attributes 'Main-Class': mainClassName
}
}
tasks.withType(GroovyCompile) {
groovyOptions.forkOptions.jvmArgs.add('-Dgroovy.parameters=true')
}
// use JUnit 5 platform
test {
useJUnitPlatform()
}
shadowJar {
mergeServiceFiles()
}
run.classpath += configurations.developmentOnly
run.jvmArgs('-noverify', '-XX:TieredStopAtLevel=1', '-Dcom.sun.management.jmxremote')
run.args('-f','test/bibliotecas_prestamos_201909.txt')
jib{
from{
image = 'gcr.io/distroless/java:11-debug'
}
to{
image = "jagedn/k8s-bibliomadrid-etl:${project.version}"
}
}
jibDockerBuild.dependsOn build
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-5.5-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
profile: cli
defaultPackage: com.puravida.biblios.etl
---
testFramework: spock
sourceLanguage: groovy
\ No newline at end of file
package com.puravida.biblios
import com.puravida.biblios.model.Prestamo
import com.puravida.biblios.model.PrestamoRepository
import com.puravida.biblios.model.Task
import com.puravida.biblios.model.TaskRepository
import io.micronaut.configuration.picocli.PicocliRunner
import picocli.CommandLine.Command
import picocli.CommandLine.Option
import javax.inject.Inject
@Command(name = 'com-puravida-bibliotecas-etl', description = '...',
mixinStandardHelpOptions = true)
class ComPuravidaBibliotecasEtlCommand implements Runnable {
@Option(names = ['-v', '--verbose'], description = '...')
boolean verbose
@Option(names = ['-y', '--year'], description = 'year', required = true)
String year
@Option(names = ['-m', '--month'], description = 'month', required = true)
String month
static void main(String[] args) throws Exception {
PicocliRunner.run(ComPuravidaBibliotecasEtlCommand.class, args)
}
@Inject
PrestamoRepository prestamoRepository
@Inject
TaskRepository taskRepository
void run() {
String url = "https://datos.madrid.es/datosabiertos/CULTURA/BIBLIOTECAS_PRESTAMOS/$year/$month/bibliotecas_prestamos_${year}${month}.txt"
String name = url.split('/').last().split('\\.').first()
if( taskRepository.findById(name) ){
println "$f.name ya ha sido cargado"
return
}
File f = File.createTempFile("prestamos","txt")
f.bytes = url.toURL().bytes
int lines=0
List<Prestamo> list = []
f.eachLine 'iso-8859-1',{ l, i ->
if( i == 1 )
return
List<String>ll = l.split('\t') as List<String>
while(ll.size() < 10)
ll.add('')
list.add( new Prestamo(
prbarc:ll[0],
prprsu:ll[1],
prcolp:ll[2],
prcocp:ll[3],
prlebi:ll[4],
prlesu:ll[5],
pradul:ll[6],
prfpre:ll[7],
prcocs:ll[8],
tititu:ll[9],
tiauto:ll[10],
tiisxn:ll[11],
))
if( list.size() > 100){
println i
prestamoRepository.saveAll(list)
list.clear()
}
lines++
}
prestamoRepository.saveAll(list)
taskRepository.save(new Task(file:name, when: new Date(), lines: lines))
}
}
micronaut:
application:
name: com-puravida-biblios-etl
datasources:
default:
url: jdbc:postgresql://${POSTGRESQL_SERVICE_HOST}/${POSTGRES_DB}
driverClassName: org.postgresql.Driver
username: ${POSTGRES_USER}
password: ${POSTGRES_PASSWORD}
dialect: POSTGRES
liquibase:
datasources:
default:
change-log: 'classpath:db/liquibase-changelog.xml'
<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<withJansi>true</withJansi>
<!-- encoders are assigned the type
ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
<encoder>
<pattern>%cyan(%d{HH:mm:ss.SSS}) %gray([%thread]) %highlight(%-5level) %magenta(%logger{36}) - %msg%n</pattern>
</encoder>
</appender>
<root level="info">
<appender-ref ref="STDOUT" />
</root>
</configuration>
plugins {
id "groovy"
}
repositories {
mavenCentral()
maven { url "https://jcenter.bintray.com" }
}
configurations {
// for dependencies that are needed for development only
developmentOnly
}
dependencies {
annotationProcessor "io.micronaut.configuration:micronaut-picocli:1.2.1"
compileOnly platform("io.micronaut:micronaut-bom:$micronautVersion")
compileOnly "io.micronaut:micronaut-inject-groovy"
implementation platform("io.micronaut:micronaut-bom:$micronautVersion")
implementation "io.micronaut:micronaut-runtime"
implementation "info.picocli:picocli"
implementation "io.micronaut.configuration:micronaut-picocli"
implementation "io.micronaut:micronaut-runtime-groovy"
implementation "io.micronaut:micronaut-validation"
runtimeOnly "ch.qos.logback:logback-classic:1.2.3"
testAnnotationProcessor "io.micronaut:micronaut-inject-java:1.2.6"
testCompileOnly platform("io.micronaut:micronaut-bom:$micronautVersion")
testImplementation platform("io.micronaut:micronaut-bom:$micronautVersion")
testImplementation("org.spockframework:spock-core") {
exclude group: "org.codehaus.groovy", module: "groovy-all"
}
testImplementation "io.micronaut:micronaut-inject-groovy"
testImplementation "io.micronaut.test:micronaut-test-spock"
testImplementation "io.micronaut.test:micronaut-test-junit5"
testImplementation "org.junit.jupiter:junit-jupiter-api"
testRuntimeOnly "org.junit.vintage:junit-vintage-engine:5.5.0"
testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine"
compile "io.micronaut.configuration:micronaut-liquibase"
compileOnly 'io.micronaut.data:micronaut-data-processor:1.0.0.M5'
compileOnly 'jakarta.persistence:jakarta.persistence-api:2.2.2'
compile 'io.micronaut.data:micronaut-data-jdbc:1.0.0.M5'
}
test.classpath += configurations.developmentOnly
jar {
manifest {
}
}
tasks.withType(GroovyCompile) {
groovyOptions.forkOptions.jvmArgs.add('-Dgroovy.parameters=true')
}
// use JUnit 5 platform
test {
useJUnitPlatform()
}
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-5.5-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
profile: cli
defaultPackage: com.puravida.biblios.etl
---
testFramework: spock
sourceLanguage: groovy
\ No newline at end of file
package com.puravida.biblios.model
import io.micronaut.data.annotation.AutoPopulated
import javax.annotation.Nullable
import javax.persistence.*
@Entity
class Prestamo {
@Id
@AutoPopulated
UUID id;
String prbarc
String prprsu
String prcolp
String prcocp
String prlebi
String prlesu
String pradul
String prfpre
String prcocs
String tititu
@Nullable
String tiauto
@Nullable
String tiisxn
}
package com.puravida.biblios.model
import io.micronaut.data.jdbc.annotation.JdbcRepository
import io.micronaut.data.model.Page
import io.micronaut.data.model.Pageable
import io.micronaut.data.model.Slice
import io.micronaut.data.model.query.builder.sql.Dialect
import io.micronaut.data.repository.CrudRepository
@JdbcRepository(dialect = Dialect.POSTGRES)
interface PrestamoRepository extends CrudRepository<Prestamo, Long> {
Page<Prestamo> list(Pageable pageable)
Page<Prestamo> findByTitituIlike(String title, Pageable pageable)
Page<Prestamo> findByTiautoIlike(String autor, Pageable pageable)
}
package com.puravida.biblios.model
import io.micronaut.data.annotation.AutoPopulated