Commit 0718d0a0 authored by Michael Ritter's avatar Michael Ritter
Browse files

Merge branch 'release-3.0'

Concludes release candidate testing for version 3.0

ingest-rest/
  update ConfigurationProperties bindings
  additional controller mappings for downloading file lists and tokens
  various db bug fixes for correct counts and better entity to model mappings
  csv processor db query performance improvements
  db migration updates to be in line with leading slash coercion

replication-shell/
  update ConfigurationProperties bindings
  finalize rsync profile names
  handle SIGTERM

tokenizer/
  expect leading slash when receiving file names

update gitlab-ci.yml for new build server config
parent c9a98cef
......@@ -94,6 +94,7 @@ deploy:jdk8:
only:
- master
- develop
- /^release-.*$/
image: maven:3.3.9-jdk-8
services:
- postgres:9
......@@ -107,6 +108,7 @@ deploy:jdk8:
only:
- master
- develop
- /^release-.*$/
image: maven:3.3.9-jdk-8
services:
- postgres:9
......@@ -116,14 +118,14 @@ rpm_ingest:
script:
- 'cd ingest-rest/rpm'
- './build.sh'
- 'for f in `find RPMS -type f`; do curl -X POST -H "Authorization: token $BUILD_TOKEN" --data-binary @$f $SERVER/artifacts/chronopolis/$CI_BUILD_REF_NAME/$(echo $CI_BUILD_REF | cut -c -7)/$(basename $f); done'
- 'for f in `find RPMS -type f`; do curl -X POST -H "Authorization: $BUILD_TOKEN" -H "Content-Type: application/octet-stream" --data-binary @$f $SERVER/resource/chronopolis/$CI_BUILD_REF_NAME/$CI_BUILD_REF/$(basename $f); done'
rpm_replication:
<<: *rpm
script:
- 'cd replication-shell/rpm'
- './build.sh'
- 'for f in `find RPMS -type f`; do curl -X POST -H "Authorization: token $BUILD_TOKEN" --data-binary @$f $SERVER/artifacts/chronopolis/$CI_BUILD_REF_NAME/$(echo $CI_BUILD_REF | cut -c -7)/$(basename $f); done'
- 'for f in `find RPMS -type f`; do curl -X POST -H "Authorization: $BUILD_TOKEN" -H "Content-Type: application/octet-stream" --data-binary @$f $SERVER/resource/chronopolis/$CI_BUILD_REF_NAME/$CI_BUILD_REF/$(basename $f); done'
deploy_tokenizer:
stage: deploy
......@@ -133,5 +135,5 @@ deploy_tokenizer:
script:
- 'cd tokenizer/deploy'
- './build.sh'
- 'for f in `find TARS -type f`; do curl -X POST -H "Authorization: token $BUILD_TOKEN" --data-binary @$f $SERVER/artifacts/chronopolis/$CI_BUILD_REF_NAME/$(echo $CI_BUILD_REF | cut -c -7)/$(basename $f); done'
- 'for f in `find TARS -type f`; do curl -X POST -H "Authorization: $BUILD_TOKEN" -H "Content-Type: application/octet-stream" --data-binary @$f $SERVER/resource/chronopolis/$CI_BUILD_REF_NAME/$CI_BUILD_REF/$(basename $f); done'
image: maven:3.3.9-jdk-8
......@@ -5,11 +5,11 @@
<parent>
<groupId>org.chronopolis</groupId>
<artifactId>chronopolis</artifactId>
<version>3.0.0-SNAPSHOT</version>
<version>3.0.0-RC8</version>
</parent>
<artifactId>chron-common</artifactId>
<version>3.0.0-SNAPSHOT</version>
<version>3.0.0-RC8</version>
<name>Common</name>
<url>http://maven.apache.org</url>
<properties>
......
......@@ -7,11 +7,11 @@
<parent>
<groupId>org.chronopolis</groupId>
<artifactId>chronopolis</artifactId>
<version>3.0.0-SNAPSHOT</version>
<version>3.0.0-RC8</version>
</parent>
<artifactId>ingest-rest</artifactId>
<version>3.0.0-SNAPSHOT</version>
<version>3.0.0-RC8</version>
<build>
<plugins>
......
package org.chronopolis.ingest;
import org.chronopolis.common.storage.Posix;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.validation.Errors;
import org.springframework.validation.Validator;
import java.io.File;
import java.nio.file.Path;
import java.nio.file.Paths;
/**
* Ingest specific configuration properties
*
* @author shake
*/
@ConfigurationProperties(prefix = "ingest")
public class IngestProperties {
public class IngestProperties implements Validator {
private Ajp ajp = new Ajp();
private Integer fileIngestBatchSize = 1000;
private Tokenizer tokenizer = new Tokenizer();
public Tokenizer getTokenizer() {
return tokenizer;
}
public IngestProperties setTokenizer(Tokenizer tokenizer) {
this.tokenizer = tokenizer;
return this;
}
public Ajp getAjp() {
return ajp;
}
......@@ -30,6 +50,69 @@ public class IngestProperties {
return this;
}
@Override
public boolean supports(Class<?> clazz) {
return IngestProperties.class == clazz;
}
@Override
public void validate(Object target, Errors errors) {
IngestProperties properties = (IngestProperties) target;
Tokenizer tokenizerProps = properties.getTokenizer();
if (tokenizerProps.enabled) {
final String key = "ingest.tokenizer.staging.path";
Path path = Paths.get(tokenizerProps.staging.getPath());
File asFile = path.toFile();
if (asFile == null) {
errors.reject(key, "Path does not exist");
} else if (!asFile.isDirectory()) {
errors.reject(key, "Path is not a directory");
} else if (!asFile.canRead() || !asFile.canExecute()) {
errors.reject(key, "Cannot read/execute on given path");
}
if (tokenizerProps.staging.getId() <= 0) {
errors.reject("ingest.tokenizer.staging.id", "Invalid id for StorageRegion");
}
}
}
public static class Tokenizer {
private Boolean enabled = true;
private String username = "admin";
private Posix staging = new Posix();
public String getUsername() {
return username;
}
public Tokenizer setUsername(String username) {
this.username = username;
return this;
}
public Boolean getEnabled() {
return enabled;
}
public Tokenizer setEnabled(Boolean enabled) {
this.enabled = enabled;
return this;
}
public Posix getStaging() {
return staging;
}
public Tokenizer setStaging(Posix staging) {
this.staging = staging;
return this;
}
}
/**
* AJP Connector configuration
*/
......
......@@ -3,8 +3,6 @@ package org.chronopolis.ingest.config;
import com.google.common.collect.ImmutableList;
import org.chronopolis.common.ace.AceConfiguration;
import org.chronopolis.common.concurrent.TrackingThreadPoolExecutor;
import org.chronopolis.common.storage.BagStagingProperties;
import org.chronopolis.common.storage.BagStagingPropertiesValidator;
import org.chronopolis.ingest.repository.dao.PagedDao;
import org.chronopolis.ingest.tokens.DatabasePredicate;
import org.chronopolis.ingest.tokens.IngestTokenRegistrar;
......@@ -14,11 +12,10 @@ import org.chronopolis.tokenize.batch.ChronopolisTokenRequestBatch;
import org.chronopolis.tokenize.filter.ProcessingFilter;
import org.chronopolis.tokenize.supervisor.DefaultSupervisor;
import org.chronopolis.tokenize.supervisor.TokenWorkSupervisor;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Profile;
import org.springframework.validation.Validator;
import java.util.Collection;
import java.util.concurrent.ExecutorService;
......@@ -33,8 +30,8 @@ import java.util.function.Predicate;
* @author shake
*/
@Configuration
@Profile("!disable-tokenizer")
@EnableConfigurationProperties({AceConfiguration.class, BagStagingProperties.class})
@EnableConfigurationProperties(AceConfiguration.class)
@ConditionalOnProperty(prefix = "ingest", name = "tokenizer.enabled", havingValue = "true")
public class TokenizeConfig {
@Bean
......@@ -72,11 +69,4 @@ public class TokenizeConfig {
service.submit(registrar);
return service;
}
@Bean
public static Validator configurationPropertiesValidator() {
return new BagStagingPropertiesValidator();
}
}
package org.chronopolis.ingest.controller;
import com.querydsl.core.QueryModifiers;
import com.querydsl.jpa.impl.JPAQueryFactory;
import org.chronopolis.ingest.IngestController;
import org.chronopolis.ingest.PageWrapper;
......@@ -13,10 +14,13 @@ import org.chronopolis.ingest.repository.dao.StagingDao;
import org.chronopolis.ingest.support.BagCreateResult;
import org.chronopolis.ingest.support.FileSizeFormatter;
import org.chronopolis.ingest.support.ReplicationCreateResult;
import org.chronopolis.ingest.tokens.TokenWriter;
import org.chronopolis.rest.entities.AceToken;
import org.chronopolis.rest.entities.Bag;
import org.chronopolis.rest.entities.Node;
import org.chronopolis.rest.entities.QAceToken;
import org.chronopolis.rest.entities.QBag;
import org.chronopolis.rest.entities.QBagFile;
import org.chronopolis.rest.entities.QNode;
import org.chronopolis.rest.entities.QReplication;
import org.chronopolis.rest.entities.Replication;
......@@ -38,8 +42,10 @@ import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
import javax.validation.Valid;
import java.nio.charset.Charset;
import java.security.Principal;
import java.util.ArrayList;
import java.util.Arrays;
......@@ -178,6 +184,56 @@ public class BagUIController extends IngestController {
.orElse("redirect:/bags/add");
}
@GetMapping("/bags/{id}/download/files")
public StreamingResponseBody getFiles(@PathVariable("id") Long id) {
JPAQueryFactory queryFactory = dao.getJPAQueryFactory();
List<String> fetch = queryFactory.select(QBagFile.bagFile.filename)
.from(QBagFile.bagFile)
.where(QBagFile.bagFile.bag.id.eq(id)).fetch();
return outputStream -> {
for (String filename : fetch) {
outputStream.write(filename.getBytes(Charset.defaultCharset())); // force UTF-8?
outputStream.write("\n".getBytes(Charset.defaultCharset()));
}
};
}
@GetMapping("/bags/{id}/download/tokens")
public StreamingResponseBody getBagTokens(@PathVariable("id") Long id) {
return outputStream -> {
// copy paste
long offset;
long page = 0;
long limit = 1000;
boolean next = true;
TokenWriter writer = new TokenWriter(outputStream);
while (next) {
offset = page * limit;
QueryModifiers queryModifiers = new QueryModifiers(limit, offset);
List<AceToken> tokens = dao.findAll(QAceToken.aceToken,
QAceToken.aceToken.bag.id.eq(id),
QAceToken.aceToken.id.asc(),
queryModifiers);
for (AceToken token : tokens) {
String tokenFilename = token.getFile().getFilename();
writer.startToken(token);
writer.addIdentifier(tokenFilename.startsWith("/")
? tokenFilename
: "/" + tokenFilename);
writer.writeTokenEntry();
}
next = tokens.size() == limit;
if (next) {
++page;
}
}
writer.close();
};
}
//
// Replication stuff
//
......
package org.chronopolis.ingest.controller;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSortedSet;
import com.google.i18n.phonenumbers.PhoneNumberUtil;
import com.querydsl.core.Tuple;
import com.querydsl.core.types.ConstructorExpression;
......@@ -50,7 +49,8 @@ import java.security.Principal;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.stream.Collectors;
/**
* Controller to handle Depositor requests and things
......@@ -400,9 +400,11 @@ public class DepositorUIController extends IngestController {
private Map<String, ?> addContactAttributes(DepositorContactCreate depositorContactCreate) {
PhoneNumberUtil util = PhoneNumberUtil.getInstance();
Set<String> regions = util.getSupportedRegions();
ImmutableSortedSet<String> supportedRegions = ImmutableSortedSet.copyOf(regions);
return ImmutableMap.of("regions", supportedRegions,
Map<String, Integer> regions = util.getSupportedRegions().stream()
.collect(Collectors.toMap(region -> region, util::getCountryCodeForRegion,
(v1, v2) -> {throw new RuntimeException("Duplicate value " + v2);},
TreeMap::new));
return ImmutableMap.of("regions", regions,
"depositorContactCreate", depositorContactCreate);
}
......
......@@ -2,7 +2,7 @@ package org.chronopolis.ingest.controller;
import org.chronopolis.tokenize.supervisor.DefaultSupervisor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Profile;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.GetMapping;
......@@ -14,7 +14,7 @@ import org.springframework.web.bind.annotation.GetMapping;
* @author shake
*/
@Controller
@Profile("!disable-tokenizer")
@ConditionalOnProperty(prefix = "ingest", name = "tokenizer.enabled", havingValue = "true")
public class TokenizerStatusController {
private final DefaultSupervisor supervisor;
......
......@@ -126,17 +126,31 @@ public class BagDao extends PagedDao {
}
}
/**
* Retrieve a List of {@link PartialBag} views which will not further query the database for
* related tables
*
* @param filter the {@link BagFilter} containing the query parameters
* @return the result of the database query
*/
public List<PartialBag> partialViews(BagFilter filter) {
QBag bag = QBag.bag;
return partialQuery(filter)
.transform(GroupBy.groupBy(bag.id).list(partialProjection()));
}
/**
* Retrieve a {@link Page} of {@link PartialBag} views
*
* @param filter the {@link BagFilter} containing the query parameters
* @return the result of the database query
*/
public Page<PartialBag> findViewAsPage(BagFilter filter) {
return PageableExecutionUtils.getPage(partialViews(filter),
// need to pass this in order to fetch the count correctly
JPAQuery<Bag> count = getJPAQueryFactory().selectFrom(QBag.bag).where(filter.getQuery());
return PageableExecutionUtils.getPage(
partialViews(filter),
filter.createPageRequest(),
partialQuery(filter)::fetchCount);
count::fetchCount);
}
private JPAQuery<?> partialQuery(BagFilter filter) {
......@@ -154,6 +168,13 @@ public class BagDao extends PagedDao {
.restrict(filter.getRestriction());
}
/**
* Retrieve a single {@link Bag} from the database projected onto a {@link CompleteBag}. This
* will create a view which maps to the API model.
*
* @param id the id of the {@link Bag} to query
* @return the {@link CompleteBag} query projection
*/
public CompleteBag findCompleteView(Long id) {
QBag bag = QBag.bag;
QNode node = new QNode(DISTRIBUTION_IDENTIFIER);
......@@ -176,7 +197,7 @@ public class BagDao extends PagedDao {
// push to function ?
.leftJoin(bag.storage, storage)
.on(storage.active.isTrue())
.innerJoin(storage.file, dataFile)
.leftJoin(storage.file, dataFile)
.where(bag.id.eq(id))
.transform(GroupBy.groupBy(bag.id)
......
......@@ -225,6 +225,12 @@ public class PagedDao {
return new JPAQueryFactory(em);
}
/**
* Constructor for a {@link ReplicationView} for use in Query Projections
*
* @return the {@link ConstructorExpression} mapping to a {@link ReplicationView}
*/
@SuppressWarnings("WeakerAccess")
public ConstructorExpression<ReplicationView> replicationProjection() {
QReplication replication = QReplication.replication;
QNode node = QNode.node;
......@@ -243,10 +249,17 @@ public class PagedDao {
);
}
/**
* Constructor for a {@link CompleteBag} for use in Query Projections
*
* @return the {@link ConstructorExpression} mapping to a {@link CompleteBag}
*/
@SuppressWarnings("WeakerAccess")
public ConstructorExpression<CompleteBag> completeProjection() {
QBag bag = QBag.bag;
QNode node = new QNode(DISTRIBUTION_IDENTIFIER);
QDataFile file = QDataFile.dataFile;
QDepositor depositor = QDepositor.depositor;
QNode node = new QNode(DISTRIBUTION_IDENTIFIER);
return Projections.constructor(CompleteBag.class,
bag.id,
bag.name,
......@@ -259,9 +272,15 @@ public class PagedDao {
depositor.namespace,
GroupBy.set(node.username),
// maybe move to Map<Dtype,FullStaging>
GroupBy.set(stagingProjection()));
GroupBy.map(file.dtype, stagingProjection()));
}
/**
* Constructor for a {@link PartialBag} for use in Query Projections
*
* @return the {@link ConstructorExpression} mapping to a {@link PartialBag}
*/
@SuppressWarnings("WeakerAccess")
public ConstructorExpression<PartialBag> partialProjection() {
QBag bag = QBag.bag;
QNode node = new QNode(DISTRIBUTION_IDENTIFIER);
......@@ -279,16 +298,22 @@ public class PagedDao {
GroupBy.set(node.username));
}
/**
* Constructor for a {@link StagingView} for use in Query Projections
*
* @return the {@link ConstructorExpression} mapping to a {@link StagingView}
*/
@SuppressWarnings("WeakerAccess")
public ConstructorExpression<StagingView> stagingProjection() {
QDataFile file = QDataFile.dataFile;
QStagingStorage storage = QStagingStorage.stagingStorage;
return Projections.constructor(StagingView.class,
storage.id,
storage.path,
file.dtype,
storage.region.id,
storage.active,
storage.totalFiles);
storage.id.coalesce(-1L),
storage.path.coalesce(""),
file.dtype.coalesce(""),
storage.region.id.coalesce(-1L),
storage.active.coalesce(false),
storage.totalFiles.coalesce(0L));
}
}
......@@ -204,7 +204,9 @@ public class ReplicationDao extends PagedDao {
JPAQueryFactory factory = getJPAQueryFactory();
JPAQuery<StagingStorage> query = factory.from(b)
.innerJoin(b.storage, storage)
.where(storage.active.isTrue().and(storage.file.dtype.eq(discriminator)))
.where(storage.bag.id.eq(bagId)
.and(storage.active.isTrue()
.and(storage.file.dtype.eq(discriminator))))
.select(storage);
return Optional.ofNullable(query.fetchFirst());
......@@ -256,6 +258,12 @@ public class ReplicationDao extends PagedDao {
":" + file.toString() + (trailingSlash ? "/" : "");
}
/**
* Query the database for a {@link Replication} projected on to a {@link ReplicationView}
*
* @param id the id of the {@link Replication}
* @return the {@link ReplicationView} projection
*/
public ReplicationView findReplicationAsView(Long id) {
QReplication replication = QReplication.replication;
// not a fan of transforming this into a map then getting the id but.. not sure how to
......@@ -266,16 +274,26 @@ public class ReplicationDao extends PagedDao {
.get(id);
}
/**
* Query the database for a set of {@link Replication}s and project the results on to
* {@link ReplicationView}s.
*
* @param filter the {@link ReplicationFilter} containing the query parameters
* @return a {@link Page} containing the results of the query
*/
public Page<ReplicationView> findViewsAsPage(ReplicationFilter filter) {
QReplication replication = QReplication.replication;
JPAQuery<?> query = createViewQuery()
.where(filter.getQuery())
.orderBy(filter.getOrderSpecifier())
.restrict(filter.getRestriction());
JPAQuery<Replication> count = getJPAQueryFactory()
.selectFrom(replication)
.where(filter.getQuery());
return PageableExecutionUtils.getPage(
query.transform(GroupBy.groupBy(replication.id).list(replicationProjection())),
filter.createPageRequest(),
query::fetchCount);
count::fetchCount);
}
private JPAQuery<?> createViewQuery() {
......@@ -296,6 +314,6 @@ public class ReplicationDao extends PagedDao {
.leftJoin(distribution.node, distributionNode)
.leftJoin(bag.storage, staging)
.on(staging.active.isTrue())
.innerJoin(staging.file, file);
.leftJoin(staging.file, file);
}
}
package org.chronopolis.ingest.support;
import com.querydsl.jpa.impl.JPAQueryFactory;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
......@@ -9,7 +10,7 @@ import org.chronopolis.rest.csv.BagFileHeaders;
import org.chronopolis.rest.entities.Bag;