Commit fe0c2a08 authored by Mat's avatar Mat

Merge branch 'feature/dockerbuild' into develop

parents a3b62d31 b2b53879
......@@ -34,6 +34,7 @@ README.html
.classpath
*/.project
*/spring-*/src/main/java/META-INF/MANIFEST.MF
.metadata/
# IDEA artifacts and output dirs
*/*.iml
......
image: ubuntu:bionic
stages:
- build
- test
- package
- docker_tag
build:
stage: build
before_script:
- apt-get update
- apt-get install -y openjdk-11-jdk maven
script:
- mvn clean install -DskipTests
test:
stage: test
dependencies:
- build
before_script:
- apt-get update
- apt-get install -y openjdk-11-jdk maven
script:
- mvn test
build:docker:builder:
stage: build
image: docker:stable
services:
- docker:dind
variables:
DOCKER_HOST: tcp://docker:2375
DOCKER_DRIVER: overlay2
script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- docker build -t $CI_REGISTRY_IMAGE/searchitect_builder:$CI_COMMIT_SHORT_SHA .
- docker push $CI_REGISTRY_IMAGE/searchitect_builder:$CI_COMMIT_SHORT_SHA
build:docker:images:
stage: package
dependencies:
- build:docker:builder
- test
image: docker:stable
services:
- docker:dind
variables:
DOCKER_HOST: tcp://docker:2375
DOCKER_DRIVER: overlay2
script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- docker pull $CI_REGISTRY_IMAGE/searchitect_builder:$CI_COMMIT_SHORT_SHA
- docker tag $CI_REGISTRY_IMAGE/searchitect_builder:$CI_COMMIT_SHORT_SHA searchitect_builder
- docker build -t $CI_REGISTRY_IMAGE/searchitect-gate:$CI_COMMIT_REF_SLUG searchitect-gate
- docker build -t $CI_REGISTRY_IMAGE/searchitect-backend-dynrh2lev:$CI_COMMIT_REF_SLUG searchitect-backend-dynrh2lev
- docker build -t $CI_REGISTRY_IMAGE/searchitect-backend-dynrh2levrocks:$CI_COMMIT_REF_SLUG searchitect-backend-dynrh2levrocks
- docker build -t $CI_REGISTRY_IMAGE/searchitect-backend-sophos:$CI_COMMIT_REF_SLUG searchitect-backend-sophos
- docker build -t $CI_REGISTRY_IMAGE/searchitect-backend-template:$CI_COMMIT_REF_SLUG searchitect-backend-template
- docker push $CI_REGISTRY_IMAGE/searchitect-gate:$CI_COMMIT_REF_SLUG
- docker push $CI_REGISTRY_IMAGE/searchitect-backend-dynrh2lev:$CI_COMMIT_REF_SLUG
- docker push $CI_REGISTRY_IMAGE/searchitect-backend-dynrh2levrocks:$CI_COMMIT_REF_SLUG
- docker push $CI_REGISTRY_IMAGE/searchitect-backend-sophos:$CI_COMMIT_REF_SLUG
- docker push $CI_REGISTRY_IMAGE/searchitect-backend-template:$CI_COMMIT_REF_SLUG
push_docker_tag:
stage: docker_tag
image: docker:stable
services:
- docker:dind
variables:
DOCKER_HOST: tcp://docker:2375
DOCKER_DRIVER: overlay2
dependencies:
- build:docker:images
script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- docker pull $CI_REGISTRY_IMAGE/searchitect-gate:$CI_COMMIT_REF_SLUG
- docker pull $CI_REGISTRY_IMAGE/searchitect-backend-dynrh2lev:$CI_COMMIT_REF_SLUG
- docker pull $CI_REGISTRY_IMAGE/searchitect-backend-dynrh2levrocks:$CI_COMMIT_REF_SLUG
- docker pull $CI_REGISTRY_IMAGE/searchitect-backend-sophos:$CI_COMMIT_REF_SLUG
- docker pull $CI_REGISTRY_IMAGE/searchitect-backend-template:$CI_COMMIT_REF_SLUG
- docker tag $CI_REGISTRY_IMAGE/searchitect-gate:$CI_COMMIT_REF_SLUG $CI_REGISTRY_IMAGE/searchitect-gate:$CI_COMMIT_TAG
- docker tag $CI_REGISTRY_IMAGE/searchitect-backend-dynrh2lev:$CI_COMMIT_REF_SLUG $CI_REGISTRY_IMAGE/searchitect-backend-dynrh2lev:$CI_COMMIT_TAG
- docker tag $CI_REGISTRY_IMAGE/searchitect-backend-dynrh2levrocks:$CI_COMMIT_REF_SLUG $CI_REGISTRY_IMAGE/searchitect-backend-dynrh2levrocks:$CI_COMMIT_TAG
- docker tag $CI_REGISTRY_IMAGE/searchitect-backend-sophos:$CI_COMMIT_REF_SLUG $CI_REGISTRY_IMAGE/searchitect-backend-sophos:$CI_COMMIT_TAG
- docker tag $CI_REGISTRY_IMAGE/searchitect-backend-template:$CI_COMMIT_REF_SLUG $CI_REGISTRY_IMAGE/searchitect-backend-template:$CI_COMMIT_TAG
- docker push $CI_REGISTRY_IMAGE/searchitect-gate:$CI_COMMIT_TAG
- docker push $CI_REGISTRY_IMAGE/searchitect-backend-dynrh2lev:$CI_COMMIT_TAG
- docker push $CI_REGISTRY_IMAGE/searchitect-backend-dynrh2levrocks:$CI_COMMIT_TAG
- docker push $CI_REGISTRY_IMAGE/searchitect-backend-sophos:$CI_COMMIT_TAG
- docker push $CI_REGISTRY_IMAGE/searchitect-backend-template:$CI_COMMIT_TAG
only:
- tags
FROM maven:3.6-jdk-11
RUN useradd -u 10000 -m builduser
RUN mkdir /home/builduser/.m2
RUN chown -R builduser /home/builduser/.m2
USER builduser
WORKDIR /home/builduser/src/
# Load the dependencies first (to cache them for future builds)
copy --chown=builduser ./pom.xml ./
copy --chown=builduser ./searchitect-backend-dynrh2lev/pom.xml ./searchitect-backend-dynrh2lev/pom.xml
copy --chown=builduser ./searchitect-backend-dynrh2levrocks/pom.xml ./searchitect-backend-dynrh2levrocks/pom.xml
copy --chown=builduser ./searchitect-backend-sophos/pom.xml ./searchitect-backend-sophos/pom.xml
copy --chown=builduser ./searchitect-backend-template/pom.xml ./searchitect-backend-template/pom.xml
copy --chown=builduser ./searchitect-client-dynrh2lev-plugin/pom.xml ./searchitect-client-dynrh2lev-plugin/pom.xml
copy --chown=builduser ./searchitect-client-dynrh2levrocks-plugin/pom.xml ./searchitect-client-dynrh2levrocks-plugin/pom.xml
copy --chown=builduser ./searchitect-client-sophos-plugin/pom.xml ./searchitect-client-sophos-plugin/pom.xml
copy --chown=builduser ./searchitect-client/pom.xml ./searchitect-client/pom.xml
copy --chown=builduser ./searchitect-common-dynrh2lev/pom.xml ./searchitect-common-dynrh2lev/pom.xml
copy --chown=builduser ./searchitect-common-sophos/pom.xml ./searchitect-common-sophos/pom.xml
copy --chown=builduser ./searchitect-common/pom.xml ./searchitect-common/pom.xml
copy --chown=builduser ./searchitect-gate/pom.xml ./searchitect-gate/pom.xml
copy --chown=builduser ./searchitect-test/pom.xml ./searchitect-test/pom.xml
copy --chown=builduser ./searchitect-testset/pom.xml ./searchitect-testset/pom.xml
# Dependency resolution will fail, because the searchitects projects ares searched but not built. '--fail-never' forces docker to continue.
RUN mvn dependency:go-offline --fail-never
# Do the actual build
COPY --chown=builduser . .
RUN mvn -e clean install -DskipTests
CMD ["echo", "Just a builder image. Nothing to see here."]
# Searchitect-Searchable Encryption Framework
## Description of the Searchitect Framework:
#### Description of the Searchitect Framework:
This framework enables the integration of SE schemes.
#### General description of SE
### General description of SE
A SE scheme enables a server to search over an encrypted database on behalf of a client without revealing the content to the server.
A SE scheme provides 3 protocols:
1. Setup - First the client is indexing a document collection contained in a directory. This plaintext index gets encrypted by a specific implementation of an encryption scheme and uploaded to the server.
1. Setup - First the client is indexing a document collection contained in a directory. This plaintext index gets encrypted by a specific implementation of an encryption scheme and uploaded to the server.
2. Search - After Setup the client is able to search over the data by passing the keyword to the search protocol, which computes a search token which is sent to the server. This search Token enables the server to search over the encrypted data and return the resulting document matches. In resource hiding schemes these are encrypted and therfore a second Resolve procedure at the client is needed to decrypt document identifiers.
3. Update - Dynamic schemes support a update of the documents contained in the encrypted index.
#### Framework Architecture:
### Framework Architecture:
* Client/server architecture based on microservices
* SOA (service oriented architecture) based on RESTful webservices
#### Implementation:
## Implementation:
* Basic implementations
* searchitect-common - classes shared between server and client
* searchitect-client - common client implementation
......@@ -27,23 +30,48 @@ A SE scheme provides 3 protocols:
* searchitect-client-scheme-plugin
* searchitect-backend-scheme
#### The interface description of the gateway is after deployment available at:
### The interface description of the gateway is after deployment available at:
https://localhost:8433/swagger-ui.html
### Compile
## Compile
Building this software has been tested on Ubuntu 18.04 using openjdk-11.
Run the following command in the top level directory
mvn clean install
### General deployment using Docker-compose
Docker enables a containerized easy deployment, the docker-compose configuration file is called docker-compose.yml.
## Deployment
### Build Docker containers
Docker containers are built using the [multistage feature](https://docs.docker.com/develop/develop-images/multistage-build/). There is a builder image in the root directory and deployment images in the subprojects. A *docker-compose* file ties it all together.
#### Build using docker-compose
docker-compose build
#### Manual build
docker build -t searchitect_builder .
docker build searchitect-backend-dynrh2lev
...
### Deploy using docker-compose
After you have built the docker containers using the commands above, you can run them on your host using
docker-compose up
### How to add a new scheme
or
docker-compose up -d
if you want to move the docker process to the background.
## How to add a new scheme
1. Implement your scheme in a new searchitect-common-scheme project
2. Create a new searchitect-client-scheme-plugin project which implements the client plugin interface. This interface can be found in searchitect.common.client.ClientScheme
3. Create a new project which implements the searchitect-backend-scheme at the server side, take a look to the other implementations the interface of the controller needs to be similar
......
version: "2"
version: "3.4"
services:
# important note: servicename MUST match backend name b_id of request urls
builder:
build:
context: ./
# important note: servicename MUST match backend name b_id of request urls
template:
build: ./searchitect-backend-template
ports:
- "8383:8383"
build:
target: backend-template
context: ./searchitect-backend-template
depends_on:
- builder
restart: always
ports:
- "8383:8383"
sophos:
build: ./searchitect-backend-sophos
build:
target: backend-sophos
context: ./searchitect-backend-sophos
depends_on:
- builder
restart: always
ports:
- "8484:8484"
dynrh2lev:
build: ./searchitect-backend-dynrh2lev
build:
context: ./searchitect-backend-dynrh2lev
target: backend-dynrh2lev
depends_on:
- builder
restart: always
ports:
- "8282:8282"
dynrh2levrocks:
build: ./searchitect-backend-dynrh2levrocks
ports:
- "8585:8585"
build:
context: ./searchitect-backend-dynrh2levrocks
target: backend-dynrh2levrocks
depends_on:
- builder
restart: always
ports:
- "8585:8585"
searchitect-gate:
build: ./searchitect-gate
gate:
build:
target: gate
context: ./searchitect-gate
depends_on:
- builder
restart: always
ports:
- "8433:8433"
FROM openjdk:10
VOLUME /tmp
COPY ./target/searchitect-backend-dynrh2lev-0.1.0.jar /tmp/searchitect-backend-dynrh2lev-0.1.0.jar
FROM searchitect_builder as builder
FROM openjdk:10-jre-slim as backend-dynrh2lev
RUN useradd -u 10000 -m searchitect
USER searchitect
COPY --from=builder /home/builduser/src/searchitect-backend-dynrh2lev/target/searchitect-backend-dynrh2lev-0.1.0.jar /tmp/searchitect-backend-dynrh2lev-0.1.0.jar
CMD ["java", "-jar","/tmp/searchitect-backend-dynrh2lev-0.1.0.jar"]
......@@ -11,7 +11,7 @@
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.0.4.RELEASE</version>
<version>2.1.6.RELEASE</version>
</parent>
<dependencies>
......@@ -69,7 +69,7 @@
</dependencies>
<properties>
<java.version>10</java.version>
<java.version>11</java.version>
<docker.image.prefix>dockerfile</docker.image.prefix>
</properties>
......@@ -105,7 +105,7 @@
<artifactId>maven-compiler-plugin</artifactId>
<version>3.7.0</version>
<configuration>
<release>${java.version}</release>
<release>10</release>
</configuration>
<dependencies>
<dependency>
......
......@@ -52,8 +52,8 @@ public class IndexImpl {
private HashMap<String, byte[]> updateDictionary;
@Lob
@Column(name = "array", columnDefinition = "BLOB")
byte[][] array;
@Column(name = "sarray", columnDefinition = "BLOB")
byte[][] sarray;
protected IndexImpl() { // jpa only
}
......@@ -86,14 +86,14 @@ public class IndexImpl {
* @return the array
*/
public byte[][] getArray() {
return array;
return sarray;
}
/**
* @param array the array to set
*/
public void setArray(byte[][] array) {
this.array = array;
public void setArray(byte[][] sarray) {
this.sarray = sarray;
}
/**
......@@ -109,7 +109,7 @@ public class IndexImpl {
throws JsonParseException, JsonMappingException, JsonProcessingException, IOException {
// test if index is valid multimap
this.dictionary = uploadIndex.getDictionary();
this.array = uploadIndex.getArray();
this.sarray = uploadIndex.getArray();
this.updateDictionary = new HashMap<String, byte[]>();
this.repositoryName = UUID.randomUUID().toString();
}
......@@ -151,10 +151,10 @@ public class IndexImpl {
NoSuchProviderException, NoSuchPaddingException, IOException, NullPointerException {
List <String> resultList;
if(updateDictionary.size()==0){
resultList = DynRH2LevModifiedMap.query(token.getSearchToken(), dictionary, array, updateDictionary);
resultList = DynRH2LevModifiedMap.query(token.getSearchToken(), dictionary, sarray, updateDictionary);
}
else{
resultList = DynRH2LevModifiedMap.queryFS(token.getSearchToken(), dictionary, array, updateDictionary);
resultList = DynRH2LevModifiedMap.queryFS(token.getSearchToken(), dictionary, sarray, updateDictionary);
}
return new SearchResult(resultList);
......
......@@ -54,7 +54,7 @@ public static void cleanup() throws IOException{
@Test
public void IndexImplConstructorSuccessTest()throws Exception {
IndexImpl index = new IndexImpl(new UploadIndexdynrh2levMap(TestUtil.dic, TestUtil.getArray()));
assertArrayEquals(index.array[0], "test1".getBytes());
assertArrayEquals(index.sarray[0], "test1".getBytes());
assertFalse(index.getRepositoryName().isEmpty());
}
......
FROM openjdk:10
VOLUME /tmp
COPY ./target/searchitect-backend-dynrh2levrocks-0.1.0.jar /tmp/searchitect-backend-dynrh2levrocks-0.1.0.jar
FROM searchitect_builder as builder
FROM openjdk:10-jre-slim as backend-dynrh2levrocks
RUN useradd -u 10000 -m searchitect
USER searchitect
COPY --from=builder /home/builduser/src/searchitect-backend-dynrh2levrocks/target/searchitect-backend-dynrh2levrocks-0.1.0.jar /tmp/searchitect-backend-dynrh2levrocks-0.1.0.jar
CMD ["java", "-jar","/tmp/searchitect-backend-dynrh2levrocks-0.1.0.jar"]
......@@ -11,7 +11,7 @@
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.0.6.RELEASE</version>
<version>2.1.6.RELEASE</version>
</parent>
<dependencies>
......@@ -68,7 +68,7 @@
</dependency>
</dependencies>
<properties>
<java.version>10</java.version>
<java.version>11</java.version>
<docker.image.prefix>dockerfile</docker.image.prefix>
</properties>
<build>
......@@ -101,7 +101,7 @@
<artifactId>maven-compiler-plugin</artifactId>
<version>3.7.0</version>
<configuration>
<release>${java.version}</release>
<release>10</release>
</configuration>
<dependencies>
<dependency>
......
FROM openjdk:10
VOLUME /tmp
COPY ./target/searchitect-backend-sophos-0.1.0.jar /tmp/searchitect-backend-sophos-0.1.0.jar
FROM searchitect_builder as builder
FROM openjdk:10-jre-slim as backend-sophos
RUN useradd -u 10000 -m searchitect
USER searchitect
COPY --from=builder /home/builduser/src/searchitect-backend-sophos/target/searchitect-backend-sophos-0.1.0.jar /tmp/searchitect-backend-sophos-0.1.0.jar
CMD ["java", "-jar","/tmp/searchitect-backend-sophos-0.1.0.jar"]
......@@ -10,7 +10,7 @@
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.0.4.RELEASE</version>
<version>2.1.6.RELEASE</version>
</parent>
<dependencies>
......@@ -68,7 +68,7 @@
</dependencies>
<properties>
<java.version>10</java.version>
<java.version>11</java.version>
<docker.image.prefix>dockerfile</docker.image.prefix>
</properties>
......@@ -102,7 +102,7 @@
<artifactId>maven-compiler-plugin</artifactId>
<version>3.7.0</version>
<configuration>
<release>${java.version}</release>
<release>10</release>
</configuration>
<dependencies>
<dependency>
......
FROM openjdk:10
VOLUME /tmp
COPY ./target/searchitect-backend-template-0.1.0.jar /tmp/searchitect-backend-template-0.1.0.jar
CMD ["java", "-jar","/tmp/searchitect-backend-template-0.1.0.jar"]
FROM searchitect_builder as builder
FROM openjdk:10-jre-slim as backend-template
RUN useradd -u 10000 -m searchitect
USER searchitect
COPY --from=builder /home/builduser/src/searchitect-backend-template/target/searchitect-backend-template-0.1.0.jar /tmp/searchitect-backend-template-0.1.0.jar
CMD ["java", "-jar","/tmp/searchitect-backend-template-0.1.0.jar"]
\ No newline at end of file
......@@ -10,7 +10,7 @@
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.0.4.RELEASE</version>
<version>2.1.6.RELEASE</version>
</parent>
<dependencies>
......@@ -52,39 +52,43 @@
</dependencies>
<properties>
<java.version>10</java.version>
<java.version>11</java.version>
<docker.image.prefix>dockerfile</docker.image.prefix>
</properties>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
<plugin>
<groupId>com.spotify</groupId>
<artifactId>docker-maven-plugin</artifactId>
<version>0.4.13</version>
<configuration>
<imageName>${docker.image.prefix}/${project.artifactId}</imageName>
<resources>
<resource>
<targetPath>/</targetPath>
<directory>${project.build.directory}</directory>
<include>${project.build.finalName}.jar</include>
</resource>
<resource>
<directory>src/main/resources</directory>
<filtering>true</filtering>
</resource>
</resources>
</configuration>
</plugin>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
<plugin>
<groupId>com.spotify</groupId>
<artifactId>docker-maven-plugin</artifactId>
<version>0.4.13</version>
<configuration>
<imageName>${docker.image.prefix}/${project.artifactId}</imageName>
<resources>
<resource>
<targetPath>/</targetPath>
<directory>${project.build.directory}</directory>
<include>${project.build.finalName}.jar</include>
</resource>
<resource>
<directory>src/main/resources</directory>
<filtering>true</filtering>
</resource>
</resources>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<release>10</release>
</configuration>
</plugin>
</plugins>
<defaultGoal>install</defaultGoal>
<defaultGoal>install</defaultGoal>
</build>
</project>
FROM openjdk:8
VOLUME /tmp
RUN useradd -u 10000 -m searchitect
USER searchitect
COPY ./target/searchitect-backend-template-0.1.0.jar /tmp/searchitect-backend-template-0.1.0.jar
CMD ["java", "-jar","/tmp/searchitect-backend-template-0.1.0.jar"]
......@@ -11,12 +11,12 @@
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.0.4.RELEASE</version>
<version>2.1.6.RELEASE</version>
</parent>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<java.version>10</java.version>
<java.version>11</java.version>
</properties>
<dependencies>
......@@ -63,4 +63,15 @@
<version>2.9.6</version>
</dependency>
</dependencies>
</project>
\ No newline at end of file
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<release>10</release>
</configuration>
</plugin>
</plugins>
</build>