Commit 907eb35c authored by Ludan Stoecklé's avatar Ludan Stoecklé

1.7.0

parent 798818e2
......@@ -19,7 +19,7 @@ stages:
# variables etc.
variables:
ROSAENLG_VERSION: 1.6.2
ROSAENLG_VERSION: 1.7.0
DOCKER_REGISTRY: registry.gitlab.com/rosaenlg-projects/rosaenlg
DOCKER_CLI_ROOT: ${DOCKER_REGISTRY}/cli
DOCKER_SERVER_ROOT: ${DOCKER_REGISTRY}/server
......
{
"name": "browser-ide-demo",
"version": "1.6.2",
"version": "1.7.0",
"description": "Client side demo of NLG templates compilation and rendering using RosaeNLG",
"main": "index.js",
"scripts": {
......@@ -34,7 +34,7 @@
"gulp-inject-string": "^1.1.2",
"gulp-rename": "^1.4.0",
"merge-stream": "^2.0.0",
"rosaenlg": "1.6.2",
"rosaenlg": "1.7.0",
"tinyify": "^2.5.2",
"vue-codemirror": "^4.0.6"
}
......
{
"name": "content-deduplicate",
"version": "0.1.2",
"version": "0.2.0",
"description": "Calculates distances between texts to prevent duplicate content",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......@@ -54,6 +54,6 @@
},
"dependencies": {
"k-medoids": "^1.0.6",
"synonym-optimizer": "2.6.2"
"synonym-optimizer": "2.7.0"
}
}
{
"name": "english-determiners",
"version": "2.6.2",
"version": "2.7.0",
"description": "English determiners",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......
{
"name": "french-adjectives",
"version": "2.6.2",
"version": "2.7.0",
"description": "Agreement of adjectives in French",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......@@ -45,7 +45,7 @@
},
"dependencies": {
"debug": "^4.1.1",
"french-h-muet-aspire": "2.6.2"
"french-h-muet-aspire": "2.7.0"
},
"keywords": [
"french",
......
{
"name": "french-determiners",
"version": "2.6.2",
"version": "2.7.0",
"description": "French determiners",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......
{
"name": "french-h-muet-aspire",
"version": "2.6.2",
"version": "2.7.0",
"description": "List of French words that are \"aspiré\" (vs. \"muet\")\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[Dstarting with a h that is \"aspiré\" vs \"muet\"",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......
{
"name": "french-ordinals",
"version": "2.6.2",
"version": "2.7.0",
"description": "Ordinal numbers in French: 12 => douzième",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......
{
"name": "french-verbs",
"version": "2.6.2",
"version": "2.7.0",
"description": "French verbs: conjugation, based on the Lefff contents, and list of transitive and intransitive verbs",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......@@ -50,7 +50,7 @@
},
"dependencies": {
"debug": "^4.1.1",
"french-h-muet-aspire": "2.6.2"
"french-h-muet-aspire": "2.7.0"
},
"devDependencies": {
"copyfiles": "^2.1.1",
......
{
"name": "french-words-gender",
"version": "2.6.2",
"version": "2.7.0",
"description": "Gender of French words, based on the Lefff",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......
{
"name": "german-adjectives",
"version": "2.6.2",
"version": "2.7.0",
"description": "German adjectives, based on german-pos-dict",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......
{
"name": "german-determiners",
"version": "2.6.2",
"version": "2.7.0",
"description": "German determiners",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......
{
"name": "german-dict-helper",
"version": "2.6.2",
"version": "2.7.0",
"description": "Helper functions on an included database derived from german-pos-dict",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......
{
"name": "german-ordinals",
"version": "2.6.2",
"version": "2.7.0",
"description": "Ordinal numbers in German: 12 => zwölfte",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......
{
"name": "german-verbs",
"version": "2.6.2",
"version": "2.7.0",
"description": "German verbs, based on german-pos-dict",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......
{
"name": "german-words",
"version": "2.6.2",
"version": "2.7.0",
"description": "Gender and case of German words, based on german-pos-dict",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......
{
"name": "gulp-rosaenlg",
"version": "2.6.2",
"version": "2.7.0",
"description": "RosaeNLG client side packaging helper",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......@@ -60,6 +60,6 @@
"browserify": "^16.5.0",
"debug": "^4.1.1",
"minify-stream": "^1.2.1",
"rosaenlg": "1.6.2"
"rosaenlg": "1.7.0"
}
}
{
"name": "italian-adjectives",
"version": "2.6.2",
"version": "2.7.0",
"description": "Italian adjectives, based on morph-it",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......@@ -58,6 +58,6 @@
"license": "MIT",
"dependencies": {
"debug": "^4.1.1",
"rosaenlg-filter": "2.6.2"
"rosaenlg-filter": "2.7.0"
}
}
{
"name": "italian-determiners",
"version": "2.6.2",
"version": "2.7.0",
"description": "Italian determiners",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......
{
"name": "italian-ordinals-cardinals",
"version": "2.6.2",
"version": "2.7.0",
"description": "Ordinal and cardinal numbers in Italian",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......
{
"name": "italian-verbs",
"version": "2.6.2",
"version": "2.7.0",
"description": "Italian verbs, based on morph-it",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......
{
"name": "italian-words",
"version": "2.6.2",
"version": "2.7.0",
"description": "Gender and plural of Italian words, based on morph-it",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......
{
"name": "lefff-helper",
"version": "2.6.2",
"version": "2.7.0",
"description": "Helper functions on an included database derived from Lefff",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......
{
"name": "morph-it-helper",
"version": "2.6.2",
"version": "2.7.0",
"description": "Helper functions on an included database derived from morph-it (Free Morphological Lexicon for the Italian Language)",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......
{
"name": "rosaenlg-cli",
"version": "1.6.2",
"version": "1.7.0",
"description": "RosaeNLG's CLI interface",
"bin": {
"rosaenlg": "./index.js",
......@@ -10,9 +10,9 @@
"dependencies": {
"chalk": "^3.0.0",
"commander": "^4.0.1",
"gulp-rosaenlg": "2.6.2",
"gulp-rosaenlg": "2.7.0",
"mkdirp": "^0.5.1",
"rosaenlg": "1.6.2"
"rosaenlg": "1.7.0"
},
"devDependencies": {
"mocha": "^6.2.2",
......
{
"name": "rosaenlg-filter",
"version": "2.6.2",
"version": "2.7.0",
"description": "Filtering feature of RosaeNLG",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......@@ -43,7 +43,7 @@
"better-title-case": "^1.0.1",
"compromise": "^11.14.3",
"debug": "^4.1.1",
"french-h-muet-aspire": "2.6.2",
"french-h-muet-aspire": "2.7.0",
"titlecase-french": "^1.0.1"
},
"keywords": [
......
......@@ -25,7 +25,7 @@ curl -X PUT \
-H 'Accept-Encoding: gzip, deflate' \
-H 'Connection: keep-alive' \
-H 'Content-Type: application/json' \
-o creation.txt \
-o creation.json \
-d '{
"templateId": "chanson",
"entryTemplate": "chanson.pug",
......@@ -40,21 +40,24 @@ curl -X PUT \
}
'
CAT_CREATION="$(cat creation.txt)"
CAT_CREATION="$(cat creation.json)"
echo "TEST CREATION ON: $CAT_CREATION"
contains "$CAT_CREATION" "CREATED" || exit 1
contains "$CAT_CREATION" "templateSha1" || exit 1
echo "TEST CREATION: OK!"
# ok this is ugly
TEMPLATE_SHA1="$(cat creation.json | grep -Eo '"templateSha1":.*?[^\\]",' | awk -F':\"' '{print $2}' | awk -F'\"' '{print $1}')"
echo "SHA1: $TEMPLATE_SHA1"
# render the template
curl -X POST \
http://docker:5000/templates/chanson/render \
http://docker:5000/templates/chanson/$TEMPLATE_SHA1/render \
-H 'Accept: */*' \
-H 'Accept-Encoding: gzip, deflate' \
-H 'Connection: keep-alive' \
-H 'Content-Type: application/json' \
-o render.txt \
-o render.json \
-d '{
"language": "fr_FR",
"chanson": {
......@@ -64,7 +67,7 @@ curl -X POST \
}'
CAT_RENDER="$(cat render.txt)"
CAT_RENDER="$(cat render.json)"
echo "TEST RENDER ON: $CAT_RENDER"
contains "$CAT_RENDER" "Il chantera" || exit 1
echo "TEST RENDER: OK!"
......
{
"name": "rosaenlg-node-server",
"version": "1.6.2",
"version": "1.7.0",
"description": "Server for RosaeNLG, written in node.js",
"main": "dist/app.js",
"bin": {
......@@ -75,7 +75,7 @@
"gulp": "^4.0.2",
"gulp-awspublish": "^4.0.1",
"gulp-rename": "^1.4.0",
"gulp-rosaenlg": "2.6.2",
"gulp-rosaenlg": "2.7.0",
"json-refs": "^3.0.13",
"mocha": "^6.2.2",
"nyc": "^14.1.1",
......@@ -92,11 +92,12 @@
"body-parser": "^1.19.0",
"express": "^4.17.1",
"node-cache": "5.0.2",
"rosaenlg": "1.6.2",
"rosaenlg": "1.7.0",
"sha1": "^1.1.1",
"swagger-ui-express": "^4.1.2",
"uuid": "^3.3.3",
"winston": "^3.2.1",
"winston-aws-cloudwatch": "^3.0.0"
"winston-aws-cloudwatch": "^3.0.0",
"yn": "^4.0.0"
}
}
import NodeCache = require('node-cache');
import { RosaeContext } from './RosaeContext';
// import winston = require('winston');
interface CacheKey {
user: string;
templateId: string;
}
export interface CacheValue {
templateSha1: string;
rosaeContext: RosaeContext;
}
export class RosaeContextsManager {
private ttl: number;
private cacheCheckPeriod: number;
private forgetTemplates: boolean;
private rosaeContextsCache: NodeCache;
constructor(forgetTemplates: boolean, specificTtl: number, specificCheckPeriod: number) {
this.ttl = specificTtl || 600; // 10 minutes
this.cacheCheckPeriod = specificCheckPeriod || 60; // 1 minute
// winston.info(`params: ttl=${this.ttl}, cacheCheckPeriod=${this.cacheCheckPeriod}`);
this.forgetTemplates = forgetTemplates;
this.rosaeContextsCache = new NodeCache({
checkperiod: this.cacheCheckPeriod,
useClones: false,
deleteOnExpire: true,
});
/*
this.rosaeContextsCache.on('del', (key, value) => {
winston.info(`del on ${key}`);
});
*/
}
private getCacheKey(user: string, templateId: string): string {
const key = JSON.stringify({
user: user,
templateId: templateId,
});
const currTtl = this.rosaeContextsCache.getTtl(key);
if (currTtl == undefined) {
// not in cache yet, don't do anything
} else if (currTtl == 0) {
// no TTl, we don't change it
} else {
// reset ttl
this.rosaeContextsCache.ttl(key, this.ttl);
}
return key;
}
public getFromCache(user: string, templateId: string): CacheValue {
return this.rosaeContextsCache.get(this.getCacheKey(user, templateId));
}
public isInCache(user: string, templateId: string): boolean {
return this.rosaeContextsCache.has(this.getCacheKey(user, templateId));
}
public setInCache(user: string, templateId: string, cacheValue: CacheValue, isTemp: boolean): void {
let ttl: number;
if (isTemp || this.forgetTemplates) {
ttl = this.ttl;
} else {
ttl = 0;
}
this.rosaeContextsCache.set(this.getCacheKey(user, templateId), cacheValue, ttl);
}
public deleteFromCache(user: string, templateId: string): void {
this.rosaeContextsCache.del(this.getCacheKey(user, templateId));
}
public getIds(user: string): string[] {
const cacheKeys = Array.from(this.rosaeContextsCache.keys());
const ids: string[] = [];
for (let i = 0; i < cacheKeys.length; i++) {
const rawCacheKey = cacheKeys[i];
const cacheKey: CacheKey = JSON.parse(rawCacheKey);
// we don't keep temp ones
if (cacheKey.user == user && this.rosaeContextsCache.getTtl(rawCacheKey) == 0) {
ids.push(cacheKey.templateId);
}
}
return ids;
}
}
// import * as express from 'express';
import App from './app';
import TemplatesController from './templates.controller';
import yn from 'yn';
let port = 5000;
......@@ -29,6 +30,11 @@ const app = new App(
secretAccessKey: process.env.AWS_CW_SECRET_ACCESS_KEY,
region: process.env.AWS_CW_REGION,
},
behavior: {
lazyStartup: process.env.ROSAENLG_LAZY_STARTUP != '' ? yn(process.env.ROSAENLG_LAZY_STARTUP) : false,
forgetTemplates:
process.env.ROSAENLG_FORGET_TEMPLATES != '' ? yn(process.env.ROSAENLG_FORGET_TEMPLATES) : false,
},
}),
],
port,
......
{
"templateId": "chanson",
"status": "CREATED"
}
\ No newline at end of file
{
"templateId": "chanson",
"templateSha1": "1bfdbcd203ec8e6f889b068fbb2d7d298b1db903",
"ms": 37.48320007324219
}
\ No newline at end of file
{
"templateId": "chanson",
"templateSha1": "1bfdbcd203ec8e6f889b068fbb2d7d298b1db903",
"templateContent": {
"templateId": "chanson",
"entryTemplate": "chanson.pug",
"compileInfo": {
"activate": false,
"compileDebug": false,
"language": "fr_FR"
},
"templates": {
"chanson.pug": "p\n | il #[+verb(getAnonMS(), {verb: 'chanter', tense:'FUTUR'} )]\n | \"#{chanson.nom}\"\n | de #{chanson.auteur}\n"
},
"user": "DEFAULT_USER"
}
}
\ No newline at end of file
{
"templateId": "chanson",
"renderedText": "<p>Il chantera \"Non, je ne regrette rien\" d'Édith Piaf</p>",
"renderOptions": {
"language": "fr_FR"
......
{
"ids": [
"chanson"
],
"timestamp": "2019-11-15T09:37:24.754Z"
]
}
\ No newline at end of file
......@@ -75,14 +75,15 @@
"description": "Health is ok."
},
"503": {
"description": "Bad health, typically cannot write to disk when persistent storage is set."
"description": "Bad health, typically cannot write to disk or S3 when persistent storage is set."
}
}
}
},
"/templates": {
"get": {
"summary": "Get the IDs of the loaded templates for user.",
"summary": "Get the IDs of the templates for user.",
"description": "Contains both the loaded templates and the ones which could be loaded",
"parameters": [
{
"$ref": "#/components/parameters/userInHeader"
......@@ -90,7 +91,7 @@
],
"responses": {
"200": {
"description": "the IDs of the loaded templates",
"description": "the IDs of the templates",
"content": {
"application/json": {
"schema": {
......@@ -103,10 +104,6 @@
"type": "string"
},
"uniqueItems": true
},
"timestamp": {
"type": "string",
"format": "date-time"
}
}