Commit c38acf30 authored by Ludan Stoecklé's avatar Ludan Stoecklé

1.5.6

parent e8b9c102
......@@ -19,7 +19,7 @@ stages:
# variables etc.
variables:
ROSAENLG_VERSION: 1.5.5
ROSAENLG_VERSION: 1.5.6
DOCKER_REGISTRY: registry.gitlab.com/rosaenlg-projects/rosaenlg
DOCKER_CLI_ROOT: ${DOCKER_REGISTRY}/cli
DOCKER_SERVER_ROOT: ${DOCKER_REGISTRY}/server
......
{
"name": "browser-ide-demo",
"version": "1.5.5",
"version": "1.5.6",
"description": "Client side demo of NLG templates compilation and rendering using RosaeNLG",
"main": "index.js",
"scripts": {
......@@ -34,7 +34,7 @@
"gulp-inject-string": "^1.1.2",
"gulp-rename": "^1.4.0",
"merge-stream": "^2.0.0",
"rosaenlg": "1.5.5",
"rosaenlg": "1.5.6",
"tinyify": "^2.5.2",
"vue-codemirror": "^4.0.6"
}
......
{
"name": "content-deduplicate",
"version": "0.0.3",
"version": "0.0.4",
"description": "Calculates distances between texts to prevent duplicate content",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......@@ -54,6 +54,6 @@
},
"dependencies": {
"k-medoids": "^1.0.6",
"synonym-optimizer": "2.5.5"
"synonym-optimizer": "2.5.6"
}
}
{
"name": "english-determiners",
"version": "2.5.5",
"version": "2.5.6",
"description": "English determiners",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......
{
"name": "french-adjectives",
"version": "2.5.5",
"version": "2.5.6",
"description": "Agreement of adjectives in French",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......@@ -45,7 +45,7 @@
},
"dependencies": {
"debug": "^4.1.1",
"french-h-muet-aspire": "2.5.5"
"french-h-muet-aspire": "2.5.6"
},
"keywords": [
"french",
......
{
"name": "french-determiners",
"version": "2.5.5",
"version": "2.5.6",
"description": "French determiners",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......
{
"name": "french-h-muet-aspire",
"version": "2.5.5",
"version": "2.5.6",
"description": "List of French words that are \"aspiré\" (vs. \"muet\")\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[D\u001b[Dstarting with a h that is \"aspiré\" vs \"muet\"",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......
{
"name": "french-ordinals",
"version": "2.5.5",
"version": "2.5.6",
"description": "Ordinal numbers in French: 12 => douzième",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......
{
"name": "french-verbs",
"version": "2.5.5",
"version": "2.5.6",
"description": "French verbs: conjugation, based on the Lefff contents, and list of transitive and intransitive verbs",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......@@ -50,7 +50,7 @@
},
"dependencies": {
"debug": "^4.1.1",
"french-h-muet-aspire": "2.5.5"
"french-h-muet-aspire": "2.5.6"
},
"devDependencies": {
"copyfiles": "^2.1.1",
......
{
"name": "french-words-gender",
"version": "2.5.5",
"version": "2.5.6",
"description": "Gender of French words, based on the Lefff",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......
{
"name": "german-adjectives",
"version": "2.5.5",
"version": "2.5.6",
"description": "German adjectives, based on german-pos-dict",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......
{
"name": "german-determiners",
"version": "2.5.5",
"version": "2.5.6",
"description": "German determiners",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......
{
"name": "german-dict-helper",
"version": "2.5.5",
"version": "2.5.6",
"description": "Helper functions on an included database derived from german-pos-dict",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......
{
"name": "german-ordinals",
"version": "2.5.5",
"version": "2.5.6",
"description": "Ordinal numbers in German: 12 => zwölfte",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......
{
"name": "german-verbs",
"version": "2.5.5",
"version": "2.5.6",
"description": "German verbs, based on german-pos-dict",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......
{
"name": "german-words",
"version": "2.5.5",
"version": "2.5.6",
"description": "Gender and case of German words, based on german-pos-dict",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......
{
"name": "gulp-rosaenlg",
"version": "2.5.5",
"version": "2.5.6",
"description": "RosaeNLG client side packaging helper",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......@@ -60,6 +60,6 @@
"browserify": "^16.5.0",
"debug": "^4.1.1",
"minify-stream": "^1.2.1",
"rosaenlg": "1.5.5"
"rosaenlg": "1.5.6"
}
}
{
"name": "italian-adjectives",
"version": "2.5.5",
"version": "2.5.6",
"description": "Italian adjectives, based on morph-it",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......@@ -58,6 +58,6 @@
"license": "MIT",
"dependencies": {
"debug": "^4.1.1",
"rosaenlg-filter": "2.5.5"
"rosaenlg-filter": "2.5.6"
}
}
{
"name": "italian-determiners",
"version": "2.5.5",
"version": "2.5.6",
"description": "Italian determiners",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......
{
"name": "italian-ordinals-cardinals",
"version": "2.5.5",
"version": "2.5.6",
"description": "Ordinal and cardinal numbers in Italian",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......
{
"name": "italian-verbs",
"version": "2.5.5",
"version": "2.5.6",
"description": "Italian verbs, based on morph-it",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......
{
"name": "italian-words",
"version": "2.5.5",
"version": "2.5.6",
"description": "Gender and plural of Italian words, based on morph-it",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......
{
"name": "lefff-helper",
"version": "2.5.5",
"version": "2.5.6",
"description": "Helper functions on an included database derived from Lefff",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......
{
"name": "morph-it-helper",
"version": "2.5.5",
"version": "2.5.6",
"description": "Helper functions on an included database derived from morph-it (Free Morphological Lexicon for the Italian Language)",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......
{
"name": "rosaenlg-cli",
"version": "1.5.5",
"version": "1.5.6",
"description": "RosaeNLG's CLI interface",
"bin": {
"rosaenlg": "./index.js",
......@@ -10,9 +10,9 @@
"dependencies": {
"chalk": "^3.0.0",
"commander": "^4.0.1",
"gulp-rosaenlg": "2.5.5",
"gulp-rosaenlg": "2.5.6",
"mkdirp": "^0.5.1",
"rosaenlg": "1.5.5"
"rosaenlg": "1.5.6"
},
"devDependencies": {
"mocha": "^6.2.2",
......
......@@ -39,3 +39,12 @@ export function cleanStructAfterUnprotect(input: string /*, lang: string*/): str
return res;
}
export function specialSpacesToNormalSpaces(input: string): string {
let res: string = input;
const specialSpaces = new RegExp('¤', 'g');
res = res.replace(specialSpaces, ' ');
return res;
}
export type Languages = 'en_US' | 'fr_FR' | 'de_DE' | 'it_IT' | string;
export const allPunctList = '\\.:!\\?;,…';
export const spaceOrNonBlockingClass = '[\\s☞☜]';
export const spaceOrNonBlockingClass = '[\\s¤☞☜]';
export const stdBetweenWithParenthesis = `(${spaceOrNonBlockingClass}+|$)`;
export const stdBeforeWithParenthesis = `([\\s☛☚☞☜${allPunctList}])`;
export const stdBeforeWithParenthesis = `([\\s¤☛☚☞☜${allPunctList}])`;
const correspondances = {
a: 'àáâãäå',
......
......@@ -23,7 +23,7 @@ export function aAnBeforeProtect(input: string): string {
//console.log('xxx' + input);
const regexA = new RegExp(
`([^${tousCaracteresMinMajRe}])([aA])${stdBetweenWithParenthesis}§([${tousCaracteresMinMajRe}]*)`,
`([^${tousCaracteresMinMajRe}])([aA])${stdBetweenWithParenthesis}§¤*([${tousCaracteresMinMajRe}]*)`,
'g',
);
res = res.replace(regexA, function(match, before, aA, between, word): string {
......@@ -70,7 +70,7 @@ export function enPossessivesBeforeProtect(input: string): string {
let res = input;
// debug("xx: "+ input);
const regexSS = new RegExp("(s\\s*§\\s*'s)([^" + tousCaracteresMinMajRe + '])', 'g');
const regexSS = new RegExp("(s\\s*§[\\s¤]*'s)([^" + tousCaracteresMinMajRe + '])', 'g');
res = res.replace(regexSS, function(corresp, first, second): string {
// debug(`AAAA ${corresp} ${first} ${offset} ${orig}`);
return `s§' ${second}`;
......
......@@ -75,11 +75,13 @@ export function filter(input: string, language: Languages): string {
res,
[
clean.joinLines,
clean.specialSpacesToNormalSpaces, // do it early so that all the rest does not have to care for ¤
punctuation.duplicatePunctuation,
contractions,
clean.cleanStruct,
punctuation.cleanSpacesPunctuation,
punctuation.parenthesis,
punctuation.quotes, // must be before cleanSpacesPunctuation as it can introduce double spaces
punctuation.cleanSpacesPunctuation,
punctuation.addCaps, // must be before contractions otherwise difficult to find words
egg,
titlecase,
......
......@@ -16,7 +16,10 @@ export function unprotect(toUnprotect: string, mappings: Mappings): string {
let res: string = toUnprotect;
for (const key in mappings) {
// debug('key/val: ' + key + '/' + mappings[key]);
res = res.replace(key, mappings[key]);
// we also just delete all the unnecessary special spaces
const specialSpaces = new RegExp('¤', 'g');
res = res.replace(key, mappings[key].replace(specialSpaces, ''));
}
return res;
......
import { tousCaracteresMinMajRe } from './constants';
import { tousCaracteresMinMajRe, stdBetweenWithParenthesis } from './constants';
import { Languages, allPunctList, spaceOrNonBlockingClass } from './constants';
export function duplicatePunctuation(input: string, lang: Languages): string {
......@@ -101,6 +101,28 @@ export function parenthesis(input: string /*, lang: string*/): string {
return res;
}
export function quotes(input: string /*, lang: string*/): string {
let res: string = input;
const regexQuotes = new RegExp(`(\\s*)"(\\s*)`, 'g');
let alreadyStarted = false;
res = res.replace(regexQuotes, function(corresp, before, after): string {
if (!alreadyStarted) {
alreadyStarted = true;
return ' "';
} else {
alreadyStarted = false;
return '" ';
}
});
// trigger a warning if an end is missing
if (alreadyStarted) {
console.log(`WARNING: did find a starting " but not the ending one`);
}
return res;
}
export function addCaps(input: string /*, lang: string*/): string {
let res: string = input;
......
{
"name": "rosaenlg-filter",
"version": "2.5.5",
"version": "2.5.6",
"description": "Filtering feature of RosaeNLG",
"main": "dist/index.js",
"types": "dist/index.d.ts",
......@@ -43,7 +43,7 @@
"better-title-case": "^1.0.1",
"compromise": "^11.14.3",
"debug": "^4.1.1",
"french-h-muet-aspire": "2.5.5",
"french-h-muet-aspire": "2.5.6",
"titlecase-french": "^1.0.1"
},
"keywords": [
......
......@@ -275,6 +275,13 @@ const testCasesList = [
'Bla <span class="toto" id="tata">bli <b>blu</b> blo</span>',
],
['<p>Only A.</p><l>first, second and third', '<p>Only A.</p><l>First, second and third'],
// "..."
['bla " bla " bla', 'Bla "bla" bla'],
['bla " bla " bla "bla "', 'Bla "bla" bla "bla"'],
['bla " bla " "bla" bla ', 'Bla "bla" "bla" bla'],
['bla"bla "bla', 'Bla "bla" bla'],
['bla " bla', 'Bla "bla'],
],
},
......
{
"name": "rosaenlg-node-server",
"version": "1.5.5",
"version": "1.5.6",
"description": "Server for RosaeNLG, written in node.js",
"main": "dist/app.js",
"bin": {
......@@ -71,7 +71,7 @@
"eslint-config-prettier": "^6.5.0",
"eslint-plugin-prettier": "^3.1.1",
"gulp": "^4.0.2",
"gulp-rosaenlg": "2.5.5",
"gulp-rosaenlg": "2.5.6",
"json-refs": "^3.0.13",
"mocha": "^6.2.2",
"nyc": "^14.1.1",
......@@ -85,7 +85,7 @@
"body-parser": "^1.19.0",
"express": "^4.17.1",
"node-cache": "5.0.2",
"rosaenlg": "1.5.5",
"rosaenlg": "1.5.6",
"sha1": "^1.1.1",
"swagger-ui-express": "^4.1.2"
}
......
......@@ -3,7 +3,6 @@ import { Languages } from 'gulp-rosaenlg';
export class RenderOptions {
public language: Languages;
public forceRandomSeed: number | undefined;
public disableFiltering: boolean | undefined;
public defaultSynoMode: string | undefined;
public defaultAmong: string | undefined;
......@@ -11,7 +10,6 @@ export class RenderOptions {
this.language = copyFrom.language;
this.forceRandomSeed = copyFrom.forceRandomSeed;
this.disableFiltering = copyFrom.disableFiltering;
this.defaultSynoMode = copyFrom.defaultSynoMode;
this.defaultAmong = copyFrom.defaultAmong;
}
......
......@@ -14,11 +14,6 @@
"description": "the random seed value, if it must be forced",
"type": "integer"
},
"disableFiltering": {
"description": "disable filtering (by default false: filtering is enabled)",
"type": "boolean",
"default": "false"
},
"defaultSynoMode": {
"description": "default mode for synonyms",
"type": "string"
......
......@@ -2,7 +2,7 @@ import * as express from 'express';
import { RosaeContext } from './RosaeContext';
import * as fs from 'fs';
import { RenderedBundle } from './RenderedBundle';
import sha1 = require('sha1');
import sha1 from 'sha1';
import NodeCache = require('node-cache');
export default class TemplatesController {
......@@ -54,6 +54,7 @@ export default class TemplatesController {
console.info(`reloading all templates...`);
if (!this.templatesPath) {
response.status(500).send(`no templates path, cannot reload!`);
return;
} else {
// forget everything that was loaded
this.rosaeContexts = new Map<string, RosaeContext>();
......@@ -99,8 +100,10 @@ export default class TemplatesController {
const rosaeContext: RosaeContext = new RosaeContext(JSON.parse(templateContent));
this.rosaeContexts.set(templateId, rosaeContext);
response.sendStatus(200);
return;
} catch (e) {
response.status(500).send(`could not load template ${e.message}`);
return;
}
};
......@@ -119,6 +122,7 @@ export default class TemplatesController {
if (!loadedExisted && !fileExisted) {
response.status(500).send(`${templateId} does not exist`);
return;
} else {
response.sendStatus(200);
}
......@@ -140,8 +144,10 @@ export default class TemplatesController {
const rosaeContext = this.rosaeContexts.get(templateId);
if (!rosaeContext) {
response.status(500).send(`${templateId} does not exist`);
return;
} else {
response.send(rosaeContext.getPackagedTemplate());
return;
}
};
......@@ -159,6 +165,7 @@ export default class TemplatesController {
const templateId = rosaeContext.getTemplateId();
if (!templateId) {
response.status(500).send(`no templateId!`);
return;
} else {
const status = this.rosaeContexts.has(templateId) ? 'UPDATED' : 'CREATED';
......@@ -190,9 +197,11 @@ export default class TemplatesController {
if (!template) {
response.status(500).send(`no template`);
return;
}
if (!data) {
response.status(500).send(`no data`);
return;
}
const templateId = sha1(JSON.stringify(template));
......@@ -223,6 +232,7 @@ export default class TemplatesController {
});
} catch (e) {
response.status(500).send(`rendering error: ${e.toString()}`);
return;
}
};
......@@ -234,6 +244,7 @@ export default class TemplatesController {
const rosaeContext = this.rosaeContexts.get(templateId);
if (!rosaeContext) {
response.status(500).send(`${templateId} does not exist`);
return;
} else {
try {
const renderedBundle: RenderedBundle = rosaeContext.render(request.body);
......@@ -244,6 +255,7 @@ export default class TemplatesController {
});
} catch (e) {
response.status(500).send(`rendering error: ${e.toString()}`);
return;
}
}
};
......
......@@ -500,6 +500,22 @@ describe('node-server', function() {
});
});
}
{
const parsedTemplate = JSON.parse(getTestTemplate('chanson_with_data'));