feat(build): #1628.9 python spellchecking

- Migrated spell checking linter to python
- Spell checks now runs over changed files
- Centralized linter errors in checkall.py
- Using python3 in scripts
- Using only one exit_code in linters
- Changed dependencies to python3

Closes #1628 and #1391
parent 5d64ee11
......@@ -58,9 +58,8 @@ content:
script:
- pre-commit run --all-files
- ./check-changed.sh
- python ci-scripts/checkall.py changes
- python ci-scripts/imgchecker.py
- ./check-spelling.sh
- python3 ci-scripts/checkall.py changes
- python3 ci-scripts/imgchecker.py
except:
- master
......
......@@ -31,6 +31,9 @@ RUN apt-get update -qq && \
python-dev \
python-pip \
python-setuptools \
python3-dev \
python3-pip \
python3-setuptools \
ruby-full \
unzip \
xz-utils \
......@@ -45,6 +48,7 @@ RUN apt-get update -qq && \
python get-pip.py && \
rm get-pip.py && \
pip install -r requirements.txt && \
pip3 install termcolor && \
yarn install && \
gem install -N sass && \
......
#!/usr/bin/env python
#!/usr/bin/env python3
# coding: utf8
"""
Script that checks structure and rules in blog entries
Replaces check-articles.sh
Author: Oscar Eduardo Prado oprado@fluidattacks.com
Version 1.0
Version 1.1
Patch notes 1.1:
- Using Python3
"""
import os
......
#!/usr/bin/env python
#!/usr/bin/env python3
# coding: utf8
"""
Script to check if defined rules are strictly applied
Replaces check-all.sh
Author: Oscar Eduardo Prado oprado@fluidattacks.com
Version 1.0
Version 1.3
Patch notes 1.3:
- Added spell checker linter
- Using Python3
- Fixed bugs in artchecker call
- Using only one EXIT_CODE
"""
import os
import sys
......@@ -14,10 +19,9 @@ import re
import genchecker
import contentrules
import articlerules
import spelling
EXIT1 = 0
EXIT2 = 0
EXIT3 = 0
EXIT_CODE = 0
if len(sys.argv) > 2:
print_helper.print_failure("Error! too many arguments\n")
......@@ -40,24 +44,26 @@ if len(sys.argv) == 2 and sys.argv[1] == 'changes':
print (FILE)
print_helper.print_warning("^ Modified files\n")
print_helper.print_success("Checking recent changes to the repo ...\n")
EXIT1 = genchecker.genchecks(EXIT1)
EXIT_CODE = genchecker.genchecks(EXIT_CODE)
for FILE in CHANGES:
if '.adoc' in FILE:
EXIT2 = contentrules.rulechecker(FILE, EXIT2)
EXIT_CODE = contentrules.rulechecker(FILE, EXIT_CODE)
EXIT_CODE = spelling.spellchecker(FILE, EXIT_CODE)
if "content/blog" in FILE:
articlerules.artchecker(FILE, EXIT3)
EXIT_CODE = articlerules.artchecker(FILE, EXIT_CODE)
print_helper.print_success("Done! \n")
if len(sys.argv) == 1:
print_helper.print_success("Checking all asciidoc files in repo ...\n")
FILES = os.popen('find content/ -iname "*.adoc"').read()
EXIT1 = genchecker.genchecks(EXIT1)
EXIT_CODE = genchecker.genchecks(EXIT_CODE)
FILES = FILES.split()
for CHECK in FILES:
EXIT2 = contentrules.rulechecker(CHECK, EXIT2)
EXIT_CODE = contentrules.rulechecker(CHECK, EXIT_CODE)
EXIT_CODE = spelling.spellchecker(CHECK, EXIT_CODE)
if "content/blog" in CHECK:
articlerules.artchecker(CHECK, EXIT3)
EXIT_CODE = articlerules.artchecker(CHECK, EXIT_CODE)
print_helper.print_success("Done! \n")
sys.exit(EXIT1 or EXIT2 or EXIT3)
sys.exit(EXIT_CODE)
#!/usr/bin/env python
# coding: utf8
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Script that defines the rules to be applied to the web repo
Author: Oscar Eduardo Prado oprado@fluidattacks.com
Version 1.0
Version 1.1
Patch notes 1.1:
- Using Python3
- Updated functions to use Python3 sintax
"""
import os
import print_helper
import sys
def rulechecker(PATH, EXIT_CODE):
#Define Rules
......@@ -103,7 +107,7 @@ def rulechecker(PATH, EXIT_CODE):
# Check for multiple main title
OUT = os.popen("pcregrep -nr '^=\s.*$' "+PATH).read()
OUT = OUT.split('\n')
OUT = filter(None, OUT)
OUT = list(filter(None, OUT))
if len(OUT) > 1:
print_helper.print_failure("Issue found in "+PATH+"\n")
print_helper.print_failure(OUT)
......
#!/usr/bin/env python
#!/usr/bin/env python3
# coding: utf8
"""
Script to run general content checks of web repo
Author: Oscar Eduardo Prado oprado@fluidattacks.com
Version 1.0
Version 1.1
Patch notes 1.1:
- Using Python3
"""
import os
......
#!/usr/bin/env python
#!/usr/bin/env python3
"""
Script to check if images inside the repo are properly optimized
in size, extension and dimensions
Author: Oscar Eduardo Prado oprado@fluidattacks.com
Version 1.0
Version 1.1
Patch notes 1.1:
- Using Python3
"""
import os
import sys
......
#!/usr/bin/env python3
# coding: utf8
"""
Script that checks strict spelling of certain words
defined in strict-words.lst
Replaces check-spelling.sh
Author: Oscar Eduardo Prado oprado@fluidattacks.com
Version 1.1:
Patch note 1.1:
- Using Python3
"""
import os
import print_helper
import re
def spellchecker(PATH, EXIT_CODE):
CORRECTWDS = os.popen("cat strict-words.lst").read()
CORRECTWDS = CORRECTWDS.split('\n')
CORRECTWDS = filter(None, CORRECTWDS)
for CORRECT in CORRECTWDS:
if " " in CORRECT:
SRCH = CORRECT.replace(' ', '(\s)?')
else:
SRCH = CORRECT
#Temporary file to run checks
os.system("./exttxt.sh "+PATH+" >> temp.txt")
OUT = os.popen("pcregrep -ioM '"+SRCH+"' temp.txt").read()
if len(OUT) > 0:
RGX = '(\s|^)[\*|\+|\(]?'+SRCH+'[\*\+\)]?[\.\:\;\,]?(\s|\\n|\])'
RAW = os.popen("pcregrep -ioM '"+RGX+"' temp.txt").read()
RAW = RAW.split('\n')
RAW = filter(None, RAW)
for WD in RAW:
WORD = re.sub('^\s|\*|\+|\(|\)|\.|\:|\;|\,|\[|\]|\s$' , '', WD)
if len(WORD) > 0:
if WORD != CORRECT:
print_helper.print_failure("Issue found in "+PATH+"\n")
print_helper.print_warning("The spelling "+WORD+" is incorrect. "\
"The only spelling admitted is "+CORRECT+"\n\n")
EXIT_CODE = 1
#Remove temporary file
os.system("rm -r temp.txt")
return EXIT_CODE
......@@ -2,6 +2,7 @@
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals
import sys
reload(sys)
sys.setdefaultencoding("utf-8")
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment